IOR
aiori-POSIX.c
Go to the documentation of this file.
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  */
4 /******************************************************************************\
5 * *
6 * Copyright (c) 2003, The Regents of the University of California *
7 * See the file COPYRIGHT for a complete copyright notice and license. *
8 * *
9 ********************************************************************************
10 *
11 * Implement of abstract I/O interface for POSIX.
12 *
13 \******************************************************************************/
14 
15 #ifdef HAVE_CONFIG_H
16 # include "config.h"
17 #endif
18 
19 #include <stdio.h>
20 #include <stdlib.h>
21 
22 #ifdef __linux__
23 # include <sys/ioctl.h> /* necessary for: */
24 # define __USE_GNU /* O_DIRECT and */
25 # include <fcntl.h> /* IO operations */
26 # undef __USE_GNU
27 #endif /* __linux__ */
28 
29 #include <errno.h>
30 #include <fcntl.h> /* IO operations */
31 #include <sys/stat.h>
32 #include <assert.h>
33 
34 
35 #ifdef HAVE_LINUX_LUSTRE_LUSTRE_USER_H
36 # include <linux/lustre/lustre_user.h>
37 #elif defined(HAVE_LUSTRE_USER)
38 # include <lustre/lustre_user.h>
39 #endif
40 #ifdef HAVE_GPFS_H
41 # include <gpfs.h>
42 #endif
43 #ifdef HAVE_GPFS_FCNTL_H
44 # include <gpfs_fcntl.h>
45 #endif
46 
47 #ifdef HAVE_BEEGFS_BEEGFS_H
48 #include <beegfs/beegfs.h>
49 #include <dirent.h>
50 #include <libgen.h>
51 #endif
52 
53 #include "ior.h"
54 #include "aiori.h"
55 #include "iordef.h"
56 #include "utilities.h"
57 
58 #include "aiori-POSIX.h"
59 
60 #ifdef HAVE_GPU_DIRECT
61 typedef long long loff_t;
62 #include <cuda_runtime.h>
63 #include <cufile.h>
64 #endif
65 
66 typedef struct {
67  int fd;
68 #ifdef HAVE_GPU_DIRECT
69  CUfileHandle_t cf_handle;
70 #endif
71 } posix_fd;
72 
73 
74 #ifndef open64 /* necessary for TRU64 -- */
75 # define open64 open /* unlikely, but may pose */
76 #endif /* not open64 */ /* conflicting prototypes */
77 
78 #ifndef lseek64 /* necessary for TRU64 -- */
79 # define lseek64 lseek /* unlikely, but may pose */
80 #endif /* not lseek64 */ /* conflicting prototypes */
81 
82 #ifndef O_BINARY /* Required on Windows */
83 # define O_BINARY 0
84 #endif
85 
86 #ifdef HAVE_GPU_DIRECT
87 static const char* cuFileGetErrorString(CUfileError_t status){
88  if(IS_CUDA_ERR(status)){
89  return cudaGetErrorString(status.err);
90  }
91  return strerror(status.err);
92 }
93 
94 static void init_cufile(posix_fd * pfd){
95  CUfileDescr_t cf_descr = (CUfileDescr_t){
96  .handle.fd = pfd->fd,
97  .type = CU_FILE_HANDLE_TYPE_OPAQUE_FD
98  };
99  CUfileError_t status = cuFileHandleRegister(& pfd->cf_handle, & cf_descr);
100  if(status.err != CU_FILE_SUCCESS){
101  EWARNF("Could not register handle %s", cuFileGetErrorString(status));
102  }
103 }
104 #endif
105 
106 /**************************** P R O T O T Y P E S *****************************/
108 static void POSIX_Finalize(aiori_mod_opt_t * options);
109 
112 
113 option_help * POSIX_options(aiori_mod_opt_t ** init_backend_options, aiori_mod_opt_t * init_values){
114  posix_options_t * o = malloc(sizeof(posix_options_t));
115 
116  if (init_values != NULL){
117  memcpy(o, init_values, sizeof(posix_options_t));
118  }else{
119  memset(o, 0, sizeof(posix_options_t));
120  o->direct_io = 0;
121  o->lustre_stripe_count = -1;
122  o->lustre_start_ost = -1;
123  o->beegfs_numTargets = -1;
124  o->beegfs_chunkSize = -1;
125  }
126 
127  *init_backend_options = (aiori_mod_opt_t*) o;
128 
129  option_help h [] = {
130  {0, "posix.odirect", "Direct I/O Mode", OPTION_FLAG, 'd', & o->direct_io},
131 #ifdef HAVE_BEEGFS_BEEGFS_H
132  {0, "posix.beegfs.NumTargets", "", OPTION_OPTIONAL_ARGUMENT, 'd', & o->beegfs_numTargets},
133  {0, "posix.beegfs.ChunkSize", "", OPTION_OPTIONAL_ARGUMENT, 'd', & o->beegfs_chunkSize},
134 #endif
135 #ifdef HAVE_GPFS_FCNTL_H
136  {0, "posix.gpfs.hintaccess", "", OPTION_FLAG, 'd', & o->gpfs_hint_access},
137  {0, "posix.gpfs.releasetoken", "", OPTION_OPTIONAL_ARGUMENT, 'd', & o->gpfs_release_token},
138 
139 #endif
140 #ifdef HAVE_LUSTRE_USER
141  {0, "posix.lustre.stripecount", "", OPTION_OPTIONAL_ARGUMENT, 'd', & o->lustre_stripe_count},
142  {0, "posix.lustre.stripesize", "", OPTION_OPTIONAL_ARGUMENT, 'd', & o->lustre_stripe_size},
143  {0, "posix.lustre.startost", "", OPTION_OPTIONAL_ARGUMENT, 'd', & o->lustre_start_ost},
144  {0, "posix.lustre.ignorelocks", "", OPTION_FLAG, 'd', & o->lustre_ignore_locks},
145 #endif
146 #ifdef HAVE_GPU_DIRECT
147  {0, "gpuDirect", "allocate I/O buffers on the GPU", OPTION_FLAG, 'd', & o->gpuDirect},
148 #endif
150  };
151  option_help * help = malloc(sizeof(h));
152  memcpy(help, h, sizeof(h));
153  return help;
154 }
155 
156 
157 /************************** D E C L A R A T I O N S ***************************/
158 
159 
161  .name = "POSIX",
162  .name_legacy = NULL,
163  .initialize = POSIX_Initialize,
164  .finalize = POSIX_Finalize,
165  .create = POSIX_Create,
166  .mknod = POSIX_Mknod,
167  .open = POSIX_Open,
168  .xfer = POSIX_Xfer,
169  .close = POSIX_Close,
170  .delete = POSIX_Delete,
171  .xfer_hints = POSIX_xfer_hints,
172  .get_version = aiori_get_version,
173  .fsync = POSIX_Fsync,
174  .get_file_size = POSIX_GetFileSize,
175  .statfs = aiori_posix_statfs,
176  .mkdir = aiori_posix_mkdir,
177  .rmdir = aiori_posix_rmdir,
178  .rename = POSIX_Rename,
179  .access = aiori_posix_access,
180  .stat = aiori_posix_stat,
181  .get_options = POSIX_options,
182  .enable_mdtest = true,
183  .sync = POSIX_Sync,
184  .check_params = POSIX_check_params
185 };
186 
187 /***************************** F U N C T I O N S ******************************/
188 
190 
192  hints = params;
193 }
194 
196  posix_options_t * o = (posix_options_t*) param;
197  if (o->beegfs_chunkSize != -1 && (!ISPOWEROFTWO(o->beegfs_chunkSize) || o->beegfs_chunkSize < (1<<16)))
198  ERR("beegfsChunkSize must be a power of two and >64k");
199  if(o->lustre_stripe_count != -1 || o->lustre_stripe_size != 0)
200  o->lustre_set_striping = 1;
201  if(o->gpuDirect && ! o->direct_io){
202  ERR("GPUDirect required direct I/O to be used!");
203  }
204 #ifndef HAVE_GPU_DIRECT
205  if(o->gpuDirect){
206  ERR("GPUDirect support is not compiled");
207  }
208 #endif
209  return 0;
210 }
211 
212 #ifdef HAVE_GPFS_FCNTL_H
213 void gpfs_free_all_locks(int fd)
214 {
215  int rc;
216  struct {
217  gpfsFcntlHeader_t header;
218  gpfsFreeRange_t release;
219  } release_all;
220  release_all.header.totalLength = sizeof(release_all);
221  release_all.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
222  release_all.header.fcntlReserved = 0;
223 
224  release_all.release.structLen = sizeof(release_all.release);
225  release_all.release.structType = GPFS_FREE_RANGE;
226  release_all.release.start = 0;
227  release_all.release.length = 0;
228 
229  rc = gpfs_fcntl(fd, &release_all);
230  if (verbose >= VERBOSE_0 && rc != 0) {
231  EWARNF("gpfs_fcntl(%d, ...) release all locks hint failed.", fd);
232  }
233 }
234 void gpfs_access_start(int fd, IOR_offset_t length, IOR_offset_t offset, int access)
235 {
236  int rc;
237  struct {
238  gpfsFcntlHeader_t header;
239  gpfsAccessRange_t access;
240  } take_locks;
241 
242  take_locks.header.totalLength = sizeof(take_locks);
243  take_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
244  take_locks.header.fcntlReserved = 0;
245 
246  take_locks.access.structLen = sizeof(take_locks.access);
247  take_locks.access.structType = GPFS_ACCESS_RANGE;
248  take_locks.access.start = offset;
249  take_locks.access.length = length;
250  take_locks.access.isWrite = (access == WRITE);
251 
252  rc = gpfs_fcntl(fd, &take_locks);
253  if (verbose >= VERBOSE_2 && rc != 0) {
254  EWARNF("gpfs_fcntl(%d, ...) access range hint failed.", fd);
255  }
256 }
257 
258 void gpfs_access_end(int fd, IOR_offset_t length, IOR_offset_t offset, int access)
259 {
260  int rc;
261  struct {
262  gpfsFcntlHeader_t header;
263  gpfsFreeRange_t free;
264  } free_locks;
265 
266 
267  free_locks.header.totalLength = sizeof(free_locks);
268  free_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
269  free_locks.header.fcntlReserved = 0;
270 
271  free_locks.free.structLen = sizeof(free_locks.free);
272  free_locks.free.structType = GPFS_FREE_RANGE;
273  free_locks.free.start = offset;
274  free_locks.free.length = length;
275 
276  rc = gpfs_fcntl(fd, &free_locks);
277  if (verbose >= VERBOSE_2 && rc != 0) {
278  EWARNF("gpfs_fcntl(%d, ...) free range hint failed.", fd);
279  }
280 }
281 
282 #endif
283 
284 #ifdef HAVE_BEEGFS_BEEGFS_H
285 
286 int mkTempInDir(char* dirPath)
287 {
288  unsigned long len = strlen(dirPath) + 8;
289  char* tmpfilename = (char*)malloc(sizeof (char)*len+1);
290  snprintf(tmpfilename, len, "%s/XXXXXX", dirPath);
291 
292  int fd = mkstemp(tmpfilename);
293  unlink(tmpfilename);
294  free(tmpfilename);
295 
296  return fd;
297 }
298 
299 bool beegfs_getStriping(char* dirPath, u_int16_t* numTargetsOut, unsigned* chunkSizeOut)
300 {
301  bool retVal = false;
302 
303  int fd = mkTempInDir(dirPath);
304  if (fd) {
305  unsigned stripePattern = 0;
306  retVal = beegfs_getStripeInfo(fd, &stripePattern, chunkSizeOut, numTargetsOut);
307  close(fd);
308  }
309 
310  return retVal;
311 }
312 
313 bool beegfs_isOptionSet(int opt) {
314  return opt != -1;
315 }
316 
317 bool beegfs_compatibleFileExists(char* filepath, int numTargets, int chunkSize)
318 {
319  int fd = open(filepath, O_RDWR);
320 
321  if (fd == -1)
322  return false;
323 
324  unsigned read_stripePattern = 0;
325  u_int16_t read_numTargets = 0;
326  int read_chunkSize = 0;
327 
328  bool retVal = beegfs_getStripeInfo(fd, &read_stripePattern, &read_chunkSize, &read_numTargets);
329 
330  close(fd);
331 
332  return retVal && read_numTargets == numTargets && read_chunkSize == chunkSize;
333 }
334 
335 /*
336  * Create a file on a BeeGFS file system with striping parameters
337  */
338 bool beegfs_createFilePath(char* filepath, mode_t mode, int numTargets, int chunkSize)
339 {
340  bool retVal = false;
341  char* dirTmp = strdup(filepath);
342  char* dir = dirname(dirTmp);
343  DIR* parentDirS = opendir(dir);
344  if (!parentDirS) {
345  ERRF("Failed to get directory: %s", dir);
346  }
347  else
348  {
349  int parentDirFd = dirfd(parentDirS);
350  if (parentDirFd < 0)
351  {
352  ERRF("Failed to get directory descriptor: %s", dir);
353  }
354  else
355  {
356  bool isBeegfs = beegfs_testIsBeeGFS(parentDirFd);
357  if (!isBeegfs)
358  {
359  WARN("Not a BeeGFS file system");
360  }
361  else
362  {
363  if ( !beegfs_isOptionSet(numTargets)
364  || !beegfs_isOptionSet(chunkSize)) {
365  u_int16_t defaultNumTargets = 0;
366  unsigned defaultChunkSize = 0;
367  bool haveDefaults = beegfs_getStriping(dir,
368  &defaultNumTargets,
369  &defaultChunkSize);
370  if (!haveDefaults)
371  ERR("Failed to get default BeeGFS striping values");
372 
373  numTargets = beegfs_isOptionSet(numTargets) ?
374  numTargets : defaultNumTargets;
375  chunkSize = beegfs_isOptionSet(chunkSize) ?
376  chunkSize : defaultChunkSize;
377  }
378 
379  char* filenameTmp = strdup(filepath);
380  char* filename = basename(filepath);
381  bool isFileCreated = beegfs_compatibleFileExists(filepath, numTargets, chunkSize)
382  || beegfs_createFile(parentDirFd, filename,
383  mode, numTargets, chunkSize);
384  if (!isFileCreated)
385  ERR("Could not create file");
386  retVal = true;
387  free(filenameTmp);
388  }
389  }
390  closedir(parentDirS);
391  }
392  free(dirTmp);
393  return retVal;
394 }
395 #endif /* HAVE_BEEGFS_BEEGFS_H */
396 
397 
398 /*
399  * Create and open a file through the POSIX interface.
400  */
401 aiori_fd_t *POSIX_Create(char *testFileName, int flags, aiori_mod_opt_t * param)
402 {
403  int fd_oflag = O_BINARY;
404  int mode = 0664;
405  posix_fd * pfd = safeMalloc(sizeof(posix_fd));
406  posix_options_t * o = (posix_options_t*) param;
407  if (o->direct_io == TRUE){
408  set_o_direct_flag(& fd_oflag);
409  }
410 
411  if(hints->dryRun)
412  return (aiori_fd_t*) 0;
413 
414 #ifdef HAVE_LUSTRE_USER
415 /* Add a #define for FASYNC if not available, as it forms part of
416  * the Lustre O_LOV_DELAY_CREATE definition. */
417 #ifndef FASYNC
418 #define FASYNC 00020000 /* fcntl, for BSD compatibility */
419 #endif
420  if (o->lustre_set_striping) {
421  /* In the single-shared-file case, task 0 has to create the
422  file with the Lustre striping options before any other
423  processes open the file */
424  if (!hints->filePerProc && rank != 0) {
425  MPI_CHECK(MPI_Barrier(testComm), "barrier error");
426  fd_oflag |= O_RDWR;
427  pfd->fd = open64(testFileName, fd_oflag, mode);
428  if (pfd->fd < 0){
429  ERRF("open64(\"%s\", %d, %#o) failed. Error: %s",
430  testFileName, fd_oflag, mode, strerror(errno));
431  }
432  } else {
433  struct lov_user_md opts = { 0 };
434 
435  /* Setup Lustre IOCTL striping pattern structure */
436  opts.lmm_magic = LOV_USER_MAGIC;
437  opts.lmm_stripe_size = o->lustre_stripe_size;
438  opts.lmm_stripe_offset = o->lustre_start_ost;
439  opts.lmm_stripe_count = o->lustre_stripe_count;
440 
441  /* File needs to be opened O_EXCL because we cannot set
442  * Lustre striping information on a pre-existing file.*/
443 
444  fd_oflag |= O_CREAT | O_EXCL | O_RDWR | O_LOV_DELAY_CREATE;
445  pfd->fd = open64(testFileName, fd_oflag, mode);
446  if (pfd->fd < 0) {
447  ERRF("Unable to open '%s': %s\n",
448  testFileName, strerror(errno));
449  } else if (ioctl(pfd->fd, LL_IOC_LOV_SETSTRIPE, &opts)) {
450  char *errmsg = "stripe already set";
451  if (errno != EEXIST && errno != EALREADY)
452  errmsg = strerror(errno);
453  ERRF("Error on ioctl for '%s' (%d): %s\n",
454  testFileName, pfd->fd, errmsg);
455  }
456  if (!hints->filePerProc)
457  MPI_CHECK(MPI_Barrier(testComm),
458  "barrier error");
459  }
460  } else {
461 #endif /* HAVE_LUSTRE_USER */
462 
463  fd_oflag |= O_CREAT | O_RDWR;
464 
465 #ifdef HAVE_BEEGFS_BEEGFS_H
466  if (beegfs_isOptionSet(o->beegfs_chunkSize)
467  || beegfs_isOptionSet(o->beegfs_numTargets)) {
468  bool result = beegfs_createFilePath(testFileName,
469  mode,
471  o->beegfs_chunkSize);
472  if (result) {
473  fd_oflag &= ~O_CREAT;
474  } else {
475  EWARN("BeeGFS tuning failed");
476  }
477  }
478 #endif /* HAVE_BEEGFS_BEEGFS_H */
479 
480  pfd->fd = open64(testFileName, fd_oflag, mode);
481  if (pfd->fd < 0){
482  ERRF("open64(\"%s\", %d, %#o) failed. Error: %s",
483  testFileName, fd_oflag, mode, strerror(errno));
484  }
485 
486 #ifdef HAVE_LUSTRE_USER
487  }
488 
489  if (o->lustre_ignore_locks) {
490  int lustre_ioctl_flags = LL_FILE_IGNORE_LOCK;
491  if (ioctl(pfd->fd, LL_IOC_SETFLAGS, &lustre_ioctl_flags) == -1)
492  ERRF("ioctl(%d, LL_IOC_SETFLAGS, ...) failed", pfd->fd);
493  }
494 #endif /* HAVE_LUSTRE_USER */
495 
496 #ifdef HAVE_GPFS_FCNTL_H
497  /* in the single shared file case, immediately release all locks, with
498  * the intent that we can avoid some byte range lock revocation:
499  * everyone will be writing/reading from individual regions */
500  if (o->gpfs_release_token ) {
501  gpfs_free_all_locks(pfd->fd);
502  }
503 #endif
504 #ifdef HAVE_GPU_DIRECT
505  if(o->gpuDirect){
506  init_cufile(pfd);
507  }
508 #endif
509  return (aiori_fd_t*) pfd;
510 }
511 
512 /*
513  * Create a file through mknod interface.
514  */
515 int POSIX_Mknod(char *testFileName)
516 {
517  int ret;
518 
519  ret = mknod(testFileName, S_IFREG | S_IRUSR, 0);
520  if (ret < 0)
521  ERR("mknod failed");
522 
523  return ret;
524 }
525 
526 /*
527  * Open a file through the POSIX interface.
528  */
529 aiori_fd_t *POSIX_Open(char *testFileName, int flags, aiori_mod_opt_t * param)
530 {
531  int fd_oflag = O_BINARY | O_RDWR;
532  posix_fd * pfd = safeMalloc(sizeof(posix_fd));
533  posix_options_t * o = (posix_options_t*) param;
534  if (o->direct_io == TRUE){
535  set_o_direct_flag(&fd_oflag);
536  }
537 
538  if(hints->dryRun)
539  return (aiori_fd_t*) 0;
540 
541  pfd->fd = open64(testFileName, fd_oflag);
542  if (pfd->fd < 0)
543  ERRF("open64(\"%s\", %d) failed: %s", testFileName, fd_oflag, strerror(errno));
544 
545 #ifdef HAVE_LUSTRE_USER
546  if (o->lustre_ignore_locks) {
547  int lustre_ioctl_flags = LL_FILE_IGNORE_LOCK;
548  if (verbose >= VERBOSE_1) {
549  EINFO("** Disabling lustre range locking **\n");
550  }
551  if (ioctl(pfd->fd, LL_IOC_SETFLAGS, &lustre_ioctl_flags) == -1)
552  ERRF("ioctl(%d, LL_IOC_SETFLAGS, ...) failed", pfd->fd);
553  }
554 #endif /* HAVE_LUSTRE_USER */
555 
556 #ifdef HAVE_GPFS_FCNTL_H
557  if(o->gpfs_release_token) {
558  gpfs_free_all_locks(pfd->fd);
559  }
560 #endif
561 #ifdef HAVE_GPU_DIRECT
562  if(o->gpuDirect){
563  init_cufile(pfd);
564  }
565 #endif
566  return (aiori_fd_t*) pfd;
567 }
568 
569 /*
570  * Write or read access to file using the POSIX interface.
571  */
572 static IOR_offset_t POSIX_Xfer(int access, aiori_fd_t *file, IOR_size_t * buffer,
574 {
575  int xferRetries = 0;
576  long long remaining = (long long)length;
577  char *ptr = (char *)buffer;
578  long long rc;
579  int fd;
580  posix_options_t * o = (posix_options_t*) param;
581 
582  if(hints->dryRun)
583  return length;
584 
585  posix_fd * pfd = (posix_fd *) file;
586  fd = pfd->fd;
587 
588 #ifdef HAVE_GPFS_FCNTL_H
589  if (o->gpfs_hint_access) {
590  gpfs_access_start(fd, length, offset, access);
591  }
592 #endif
593 
594 
595  /* seek to offset */
596  if (lseek64(fd, offset, SEEK_SET) == -1)
597  ERRF("lseek64(%d, %lld, SEEK_SET) failed", fd, offset);
598  off_t mem_offset = 0;
599  while (remaining > 0) {
600  /* write/read file */
601  if (access == WRITE) { /* WRITE */
602  if (verbose >= VERBOSE_4) {
603  EINFO("task %d writing to offset %lld\n",
604  rank,
605  offset + length - remaining);
606  }
607 #ifdef HAVE_GPU_DIRECT
608  if(o->gpuDirect){
609  rc = cuFileWrite(pfd->cf_handle, ptr, remaining, offset + mem_offset, mem_offset);
610  }else{
611 #endif
612  rc = write(fd, ptr, remaining);
613 #ifdef HAVE_GPU_DIRECT
614  }
615 #endif
616  if (rc == -1)
617  ERRF("write(%d, %p, %lld) failed",
618  fd, (void*)ptr, remaining);
619  if (hints->fsyncPerWrite == TRUE){
620  POSIX_Fsync((aiori_fd_t*) &fd, param);
621  }
622  } else { /* READ or CHECK */
623  if (verbose >= VERBOSE_4) {
624  EINFO("task %d reading from offset %lld\n",
625  rank,
626  offset + length - remaining);
627  }
628 #ifdef HAVE_GPU_DIRECT
629  if(o->gpuDirect){
630  rc = cuFileRead(pfd->cf_handle, ptr, remaining, offset + mem_offset, mem_offset);
631  }else{
632 #endif
633  rc = read(fd, ptr, remaining);
634 #ifdef HAVE_GPU_DIRECT
635  }
636 #endif
637  if (rc == 0)
638  ERRF("read(%d, %p, %lld) returned EOF prematurely",
639  fd, (void*)ptr, remaining);
640  if (rc == -1)
641  ERRF("read(%d, %p, %lld) failed",
642  fd, (void*)ptr, remaining);
643  }
644  if (rc < remaining) {
645  EWARNF("task %d, partial %s, %lld of %lld bytes at offset %lld\n",
646  rank,
647  access == WRITE ? "write()" : "read()",
648  rc, remaining,
649  offset + length - remaining);
650  if (xferRetries > MAX_RETRY || hints->singleXferAttempt)
651  ERR("too many retries -- aborting");
652  }
653  assert(rc >= 0);
654  assert(rc <= remaining);
655  remaining -= rc;
656  ptr += rc;
657  mem_offset += rc;
658  xferRetries++;
659  }
660 #ifdef HAVE_GPFS_FCNTL_H
661  if (o->gpfs_hint_access) {
662  gpfs_access_end(fd, length, offset, access);
663  }
664 #endif
665  return (length);
666 }
667 
669 {
670  int fd = ((posix_fd*) afd)->fd;
671  if (fsync(fd) != 0)
672  EWARNF("fsync(%d) failed", fd);
673 }
674 
675 
677 {
678  int ret = system("sync");
679  if (ret != 0){
680  FAIL("Error executing the sync command, ensure it exists.");
681  }
682 }
683 
684 
685 /*
686  * Close a file through the POSIX interface.
687  */
689 {
690  if(hints->dryRun)
691  return;
692  posix_options_t * o = (posix_options_t*) param;
693  int fd = ((posix_fd*) afd)->fd;
694 #ifdef HAVE_GPU_DIRECT
695  if(o->gpuDirect){
696  cuFileHandleDeregister(((posix_fd*) afd)->cf_handle);
697  }
698 #endif
699  if (close(fd) != 0){
700  ERRF("close(%d) failed", fd);
701  }
702  free(afd);
703 }
704 
705 /*
706  * Delete a file through the POSIX interface.
707  */
708 void POSIX_Delete(char *testFileName, aiori_mod_opt_t * param)
709 {
710  if(hints->dryRun)
711  return;
712  if (unlink(testFileName) != 0){
713  EWARNF("[RANK %03d]: unlink() of file \"%s\" failed", rank, testFileName);
714  }
715 }
716 
717 int POSIX_Rename(const char * oldfile, const char * newfile, aiori_mod_opt_t * module_options){
718  if(hints->dryRun)
719  return 0;
720 
721  if(rename(oldfile, newfile) != 0){
722  EWARNF("[RANK %03d]: rename() of file \"%s\" to \"%s\" failed", rank, oldfile, newfile);
723  return -1;
724  }
725  return 0;
726 }
727 
728 /*
729  * Use POSIX stat() to return aggregate file size.
730  */
731 IOR_offset_t POSIX_GetFileSize(aiori_mod_opt_t * test, char *testFileName)
732 {
733  if(hints->dryRun)
734  return 0;
735  struct stat stat_buf;
736  IOR_offset_t aggFileSizeFromStat, tmpMin, tmpMax, tmpSum;
737 
738  if (stat(testFileName, &stat_buf) != 0) {
739  ERRF("stat(\"%s\", ...) failed", testFileName);
740  }
741  aggFileSizeFromStat = stat_buf.st_size;
742 
743  return (aggFileSizeFromStat);
744 }
745 
747 #ifdef HAVE_GPU_DIRECT
748  CUfileError_t err = cuFileDriverOpen();
749 #endif
750 }
751 
753 #ifdef HAVE_GPU_DIRECT
754  CUfileError_t err = cuFileDriverClose();
755 #endif
756 }
int gpfs_release_token
Definition: aiori-POSIX.h:20
#define ERRF(FORMAT,...)
Definition: aiori-debug.h:77
#define ISPOWEROFTWO(x)
Definition: ior.h:48
static void POSIX_Finalize(aiori_mod_opt_t *options)
Definition: aiori-POSIX.c:752
#define LAST_OPTION
Definition: option.h:39
#define VERBOSE_0
Definition: iordef.h:92
CURLcode rc
Definition: aiori-S3-4c.c:111
int errno
struct benchmark_options o
Definition: md-workbench.c:128
int POSIX_Mknod(char *testFileName)
Definition: aiori-POSIX.c:515
static IOR_offset_t POSIX_Xfer(int, aiori_fd_t *, IOR_size_t *, IOR_offset_t, IOR_offset_t, aiori_mod_opt_t *)
Definition: aiori-POSIX.c:572
static void POSIX_Initialize(aiori_mod_opt_t *options)
Definition: aiori-POSIX.c:746
#define EINFO(FORMAT,...)
Definition: aiori-debug.h:65
void POSIX_Close(aiori_fd_t *afd, aiori_mod_opt_t *param)
Definition: aiori-POSIX.c:688
ior_aiori_t posix_aiori
Definition: aiori-POSIX.c:160
#define FAIL(...)
Definition: aiori-debug.h:12
int POSIX_Rename(const char *oldfile, const char *newfile, aiori_mod_opt_t *module_options)
Definition: aiori-POSIX.c:717
int lustre_set_striping
Definition: aiori-POSIX.h:12
#define MPI_CHECK(MPI_STATUS, MSG)
Definition: aiori-debug.h:127
#define WRITE
Definition: iordef.h:86
int aiori_posix_stat(const char *path, struct stat *buf, aiori_mod_opt_t *module_options)
Definition: aiori.c:227
aiori_fd_t * POSIX_Open(char *testFileName, int flags, aiori_mod_opt_t *param)
Definition: aiori-POSIX.c:529
char * aiori_get_version()
Definition: aiori.c:232
#define O_BINARY
Definition: aiori-POSIX.c:83
aiori_fd_t * POSIX_Create(char *testFileName, int flags, aiori_mod_opt_t *param)
Definition: aiori-POSIX.c:401
MPI_Comm testComm
Definition: utilities.c:71
static option_help options[]
Definition: aiori-CEPHFS.c:54
void POSIX_Delete(char *testFileName, aiori_mod_opt_t *param)
Definition: aiori-POSIX.c:708
static aiori_xfer_hint_t * hints
Definition: aiori-POSIX.c:189
#define WARN(MSG)
Definition: aiori-debug.h:32
void POSIX_Sync(aiori_mod_opt_t *param)
Definition: aiori-POSIX.c:676
int lustre_stripe_count
Definition: aiori-POSIX.h:13
int singleXferAttempt
Definition: aiori.h:75
#define open64
Definition: aiori-POSIX.c:75
Definition: ior.h:56
#define MAX_RETRY
Definition: iordef.h:101
int POSIX_check_params(aiori_mod_opt_t *param)
Definition: aiori-POSIX.c:195
#define EWARN(MSG)
Definition: aiori-debug.h:59
#define EWARNF(FORMAT,...)
Definition: aiori-debug.h:45
int aiori_posix_access(const char *path, int mode, aiori_mod_opt_t *module_options)
Definition: aiori.c:222
long long int IOR_size_t
Definition: iordef.h:110
int aiori_posix_rmdir(const char *path, aiori_mod_opt_t *module_options)
Definition: aiori.c:217
void POSIX_Fsync(aiori_fd_t *afd, aiori_mod_opt_t *param)
Definition: aiori-POSIX.c:668
#define VERBOSE_2
Definition: iordef.h:94
void POSIX_xfer_hints(aiori_xfer_hint_t *params)
Definition: aiori-POSIX.c:191
int aiori_posix_mkdir(const char *path, mode_t mode, aiori_mod_opt_t *module_options)
Definition: aiori.c:212
#define lseek64
Definition: aiori-POSIX.c:79
int verbose
Definition: utilities.c:70
#define VERBOSE_4
Definition: iordef.h:96
int aiori_posix_statfs(const char *path, ior_aiori_statfs_t *stat_buf, aiori_mod_opt_t *module_options)
Definition: aiori.c:166
void set_o_direct_flag(int *flag)
Definition: utilities.c:217
#define ERR(MSG)
Definition: aiori-debug.h:92
#define VERBOSE_1
Definition: iordef.h:93
option_help * POSIX_options(aiori_mod_opt_t **init_backend_options, aiori_mod_opt_t *init_values)
Definition: aiori-POSIX.c:113
int fsyncPerWrite
Definition: aiori.h:70
char * name
Definition: aiori.h:88
int filePerProc
Definition: aiori.h:65
int lustre_stripe_size
Definition: aiori-POSIX.h:14
long long int IOR_offset_t
Definition: iordef.h:109
int rank
Definition: utilities.c:68
int lustre_ignore_locks
Definition: aiori-POSIX.h:16
#define TRUE
Definition: iordef.h:66
IOR_offset_t POSIX_GetFileSize(aiori_mod_opt_t *test, char *testFileName)
Definition: aiori-POSIX.c:731
void * safeMalloc(uint64_t size)
Definition: utilities.c:125
#define NULL
Definition: iordef.h:70