23 # include <sys/ioctl.h> 35 #ifdef HAVE_LINUX_LUSTRE_LUSTRE_USER_H 36 # include <linux/lustre/lustre_user.h> 37 #elif defined(HAVE_LUSTRE_LUSTRE_USER_H) 38 # include <lustre/lustre_user.h> 43 #ifdef HAVE_GPFS_FCNTL_H 44 # include <gpfs_fcntl.h> 47 #ifdef HAVE_BEEGFS_BEEGFS_H 48 #include <beegfs/beegfs.h> 63 # define lseek64 lseek 86 if (init_values !=
NULL){
92 *init_backend_options =
o;
99 memcpy(help, h,
sizeof(h));
125 .enable_mdtest =
true,
132 #ifdef HAVE_GPFS_FCNTL_H 133 void gpfs_free_all_locks(
int fd)
137 gpfsFcntlHeader_t header;
138 gpfsFreeRange_t release;
140 release_all.header.totalLength =
sizeof(release_all);
141 release_all.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
142 release_all.header.fcntlReserved = 0;
144 release_all.release.structLen =
sizeof(release_all.release);
145 release_all.release.structType = GPFS_FREE_RANGE;
146 release_all.release.start = 0;
147 release_all.release.length = 0;
149 rc = gpfs_fcntl(fd, &release_all);
151 EWARNF(
"gpfs_fcntl(%d, ...) release all locks hint failed.", fd);
158 gpfsFcntlHeader_t header;
159 gpfsAccessRange_t access;
162 take_locks.header.totalLength =
sizeof(take_locks);
163 take_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
164 take_locks.header.fcntlReserved = 0;
166 take_locks.access.structLen =
sizeof(take_locks.access);
167 take_locks.access.structType = GPFS_ACCESS_RANGE;
168 take_locks.access.start = param->
offset;
169 take_locks.access.length = length;
170 take_locks.access.isWrite = (access ==
WRITE);
172 rc = gpfs_fcntl(fd, &take_locks);
174 EWARNF(
"gpfs_fcntl(%d, ...) access range hint failed.", fd);
182 gpfsFcntlHeader_t header;
183 gpfsFreeRange_t free;
187 free_locks.header.totalLength =
sizeof(free_locks);
188 free_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
189 free_locks.header.fcntlReserved = 0;
191 free_locks.free.structLen =
sizeof(free_locks.free);
192 free_locks.free.structType = GPFS_FREE_RANGE;
193 free_locks.free.start = param->
offset;
194 free_locks.free.length = length;
196 rc = gpfs_fcntl(fd, &free_locks);
198 EWARNF(
"gpfs_fcntl(%d, ...) free range hint failed.", fd);
204 #ifdef HAVE_BEEGFS_BEEGFS_H 206 int mkTempInDir(
char* dirPath)
208 unsigned long len = strlen(dirPath) + 8;
209 char* tmpfilename = (
char*)malloc(
sizeof (
char)*len+1);
210 snprintf(tmpfilename, len,
"%s/XXXXXX", dirPath);
212 int fd = mkstemp(tmpfilename);
219 bool beegfs_getStriping(
char* dirPath, u_int16_t* numTargetsOut,
unsigned* chunkSizeOut)
223 int fd = mkTempInDir(dirPath);
225 unsigned stripePattern = 0;
226 retVal = beegfs_getStripeInfo(fd, &stripePattern, chunkSizeOut, numTargetsOut);
233 bool beegfs_isOptionSet(
int opt) {
237 bool beegfs_compatibleFileExists(
char* filepath,
int numTargets,
int chunkSize)
239 int fd = open(filepath, O_RDWR);
244 unsigned read_stripePattern = 0;
245 u_int16_t read_numTargets = 0;
246 int read_chunkSize = 0;
248 bool retVal = beegfs_getStripeInfo(fd, &read_stripePattern, &read_chunkSize, &read_numTargets);
252 return retVal && read_numTargets == numTargets && read_chunkSize == chunkSize;
258 bool beegfs_createFilePath(
char* filepath, mode_t mode,
int numTargets,
int chunkSize)
261 char* dirTmp = strdup(filepath);
262 char* dir = dirname(dirTmp);
263 DIR* parentDirS = opendir(dir);
265 ERRF(
"Failed to get directory: %s", dir);
269 int parentDirFd = dirfd(parentDirS);
272 ERRF(
"Failed to get directory descriptor: %s", dir);
276 bool isBeegfs = beegfs_testIsBeeGFS(parentDirFd);
279 WARN(
"Not a BeeGFS file system");
283 if ( !beegfs_isOptionSet(numTargets)
284 || !beegfs_isOptionSet(chunkSize)) {
285 u_int16_t defaultNumTargets = 0;
286 unsigned defaultChunkSize = 0;
287 bool haveDefaults = beegfs_getStriping(dir,
291 ERR(
"Failed to get default BeeGFS striping values");
293 numTargets = beegfs_isOptionSet(numTargets) ?
294 numTargets : defaultNumTargets;
295 chunkSize = beegfs_isOptionSet(chunkSize) ?
296 chunkSize : defaultChunkSize;
299 char* filenameTmp = strdup(filepath);
300 char* filename = basename(filepath);
301 bool isFileCreated = beegfs_compatibleFileExists(filepath, numTargets, chunkSize)
302 || beegfs_createFile(parentDirFd, filename,
303 mode, numTargets, chunkSize);
305 ERR(
"Could not create file");
310 closedir(parentDirS);
327 fd = (
int *)malloc(
sizeof(
int));
329 ERR(
"Unable to malloc file descriptor");
338 #ifdef HAVE_LUSTRE_LUSTRE_USER_H 342 #define FASYNC 00020000 352 *fd =
open64(testFileName, fd_oflag, mode);
354 ERRF(
"open64(\"%s\", %d, %#o) failed",
355 testFileName, fd_oflag, mode);
357 struct lov_user_md opts = { 0 };
360 opts.lmm_magic = LOV_USER_MAGIC;
369 O_CREAT | O_EXCL | O_RDWR | O_LOV_DELAY_CREATE;
370 *fd =
open64(testFileName, fd_oflag, mode);
372 fprintf(stdout,
"\nUnable to open '%s': %s\n",
373 testFileName, strerror(
errno));
375 "MPI_Abort() error");
376 }
else if (ioctl(*fd, LL_IOC_LOV_SETSTRIPE, &opts)) {
377 char *errmsg =
"stripe already set";
379 errmsg = strerror(
errno);
381 "\nError on ioctl for '%s' (%d): %s\n",
382 testFileName, *fd, errmsg);
384 "MPI_Abort() error");
393 fd_oflag |= O_CREAT | O_RDWR;
395 #ifdef HAVE_BEEGFS_BEEGFS_H 398 bool result = beegfs_createFilePath(testFileName,
403 fd_oflag &= ~O_CREAT;
405 EWARN(
"BeeGFS tuning failed");
410 *fd =
open64(testFileName, fd_oflag, mode);
412 ERRF(
"open64(\"%s\", %d, %#o) failed",
413 testFileName, fd_oflag, mode);
415 #ifdef HAVE_LUSTRE_LUSTRE_USER_H 419 int lustre_ioctl_flags = LL_FILE_IGNORE_LOCK;
420 if (ioctl(*fd, LL_IOC_SETFLAGS, &lustre_ioctl_flags) == -1)
421 ERRF(
"ioctl(%d, LL_IOC_SETFLAGS, ...) failed", *fd);
425 #ifdef HAVE_GPFS_FCNTL_H 430 gpfs_free_all_locks(*fd);
443 ret = mknod(testFileName, S_IFREG | S_IRUSR, 0);
458 fd = (
int *)malloc(
sizeof(
int));
460 ERR(
"Unable to malloc file descriptor");
471 *fd =
open64(testFileName, fd_oflag);
473 ERRF(
"open64(\"%s\", %d) failed", testFileName, fd_oflag);
475 #ifdef HAVE_LUSTRE_LUSTRE_USER_H 477 int lustre_ioctl_flags = LL_FILE_IGNORE_LOCK;
480 "** Disabling lustre range locking **\n");
482 if (ioctl(*fd, LL_IOC_SETFLAGS, &lustre_ioctl_flags) == -1)
483 ERRF(
"ioctl(%d, LL_IOC_SETFLAGS, ...) failed", *fd);
487 #ifdef HAVE_GPFS_FCNTL_H 489 gpfs_free_all_locks(*fd);
502 long long remaining = (
long long)length;
503 char *ptr = (
char *)buffer;
512 #ifdef HAVE_GPFS_FCNTL_H 514 gpfs_access_start(fd, length, param, access);
521 ERRF(
"lseek64(%d, %lld, SEEK_SET) failed", fd, param->
offset);
523 while (remaining > 0) {
525 if (access ==
WRITE) {
528 "task %d writing to offset %lld\n",
530 param->
offset + length - remaining);
532 rc = write(fd, ptr, remaining);
534 ERRF(
"write(%d, %p, %lld) failed",
535 fd, (
void*)ptr, remaining);
541 "task %d reading from offset %lld\n",
543 param->
offset + length - remaining);
545 rc = read(fd, ptr, remaining);
547 ERRF(
"read(%d, %p, %lld) returned EOF prematurely",
548 fd, (
void*)ptr, remaining);
550 ERRF(
"read(%d, %p, %lld) failed",
551 fd, (
void*)ptr, remaining);
553 if (rc < remaining) {
555 "WARNING: Task %d, partial %s, %lld of %lld bytes at offset %lld\n",
557 access ==
WRITE ?
"write()" :
"read()",
559 param->
offset + length - remaining);
564 ERR(
"too many retries -- aborting");
567 assert(rc <= remaining);
572 #ifdef HAVE_GPFS_FCNTL_H 574 gpfs_access_end(fd, length, param, access);
585 if (fsync(*(
int *)fd) != 0)
586 EWARNF(
"fsync(%d) failed", *(
int *)fd);
592 int ret = system(
"sync");
594 FAIL(
"Error executing the sync command, ensure it exists.");
606 if (close(*(
int *)fd) != 0)
607 ERRF(
"close(%d) failed", *(
int *)fd);
618 if (unlink(testFileName) != 0){
619 EWARNF(
"[RANK %03d]: unlink() of file \"%s\" failed\n",
632 struct stat stat_buf;
633 IOR_offset_t aggFileSizeFromStat, tmpMin, tmpMax, tmpSum;
635 if (stat(testFileName, &stat_buf) != 0) {
636 ERRF(
"stat(\"%s\", ...) failed", testFileName);
638 aggFileSizeFromStat = stat_buf.st_size;
641 MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpSum, 1,
642 MPI_LONG_LONG_INT, MPI_SUM, testComm),
643 "cannot total data moved");
644 aggFileSizeFromStat = tmpSum;
646 MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMin, 1,
647 MPI_LONG_LONG_INT, MPI_MIN, testComm),
648 "cannot total data moved");
649 MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMax, 1,
650 MPI_LONG_LONG_INT, MPI_MAX, testComm),
651 "cannot total data moved");
652 if (tmpMin != tmpMax) {
654 WARN(
"inconsistent file size by different tasks");
657 aggFileSizeFromStat = tmpMin;
661 return (aggFileSizeFromStat);
static void POSIX_Fsync(void *, IOR_param_t *)
void set_o_direct_flag(int *fd)
void * POSIX_Open(char *testFileName, IOR_param_t *param)
int POSIX_Mknod(char *testFileName)
int aiori_posix_rmdir(const char *path, IOR_param_t *param)
#define EWARNF(FORMAT,...)
int aiori_posix_mkdir(const char *path, mode_t mode, IOR_param_t *param)
int aiori_posix_statfs(const char *path, ior_aiori_statfs_t *stat_buf, IOR_param_t *param)
void * POSIX_Create(char *testFileName, IOR_param_t *param)
char * aiori_get_version()
void POSIX_Delete(char *testFileName, IOR_param_t *param)
option_help * POSIX_options(void **init_backend_options, void *init_values)
#define MPI_CHECK(MPI_STATUS, MSG)
IOR_offset_t POSIX_GetFileSize(IOR_param_t *test, MPI_Comm testComm, char *testFileName)
static void POSIX_Sync(IOR_param_t *)
void POSIX_Close(void *fd, IOR_param_t *param)
int aiori_posix_stat(const char *path, struct stat *buf, IOR_param_t *param)
int aiori_posix_access(const char *path, int mode, IOR_param_t *param)
long long int IOR_offset_t
static struct cephfs_options o
static IOR_offset_t POSIX_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)