IOR
aiori-IME.c
Go to the documentation of this file.
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  */
4 /******************************************************************************\
5 * *
6 * Copyright (c) 2003, The Regents of the University of California. *
7 * Copyright (c) 2018, DataDirect Networks. *
8 * See the file COPYRIGHT for a complete copyright notice and license. *
9 * *
10 ********************************************************************************
11 *
12 * Implement abstract I/O interface for DDN Infinite Memory Engine (IME).
13 *
14 \******************************************************************************/
15 
16 #ifdef HAVE_CONFIG_H
17 #include "config.h"
18 #endif
19 
20 #include <assert.h>
21 #include <stdio.h>
22 #include <stdlib.h>
23 #include <sys/stat.h>
24 #include <errno.h> /* sys_errlist */
25 #include <fcntl.h> /* IO operations */
26 
27 #include "ior.h"
28 #include "iordef.h"
29 #include "aiori.h"
30 #include "utilities.h"
31 #include "ime_native.h"
32 
33 #ifndef O_BINARY /* Required on Windows */
34 # define O_BINARY 0
35 #endif
36 
37 /**************************** P R O T O T Y P E S *****************************/
38 
39 static void *IME_Create(char *, IOR_param_t *);
40 static void *IME_Open(char *, IOR_param_t *);
41 static void IME_Close(void *, IOR_param_t *);
42 static void IME_Delete(char *, IOR_param_t *);
43 static char *IME_GetVersion();
44 static void IME_Fsync(void *, IOR_param_t *);
45 static int IME_Access(const char *, int, IOR_param_t *);
46 static IOR_offset_t IME_GetFileSize(IOR_param_t *, MPI_Comm, char *);
47 static IOR_offset_t IME_Xfer(int, void *, IOR_size_t *,
49 static int IME_StatFS(const char *, ior_aiori_statfs_t *,
50  IOR_param_t *);
51 static int IME_RmDir(const char *, IOR_param_t *);
52 static int IME_MkDir(const char *, mode_t, IOR_param_t *);
53 static int IME_Stat(const char *, struct stat *, IOR_param_t *);
54 
55 #if (IME_NATIVE_API_VERSION >= 132)
56 static int IME_Mknod(char *);
57 static void IME_Sync(IOR_param_t *);
58 #endif
59 
60 static void IME_Initialize();
61 static void IME_Finalize();
62 
63 
64 /************************** O P T I O N S *****************************/
65 typedef struct{
66  int direct_io;
68 
69 
70 option_help * IME_options(void ** init_backend_options, void * init_values){
71  ime_options_t * o = malloc(sizeof(ime_options_t));
72 
73  if (init_values != NULL){
74  memcpy(o, init_values, sizeof(ime_options_t));
75  }else{
76  o->direct_io = 0;
77  }
78 
79  *init_backend_options = o;
80 
81  option_help h [] = {
82  {0, "ime.odirect", "Direct I/O Mode", OPTION_FLAG, 'd', & o->direct_io},
84  };
85  option_help * help = malloc(sizeof(h));
86  memcpy(help, h, sizeof(h));
87  return help;
88 }
89 
90 /************************** D E C L A R A T I O N S ***************************/
91 
92 extern int rank;
93 extern int rankOffset;
94 extern int verbose;
95 extern MPI_Comm testComm;
96 
98  .name = "IME",
99  .name_legacy = "IM",
100  .create = IME_Create,
101  .open = IME_Open,
102  .xfer = IME_Xfer,
103  .close = IME_Close,
104  .delete = IME_Delete,
105  .get_version = IME_GetVersion,
106  .fsync = IME_Fsync,
107  .get_file_size = IME_GetFileSize,
108  .access = IME_Access,
109  .statfs = IME_StatFS,
110  .rmdir = IME_RmDir,
111  .mkdir = IME_MkDir,
112  .stat = IME_Stat,
113  .initialize = IME_Initialize,
114  .finalize = IME_Finalize,
115  .get_options = IME_options,
116 #if (IME_NATIVE_API_VERSION >= 132)
117  .sync = IME_Sync,
118  .mknod = IME_Mknod,
119 #endif
120  .enable_mdtest = true,
121 };
122 
123 /***************************** F U N C T I O N S ******************************/
124 
125 /*
126  * Initialize IME (before MPI is started).
127  */
128 static void IME_Initialize()
129 {
130  ime_native_init();
131 }
132 
133 /*
134  * Finlize IME (after MPI is shutdown).
135  */
136 static void IME_Finalize()
137 {
138  (void)ime_native_finalize();
139 }
140 
141 /*
142  * Try to access a file through the IME interface.
143  */
144 static int IME_Access(const char *path, int mode, IOR_param_t *param)
145 {
146  (void)param;
147 
148  return ime_native_access(path, mode);
149 }
150 
151 /*
152  * Creat and open a file through the IME interface.
153  */
154 static void *IME_Create(char *testFileName, IOR_param_t *param)
155 {
156  return IME_Open(testFileName, param);
157 }
158 
159 /*
160  * Open a file through the IME interface.
161  */
162 static void *IME_Open(char *testFileName, IOR_param_t *param)
163 {
164  int fd_oflag = O_BINARY;
165  int *fd;
166 
167  fd = (int *)malloc(sizeof(int));
168  if (fd == NULL)
169  ERR("Unable to malloc file descriptor");
170 
172  if (o->direct_io == TRUE){
173  set_o_direct_flag(&fd_oflag);
174  }
175 
176  if (param->openFlags & IOR_RDONLY)
177  fd_oflag |= O_RDONLY;
178  if (param->openFlags & IOR_WRONLY)
179  fd_oflag |= O_WRONLY;
180  if (param->openFlags & IOR_RDWR)
181  fd_oflag |= O_RDWR;
182  if (param->openFlags & IOR_APPEND)
183  fd_oflag |= O_APPEND;
184  if (param->openFlags & IOR_CREAT)
185  fd_oflag |= O_CREAT;
186  if (param->openFlags & IOR_EXCL)
187  fd_oflag |= O_EXCL;
188  if (param->openFlags & IOR_TRUNC)
189  fd_oflag |= O_TRUNC;
190 
191  *fd = ime_native_open(testFileName, fd_oflag, 0664);
192  if (*fd < 0) {
193  free(fd);
194  ERR("cannot open file");
195  }
196 
197  return((void *)fd);
198 }
199 
200 /*
201  * Write or read access to file using the IM interface.
202  */
203 static IOR_offset_t IME_Xfer(int access, void *file, IOR_size_t *buffer,
204  IOR_offset_t length, IOR_param_t *param)
205 {
206  int xferRetries = 0;
207  long long remaining = (long long)length;
208  char *ptr = (char *)buffer;
209  int fd = *(int *)file;
210  long long rc;
211 
212  while (remaining > 0) {
213  /* write/read file */
214  if (access == WRITE) { /* WRITE */
215  if (verbose >= VERBOSE_4) {
216  fprintf(stdout, "task %d writing to offset %lld\n",
217  rank, param->offset + length - remaining);
218  }
219 
220  rc = ime_native_pwrite(fd, ptr, remaining, param->offset);
221 
222  if (param->fsyncPerWrite)
223  IME_Fsync(&fd, param);
224  } else { /* READ or CHECK */
225  if (verbose >= VERBOSE_4) {
226  fprintf(stdout, "task %d reading from offset %lld\n",
227  rank, param->offset + length - remaining);
228  }
229 
230  rc = ime_native_pread(fd, ptr, remaining, param->offset);
231  if (rc == 0)
232  ERR("hit EOF prematurely");
233  else if (rc < 0)
234  ERR("read failed");
235  }
236 
237  if (rc < remaining) {
238  fprintf(stdout, "WARNING: Task %d, partial %s, %lld of "
239  "%lld bytes at offset %lld\n",
240  rank, access == WRITE ? "write" : "read", rc,
241  remaining, param->offset + length - remaining );
242 
243  if (param->singleXferAttempt) {
244  MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1),
245  "barrier error");
246  }
247 
248  if (xferRetries > MAX_RETRY) {
249  ERR( "too many retries -- aborting" );
250  }
251  } else if (rc > remaining) /* this should never happen */
252  ERR("too many bytes transferred!?!");
253 
254  assert(rc >= 0);
255  assert(rc <= remaining);
256  remaining -= rc;
257  ptr += rc;
258  xferRetries++;
259  }
260 
261  return(length);
262 }
263 
264 /*
265  * Perform fsync().
266  */
267 static void IME_Fsync(void *fd, IOR_param_t *param)
268 {
269  if (ime_native_fsync(*(int *)fd) != 0)
270  WARN("cannot perform fsync on file");
271 }
272 
273 /*
274  * Close a file through the IME interface.
275  */
276 static void IME_Close(void *fd, IOR_param_t *param)
277 {
278  if (ime_native_close(*(int *)fd) != 0)
279  {
280  free(fd);
281  ERR("cannot close file");
282  }
283  else
284  free(fd);
285 }
286 
287 /*
288  * Delete a file through the IME interface.
289  */
290 static void IME_Delete(char *testFileName, IOR_param_t *param)
291 {
292  char errmsg[256];
293  sprintf(errmsg, "[RANK %03d]:cannot delete file %s\n",
294  rank, testFileName);
295  if (ime_native_unlink(testFileName) != 0)
296  WARN(errmsg);
297 }
298 
299 /*
300  * Determine API version.
301  */
302 static char *IME_GetVersion()
303 {
304  static char ver[1024] = {};
305 #if (IME_NATIVE_API_VERSION >= 120)
306  strcpy(ver, ime_native_version());
307 #else
308  strcpy(ver, "not supported");
309 #endif
310  return ver;
311 }
312 
313 static int IME_StatFS(const char *path, ior_aiori_statfs_t *stat_buf,
315 {
316  (void)param;
317 
318 #if (IME_NATIVE_API_VERSION >= 130)
319  struct statvfs statfs_buf;
320 
321  int ret = ime_native_statvfs(path, &statfs_buf);
322  if (ret)
323  return ret;
324 
325  stat_buf->f_bsize = statfs_buf.f_bsize;
326  stat_buf->f_blocks = statfs_buf.f_blocks;
327  stat_buf->f_bfree = statfs_buf.f_bfree;
328  stat_buf->f_files = statfs_buf.f_files;
329  stat_buf->f_ffree = statfs_buf.f_ffree;
330 
331  return 0;
332 #else
333  (void)path;
334  (void)stat_buf;
335 
336  WARN("statfs is currently not supported in IME backend!");
337  return -1;
338 #endif
339 }
340 
341 
342 static int IME_MkDir(const char *path, mode_t mode, IOR_param_t *param)
343 {
344  (void)param;
345 
346 #if (IME_NATIVE_API_VERSION >= 130)
347  return ime_native_mkdir(path, mode);
348 #else
349  (void)path;
350  (void)mode;
351 
352  WARN("mkdir not supported in IME backend!");
353  return -1;
354 #endif
355 }
356 
357 static int IME_RmDir(const char *path, IOR_param_t *param)
358 {
359  (void)param;
360 
361 #if (IME_NATIVE_API_VERSION >= 130)
362  return ime_native_rmdir(path);
363 #else
364  (void)path;
365 
366  WARN("rmdir not supported in IME backend!");
367  return -1;
368 #endif
369 }
370 
371 /*
372  * Perform stat() through the IME interface.
373  */
374 static int IME_Stat(const char *path, struct stat *buf, IOR_param_t *param)
375 {
376  (void)param;
377 
378  return ime_native_stat(path, buf);
379 }
380 
381 /*
382  * Use IME stat() to return aggregate file size.
383  */
385  char *testFileName)
386 {
387  struct stat stat_buf;
388  IOR_offset_t aggFileSizeFromStat, tmpMin, tmpMax, tmpSum;
389 
390  if (ime_native_stat(testFileName, &stat_buf) != 0) {
391  ERR("cannot get status of written file");
392  }
393  aggFileSizeFromStat = stat_buf.st_size;
394 
395  if (test->filePerProc) {
396  MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpSum, 1,
397  MPI_LONG_LONG_INT, MPI_SUM, testComm),
398  "cannot total data moved");
399  aggFileSizeFromStat = tmpSum;
400  } else {
401  MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMin, 1,
402  MPI_LONG_LONG_INT, MPI_MIN, testComm),
403  "cannot total data moved");
404  MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMax, 1,
405  MPI_LONG_LONG_INT, MPI_MAX, testComm),
406  "cannot total data moved");
407 
408  if (tmpMin != tmpMax) {
409  if (rank == 0) {
410  WARN("inconsistent file size by different tasks");
411  }
412  /* incorrect, but now consistent across tasks */
413  aggFileSizeFromStat = tmpMin;
414  }
415  }
416 
417  return(aggFileSizeFromStat);
418 }
419 
420 #if (IME_NATIVE_API_VERSION >= 132)
421 /*
422  * Create a file through mknod interface.
423  */
424 static int IME_Mknod(char *testFileName)
425 {
426  int ret = ime_native_mknod(testFileName, S_IFREG | S_IRUSR, 0);
427  if (ret < 0)
428  ERR("mknod failed");
429 
430  return ret;
431 }
432 
433 /*
434  * Use IME sync to flush page cache of all opened files.
435  */
436 static void IME_Sync(IOR_param_t * param)
437 {
438  int ret = ime_native_sync(0);
439  if (ret != 0)
440  FAIL("Error executing the sync command.");
441 }
442 #endif
static void * IME_Create(char *, IOR_param_t *)
Definition: aiori-IME.c:154
static void * IME_Open(char *, IOR_param_t *)
Definition: aiori-IME.c:162
uint64_t f_blocks
Definition: aiori.h:59
#define O_BINARY
Definition: aiori-IME.c:34
void set_o_direct_flag(int *fd)
Definition: utilities.c:160
static void IME_Initialize()
Definition: aiori-IME.c:128
uint64_t f_bfree
Definition: aiori.h:60
#define ERR(MSG)
Definition: iordef.h:184
#define LAST_OPTION
Definition: option.h:37
int filePerProc
Definition: ior.h:111
static void IME_Fsync(void *, IOR_param_t *)
Definition: aiori-IME.c:267
CURLcode rc
Definition: aiori-S3.c:121
static int IME_RmDir(const char *, IOR_param_t *)
Definition: aiori-IME.c:357
uint64_t f_ffree
Definition: aiori.h:63
static void IME_Finalize()
Definition: aiori-IME.c:136
static int IME_Stat(const char *, struct stat *, IOR_param_t *)
Definition: aiori-IME.c:374
#define IOR_APPEND
Definition: aiori.h:37
static IOR_offset_t IME_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori-IME.c:203
#define IOR_RDONLY
Definition: aiori.h:34
unsigned int openFlags
Definition: ior.h:88
int fsyncPerWrite
Definition: ior.h:162
#define WRITE
Definition: iordef.h:95
void * backend_options
Definition: ior.h:158
#define IOR_CREAT
Definition: aiori.h:38
option_help * IME_options(void **init_backend_options, void *init_values)
Definition: aiori-IME.c:70
#define IOR_EXCL
Definition: aiori.h:40
static int IME_Access(const char *, int, IOR_param_t *)
Definition: aiori-IME.c:144
static void IME_Close(void *, IOR_param_t *)
Definition: aiori-IME.c:276
uint64_t f_files
Definition: aiori.h:62
MPI_Comm testComm
Definition: utilities.c:60
#define IOR_TRUNC
Definition: aiori.h:39
uint64_t f_bsize
Definition: aiori.h:58
#define MPI_CHECK(MPI_STATUS, MSG)
Definition: iordef.h:224
int singleXferAttempt
Definition: ior.h:161
#define FAIL(...)
Definition: utilities.h:42
#define MAX_RETRY
Definition: iordef.h:110
static IOR_param_t param
Definition: mdtest.c:170
static IOR_offset_t IME_GetFileSize(IOR_param_t *, MPI_Comm, char *)
Definition: aiori-IME.c:384
static void IME_Delete(char *, IOR_param_t *)
Definition: aiori-IME.c:290
#define IOR_WRONLY
Definition: aiori.h:35
int rankOffset
Definition: utilities.c:58
long long int IOR_size_t
Definition: iordef.h:123
#define WARN(MSG)
Definition: iordef.h:144
static int IME_MkDir(const char *, mode_t, IOR_param_t *)
Definition: aiori-IME.c:342
IOR_offset_t offset
Definition: ior.h:126
#define VERBOSE_4
Definition: iordef.h:105
#define IOR_RDWR
Definition: aiori.h:36
int verbose
Definition: utilities.c:59
char * name
Definition: aiori.h:68
long long int IOR_offset_t
Definition: iordef.h:122
ior_aiori_t ime_aiori
Definition: aiori-IME.c:97
static char * IME_GetVersion()
Definition: aiori-IME.c:302
int rank
Definition: utilities.c:57
#define TRUE
Definition: iordef.h:75
static struct cephfs_options o
Definition: aiori-CEPHFS.c:48
static int IME_StatFS(const char *, ior_aiori_statfs_t *, IOR_param_t *)
Definition: aiori-IME.c:313
#define NULL
Definition: iordef.h:79