IOR
aiori-IME.c
Go to the documentation of this file.
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  */
4 /******************************************************************************\
5 * *
6 * Copyright (c) 2003, The Regents of the University of California. *
7 * Copyright (c) 2018, DataDirect Networks. *
8 * See the file COPYRIGHT for a complete copyright notice and license. *
9 * *
10 ********************************************************************************
11 *
12 * Implement abstract I/O interface for DDN Infinite Memory Engine (IME).
13 *
14 \******************************************************************************/
15 
16 #ifdef HAVE_CONFIG_H
17 #include "config.h"
18 #endif
19 
20 #include <assert.h>
21 #include <stdio.h>
22 #include <stdlib.h>
23 #include <sys/stat.h>
24 #include <errno.h> /* sys_errlist */
25 #include <fcntl.h> /* IO operations */
26 
27 #include "ior.h"
28 #include "iordef.h"
29 #include "aiori.h"
30 #include "utilities.h"
31 #include "ime_native.h"
32 
33 #define IME_UNUSED(x) (void)(x) /* Silence compiler warnings */
34 
35 #ifndef O_BINARY /* Required on Windows */
36 # define O_BINARY 0
37 #endif
38 
39 /**************************** P R O T O T Y P E S *****************************/
40 
41 aiori_fd_t *IME_Create(char *, int, aiori_mod_opt_t *);
42 aiori_fd_t *IME_Open(char *, int, aiori_mod_opt_t *);
44 void IME_Delete(char *, aiori_mod_opt_t *);
45 char *IME_GetVersion();
47 int IME_Access(const char *, int, aiori_mod_opt_t *);
51 int IME_Statfs(const char *, ior_aiori_statfs_t *,
52  aiori_mod_opt_t *);
53 int IME_Rmdir(const char *, aiori_mod_opt_t *);
54 int IME_Mkdir(const char *, mode_t, aiori_mod_opt_t *);
55 int IME_Stat(const char *, struct stat *, aiori_mod_opt_t *);
56 void IME_Xferhints(aiori_xfer_hint_t *params);
57 
58 #if (IME_NATIVE_API_VERSION >= 132)
59 int IME_Mknod(char *);
60 void IME_Sync(aiori_mod_opt_t *param);
61 #endif
62 
63 void IME_Initialize();
64 void IME_Finalize();
65 
66 /****************************** O P T I O N S *********************************/
67 
68 typedef struct{
69  int direct_io;
71 
72 option_help *IME_Options(aiori_mod_opt_t **init_backend_options,
73  aiori_mod_opt_t *init_values)
74 {
75  ime_options_t *o = malloc(sizeof(ime_options_t));
76 
77  if (init_values != NULL)
78  memcpy(o, init_values, sizeof(ime_options_t));
79  else
80  o->direct_io = 0;
81 
82  *init_backend_options = (aiori_mod_opt_t*)o;
83 
84  option_help h[] = {
85  {0, "ime.odirect", "Direct I/O Mode", OPTION_FLAG, 'd', & o->direct_io},
87  };
88  option_help *help = malloc(sizeof(h));
89  memcpy(help, h, sizeof(h));
90 
91  return help;
92 }
93 
94 
95 /************************** D E C L A R A T I O N S ***************************/
96 
97 extern int rank;
98 extern int rankOffset;
99 extern int verbose;
100 extern MPI_Comm testComm;
101 
103  .name = "IME",
104  .name_legacy = "IM",
105  .create = IME_Create,
106  .open = IME_Open,
107  .xfer = IME_Xfer,
108  .xfer_hints = IME_Xferhints,
109  .close = IME_Close,
110  .delete = IME_Delete,
111  .get_version = IME_GetVersion,
112  .fsync = IME_Fsync,
113  .get_file_size = IME_GetFileSize,
114  .access = IME_Access,
115  .statfs = IME_Statfs,
116  .rmdir = IME_Rmdir,
117  .mkdir = IME_Mkdir,
118  .stat = IME_Stat,
119  .initialize = IME_Initialize,
120  .finalize = IME_Finalize,
121  .get_options = IME_Options,
122 #if (IME_NATIVE_API_VERSION >= 132)
123  .sync = IME_Sync,
124  .mknod = IME_Mknod,
125 #endif
126  .enable_mdtest = true,
127 };
128 
130 static bool ime_initialized = false;
131 
132 
133 /***************************** F U N C T I O N S ******************************/
134 
136 {
137  hints = params;
138 }
139 
140 /*
141  * Initialize IME (before MPI is started).
142  */
144 {
145  if (ime_initialized)
146  return;
147 
148  ime_native_init();
149  ime_initialized = true;
150 }
151 
152 /*
153  * Finlize IME (after MPI is shutdown).
154  */
156 {
157  if (!ime_initialized)
158  return;
159 
160  (void)ime_native_finalize();
161  ime_initialized = false;
162 }
163 
164 /*
165  * Try to access a file through the IME interface.
166  */
167 
168 int IME_Access(const char *path, int mode, aiori_mod_opt_t *module_options)
169 {
170  IME_UNUSED(module_options);
171 
172  return ime_native_access(path, mode);
173 }
174 
175 /*
176  * Create and open a file through the IME interface.
177  */
178 aiori_fd_t *IME_Create(char *testFileName, int flags, aiori_mod_opt_t *param)
179 {
180  return IME_Open(testFileName, flags, param);
181 }
182 
183 /*
184  * Open a file through the IME interface.
185  */
186 aiori_fd_t *IME_Open(char *testFileName, int flags, aiori_mod_opt_t *param)
187 {
188  int fd_oflag = O_BINARY;
189  int *fd;
190 
191  if (hints->dryRun)
192  return NULL;
193 
194  fd = (int *)malloc(sizeof(int));
195  if (fd == NULL)
196  ERR("Unable to malloc file descriptor");
197 
198  ime_options_t *o = (ime_options_t*) param;
199  if (o->direct_io == TRUE)
200  set_o_direct_flag(&fd_oflag);
201 
202  if (flags & IOR_RDONLY)
203  fd_oflag |= O_RDONLY;
204  if (flags & IOR_WRONLY)
205  fd_oflag |= O_WRONLY;
206  if (flags & IOR_RDWR)
207  fd_oflag |= O_RDWR;
208  if (flags & IOR_APPEND)
209  fd_oflag |= O_APPEND;
210  if (flags & IOR_CREAT)
211  fd_oflag |= O_CREAT;
212  if (flags & IOR_EXCL)
213  fd_oflag |= O_EXCL;
214  if (flags & IOR_TRUNC)
215  fd_oflag |= O_TRUNC;
216 
217  *fd = ime_native_open(testFileName, fd_oflag, 0664);
218  if (*fd < 0) {
219  free(fd);
220  ERR("cannot open file");
221  }
222 
223  return (aiori_fd_t*) fd;
224 }
225 
226 /*
227  * Write or read access to file using the IM interface.
228  */
229 IOR_offset_t IME_Xfer(int access, aiori_fd_t *file, IOR_size_t *buffer,
231 {
232  int xferRetries = 0;
233  long long remaining = (long long)length;
234  char *ptr = (char *)buffer;
235  int fd = *(int *)file;
236  long long rc;
237 
238  if (hints->dryRun)
239  return length;
240 
241  while (remaining > 0) {
242  /* write/read file */
243  if (access == WRITE) { /* WRITE */
244  if (verbose >= VERBOSE_4) {
245  fprintf(stdout, "task %d writing to offset %lld\n",
246  rank, offset + length - remaining);
247  }
248 
249  rc = ime_native_pwrite(fd, ptr, remaining, offset);
250 
251  if (hints->fsyncPerWrite)
252  IME_Fsync(file, param);
253  } else { /* READ or CHECK */
254  if (verbose >= VERBOSE_4) {
255  fprintf(stdout, "task %d reading from offset %lld\n",
256  rank, offset + length - remaining);
257  }
258 
259  rc = ime_native_pread(fd, ptr, remaining, offset);
260  if (rc == 0)
261  ERR("hit EOF prematurely");
262  else if (rc < 0)
263  ERR("read failed");
264  }
265 
266  if (rc < remaining) {
267  fprintf(stdout, "WARNING: Task %d, partial %s, %lld of "
268  "%lld bytes at offset %lld\n",
269  rank, access == WRITE ? "write" : "read", rc,
270  remaining, offset + length - remaining );
271 
272  if (hints->singleXferAttempt) {
273  MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1),
274  "barrier error");
275  }
276 
277  if (xferRetries > MAX_RETRY) {
278  ERR( "too many retries -- aborting" );
279  }
280  } else if (rc > remaining) /* this should never happen */
281  ERR("too many bytes transferred!?!");
282 
283  assert(rc >= 0);
284  assert(rc <= remaining);
285  remaining -= rc;
286  ptr += rc;
287  xferRetries++;
288  }
289 
290  return(length);
291 }
292 
293 /*
294  * Perform fsync().
295  */
297 {
298  if (ime_native_fsync(*(int *)fd) != 0)
299  WARN("cannot perform fsync on file");
300 }
301 
302 /*
303  * Close a file through the IME interface.
304  */
306 {
307  if (hints->dryRun)
308  return;
309 
310  if (ime_native_close(*(int*)file) != 0)
311  ERRF("Cannot close file descriptor: %d", *(int*)file);
312 
313  free(file);
314 }
315 
316 /*
317  * Delete a file through the IME interface.
318  */
319 void IME_Delete(char *testFileName, aiori_mod_opt_t *param)
320 {
321  if (hints->dryRun)
322  return;
323 
324  if (ime_native_unlink(testFileName) != 0)
325  EWARNF("[RANK %03d]: cannot delete file \"%s\"\n",
326  rank, testFileName);
327 }
328 
329 /*
330  * Determine API version.
331  */
333 {
334  static char ver[1024] = {};
335 #if (IME_NATIVE_API_VERSION >= 120)
336  strcpy(ver, ime_native_version());
337 #else
338  strcpy(ver, "not supported");
339 #endif
340  return ver;
341 }
342 
343 int IME_Statfs(const char *path, ior_aiori_statfs_t *stat_buf,
344  aiori_mod_opt_t *module_options)
345 {
346  IME_UNUSED(module_options);
347 
348 #if (IME_NATIVE_API_VERSION >= 130)
349  struct statvfs statfs_buf;
350 
351  int ret = ime_native_statvfs(path, &statfs_buf);
352  if (ret)
353  return ret;
354  stat_buf->f_bsize = statfs_buf.f_bsize;
355  stat_buf->f_blocks = statfs_buf.f_blocks;
356  stat_buf->f_bfree = statfs_buf.f_bfree;
357  stat_buf->f_files = statfs_buf.f_files;
358  stat_buf->f_ffree = statfs_buf.f_ffree;
359 
360  return 0;
361 #else
362  IME_UNUSED(path);
363  IME_UNUSED(stat_buf);
364 
365  WARN("statfs is currently not supported in IME backend!");
366  return -1;
367 #endif
368 }
369 
370 int IME_Mkdir(const char *path, mode_t mode, aiori_mod_opt_t * module_options)
371 {
372  IME_UNUSED(module_options);
373 
374 #if (IME_NATIVE_API_VERSION >= 130)
375  return ime_native_mkdir(path, mode);
376 #else
377  IME_UNUSED(path);
378  IME_UNUSED(mode);
379 
380  WARN("mkdir not supported in IME backend!");
381  return -1;
382 #endif
383 }
384 
385 int IME_Rmdir(const char *path, aiori_mod_opt_t *module_options)
386 {
387  IME_UNUSED(module_options);
388 
389 #if (IME_NATIVE_API_VERSION >= 130)
390  return ime_native_rmdir(path);
391 #else
392  IME_UNUSED(path);
393 
394  WARN("rmdir not supported in IME backend!");
395  return -1;
396 #endif
397 }
398 
399 /*
400  * Perform stat() through the IME interface.
401  */
402 int IME_Stat(const char *path, struct stat *buf,
403  aiori_mod_opt_t *module_options)
404 {
405  IME_UNUSED(module_options);
406 
407  return ime_native_stat(path, buf);
408 }
409 
410 /*
411  * Use IME stat() to return aggregate file size.
412  */
413 IOR_offset_t IME_GetFileSize(aiori_mod_opt_t *test, char *testFileName)
414 {
415  struct stat stat_buf;
416 
417  if (hints->dryRun)
418  return 0;
419 
420  if (ime_native_stat(testFileName, &stat_buf) != 0)
421  ERRF("cannot get status of written file %s",
422  testFileName);
423  return stat_buf.st_size;
424 }
425 
426 #if (IME_NATIVE_API_VERSION >= 132)
427 /*
428  * Create a file through mknod interface.
429  */
430 int IME_Mknod(char *testFileName)
431 {
432  int ret = ime_native_mknod(testFileName, S_IFREG | S_IRUSR, 0);
433  if (ret < 0)
434  ERR("mknod failed");
435 
436  return ret;
437 }
438 
439 /*
440  * Use IME sync to flush page cache of all opened files.
441  */
442 void IME_Sync(aiori_mod_opt_t *param)
443 {
444  int ret = ime_native_sync(0);
445  if (ret != 0)
446  FAIL("Error executing the sync command.");
447 }
448 #endif
#define ERRF(FORMAT,...)
Definition: aiori-debug.h:77
uint64_t f_blocks
Definition: aiori.h:53
#define O_BINARY
Definition: aiori-IME.c:36
uint64_t f_bfree
Definition: aiori.h:54
#define LAST_OPTION
Definition: option.h:39
CURLcode rc
Definition: aiori-S3-4c.c:111
static aiori_xfer_hint_t * hints
Definition: aiori-IME.c:129
option_help * IME_Options(aiori_mod_opt_t **init_backend_options, aiori_mod_opt_t *init_values)
Definition: aiori-IME.c:72
struct benchmark_options o
Definition: md-workbench.c:128
int IME_Rmdir(const char *, aiori_mod_opt_t *)
Definition: aiori-IME.c:385
uint64_t f_ffree
Definition: aiori.h:57
#define FAIL(...)
Definition: aiori-debug.h:12
#define IOR_APPEND
Definition: aiori.h:31
int IME_Stat(const char *, struct stat *, aiori_mod_opt_t *)
Definition: aiori-IME.c:402
#define IME_UNUSED(x)
Definition: aiori-IME.c:33
char * IME_GetVersion()
Definition: aiori-IME.c:332
#define IOR_RDONLY
Definition: aiori.h:28
#define MPI_CHECK(MPI_STATUS, MSG)
Definition: aiori-debug.h:127
#define WRITE
Definition: iordef.h:86
static bool ime_initialized
Definition: aiori-IME.c:130
void IME_Initialize()
Definition: aiori-IME.c:143
#define IOR_CREAT
Definition: aiori.h:32
#define IOR_EXCL
Definition: aiori.h:34
void IME_Finalize()
Definition: aiori-IME.c:155
uint64_t f_files
Definition: aiori.h:56
MPI_Comm testComm
Definition: utilities.c:71
#define IOR_TRUNC
Definition: aiori.h:33
uint64_t f_bsize
Definition: aiori.h:52
#define WARN(MSG)
Definition: aiori-debug.h:32
int singleXferAttempt
Definition: aiori.h:75
Definition: ior.h:56
void IME_Close(aiori_fd_t *, aiori_mod_opt_t *)
Definition: aiori-IME.c:305
int IME_Statfs(const char *, ior_aiori_statfs_t *, aiori_mod_opt_t *)
Definition: aiori-IME.c:343
#define MAX_RETRY
Definition: iordef.h:101
#define EWARNF(FORMAT,...)
Definition: aiori-debug.h:45
#define IOR_WRONLY
Definition: aiori.h:29
IOR_offset_t IME_Xfer(int, aiori_fd_t *, IOR_size_t *, IOR_offset_t, IOR_offset_t, aiori_mod_opt_t *)
Definition: aiori-IME.c:229
int IME_Mkdir(const char *, mode_t, aiori_mod_opt_t *)
Definition: aiori-IME.c:370
aiori_fd_t * IME_Open(char *, int, aiori_mod_opt_t *)
Definition: aiori-IME.c:186
int IME_Access(const char *, int, aiori_mod_opt_t *)
Definition: aiori-IME.c:168
int rankOffset
Definition: utilities.c:69
long long int IOR_size_t
Definition: iordef.h:110
IOR_offset_t IME_GetFileSize(aiori_mod_opt_t *, char *)
Definition: aiori-IME.c:413
aiori_fd_t * IME_Create(char *, int, aiori_mod_opt_t *)
Definition: aiori-IME.c:178
#define VERBOSE_4
Definition: iordef.h:96
void set_o_direct_flag(int *flag)
Definition: utilities.c:217
#define ERR(MSG)
Definition: aiori-debug.h:92
#define IOR_RDWR
Definition: aiori.h:30
int verbose
Definition: utilities.c:70
int fsyncPerWrite
Definition: aiori.h:70
char * name
Definition: aiori.h:88
void IME_Delete(char *, aiori_mod_opt_t *)
Definition: aiori-IME.c:319
long long int IOR_offset_t
Definition: iordef.h:109
ior_aiori_t ime_aiori
Definition: aiori-IME.c:102
void IME_Xferhints(aiori_xfer_hint_t *params)
Definition: aiori-IME.c:135
int rank
Definition: utilities.c:68
#define TRUE
Definition: iordef.h:66
void IME_Fsync(aiori_fd_t *, aiori_mod_opt_t *)
Definition: aiori-IME.c:296
#define NULL
Definition: iordef.h:70