IOR
aiori-NCMPI.c
Go to the documentation of this file.
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  */
4 /******************************************************************************\
5 * *
6 * Copyright (c) 2003, The Regents of the University of California *
7 * See the file COPYRIGHT for a complete copyright notice and license. *
8 * *
9 ********************************************************************************
10 *
11 * Implement abstract I/O interface for Parallel NetCDF (NCMPI).
12 *
13 \******************************************************************************/
14 
15 #ifdef HAVE_CONFIG_H
16 #include "config.h"
17 #endif
18 
19 #include <stdio.h>
20 #include <stdlib.h>
21 #include <sys/stat.h>
22 #include <pnetcdf.h>
23 
24 #include "ior.h"
25 #include "iordef.h"
26 #include "aiori.h"
27 #include "utilities.h"
28 
29 #define NUM_DIMS 3 /* number of dimensions to data set */
30 
31 /*
32  * NCMPI_CHECK will display a custom error message and then exit the program
33  */
34 #define NCMPI_CHECK(NCMPI_RETURN, MSG) do { \
35  \
36  if (NCMPI_RETURN < 0) { \
37  fprintf(stdout, "** error **\n"); \
38  fprintf(stdout, "ERROR in %s (line %d): %s.\n", \
39  __FILE__, __LINE__, MSG); \
40  fprintf(stdout, "ERROR: %s.\n", ncmpi_strerror(NCMPI_RETURN)); \
41  fprintf(stdout, "** exiting **\n"); \
42  exit(-1); \
43  } \
44 } while(0)
45 
46 /**************************** P R O T O T Y P E S *****************************/
47 
48 static int GetFileMode(int flags);
49 
50 static aiori_fd_t *NCMPI_Create(char *, int iorflags, aiori_mod_opt_t *);
51 static aiori_fd_t *NCMPI_Open(char *, int iorflags, aiori_mod_opt_t *);
54 static void NCMPI_Close(aiori_fd_t *, aiori_mod_opt_t *);
55 static void NCMPI_Delete(char *, aiori_mod_opt_t *);
56 static char *NCMPI_GetVersion();
57 static void NCMPI_Fsync(aiori_fd_t *, aiori_mod_opt_t *);
59 static int NCMPI_Access(const char *, int, aiori_mod_opt_t *);
60 
61 /************************** D E C L A R A T I O N S ***************************/
63 
64 static void NCMPI_xfer_hints(aiori_xfer_hint_t * params){
65  hints = params;
66 
67  MPIIO_xfer_hints(params);
68 }
69 
70 typedef struct {
71  int showHints; /* show hints */
72  char * hintsFileName; /* full name for hints file */
73 
74  /* runtime variables */
75  int var_id; /* variable id handle for data set */
79 
80 
81 static option_help * NCMPI_options(aiori_mod_opt_t ** init_backend_options, aiori_mod_opt_t * init_values){
82  ncmpi_options_t * o = malloc(sizeof(ncmpi_options_t));
83  if (init_values != NULL){
84  memcpy(o, init_values, sizeof(ncmpi_options_t));
85  }else{
86  memset(o, 0, sizeof(ncmpi_options_t));
87  }
88  *init_backend_options = (aiori_mod_opt_t*) o;
89 
90  option_help h [] = {
91  {0, "mpiio.hintsFileName","Full name for hints file", OPTION_OPTIONAL_ARGUMENT, 's', & o->hintsFileName},
92  {0, "mpiio.showHints", "Show MPI hints", OPTION_FLAG, 'd', & o->showHints},
94  };
95  option_help * help = malloc(sizeof(h));
96  memcpy(help, h, sizeof(h));
97  return help;
98 }
99 
101  .name = "NCMPI",
102  .name_legacy = NULL,
103  .create = NCMPI_Create,
104  .open = NCMPI_Open,
105  .xfer = NCMPI_Xfer,
106  .close = NCMPI_Close,
107  .delete = NCMPI_Delete,
108  .get_version = NCMPI_GetVersion,
109  .fsync = NCMPI_Fsync,
110  .get_file_size = NCMPI_GetFileSize,
111  .statfs = aiori_posix_statfs,
112  .mkdir = aiori_posix_mkdir,
113  .rmdir = aiori_posix_rmdir,
114  .access = NCMPI_Access,
115  .stat = aiori_posix_stat,
116  .get_options = NCMPI_options,
117  .xfer_hints = NCMPI_xfer_hints,
118 };
119 
120 /***************************** F U N C T I O N S ******************************/
121 
122 /*
123  * Create and open a file through the NCMPI interface.
124  */
125 static aiori_fd_t *NCMPI_Create(char *testFileName, int iorflags, aiori_mod_opt_t * param)
126 {
127  int *fd;
128  int fd_mode;
129  MPI_Info mpiHints = MPI_INFO_NULL;
130  ncmpi_options_t * o = (ncmpi_options_t*) param;
131 
132  /* read and set MPI file hints from hintsFile */
133  SetHints(&mpiHints, o->hintsFileName);
134  if (rank == 0 && o->showHints) {
135  fprintf(stdout, "\nhints passed to MPI_File_open() {\n");
136  ShowHints(&mpiHints);
137  fprintf(stdout, "}\n");
138  }
139 
140  fd = (int *)malloc(sizeof(int));
141  if (fd == NULL)
142  ERR("malloc() failed");
143 
144  fd_mode = GetFileMode(iorflags);
145  NCMPI_CHECK(ncmpi_create(testComm, testFileName, fd_mode,
146  mpiHints, fd), "cannot create file");
147 
148  /* free up the mpiHints object */
149  if (mpiHints != MPI_INFO_NULL)
150  MPI_CHECK(MPI_Info_free(&mpiHints), "cannot free file info");
151 
152 #if defined(PNETCDF_VERSION_MAJOR) && (PNETCDF_VERSION_MAJOR > 1 || PNETCDF_VERSION_MINOR >= 2)
153  /* ncmpi_get_file_info is first available in 1.2.0 */
154  if (rank == 0 && o->showHints) {
155  MPI_Info info_used;
156  MPI_CHECK(ncmpi_get_file_info(*fd, &info_used),
157  "cannot inquire file info");
158  /* print the MPI file hints currently used */
159  fprintf(stdout, "\nhints returned from opened file {\n");
160  ShowHints(&info_used);
161  fprintf(stdout, "}\n");
162  MPI_CHECK(MPI_Info_free(&info_used), "cannot free file info");
163  }
164 #endif
165 
166  return (aiori_fd_t*)(fd);
167 }
168 
169 /*
170  * Open a file through the NCMPI interface.
171  */
172 static aiori_fd_t *NCMPI_Open(char *testFileName, int iorflags, aiori_mod_opt_t * param)
173 {
174  int *fd;
175  int fd_mode;
176  MPI_Info mpiHints = MPI_INFO_NULL;
177  ncmpi_options_t * o = (ncmpi_options_t*) param;
178 
179  /* read and set MPI file hints from hintsFile */
180  SetHints(&mpiHints, o->hintsFileName);
181  if (rank == 0 && o->showHints) {
182  fprintf(stdout, "\nhints passed to MPI_File_open() {\n");
183  ShowHints(&mpiHints);
184  fprintf(stdout, "}\n");
185  }
186 
187  fd = (int *)malloc(sizeof(int));
188  if (fd == NULL)
189  ERR("malloc() failed");
190 
191  fd_mode = GetFileMode(iorflags);
192  NCMPI_CHECK(ncmpi_open(testComm, testFileName, fd_mode,
193  mpiHints, fd), "cannot open file");
194 
195  /* free up the mpiHints object */
196  if (mpiHints != MPI_INFO_NULL)
197  MPI_CHECK(MPI_Info_free(&mpiHints), "cannot free file info");
198 
199 #if defined(PNETCDF_VERSION_MAJOR) && (PNETCDF_VERSION_MAJOR > 1 || PNETCDF_VERSION_MINOR >= 2)
200  /* ncmpi_get_file_info is first available in 1.2.0 */
201  if (rank == 0 && o->showHints) {
202  MPI_Info info_used;
203  MPI_CHECK(ncmpi_get_file_info(*fd, &info_used),
204  "cannot inquire file info");
205  /* print the MPI file hints currently used */
206  fprintf(stdout, "\nhints returned from opened file {\n");
207  ShowHints(&info_used);
208  fprintf(stdout, "}\n");
209  MPI_CHECK(MPI_Info_free(&info_used), "cannot free file info");
210  }
211 #endif
212 
213  return (aiori_fd_t*)(fd);
214 }
215 
216 /*
217  * Write or read access to file using the NCMPI interface.
218  */
219 static IOR_offset_t NCMPI_Xfer(int access, aiori_fd_t *fd, IOR_size_t * buffer, IOR_offset_t transferSize, IOR_offset_t offset, aiori_mod_opt_t * param)
220 {
221  signed char *bufferPtr = (signed char *)buffer;
222  ncmpi_options_t * o = (ncmpi_options_t*) param;
223  int var_id, dim_id[NUM_DIMS];
224  MPI_Offset bufSize[NUM_DIMS], offsets[NUM_DIMS];
225  IOR_offset_t segmentPosition;
226  int segmentNum, transferNum;
227 
228  /* determine by offset if need to start data set */
229  if (hints->filePerProc == TRUE) {
230  segmentPosition = (IOR_offset_t) 0;
231  } else {
232  segmentPosition = (IOR_offset_t) ((rank + rankOffset) % hints->numTasks) * hints->blockSize;
233  }
234  if ((int)(offset - segmentPosition) == 0) {
235  o->startDataSet = TRUE;
236  /*
237  * this toggle is for the read check operation, which passes through
238  * this function twice; note that this function will open a data set
239  * only on the first read check and close only on the second
240  */
241  if (access == READCHECK) {
242  o->firstReadCheck = ! o->firstReadCheck;
243  }
244  }
245 
246  if (o->startDataSet == TRUE &&
247  (access != READCHECK || o->firstReadCheck == TRUE)) {
248  if (access == WRITE) {
249  int numTransfers = hints->blockSize / hints->transferSize;
250 
251  /* reshape 1D array to 3D array:
252  [segmentCount*numTasks][numTransfers][transferSize]
253  Requirement: none of these dimensions should be > 4G,
254  */
255  NCMPI_CHECK(ncmpi_def_dim
256  (*(int *)fd, "segments_times_np",
257  NC_UNLIMITED, &dim_id[0]),
258  "cannot define data set dimensions");
259  NCMPI_CHECK(ncmpi_def_dim
260  (*(int *)fd, "number_of_transfers",
261  numTransfers, &dim_id[1]),
262  "cannot define data set dimensions");
263  NCMPI_CHECK(ncmpi_def_dim
264  (*(int *)fd, "transfer_size",
265  hints->transferSize, &dim_id[2]),
266  "cannot define data set dimensions");
267  NCMPI_CHECK(ncmpi_def_var
268  (*(int *)fd, "data_var", NC_BYTE, NUM_DIMS,
269  dim_id, &var_id),
270  "cannot define data set variables");
271  NCMPI_CHECK(ncmpi_enddef(*(int *)fd),
272  "cannot close data set define mode");
273 
274  } else {
275  NCMPI_CHECK(ncmpi_inq_varid
276  (*(int *)fd, "data_var", &var_id),
277  "cannot retrieve data set variable");
278  }
279 
280  if (hints->collective == FALSE) {
281  NCMPI_CHECK(ncmpi_begin_indep_data(*(int *)fd),
282  "cannot enable independent data mode");
283  }
284 
285  o->var_id = var_id;
286  o->startDataSet = FALSE;
287  }
288 
289  var_id = o->var_id;
290 
291  /* calculate the segment number */
292  segmentNum = offset / (hints->numTasks * hints->blockSize);
293 
294  /* calculate the transfer number in each block */
295  transferNum = offset % hints->blockSize / hints->transferSize;
296 
297  /* read/write the 3rd dim of the dataset, each is of
298  amount param->transferSize */
299  bufSize[0] = 1;
300  bufSize[1] = 1;
301  bufSize[2] = transferSize;
302 
303  offsets[0] = segmentNum * hints->numTasks + rank;
304  offsets[1] = transferNum;
305  offsets[2] = 0;
306 
307  /* access the file */
308  if (access == WRITE) { /* WRITE */
309  if (hints->collective) {
310  NCMPI_CHECK(ncmpi_put_vara_schar_all
311  (*(int *)fd, var_id, offsets, bufSize, bufferPtr),
312  "cannot write to data set");
313  } else {
314  NCMPI_CHECK(ncmpi_put_vara_schar
315  (*(int *)fd, var_id, offsets, bufSize, bufferPtr),
316  "cannot write to data set");
317  }
318  } else { /* READ or CHECK */
319  if (hints->collective == TRUE) {
320  NCMPI_CHECK(ncmpi_get_vara_schar_all
321  (*(int *)fd, var_id, offsets, bufSize, bufferPtr),
322  "cannot read from data set");
323  } else {
324  NCMPI_CHECK(ncmpi_get_vara_schar
325  (*(int *)fd, var_id, offsets, bufSize, bufferPtr),
326  "cannot read from data set");
327  }
328  }
329 
330  return (transferSize);
331 }
332 
333 /*
334  * Perform fsync().
335  */
336 static void NCMPI_Fsync(aiori_fd_t *fd, aiori_mod_opt_t * param)
337 {
338 }
339 
340 /*
341  * Close a file through the NCMPI interface.
342  */
343 static void NCMPI_Close(aiori_fd_t *fd, aiori_mod_opt_t * param)
344 {
345  if (hints->collective == FALSE) {
346  NCMPI_CHECK(ncmpi_end_indep_data(*(int *)fd),
347  "cannot disable independent data mode");
348  }
349  NCMPI_CHECK(ncmpi_close(*(int *)fd), "cannot close file");
350  free(fd);
351 }
352 
353 /*
354  * Delete a file through the NCMPI interface.
355  */
356 static void NCMPI_Delete(char *testFileName, aiori_mod_opt_t * param)
357 {
358  return(MPIIO_Delete(testFileName, param));
359 }
360 
361 /*
362  * Determine api version.
363  */
364 static char* NCMPI_GetVersion()
365 {
366  return (char *)ncmpi_inq_libvers();
367 }
368 
369 /*
370  * Return the correct file mode for NCMPI.
371  */
372 static int GetFileMode(int flags)
373 {
374  int fd_mode = 0;
375 
376  /* set IOR file flags to NCMPI flags */
377  /* -- file open flags -- */
378  if (flags & IOR_RDONLY) {
379  fd_mode |= NC_NOWRITE;
380  }
381  if (flags & IOR_WRONLY) {
382  WARN("File write only not implemented in NCMPI");
383  }
384  if (flags & IOR_RDWR) {
385  fd_mode |= NC_WRITE;
386  }
387  if (flags & IOR_APPEND) {
388  WARN("File append not implemented in NCMPI");
389  }
390  if (flags & IOR_CREAT) {
391  fd_mode |= NC_CLOBBER;
392  }
393  if (flags & IOR_EXCL) {
394  WARN("Exclusive access not implemented in NCMPI");
395  }
396  if (flags & IOR_TRUNC) {
397  WARN("File truncation not implemented in NCMPI");
398  }
399  if (flags & IOR_DIRECT) {
400  WARN("O_DIRECT not implemented in NCMPI");
401  }
402 
403  /* to enable > 4GB file size */
404  fd_mode |= NC_64BIT_OFFSET;
405 
406  return (fd_mode);
407 }
408 
409 /*
410  * Use MPIIO call to get file size.
411  */
413  char *testFileName)
414 {
415  return(MPIIO_GetFileSize(opt, testFileName));
416 }
417 
418 /*
419  * Use MPIIO call to check for access.
420  */
421 static int NCMPI_Access(const char *path, int mode, aiori_mod_opt_t *param)
422 {
423  return(MPIIO_Access(path, mode, param));
424 }
static char * NCMPI_GetVersion()
Definition: aiori-NCMPI.c:364
void ShowHints(MPI_Info *mpiHints)
Definition: utilities.c:572
#define LAST_OPTION
Definition: option.h:39
struct benchmark_options o
Definition: md-workbench.c:128
IOR_offset_t blockSize
Definition: aiori.h:72
static aiori_fd_t * NCMPI_Create(char *, int iorflags, aiori_mod_opt_t *)
Definition: aiori-NCMPI.c:125
static void NCMPI_Delete(char *, aiori_mod_opt_t *)
Definition: aiori-NCMPI.c:356
#define READCHECK
Definition: iordef.h:89
static aiori_xfer_hint_t * hints
Definition: aiori-NCMPI.c:62
#define IOR_APPEND
Definition: aiori.h:31
IOR_offset_t MPIIO_GetFileSize(aiori_mod_opt_t *module_options, char *testFileName)
Definition: aiori-MPIIO.c:588
ior_aiori_t ncmpi_aiori
Definition: aiori-NCMPI.c:100
#define IOR_RDONLY
Definition: aiori.h:28
#define MPI_CHECK(MPI_STATUS, MSG)
Definition: aiori-debug.h:127
#define WRITE
Definition: iordef.h:86
int aiori_posix_stat(const char *path, struct stat *buf, aiori_mod_opt_t *module_options)
Definition: aiori.c:227
#define IOR_CREAT
Definition: aiori.h:32
#define IOR_EXCL
Definition: aiori.h:34
static aiori_fd_t * NCMPI_Open(char *, int iorflags, aiori_mod_opt_t *)
Definition: aiori-NCMPI.c:172
int MPIIO_Access(const char *path, int mode, aiori_mod_opt_t *module_options)
Definition: aiori-MPIIO.c:139
MPI_Comm testComm
Definition: utilities.c:71
#define IOR_TRUNC
Definition: aiori.h:33
#define WARN(MSG)
Definition: aiori-debug.h:32
int collective
Definition: aiori.h:66
void MPIIO_xfer_hints(aiori_xfer_hint_t *params)
Definition: aiori-MPIIO.c:111
static void NCMPI_Fsync(aiori_fd_t *, aiori_mod_opt_t *)
Definition: aiori-NCMPI.c:336
static void NCMPI_Close(aiori_fd_t *, aiori_mod_opt_t *)
Definition: aiori-NCMPI.c:343
Definition: ior.h:56
IOR_offset_t transferSize
Definition: aiori.h:73
#define NUM_DIMS
Definition: aiori-NCMPI.c:29
#define IOR_WRONLY
Definition: aiori.h:29
#define FALSE
Definition: iordef.h:62
int rankOffset
Definition: utilities.c:69
long long int IOR_size_t
Definition: iordef.h:110
int aiori_posix_rmdir(const char *path, aiori_mod_opt_t *module_options)
Definition: aiori.c:217
static option_help * NCMPI_options(aiori_mod_opt_t **init_backend_options, aiori_mod_opt_t *init_values)
Definition: aiori-NCMPI.c:81
static IOR_offset_t NCMPI_Xfer(int, aiori_fd_t *, IOR_size_t *, IOR_offset_t, IOR_offset_t, aiori_mod_opt_t *)
Definition: aiori-NCMPI.c:219
int aiori_posix_mkdir(const char *path, mode_t mode, aiori_mod_opt_t *module_options)
Definition: aiori.c:212
static int GetFileMode(int flags)
Definition: aiori-NCMPI.c:372
int aiori_posix_statfs(const char *path, ior_aiori_statfs_t *stat_buf, aiori_mod_opt_t *module_options)
Definition: aiori.c:166
void SetHints(MPI_Info *mpiHints, char *hintsFileName)
Definition: utilities.c:511
#define ERR(MSG)
Definition: aiori-debug.h:92
#define IOR_RDWR
Definition: aiori.h:30
void MPIIO_Delete(char *testFileName, aiori_mod_opt_t *module_options)
Definition: aiori-MPIIO.c:521
static IOR_offset_t NCMPI_GetFileSize(aiori_mod_opt_t *, char *)
Definition: aiori-NCMPI.c:412
static int NCMPI_Access(const char *, int, aiori_mod_opt_t *)
Definition: aiori-NCMPI.c:421
char * hintsFileName
Definition: aiori-NCMPI.c:72
#define NCMPI_CHECK(NCMPI_RETURN, MSG)
Definition: aiori-NCMPI.c:34
char * name
Definition: aiori.h:88
int filePerProc
Definition: aiori.h:65
long long int IOR_offset_t
Definition: iordef.h:109
int rank
Definition: utilities.c:68
#define TRUE
Definition: iordef.h:66
#define IOR_DIRECT
Definition: aiori.h:35
static void NCMPI_xfer_hints(aiori_xfer_hint_t *params)
Definition: aiori-NCMPI.c:64
#define NULL
Definition: iordef.h:70