IOR
aiori-NCMPI.c
Go to the documentation of this file.
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  */
4 /******************************************************************************\
5 * *
6 * Copyright (c) 2003, The Regents of the University of California *
7 * See the file COPYRIGHT for a complete copyright notice and license. *
8 * *
9 ********************************************************************************
10 *
11 * Implement abstract I/O interface for Parallel NetCDF (NCMPI).
12 *
13 \******************************************************************************/
14 
15 #ifdef HAVE_CONFIG_H
16 #include "config.h"
17 #endif
18 
19 #include <stdio.h>
20 #include <stdlib.h>
21 #include <sys/stat.h>
22 #include <pnetcdf.h>
23 
24 #include "ior.h"
25 #include "iordef.h"
26 #include "aiori.h"
27 #include "utilities.h"
28 
29 #define NUM_DIMS 3 /* number of dimensions to data set */
30 
31 /*
32  * NCMPI_CHECK will display a custom error message and then exit the program
33  */
34 #define NCMPI_CHECK(NCMPI_RETURN, MSG) do { \
35  \
36  if (NCMPI_RETURN < 0) { \
37  fprintf(stdout, "** error **\n"); \
38  fprintf(stdout, "ERROR in %s (line %d): %s.\n", \
39  __FILE__, __LINE__, MSG); \
40  fprintf(stdout, "ERROR: %s.\n", ncmpi_strerror(NCMPI_RETURN)); \
41  fprintf(stdout, "** exiting **\n"); \
42  exit(-1); \
43  } \
44 } while(0)
45 
46 /**************************** P R O T O T Y P E S *****************************/
47 
48 static int GetFileMode(IOR_param_t *);
49 
50 static void *NCMPI_Create(char *, IOR_param_t *);
51 static void *NCMPI_Open(char *, IOR_param_t *);
52 static IOR_offset_t NCMPI_Xfer(int, void *, IOR_size_t *,
54 static void NCMPI_Close(void *, IOR_param_t *);
55 static void NCMPI_Delete(char *, IOR_param_t *);
56 static char *NCMPI_GetVersion();
57 static void NCMPI_Fsync(void *, IOR_param_t *);
58 static IOR_offset_t NCMPI_GetFileSize(IOR_param_t *, MPI_Comm, char *);
59 static int NCMPI_Access(const char *, int, IOR_param_t *);
60 
61 /************************** D E C L A R A T I O N S ***************************/
62 
64  .name = "NCMPI",
65  .name_legacy = NULL,
66  .create = NCMPI_Create,
67  .open = NCMPI_Open,
68  .xfer = NCMPI_Xfer,
69  .close = NCMPI_Close,
70  .delete = NCMPI_Delete,
71  .get_version = NCMPI_GetVersion,
72  .fsync = NCMPI_Fsync,
73  .get_file_size = NCMPI_GetFileSize,
74  .statfs = aiori_posix_statfs,
75  .mkdir = aiori_posix_mkdir,
76  .rmdir = aiori_posix_rmdir,
77  .access = NCMPI_Access,
78  .stat = aiori_posix_stat,
79 };
80 
81 /***************************** F U N C T I O N S ******************************/
82 
83 /*
84  * Create and open a file through the NCMPI interface.
85  */
86 static void *NCMPI_Create(char *testFileName, IOR_param_t * param)
87 {
88  int *fd;
89  int fd_mode;
90  MPI_Info mpiHints = MPI_INFO_NULL;
91 
92  /* read and set MPI file hints from hintsFile */
93  SetHints(&mpiHints, param->hintsFileName);
94  if (rank == 0 && param->showHints) {
95  fprintf(stdout, "\nhints passed to MPI_File_open() {\n");
96  ShowHints(&mpiHints);
97  fprintf(stdout, "}\n");
98  }
99 
100  fd = (int *)malloc(sizeof(int));
101  if (fd == NULL)
102  ERR("malloc() failed");
103 
104  fd_mode = GetFileMode(param);
105  NCMPI_CHECK(ncmpi_create(testComm, testFileName, fd_mode,
106  mpiHints, fd), "cannot create file");
107 
108  /* free up the mpiHints object */
109  if (mpiHints != MPI_INFO_NULL)
110  MPI_CHECK(MPI_Info_free(&mpiHints), "cannot free file info");
111 
112 #if defined(PNETCDF_VERSION_MAJOR) && (PNETCDF_VERSION_MAJOR > 1 || PNETCDF_VERSION_MINOR >= 2)
113  /* ncmpi_get_file_info is first available in 1.2.0 */
114  if (rank == 0 && param->showHints) {
115  MPI_Info info_used;
116  MPI_CHECK(ncmpi_get_file_info(*fd, &info_used),
117  "cannot inquire file info");
118  /* print the MPI file hints currently used */
119  fprintf(stdout, "\nhints returned from opened file {\n");
120  ShowHints(&info_used);
121  fprintf(stdout, "}\n");
122  MPI_CHECK(MPI_Info_free(&info_used), "cannot free file info");
123  }
124 #endif
125 
126  return (fd);
127 }
128 
129 /*
130  * Open a file through the NCMPI interface.
131  */
132 static void *NCMPI_Open(char *testFileName, IOR_param_t * param)
133 {
134  int *fd;
135  int fd_mode;
136  MPI_Info mpiHints = MPI_INFO_NULL;
137 
138  /* read and set MPI file hints from hintsFile */
139  SetHints(&mpiHints, param->hintsFileName);
140  if (rank == 0 && param->showHints) {
141  fprintf(stdout, "\nhints passed to MPI_File_open() {\n");
142  ShowHints(&mpiHints);
143  fprintf(stdout, "}\n");
144  }
145 
146  fd = (int *)malloc(sizeof(int));
147  if (fd == NULL)
148  ERR("malloc() failed");
149 
150  fd_mode = GetFileMode(param);
151  NCMPI_CHECK(ncmpi_open(testComm, testFileName, fd_mode,
152  mpiHints, fd), "cannot open file");
153 
154  /* free up the mpiHints object */
155  if (mpiHints != MPI_INFO_NULL)
156  MPI_CHECK(MPI_Info_free(&mpiHints), "cannot free file info");
157 
158 #if defined(PNETCDF_VERSION_MAJOR) && (PNETCDF_VERSION_MAJOR > 1 || PNETCDF_VERSION_MINOR >= 2)
159  /* ncmpi_get_file_info is first available in 1.2.0 */
160  if (rank == 0 && param->showHints) {
161  MPI_Info info_used;
162  MPI_CHECK(ncmpi_get_file_info(*fd, &info_used),
163  "cannot inquire file info");
164  /* print the MPI file hints currently used */
165  fprintf(stdout, "\nhints returned from opened file {\n");
166  ShowHints(&info_used);
167  fprintf(stdout, "}\n");
168  MPI_CHECK(MPI_Info_free(&info_used), "cannot free file info");
169  }
170 #endif
171 
172  return (fd);
173 }
174 
175 /*
176  * Write or read access to file using the NCMPI interface.
177  */
178 static IOR_offset_t NCMPI_Xfer(int access, void *fd, IOR_size_t * buffer,
179  IOR_offset_t length, IOR_param_t * param)
180 {
181  signed char *bufferPtr = (signed char *)buffer;
182  static int firstReadCheck = FALSE, startDataSet;
183  int var_id, dim_id[NUM_DIMS];
184  MPI_Offset bufSize[NUM_DIMS], offset[NUM_DIMS];
185  IOR_offset_t segmentPosition;
186  int segmentNum, transferNum;
187 
188  /* determine by offset if need to start data set */
189  if (param->filePerProc == TRUE) {
190  segmentPosition = (IOR_offset_t) 0;
191  } else {
192  segmentPosition =
193  (IOR_offset_t) ((rank + rankOffset) % param->numTasks)
194  * param->blockSize;
195  }
196  if ((int)(param->offset - segmentPosition) == 0) {
197  startDataSet = TRUE;
198  /*
199  * this toggle is for the read check operation, which passes through
200  * this function twice; note that this function will open a data set
201  * only on the first read check and close only on the second
202  */
203  if (access == READCHECK) {
204  if (firstReadCheck == TRUE) {
205  firstReadCheck = FALSE;
206  } else {
207  firstReadCheck = TRUE;
208  }
209  }
210  }
211 
212  if (startDataSet == TRUE &&
213  (access != READCHECK || firstReadCheck == TRUE)) {
214  if (access == WRITE) {
215  int numTransfers =
216  param->blockSize / param->transferSize;
217 
218  /* reshape 1D array to 3D array:
219  [segmentCount*numTasks][numTransfers][transferSize]
220  Requirement: none of these dimensions should be > 4G,
221  */
222  NCMPI_CHECK(ncmpi_def_dim
223  (*(int *)fd, "segments_times_np",
224  NC_UNLIMITED, &dim_id[0]),
225  "cannot define data set dimensions");
226  NCMPI_CHECK(ncmpi_def_dim
227  (*(int *)fd, "number_of_transfers",
228  numTransfers, &dim_id[1]),
229  "cannot define data set dimensions");
230  NCMPI_CHECK(ncmpi_def_dim
231  (*(int *)fd, "transfer_size",
232  param->transferSize, &dim_id[2]),
233  "cannot define data set dimensions");
234  NCMPI_CHECK(ncmpi_def_var
235  (*(int *)fd, "data_var", NC_BYTE, NUM_DIMS,
236  dim_id, &var_id),
237  "cannot define data set variables");
238  NCMPI_CHECK(ncmpi_enddef(*(int *)fd),
239  "cannot close data set define mode");
240 
241  } else {
242  NCMPI_CHECK(ncmpi_inq_varid
243  (*(int *)fd, "data_var", &var_id),
244  "cannot retrieve data set variable");
245  }
246 
247  if (param->collective == FALSE) {
248  NCMPI_CHECK(ncmpi_begin_indep_data(*(int *)fd),
249  "cannot enable independent data mode");
250  }
251 
252  param->var_id = var_id;
253  startDataSet = FALSE;
254  }
255 
256  var_id = param->var_id;
257 
258  /* calculate the segment number */
259  segmentNum = param->offset / (param->numTasks * param->blockSize);
260 
261  /* calculate the transfer number in each block */
262  transferNum = param->offset % param->blockSize / param->transferSize;
263 
264  /* read/write the 3rd dim of the dataset, each is of
265  amount param->transferSize */
266  bufSize[0] = 1;
267  bufSize[1] = 1;
268  bufSize[2] = param->transferSize;
269 
270  offset[0] = segmentNum * param->numTasks + rank;
271  offset[1] = transferNum;
272  offset[2] = 0;
273 
274  /* access the file */
275  if (access == WRITE) { /* WRITE */
276  if (param->collective) {
277  NCMPI_CHECK(ncmpi_put_vara_schar_all
278  (*(int *)fd, var_id, offset, bufSize,
279  bufferPtr),
280  "cannot write to data set");
281  } else {
282  NCMPI_CHECK(ncmpi_put_vara_schar
283  (*(int *)fd, var_id, offset, bufSize,
284  bufferPtr),
285  "cannot write to data set");
286  }
287  } else { /* READ or CHECK */
288  if (param->collective == TRUE) {
289  NCMPI_CHECK(ncmpi_get_vara_schar_all
290  (*(int *)fd, var_id, offset, bufSize,
291  bufferPtr),
292  "cannot read from data set");
293  } else {
294  NCMPI_CHECK(ncmpi_get_vara_schar
295  (*(int *)fd, var_id, offset, bufSize,
296  bufferPtr),
297  "cannot read from data set");
298  }
299  }
300 
301  return (length);
302 }
303 
304 /*
305  * Perform fsync().
306  */
307 static void NCMPI_Fsync(void *fd, IOR_param_t * param)
308 {
309  ;
310 }
311 
312 /*
313  * Close a file through the NCMPI interface.
314  */
315 static void NCMPI_Close(void *fd, IOR_param_t * param)
316 {
317  if (param->collective == FALSE) {
318  NCMPI_CHECK(ncmpi_end_indep_data(*(int *)fd),
319  "cannot disable independent data mode");
320  }
321  NCMPI_CHECK(ncmpi_close(*(int *)fd), "cannot close file");
322  free(fd);
323 }
324 
325 /*
326  * Delete a file through the NCMPI interface.
327  */
328 static void NCMPI_Delete(char *testFileName, IOR_param_t * param)
329 {
330  return(MPIIO_Delete(testFileName, param));
331 }
332 
333 /*
334  * Determine api version.
335  */
336 static char* NCMPI_GetVersion()
337 {
338  return (char *)ncmpi_inq_libvers();
339 }
340 
341 /*
342  * Return the correct file mode for NCMPI.
343  */
345 {
346  int fd_mode = 0;
347 
348  /* set IOR file flags to NCMPI flags */
349  /* -- file open flags -- */
350  if (param->openFlags & IOR_RDONLY) {
351  fd_mode |= NC_NOWRITE;
352  }
353  if (param->openFlags & IOR_WRONLY) {
354  fprintf(stdout, "File write only not implemented in NCMPI\n");
355  }
356  if (param->openFlags & IOR_RDWR) {
357  fd_mode |= NC_WRITE;
358  }
359  if (param->openFlags & IOR_APPEND) {
360  fprintf(stdout, "File append not implemented in NCMPI\n");
361  }
362  if (param->openFlags & IOR_CREAT) {
363  fd_mode |= NC_CLOBBER;
364  }
365  if (param->openFlags & IOR_EXCL) {
366  fprintf(stdout, "Exclusive access not implemented in NCMPI\n");
367  }
368  if (param->openFlags & IOR_TRUNC) {
369  fprintf(stdout, "File truncation not implemented in NCMPI\n");
370  }
371  if (param->openFlags & IOR_DIRECT) {
372  fprintf(stdout, "O_DIRECT not implemented in NCMPI\n");
373  }
374 
375  /* to enable > 4GB file size */
376  fd_mode |= NC_64BIT_OFFSET;
377 
378  return (fd_mode);
379 }
380 
381 /*
382  * Use MPIIO call to get file size.
383  */
385  char *testFileName)
386 {
387  return(MPIIO_GetFileSize(test, testComm, testFileName));
388 }
389 
390 /*
391  * Use MPIIO call to check for access.
392  */
393 static int NCMPI_Access(const char *path, int mode, IOR_param_t *param)
394 {
395  return(MPIIO_Access(path, mode, param));
396 }
int showHints
Definition: ior.h:132
static char * NCMPI_GetVersion()
Definition: aiori-NCMPI.c:336
#define ERR(MSG)
Definition: iordef.h:184
void ShowHints(MPI_Info *mpiHints)
Definition: utilities.c:515
int filePerProc
Definition: ior.h:111
IOR_offset_t transferSize
Definition: ior.h:125
int aiori_posix_rmdir(const char *path, IOR_param_t *param)
Definition: aiori.c:185
static IOR_offset_t NCMPI_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori-NCMPI.c:178
#define READCHECK
Definition: iordef.h:98
static int GetFileMode(IOR_param_t *)
Definition: aiori-NCMPI.c:344
#define IOR_APPEND
Definition: aiori.h:37
int aiori_posix_mkdir(const char *path, mode_t mode, IOR_param_t *param)
Definition: aiori.c:180
ior_aiori_t ncmpi_aiori
Definition: aiori-NCMPI.c:63
#define IOR_RDONLY
Definition: aiori.h:34
unsigned int openFlags
Definition: ior.h:88
#define WRITE
Definition: iordef.h:95
static IOR_offset_t NCMPI_GetFileSize(IOR_param_t *, MPI_Comm, char *)
Definition: aiori-NCMPI.c:384
int aiori_posix_statfs(const char *path, ior_aiori_statfs_t *stat_buf, IOR_param_t *param)
Definition: aiori.c:155
#define IOR_CREAT
Definition: aiori.h:38
#define IOR_EXCL
Definition: aiori.h:40
char * hintsFileName
Definition: ior.h:95
MPI_Comm testComm
Definition: utilities.c:60
#define IOR_TRUNC
Definition: aiori.h:39
#define MPI_CHECK(MPI_STATUS, MSG)
Definition: iordef.h:224
Definition: ior.h:48
void MPIIO_Delete(char *testFileName, IOR_param_t *param)
Definition: aiori-MPIIO.c:431
static IOR_param_t param
Definition: mdtest.c:170
#define NUM_DIMS
Definition: aiori-NCMPI.c:29
int MPIIO_Access(const char *path, int mode, IOR_param_t *param)
Definition: aiori-MPIIO.c:70
#define IOR_WRONLY
Definition: aiori.h:35
#define FALSE
Definition: iordef.h:71
int rankOffset
Definition: utilities.c:58
long long int IOR_size_t
Definition: iordef.h:123
int numTasks
Definition: ior.h:99
static void * NCMPI_Create(char *, IOR_param_t *)
Definition: aiori-NCMPI.c:86
int collective
Definition: ior.h:122
IOR_offset_t offset
Definition: ior.h:126
int aiori_posix_stat(const char *path, struct stat *buf, IOR_param_t *param)
Definition: aiori.c:195
static void NCMPI_Fsync(void *, IOR_param_t *)
Definition: aiori-NCMPI.c:307
void SetHints(MPI_Info *mpiHints, char *hintsFileName)
Definition: utilities.c:454
int var_id
Definition: ior.h:192
#define IOR_RDWR
Definition: aiori.h:36
static void * NCMPI_Open(char *, IOR_param_t *)
Definition: aiori-NCMPI.c:132
#define NCMPI_CHECK(NCMPI_RETURN, MSG)
Definition: aiori-NCMPI.c:34
char * name
Definition: aiori.h:68
long long int IOR_offset_t
Definition: iordef.h:122
static void NCMPI_Delete(char *, IOR_param_t *)
Definition: aiori-NCMPI.c:328
int rank
Definition: utilities.c:57
static void NCMPI_Close(void *, IOR_param_t *)
Definition: aiori-NCMPI.c:315
IOR_offset_t blockSize
Definition: ior.h:124
#define TRUE
Definition: iordef.h:75
#define IOR_DIRECT
Definition: aiori.h:41
IOR_offset_t MPIIO_GetFileSize(IOR_param_t *test, MPI_Comm testComm, char *testFileName)
Definition: aiori-MPIIO.c:496
#define NULL
Definition: iordef.h:79
static int NCMPI_Access(const char *, int, IOR_param_t *)
Definition: aiori-NCMPI.c:393