IOR
mdtest.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2003, The Regents of the University of California.
3  * Produced at the Lawrence Livermore National Laboratory.
4  * Written by Christopher J. Morrone <morrone@llnl.gov>,
5  * Bill Loewe <loewe@loewe.net>, Tyce McLarty <mclarty@llnl.gov>,
6  * and Ryan Kroiss <rrkroiss@lanl.gov>.
7  * All rights reserved.
8  * UCRL-CODE-155800
9  *
10  * Please read the COPYRIGHT file.
11  *
12  * This program is free software; you can redistribute it and/or modify
13  * it under the terms of the GNU General Public License (as published by
14  * the Free Software Foundation) version 2, dated June 1991.
15  *
16  * This program is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19  * terms and conditions of the GNU General Public License for more details.
20  *
21  * You should have received a copy of the GNU General Public License
22  * along with this program; if not, write to the Free Software
23  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24  *
25  * CVS info:
26  * $RCSfile: mdtest.c,v $
27  * $Revision: 1.4 $
28  * $Date: 2013/11/27 17:05:31 $
29  * $Author: brettkettering $
30  */
31 #include <limits.h>
32 #include <math.h>
33 #include <stdio.h>
34 #include <stdlib.h>
35 #include <stdbool.h>
36 #include <inttypes.h>
37 #include <sys/types.h>
38 #include <sys/stat.h>
39 #include <stdarg.h>
40 
41 #include "option.h"
42 #include "utilities.h"
43 
44 #if HAVE_SYS_PARAM_H
45 #include <sys/param.h>
46 #endif
47 
48 #if HAVE_SYS_MOUNT_H
49 #include <sys/mount.h>
50 #endif
51 
52 #if HAVE_SYS_STATFS_H
53 #include <sys/statfs.h>
54 #endif
55 
56 #if HAVE_SYS_STATVFS_H
57 #include <sys/statvfs.h>
58 #endif
59 
60 #include <fcntl.h>
61 #include <string.h>
62 
63 #if HAVE_STRINGS_H
64 #include <strings.h>
65 #endif
66 
67 #include <unistd.h>
68 #include <dirent.h>
69 #include <errno.h>
70 #include <time.h>
71 #include <sys/time.h>
72 
73 #include "aiori.h"
74 #include "ior.h"
75 #include "mdtest.h"
76 
77 #include <mpi.h>
78 
79 #ifdef HAVE_LUSTRE_LUSTREAPI
80 #include <lustre/lustreapi.h>
81 #endif /* HAVE_LUSTRE_LUSTREAPI */
82 
83 #define FILEMODE S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH
84 #define DIRMODE S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IXOTH
85 #define RELEASE_VERS META_VERSION
86 #define TEST_DIR "test-dir"
87 #define ITEM_COUNT 25000
88 
89 #define LLU "%lu"
90 
91 static int size;
92 static uint64_t *rand_array;
93 static char testdir[MAX_PATHLEN];
94 static char testdirpath[MAX_PATHLEN];
96 static char **filenames;
97 static char hostname[MAX_PATHLEN];
98 static char mk_name[MAX_PATHLEN];
99 static char stat_name[MAX_PATHLEN];
100 static char read_name[MAX_PATHLEN];
101 static char rm_name[MAX_PATHLEN];
108 static char *write_buffer;
109 static char *read_buffer;
110 static char *verify_read_buffer;
112 
113 
114 static int barriers;
115 static int create_only;
116 static int stat_only;
117 static int read_only;
118 static int verify_read;
120 static int remove_only;
121 static int leaf_only;
122 static unsigned branch_factor;
123 static int depth;
124 
125 /*
126  * This is likely a small value, but it's sometimes computed by
127  * branch_factor^(depth+1), so we'll make it a larger variable,
128  * just in case.
129  */
130 static uint64_t num_dirs_in_tree;
131 /*
132  * As we start moving towards Exascale, we could have billions
133  * of files in a directory. Make room for that possibility with
134  * a larger variable.
135  */
136 static uint64_t items;
137 static uint64_t items_per_dir;
138 static uint64_t num_dirs_in_tree_calc; /* this is a workaround until the overal code is refactored */
139 static int directory_loops;
140 static int print_time;
142 static int random_seed;
143 static int shared_file;
144 static int files_only;
145 static int dirs_only;
146 static int pre_delay;
149 static int throttle;
151 static size_t write_bytes;
153 static size_t read_bytes;
154 static int sync_file;
155 static int call_sync;
156 static int path_count;
157 static int nstride; /* neighbor stride */
158 static int make_node = 0;
159 #ifdef HAVE_LUSTRE_LUSTREAPI
160 static int global_dir_layout;
161 #endif /* HAVE_LUSTRE_LUSTREAPI */
162 
164 static pid_t pid;
165 static uid_t uid;
166 
167 /* Use the POSIX backend by default */
168 static const ior_aiori_t *backend;
169 
171 
172 /* This structure describes the processing status for stonewalling */
173 typedef struct{
174  double start_time;
175 
177 
178  uint64_t items_start;
179  uint64_t items_done;
180 
181  uint64_t items_per_dir;
183 
184 #define CHECK_STONE_WALL(p) (((p)->stone_wall_timer_seconds != 0) && ((GetTimeStamp() - (p)->start_time) > (p)->stone_wall_timer_seconds))
185 
186 /* for making/removing unique directory && stating/deleting subdirectory */
188 
189 /* a helper function for passing debug and verbose messages.
190  use the MACRO as it will insert __LINE__ for you.
191  Pass the verbose level for root to print, then the verbose level for anyone to print.
192  Pass -1 to suppress the print for anyone.
193  Then do the standard printf stuff. This function adds the newline for you.
194 */
195 #define VERBOSE(root,any,...) VerboseMessage(root,any,__LINE__,__VA_ARGS__)
196 void VerboseMessage (int root_level, int any_level, int line, char * format, ...) {
197  if ((rank==0 && verbose >= root_level) || (any_level > 0 && verbose >= any_level)) {
198  char buffer[1024];
199  va_list args;
200  va_start (args, format);
201  vsnprintf (buffer, 1024, format, args);
202  va_end (args);
203  if (root_level == 0 && any_level == -1) {
204  /* No header when it is just the standard output */
205  fprintf( out_logfile, "%s\n", buffer );
206  } else {
207  /* add a header when the verbose is greater than 0 */
208  fprintf( out_logfile, "V-%d: Rank %3d Line %5d %s\n", root_level, rank, line, buffer );
209  }
210  fflush(out_logfile);
211  }
212 }
213 
214 void generate_memory_pattern(char * buffer, size_t bytes){
215  for(int i=0; i < bytes; i++){
216  buffer[i] = i + 1;
217  }
218 }
219 
220 void offset_timers(double * t, int tcount) {
221  double toffset;
222  int i;
223 
224 
225  VERBOSE(1,-1,"V-1: Entering offset_timers..." );
226 
227  toffset = GetTimeStamp() - t[tcount];
228  for (i = 0; i < tcount+1; i++) {
229  t[i] += toffset;
230  }
231 }
232 
233 void parse_dirpath(char *dirpath_arg) {
234  char * tmp, * token;
235  char delimiter_string[3] = { '@', '\n', '\0' };
236  int i = 0;
237 
238 
239  VERBOSE(1,-1, "Entering parse_dirpath on %s...", dirpath_arg );
240 
241  tmp = dirpath_arg;
242 
243  if (* tmp != '\0') path_count++;
244  while (* tmp != '\0') {
245  if (* tmp == '@') {
246  path_count++;
247  }
248  tmp++;
249  }
250  // prevent changes to the original dirpath_arg
251  dirpath_arg = strdup(dirpath_arg);
252  filenames = (char **)malloc(path_count * sizeof(char **));
253  if (filenames == NULL || dirpath_arg == NULL) {
254  FAIL("out of memory");
255  }
256 
257  token = strtok(dirpath_arg, delimiter_string);
258  while (token != NULL) {
259  filenames[i] = token;
260  token = strtok(NULL, delimiter_string);
261  i++;
262  }
263 }
264 
265 static void prep_testdir(int j, int dir_iter){
266  int pos = sprintf(testdir, "%s", testdirpath);
267  if ( testdir[strlen( testdir ) - 1] != '/' ) {
268  pos += sprintf(& testdir[pos], "/");
269  }
270  pos += sprintf(& testdir[pos], "%s", TEST_DIR);
271  pos += sprintf(& testdir[pos], ".%d-%d", j, dir_iter);
272 }
273 
274 static void phase_end(){
275  if (call_sync){
276  if(! backend->sync){
277  FAIL("Error, backend does not provide the sync method, but you requested to use sync.\n");
278  }
279  backend->sync(& param);
280  }
281 
282  if (barriers) {
283  MPI_Barrier(testComm);
284  }
285 }
286 
287 /*
288  * This function copies the unique directory name for a given option to
289  * the "to" parameter. Some memory must be allocated to the "to" parameter.
290  */
291 
292 void unique_dir_access(int opt, char *to) {
293  if (opt == MK_UNI_DIR) {
294  MPI_Barrier(testComm);
295  sprintf( to, "%s/%s", testdir, unique_chdir_dir );
296  } else if (opt == STAT_SUB_DIR) {
297  sprintf( to, "%s/%s", testdir, unique_stat_dir );
298  } else if (opt == READ_SUB_DIR) {
299  sprintf( to, "%s/%s", testdir, unique_read_dir );
300  } else if (opt == RM_SUB_DIR) {
301  sprintf( to, "%s/%s", testdir, unique_rm_dir );
302  } else if (opt == RM_UNI_DIR) {
303  sprintf( to, "%s/%s", testdir, unique_rm_uni_dir );
304  }
305  VERBOSE(1,-1,"Entering unique_dir_access, set it to %s", to );
306 }
307 
308 static void create_remove_dirs (const char *path, bool create, uint64_t itemNum) {
309  char curr_item[MAX_PATHLEN];
310  const char *operation = create ? "create" : "remove";
311 
312  if ( (itemNum % ITEM_COUNT==0 && (itemNum != 0))) {
313  VERBOSE(3,5,"dir: "LLU"", operation, itemNum);
314  }
315 
316  //create dirs
317  sprintf(curr_item, "%s/dir.%s%" PRIu64, path, create ? mk_name : rm_name, itemNum);
318  VERBOSE(3,5,"create_remove_items_helper (dirs %s): curr_item is '%s'", operation, curr_item);
319 
320  if (create) {
321  if (backend->mkdir(curr_item, DIRMODE, &param) == -1) {
322  FAIL("unable to create directory %s", curr_item);
323  }
324  } else {
325  if (backend->rmdir(curr_item, &param) == -1) {
326  FAIL("unable to remove directory %s", curr_item);
327  }
328  }
329 }
330 
331 static void remove_file (const char *path, uint64_t itemNum) {
332  char curr_item[MAX_PATHLEN];
333 
334  if ( (itemNum % ITEM_COUNT==0 && (itemNum != 0))) {
335  VERBOSE(3,5,"remove file: "LLU"\n", itemNum);
336  }
337 
338  //remove files
339  sprintf(curr_item, "%s/file.%s"LLU"", path, rm_name, itemNum);
340  VERBOSE(3,5,"create_remove_items_helper (non-dirs remove): curr_item is '%s'", curr_item);
341  if (!(shared_file && rank != 0)) {
342  backend->delete (curr_item, &param);
343  }
344 }
345 
346 static void create_file (const char *path, uint64_t itemNum) {
347  char curr_item[MAX_PATHLEN];
348  void *aiori_fh = NULL;
349 
350  if ( (itemNum % ITEM_COUNT==0 && (itemNum != 0))) {
351  VERBOSE(3,5,"create file: "LLU"", itemNum);
352  }
353 
354  //create files
355  sprintf(curr_item, "%s/file.%s"LLU"", path, mk_name, itemNum);
356  VERBOSE(3,5,"create_remove_items_helper (non-dirs create): curr_item is '%s'", curr_item);
357 
358  param.openFlags = IOR_WRONLY;
359 
360  if (make_node) {
361  int ret;
362  VERBOSE(3,5,"create_remove_items_helper : mknod..." );
363 
364  ret = backend->mknod (curr_item);
365  if (ret != 0)
366  FAIL("unable to mknode file %s", curr_item);
367 
368  return;
369  } else if (collective_creates) {
370  VERBOSE(3,5,"create_remove_items_helper (collective): open..." );
371 
372  aiori_fh = backend->open (curr_item, &param);
373  if (NULL == aiori_fh)
374  FAIL("unable to open file %s", curr_item);
375 
376  /*
377  * !collective_creates
378  */
379  } else {
380  param.openFlags |= IOR_CREAT;
381  param.filePerProc = !shared_file;
382  param.mode = FILEMODE;
383  VERBOSE(3,5,"create_remove_items_helper (non-collective, shared): open..." );
384 
385  aiori_fh = backend->create (curr_item, &param);
386  if (NULL == aiori_fh)
387  FAIL("unable to create file %s", curr_item);
388  }
389 
390  if (write_bytes > 0) {
391  VERBOSE(3,5,"create_remove_items_helper: write..." );
392 
393  /*
394  * According to Bill Loewe, writes are only done one time, so they are always at
395  * offset 0 (zero).
396  */
397  param.offset = 0;
398  param.fsyncPerWrite = sync_file;
399  if ( write_bytes != (size_t) backend->xfer (WRITE, aiori_fh, (IOR_size_t *) write_buffer, write_bytes, &param)) {
400  FAIL("unable to write file %s", curr_item);
401  }
402  }
403 
404  VERBOSE(3,5,"create_remove_items_helper: close..." );
405  backend->close (aiori_fh, &param);
406 }
407 
408 /* helper for creating/removing items */
409 void create_remove_items_helper(const int dirs, const int create, const char *path,
410  uint64_t itemNum, rank_progress_t * progress) {
411 
412  VERBOSE(1,-1,"Entering create_remove_items_helper on %s", path );
413 
414  for (uint64_t i = progress->items_start; i < progress->items_per_dir ; ++i) {
415  if (!dirs) {
416  if (create) {
417  create_file (path, itemNum + i);
418  } else {
419  remove_file (path, itemNum + i);
420  }
421  } else {
422  create_remove_dirs (path, create, itemNum + i);
423  }
424  if(CHECK_STONE_WALL(progress)){
425  if(progress->items_done == 0){
426  progress->items_done = i + 1;
427  }
428  return;
429  }
430  }
431  progress->items_done = progress->items_per_dir;
432 }
433 
434 /* helper function to do collective operations */
435 void collective_helper(const int dirs, const int create, const char* path, uint64_t itemNum, rank_progress_t * progress) {
436  char curr_item[MAX_PATHLEN];
437 
438  VERBOSE(1,-1,"Entering collective_helper on %s", path );
439  for (uint64_t i = progress->items_start ; i < progress->items_per_dir ; ++i) {
440  if (dirs) {
441  create_remove_dirs (path, create, itemNum + i);
442  continue;
443  }
444 
445  sprintf(curr_item, "%s/file.%s"LLU"", path, create ? mk_name : rm_name, itemNum+i);
446  VERBOSE(3,5,"create file: %s", curr_item);
447 
448  if (create) {
449  void *aiori_fh;
450 
451  //create files
452  param.openFlags = IOR_WRONLY | IOR_CREAT;
453  param.mode = FILEMODE;
454  aiori_fh = backend->create (curr_item, &param);
455  if (NULL == aiori_fh) {
456  FAIL("unable to create file %s", curr_item);
457  }
458 
459  backend->close (aiori_fh, &param);
460  } else if (!(shared_file && rank != 0)) {
461  //remove files
462  backend->delete (curr_item, &param);
463  }
464  if(CHECK_STONE_WALL(progress)){
465  progress->items_done = i + 1;
466  return;
467  }
468  }
469  progress->items_done = progress->items_per_dir;
470 }
471 
472 /* recusive function to create and remove files/directories from the
473  directory tree */
474 void create_remove_items(int currDepth, const int dirs, const int create, const int collective, const char *path, uint64_t dirNum, rank_progress_t * progress) {
475  unsigned i;
476  char dir[MAX_PATHLEN];
477  char temp_path[MAX_PATHLEN];
478  unsigned long long currDir = dirNum;
479 
480 
481  VERBOSE(1,-1,"Entering create_remove_items on %s, currDepth = %d...", path, currDepth );
482 
483 
484  memset(dir, 0, MAX_PATHLEN);
485  strcpy(temp_path, path);
486 
487  VERBOSE(3,5,"create_remove_items (start): temp_path is '%s'", temp_path );
488 
489  if (currDepth == 0) {
490  /* create items at this depth */
491  if (!leaf_only || (depth == 0 && leaf_only)) {
492  if (collective) {
493  collective_helper(dirs, create, temp_path, 0, progress);
494  } else {
495  create_remove_items_helper(dirs, create, temp_path, 0, progress);
496  }
497  }
498 
499  if (depth > 0) {
500  create_remove_items(++currDepth, dirs, create,
501  collective, temp_path, ++dirNum, progress);
502  }
503 
504  } else if (currDepth <= depth) {
505  /* iterate through the branches */
506  for (i=0; i<branch_factor; i++) {
507 
508  /* determine the current branch and append it to the path */
509  sprintf(dir, "%s.%llu/", base_tree_name, currDir);
510  strcat(temp_path, "/");
511  strcat(temp_path, dir);
512 
513  VERBOSE(3,5,"create_remove_items (for loop): temp_path is '%s'", temp_path );
514 
515  /* create the items in this branch */
516  if (!leaf_only || (leaf_only && currDepth == depth)) {
517  if (collective) {
518  collective_helper(dirs, create, temp_path, currDir*items_per_dir, progress);
519  } else {
520  create_remove_items_helper(dirs, create, temp_path, currDir*items_per_dir, progress);
521  }
522  }
523 
524  /* make the recursive call for the next level below this branch */
526  ++currDepth,
527  dirs,
528  create,
529  collective,
530  temp_path,
531  ( currDir * ( unsigned long long )branch_factor ) + 1,
532  progress
533  );
534  currDepth--;
535 
536  /* reset the path */
537  strcpy(temp_path, path);
538  currDir++;
539  }
540  }
541 }
542 
543 /* stats all of the items created as specified by the input parameters */
544 void mdtest_stat(const int random, const int dirs, const long dir_iter, const char *path, rank_progress_t * progress) {
545  struct stat buf;
546  uint64_t parent_dir, item_num = 0;
547  char item[MAX_PATHLEN], temp[MAX_PATHLEN];
548 
549  VERBOSE(1,-1,"Entering mdtest_stat on %s", path );
550 
551  uint64_t stop_items = items;
552 
553  if( directory_loops != 1 ){
554  stop_items = items_per_dir;
555  }
556 
557  /* iterate over all of the item IDs */
558  for (uint64_t i = 0 ; i < stop_items ; ++i) {
559  /*
560  * It doesn't make sense to pass the address of the array because that would
561  * be like passing char **. Tested it on a Cray and it seems to work either
562  * way, but it seems that it is correct without the "&".
563  *
564  memset(&item, 0, MAX_PATHLEN);
565  */
566  memset(item, 0, MAX_PATHLEN);
567  memset(temp, 0, MAX_PATHLEN);
568 
569 
570  /* determine the item number to stat */
571  if (random) {
572  item_num = rand_array[i];
573  } else {
574  item_num = i;
575  }
576 
577  /* make adjustments if in leaf only mode*/
578  if (leaf_only) {
579  item_num += items_per_dir *
580  (num_dirs_in_tree - (uint64_t) pow( branch_factor, depth ));
581  }
582 
583  /* create name of file/dir to stat */
584  if (dirs) {
585  if ( (i % ITEM_COUNT == 0) && (i != 0)) {
586  VERBOSE(3,5,"stat dir: "LLU"", i);
587  }
588  sprintf(item, "dir.%s"LLU"", stat_name, item_num);
589  } else {
590  if ( (i % ITEM_COUNT == 0) && (i != 0)) {
591  VERBOSE(3,5,"stat file: "LLU"", i);
592  }
593  sprintf(item, "file.%s"LLU"", stat_name, item_num);
594  }
595 
596  /* determine the path to the file/dir to be stat'ed */
597  parent_dir = item_num / items_per_dir;
598 
599  if (parent_dir > 0) { //item is not in tree's root directory
600 
601  /* prepend parent directory to item's path */
602  sprintf(temp, "%s."LLU"/%s", base_tree_name, parent_dir, item);
603  strcpy(item, temp);
604 
605  //still not at the tree's root dir
606  while (parent_dir > branch_factor) {
607  parent_dir = (uint64_t) ((parent_dir-1) / branch_factor);
608  sprintf(temp, "%s."LLU"/%s", base_tree_name, parent_dir, item);
609  strcpy(item, temp);
610  }
611  }
612 
613  /* Now get item to have the full path */
614  sprintf( temp, "%s/%s", path, item );
615  strcpy( item, temp );
616 
617  /* below temp used to be hiername */
618  VERBOSE(3,5,"mdtest_stat %4s: %s", (dirs ? "dir" : "file"), item);
619  if (-1 == backend->stat (item, &buf, &param)) {
620  FAIL("unable to stat %s %s", dirs ? "directory" : "file", item);
621  }
622  }
623 }
624 
625 
626 /* reads all of the items created as specified by the input parameters */
627 void mdtest_read(int random, int dirs, const long dir_iter, char *path) {
628  uint64_t parent_dir, item_num = 0;
629  char item[MAX_PATHLEN], temp[MAX_PATHLEN];
630  void *aiori_fh;
631 
632  VERBOSE(1,-1,"Entering mdtest_read on %s", path );
633 
634  /* allocate read buffer */
635  if (read_bytes > 0) {
636  int alloc_res = posix_memalign((void**)&read_buffer, sysconf(_SC_PAGESIZE), read_bytes);
637  if (alloc_res) {
638  FAIL("out of memory");
639  }
640 
641  if (verify_read > 0) {
642  verify_read_buffer = (char *)malloc(read_bytes);
643  if (verify_read_buffer == NULL) {
644  FAIL("out of memory");
645  }
647  }
648  }
649 
650  uint64_t stop_items = items;
651 
652  if( directory_loops != 1 ){
653  stop_items = items_per_dir;
654  }
655 
656  /* iterate over all of the item IDs */
657  for (uint64_t i = 0 ; i < stop_items ; ++i) {
658  /*
659  * It doesn't make sense to pass the address of the array because that would
660  * be like passing char **. Tested it on a Cray and it seems to work either
661  * way, but it seems that it is correct without the "&".
662  *
663  * NTH: Both are technically correct in C.
664  *
665  * memset(&item, 0, MAX_PATHLEN);
666  */
667  memset(item, 0, MAX_PATHLEN);
668  memset(temp, 0, MAX_PATHLEN);
669 
670  /* determine the item number to read */
671  if (random) {
672  item_num = rand_array[i];
673  } else {
674  item_num = i;
675  }
676 
677  /* make adjustments if in leaf only mode*/
678  if (leaf_only) {
679  item_num += items_per_dir *
680  (num_dirs_in_tree - (uint64_t) pow (branch_factor, depth));
681  }
682 
683  /* create name of file to read */
684  if (!dirs) {
685  if ((i%ITEM_COUNT == 0) && (i != 0)) {
686  VERBOSE(3,5,"read file: "LLU"", i);
687  }
688  sprintf(item, "file.%s"LLU"", read_name, item_num);
689  }
690 
691  /* determine the path to the file/dir to be read'ed */
692  parent_dir = item_num / items_per_dir;
693 
694  if (parent_dir > 0) { //item is not in tree's root directory
695 
696  /* prepend parent directory to item's path */
697  sprintf(temp, "%s."LLU"/%s", base_tree_name, parent_dir, item);
698  strcpy(item, temp);
699 
700  /* still not at the tree's root dir */
701  while (parent_dir > branch_factor) {
702  parent_dir = (unsigned long long) ((parent_dir-1) / branch_factor);
703  sprintf(temp, "%s."LLU"/%s", base_tree_name, parent_dir, item);
704  strcpy(item, temp);
705  }
706  }
707 
708  /* Now get item to have the full path */
709  sprintf( temp, "%s/%s", path, item );
710  strcpy( item, temp );
711 
712  /* below temp used to be hiername */
713  VERBOSE(3,5,"mdtest_read file: %s", item);
714 
715  /* open file for reading */
716  param.openFlags = O_RDONLY;
717  aiori_fh = backend->open (item, &param);
718  if (NULL == aiori_fh) {
719  FAIL("unable to open file %s", item);
720  }
721 
722  /* read file */
723  if (read_bytes > 0) {
724  read_buffer[0] = 42; /* use a random value to ensure that the read_buffer is now different from the expected buffer and read isn't sometimes NOOP */
725  if (read_bytes != (size_t) backend->xfer (READ, aiori_fh, (IOR_size_t *) read_buffer, read_bytes, &param)) {
726  FAIL("unable to read file %s", item);
727  }
728  if(verify_read){
729  if (memcmp(read_buffer, verify_read_buffer, read_bytes) != 0){
730  VERBOSE(2, -1, "Error verifying %s", item);
732  }
733  }
734  }
735 
736  /* close file */
737  backend->close (aiori_fh, &param);
738  }
739 }
740 
741 /* This method should be called by rank 0. It subsequently does all of
742  the creates and removes for the other ranks */
743 void collective_create_remove(const int create, const int dirs, const int ntasks, const char *path, rank_progress_t * progress) {
744  char temp[MAX_PATHLEN];
745 
746  VERBOSE(1,-1,"Entering collective_create_remove on %s", path );
747 
748  /* rank 0 does all of the creates and removes for all of the ranks */
749  for (int i = 0 ; i < ntasks ; ++i) {
750  memset(temp, 0, MAX_PATHLEN);
751 
752  strcpy(temp, testdir);
753  strcat(temp, "/");
754 
755  /* set the base tree name appropriately */
756  if (unique_dir_per_task) {
757  sprintf(base_tree_name, "mdtest_tree.%d", i);
758  } else {
759  sprintf(base_tree_name, "mdtest_tree");
760  }
761 
762  /* Setup to do I/O to the appropriate test dir */
763  strcat(temp, base_tree_name);
764  strcat(temp, ".0");
765 
766  /* set all item names appropriately */
767  if (!shared_file) {
768  sprintf(mk_name, "mdtest.%d.", (i+(0*nstride))%ntasks);
769  sprintf(stat_name, "mdtest.%d.", (i+(1*nstride))%ntasks);
770  sprintf(read_name, "mdtest.%d.", (i+(2*nstride))%ntasks);
771  sprintf(rm_name, "mdtest.%d.", (i+(3*nstride))%ntasks);
772  }
773  if (unique_dir_per_task) {
774  VERBOSE(3,5,"i %d nstride %d ntasks %d", i, nstride, ntasks);
775  sprintf(unique_mk_dir, "%s/mdtest_tree.%d.0", testdir,
776  (i+(0*nstride))%ntasks);
777  sprintf(unique_chdir_dir, "%s/mdtest_tree.%d.0", testdir,
778  (i+(1*nstride))%ntasks);
779  sprintf(unique_stat_dir, "%s/mdtest_tree.%d.0", testdir,
780  (i+(2*nstride))%ntasks);
781  sprintf(unique_read_dir, "%s/mdtest_tree.%d.0", testdir,
782  (i+(3*nstride))%ntasks);
783  sprintf(unique_rm_dir, "%s/mdtest_tree.%d.0", testdir,
784  (i+(4*nstride))%ntasks);
785  sprintf(unique_rm_uni_dir, "%s", testdir);
786  }
787 
788  /* Now that everything is set up as it should be, do the create or remove */
789  VERBOSE(3,5,"collective_create_remove (create_remove_items): temp is '%s'", temp);
790 
791  create_remove_items(0, dirs, create, 1, temp, 0, progress);
792  }
793 
794  /* reset all of the item names */
795  if (unique_dir_per_task) {
796  sprintf(base_tree_name, "mdtest_tree.0");
797  } else {
798  sprintf(base_tree_name, "mdtest_tree");
799  }
800  if (!shared_file) {
801  sprintf(mk_name, "mdtest.%d.", (0+(0*nstride))%ntasks);
802  sprintf(stat_name, "mdtest.%d.", (0+(1*nstride))%ntasks);
803  sprintf(read_name, "mdtest.%d.", (0+(2*nstride))%ntasks);
804  sprintf(rm_name, "mdtest.%d.", (0+(3*nstride))%ntasks);
805  }
806  if (unique_dir_per_task) {
807  sprintf(unique_mk_dir, "%s/mdtest_tree.%d.0", testdir,
808  (0+(0*nstride))%ntasks);
809  sprintf(unique_chdir_dir, "%s/mdtest_tree.%d.0", testdir,
810  (0+(1*nstride))%ntasks);
811  sprintf(unique_stat_dir, "%s/mdtest_tree.%d.0", testdir,
812  (0+(2*nstride))%ntasks);
813  sprintf(unique_read_dir, "%s/mdtest_tree.%d.0", testdir,
814  (0+(3*nstride))%ntasks);
815  sprintf(unique_rm_dir, "%s/mdtest_tree.%d.0", testdir,
816  (0+(4*nstride))%ntasks);
817  sprintf(unique_rm_uni_dir, "%s", testdir);
818  }
819 }
820 
821 void directory_test(const int iteration, const int ntasks, const char *path, rank_progress_t * progress) {
822  int size;
823  double t[5] = {0};
824  char temp_path[MAX_PATHLEN];
825 
826  MPI_Comm_size(testComm, &size);
827 
828  VERBOSE(1,-1,"Entering directory_test on %s", path );
829 
830  MPI_Barrier(testComm);
831  t[0] = GetTimeStamp();
832 
833  /* create phase */
834  if(create_only) {
835  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
836  prep_testdir(iteration, dir_iter);
837  if (unique_dir_per_task) {
838  unique_dir_access(MK_UNI_DIR, temp_path);
840  offset_timers(t, 0);
841  }
842  } else {
843  sprintf( temp_path, "%s/%s", testdir, path );
844  }
845 
846  VERBOSE(3,-1,"directory_test: create path is '%s'", temp_path );
847 
848  /* "touch" the files */
849  if (collective_creates) {
850  if (rank == 0) {
851  collective_create_remove(1, 1, ntasks, temp_path, progress);
852  }
853  } else {
854  /* create directories */
855  create_remove_items(0, 1, 1, 0, temp_path, 0, progress);
856  }
857  }
858  }
859 
860  phase_end();
861  t[1] = GetTimeStamp();
862 
863  /* stat phase */
864  if (stat_only) {
865  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
866  prep_testdir(iteration, dir_iter);
867  if (unique_dir_per_task) {
868  unique_dir_access(STAT_SUB_DIR, temp_path);
870  offset_timers(t, 1);
871  }
872  } else {
873  sprintf( temp_path, "%s/%s", testdir, path );
874  }
875 
876  VERBOSE(3,5,"stat path is '%s'", temp_path );
877 
878  /* stat directories */
879  if (random_seed > 0) {
880  mdtest_stat(1, 1, dir_iter, temp_path, progress);
881  } else {
882  mdtest_stat(0, 1, dir_iter, temp_path, progress);
883  }
884  }
885  }
886  phase_end();
887  t[2] = GetTimeStamp();
888 
889  /* read phase */
890  if (read_only) {
891  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
892  prep_testdir(iteration, dir_iter);
893  if (unique_dir_per_task) {
894  unique_dir_access(READ_SUB_DIR, temp_path);
896  offset_timers(t, 2);
897  }
898  } else {
899  sprintf( temp_path, "%s/%s", testdir, path );
900  }
901 
902  VERBOSE(3,5,"directory_test: read path is '%s'", temp_path );
903 
904  /* read directories */
905  if (random_seed > 0) {
906  ; /* N/A */
907  } else {
908  ; /* N/A */
909  }
910  }
911  }
912 
913  phase_end();
914  t[3] = GetTimeStamp();
915 
916  if (remove_only) {
917  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
918  prep_testdir(iteration, dir_iter);
919  if (unique_dir_per_task) {
920  unique_dir_access(RM_SUB_DIR, temp_path);
922  offset_timers(t, 3);
923  }
924  } else {
925  sprintf( temp_path, "%s/%s", testdir, path );
926  }
927 
928  VERBOSE(3,5,"directory_test: remove directories path is '%s'", temp_path );
929 
930  /* remove directories */
931  if (collective_creates) {
932  if (rank == 0) {
933  collective_create_remove(0, 1, ntasks, temp_path, progress);
934  }
935  } else {
936  create_remove_items(0, 1, 0, 0, temp_path, 0, progress);
937  }
938  }
939  }
940 
941  phase_end();
942  t[4] = GetTimeStamp();
943 
944  if (remove_only) {
945  if (unique_dir_per_task) {
946  unique_dir_access(RM_UNI_DIR, temp_path);
947  } else {
948  sprintf( temp_path, "%s/%s", testdir, path );
949  }
950 
951  VERBOSE(3,5,"directory_test: remove unique directories path is '%s'\n", temp_path );
952  }
953 
955  offset_timers(t, 4);
956  }
957 
958  /* calculate times */
959  if (create_only) {
960  summary_table[iteration].rate[0] = items*size/(t[1] - t[0]);
961  summary_table[iteration].time[0] = t[1] - t[0];
962  summary_table[iteration].items[0] = items*size;
963  summary_table[iteration].stonewall_last_item[0] = items;
964  }
965  if (stat_only) {
966  summary_table[iteration].rate[1] = items*size/(t[2] - t[1]);
967  summary_table[iteration].time[1] = t[2] - t[1];
968  summary_table[iteration].items[1] = items*size;
969  summary_table[iteration].stonewall_last_item[1] = items;
970  }
971  if (read_only) {
972  summary_table[iteration].rate[2] = items*size/(t[3] - t[2]);
973  summary_table[iteration].time[2] = t[3] - t[2];
974  summary_table[iteration].items[2] = items*size;
975  summary_table[iteration].stonewall_last_item[2] = items;
976  }
977  if (remove_only) {
978  summary_table[iteration].rate[3] = items*size/(t[4] - t[3]);
979  summary_table[iteration].time[3] = t[4] - t[3];
980  summary_table[iteration].items[3] = items*size;
981  summary_table[iteration].stonewall_last_item[3] = items;
982  }
983 
984  VERBOSE(1,-1," Directory creation: %14.3f sec, %14.3f ops/sec", t[1] - t[0], summary_table[iteration].rate[0]);
985  VERBOSE(1,-1," Directory stat : %14.3f sec, %14.3f ops/sec", t[2] - t[1], summary_table[iteration].rate[1]);
986  /* N/A
987  VERBOSE(1,-1," Directory read : %14.3f sec, %14.3f ops/sec", t[3] - t[2], summary_table[iteration].rate[2]);
988  */
989  VERBOSE(1,-1," Directory removal : %14.3f sec, %14.3f ops/sec", t[4] - t[3], summary_table[iteration].rate[3]);
990 }
991 
992 /* Returns if the stonewall was hit */
993 int updateStoneWallIterations(int iteration, rank_progress_t * progress, double tstart){
994  int hit = 0;
995  uint64_t done = progress->items_done;
996  long long unsigned max_iter = 0;
997 
998  VERBOSE(1,1,"stonewall hit with %lld items", (long long) progress->items_done );
999  MPI_Allreduce(& progress->items_done, & max_iter, 1, MPI_LONG_LONG_INT, MPI_MAX, testComm);
1000  summary_table[iteration].stonewall_time[MDTEST_FILE_CREATE_NUM] = GetTimeStamp() - tstart;
1001 
1002  // continue to the maximum...
1003  long long min_accessed = 0;
1004  MPI_Reduce(& progress->items_done, & min_accessed, 1, MPI_LONG_LONG_INT, MPI_MIN, 0, testComm);
1005  long long sum_accessed = 0;
1006  MPI_Reduce(& progress->items_done, & sum_accessed, 1, MPI_LONG_LONG_INT, MPI_SUM, 0, testComm);
1007  summary_table[iteration].stonewall_item_sum[MDTEST_FILE_CREATE_NUM] = sum_accessed;
1008  summary_table[iteration].stonewall_item_min[MDTEST_FILE_CREATE_NUM] = min_accessed * size;
1009 
1010  if(items != (sum_accessed / size)){
1011  VERBOSE(0,-1, "Continue stonewall hit min: %lld max: %lld avg: %.1f \n", min_accessed, max_iter, ((double) sum_accessed) / size);
1012  hit = 1;
1013  }
1014  progress->items_start = done;
1015  progress->items_per_dir = max_iter;
1016 
1017  return hit;
1018 }
1019 
1020 void file_test(const int iteration, const int ntasks, const char *path, rank_progress_t * progress) {
1021  int size;
1022  double t[5] = {0};
1023  char temp_path[MAX_PATHLEN];
1024  MPI_Comm_size(testComm, &size);
1025 
1026  VERBOSE(3,5,"Entering file_test on %s", path);
1027 
1028  MPI_Barrier(testComm);
1029  t[0] = GetTimeStamp();
1030 
1031  /* create phase */
1032  if (create_only ) {
1033  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
1034  prep_testdir(iteration, dir_iter);
1035 
1036  if (unique_dir_per_task) {
1037  unique_dir_access(MK_UNI_DIR, temp_path);
1038  VERBOSE(5,5,"operating on %s", temp_path);
1039  if (!time_unique_dir_overhead) {
1040  offset_timers(t, 0);
1041  }
1042  } else {
1043  sprintf( temp_path, "%s/%s", testdir, path );
1044  }
1045 
1046 
1047 
1048  VERBOSE(3,-1,"file_test: create path is '%s'", temp_path );
1049 
1050  /* "touch" the files */
1051  if (collective_creates) {
1052  if (rank == 0) {
1053  collective_create_remove(1, 0, ntasks, temp_path, progress);
1054  }
1055  MPI_Barrier(testComm);
1056  }
1057 
1058  /* create files */
1059  create_remove_items(0, 0, 1, 0, temp_path, 0, progress);
1061  int hit = updateStoneWallIterations(iteration, progress, t[0]);
1062 
1063  if (hit){
1064  progress->stone_wall_timer_seconds = 0;
1065  VERBOSE(1,1,"stonewall: %lld of %lld", (long long) progress->items_start, (long long) progress->items_per_dir);
1066  create_remove_items(0, 0, 1, 0, temp_path, 0, progress);
1067  // now reset the values
1069  items = progress->items_done;
1070  }
1073  }
1074  // reset stone wall timer to allow proper cleanup
1075  progress->stone_wall_timer_seconds = 0;
1076  }
1077  }
1078  }else{
1080  int64_t expected_items;
1081  /* The number of items depends on the stonewalling file */
1083  if(expected_items >= 0){
1084  items = expected_items;
1085  progress->items_per_dir = items;
1086  }
1087  if (rank == 0) {
1088  if(expected_items == -1){
1089  fprintf(out_logfile, "WARNING: could not read stonewall status file\n");
1090  }else {
1091  VERBOSE(1,1, "Read stonewall status; items: "LLU"\n", items);
1092  }
1093  }
1094  }
1095  }
1096 
1097  phase_end();
1098  t[1] = GetTimeStamp();
1099 
1100  /* stat phase */
1101  if (stat_only ) {
1102  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
1103  prep_testdir(iteration, dir_iter);
1104  if (unique_dir_per_task) {
1105  unique_dir_access(STAT_SUB_DIR, temp_path);
1106  if (!time_unique_dir_overhead) {
1107  offset_timers(t, 1);
1108  }
1109  } else {
1110  sprintf( temp_path, "%s/%s", testdir, path );
1111  }
1112 
1113  VERBOSE(3,5,"file_test: stat path is '%s'", temp_path );
1114 
1115  /* stat files */
1116  mdtest_stat((random_seed > 0 ? 1 : 0), 0, dir_iter, temp_path, progress);
1117  }
1118  }
1119 
1120  phase_end();
1121  t[2] = GetTimeStamp();
1122 
1123  /* read phase */
1124  if (read_only ) {
1125  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
1126  prep_testdir(iteration, dir_iter);
1127  if (unique_dir_per_task) {
1128  unique_dir_access(READ_SUB_DIR, temp_path);
1129  if (!time_unique_dir_overhead) {
1130  offset_timers(t, 2);
1131  }
1132  } else {
1133  sprintf( temp_path, "%s/%s", testdir, path );
1134  }
1135 
1136  VERBOSE(3,5,"file_test: read path is '%s'", temp_path );
1137 
1138  /* read files */
1139  if (random_seed > 0) {
1140  mdtest_read(1,0, dir_iter, temp_path);
1141  } else {
1142  mdtest_read(0,0, dir_iter, temp_path);
1143  }
1144  }
1145  }
1146 
1147  phase_end();
1148  t[3] = GetTimeStamp();
1149 
1150  if (remove_only) {
1151  progress->items_start = 0;
1152 
1153  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
1154  prep_testdir(iteration, dir_iter);
1155  if (unique_dir_per_task) {
1156  unique_dir_access(RM_SUB_DIR, temp_path);
1157  if (!time_unique_dir_overhead) {
1158  offset_timers(t, 3);
1159  }
1160  } else {
1161  sprintf( temp_path, "%s/%s", testdir, path );
1162  }
1163 
1164  VERBOSE(3,5,"file_test: rm directories path is '%s'", temp_path );
1165 
1166  if (collective_creates) {
1167  if (rank == 0) {
1168  collective_create_remove(0, 0, ntasks, temp_path, progress);
1169  }
1170  } else {
1171  VERBOSE(3,5,"gonna create %s", temp_path);
1172  create_remove_items(0, 0, 0, 0, temp_path, 0, progress);
1173  }
1174  }
1175  }
1176 
1177  phase_end();
1178  t[4] = GetTimeStamp();
1179  if (remove_only) {
1180  if (unique_dir_per_task) {
1181  unique_dir_access(RM_UNI_DIR, temp_path);
1182  } else {
1183  strcpy( temp_path, path );
1184  }
1185 
1186  VERBOSE(3,5,"file_test: rm unique directories path is '%s'", temp_path );
1187  }
1188 
1190  offset_timers(t, 4);
1191  }
1192 
1193  if(num_dirs_in_tree_calc){ /* this is temporary fix needed when using -n and -i together */
1195  }
1196 
1197  /* calculate times */
1198  if (create_only) {
1199  summary_table[iteration].rate[4] = items*size/(t[1] - t[0]);
1200  summary_table[iteration].time[4] = t[1] - t[0];
1201  summary_table[iteration].items[4] = items*size;
1202  summary_table[iteration].stonewall_last_item[4] = items;
1203  }
1204  if (stat_only) {
1205  summary_table[iteration].rate[5] = items*size/(t[2] - t[1]);
1206  summary_table[iteration].time[5] = t[2] - t[1];
1207  summary_table[iteration].items[5] = items*size;
1208  summary_table[iteration].stonewall_last_item[5] = items;
1209  }
1210  if (read_only) {
1211  summary_table[iteration].rate[6] = items*size/(t[3] - t[2]);
1212  summary_table[iteration].time[6] = t[3] - t[2];
1213  summary_table[iteration].items[6] = items*size;
1214  summary_table[iteration].stonewall_last_item[6] = items;
1215  }
1216  if (remove_only) {
1217  summary_table[iteration].rate[7] = items*size/(t[4] - t[3]);
1218  summary_table[iteration].time[7] = t[4] - t[3];
1219  summary_table[iteration].items[7] = items*size;
1220  summary_table[iteration].stonewall_last_item[7] = items;
1221  }
1222 
1223  VERBOSE(1,-1," File creation : %14.3f sec, %14.3f ops/sec", t[1] - t[0], summary_table[iteration].rate[4]);
1224  if(summary_table[iteration].stonewall_time[MDTEST_FILE_CREATE_NUM]){
1225  VERBOSE(1,-1," File creation (stonewall): %14.3f sec, %14.3f ops/sec", summary_table[iteration].stonewall_time[MDTEST_FILE_CREATE_NUM], summary_table[iteration].stonewall_item_sum[MDTEST_FILE_CREATE_NUM]);
1226  }
1227  VERBOSE(1,-1," File stat : %14.3f sec, %14.3f ops/sec", t[2] - t[1], summary_table[iteration].rate[5]);
1228  VERBOSE(1,-1," File read : %14.3f sec, %14.3f ops/sec", t[3] - t[2], summary_table[iteration].rate[6]);
1229  VERBOSE(1,-1," File removal : %14.3f sec, %14.3f ops/sec", t[4] - t[3], summary_table[iteration].rate[7]);
1230 }
1231 
1232 int calc_allreduce_index(int iter, int rank, int op){
1233  int tableSize = MDTEST_LAST_NUM;
1234  return iter * tableSize * size + rank * tableSize + op;
1235 }
1236 
1237 void summarize_results(int iterations, int print_time) {
1238  char access[MAX_PATHLEN];
1239  int i, j, k;
1240  int start, stop, tableSize = MDTEST_LAST_NUM;
1241  double min, max, mean, sd, sum = 0, var = 0, curr = 0;
1242 
1243  double all[iterations * size * tableSize];
1244 
1245 
1246  VERBOSE(1,-1,"Entering summarize_results..." );
1247 
1248  MPI_Barrier(testComm);
1249  for(int i=0; i < iterations; i++){
1250  if(print_time){
1251  MPI_Gather(& summary_table[i].time[0], tableSize, MPI_DOUBLE, & all[i*tableSize*size], tableSize, MPI_DOUBLE, 0, testComm);
1252  }else{
1253  MPI_Gather(& summary_table[i].rate[0], tableSize, MPI_DOUBLE, & all[i*tableSize*size], tableSize, MPI_DOUBLE, 0, testComm);
1254  }
1255  }
1256 
1257  if (rank != 0) {
1258  return;
1259  }
1260 
1261  VERBOSE(0,-1,"\nSUMMARY %s: (of %d iterations)", print_time ? "time": "rate", iterations);
1262  VERBOSE(0,-1," Operation Max Min Mean Std Dev");
1263  VERBOSE(0,-1," --------- --- --- ---- -------");
1264 
1265  /* if files only access, skip entries 0-3 (the dir tests) */
1266  if (files_only && !dirs_only) {
1267  start = 4;
1268  } else {
1269  start = 0;
1270  }
1271 
1272  /* if directories only access, skip entries 4-7 (the file tests) */
1273  if (dirs_only && !files_only) {
1274  stop = 4;
1275  } else {
1276  stop = 8;
1277  }
1278 
1279  /* special case: if no directory or file tests, skip all */
1280  if (!dirs_only && !files_only) {
1281  start = stop = 0;
1282  }
1283 
1284  for (i = start; i < stop; i++) {
1285  min = max = all[i];
1286  for (k=0; k < size; k++) {
1287  for (j = 0; j < iterations; j++) {
1288  curr = all[calc_allreduce_index(j, k, i)];
1289  if (min > curr) {
1290  min = curr;
1291  }
1292  if (max < curr) {
1293  max = curr;
1294  }
1295  sum += curr;
1296  }
1297  }
1298  mean = sum / (iterations * size);
1299  for (k=0; k<size; k++) {
1300  for (j = 0; j < iterations; j++) {
1301  var += pow((mean - all[calc_allreduce_index(j, k, i)]), 2);
1302  }
1303  }
1304  var = var / (iterations * size);
1305  sd = sqrt(var);
1306  switch (i) {
1307  case 0: strcpy(access, "Directory creation :"); break;
1308  case 1: strcpy(access, "Directory stat :"); break;
1309  /* case 2: strcpy(access, "Directory read :"); break; */
1310  case 2: ; break; /* N/A */
1311  case 3: strcpy(access, "Directory removal :"); break;
1312  case 4: strcpy(access, "File creation :"); break;
1313  case 5: strcpy(access, "File stat :"); break;
1314  case 6: strcpy(access, "File read :"); break;
1315  case 7: strcpy(access, "File removal :"); break;
1316  default: strcpy(access, "ERR"); break;
1317  }
1318  if (i != 2) {
1319  fprintf(out_logfile, " %s ", access);
1320  fprintf(out_logfile, "%14.3f ", max);
1321  fprintf(out_logfile, "%14.3f ", min);
1322  fprintf(out_logfile, "%14.3f ", mean);
1323  fprintf(out_logfile, "%14.3f\n", sd);
1324  fflush(out_logfile);
1325  }
1326  sum = var = 0;
1327 
1328  }
1329 
1330  // TODO generalize once more stonewall timers are supported
1331  double stonewall_time = 0;
1332  uint64_t stonewall_items = 0;
1333  for(int i=0; i < iterations; i++){
1334  if(summary_table[i].stonewall_time[MDTEST_FILE_CREATE_NUM]){
1335  stonewall_time += summary_table[i].stonewall_time[MDTEST_FILE_CREATE_NUM];
1336  stonewall_items += summary_table[i].stonewall_item_sum[MDTEST_FILE_CREATE_NUM];
1337  }
1338  }
1339  if(stonewall_items != 0){
1340  fprintf(out_logfile, " File create (stonewall) : ");
1341  fprintf(out_logfile, "%14s %14s %14.3f %14s\n", "NA", "NA", print_time ? stonewall_time : stonewall_items / stonewall_time, "NA");
1342  }
1343 
1344  /* calculate tree create/remove rates */
1345  for (i = 8; i < tableSize; i++) {
1346  min = max = all[i];
1347  for (j = 0; j < iterations; j++) {
1348  if(print_time){
1349  curr = summary_table[j].time[i];
1350  }else{
1351  curr = summary_table[j].rate[i];
1352  }
1353 
1354  if (min > curr) {
1355  min = curr;
1356  }
1357  if (max < curr) {
1358  max = curr;
1359  }
1360  sum += curr;
1361  }
1362  mean = sum / (iterations);
1363  for (j = 0; j < iterations; j++) {
1364  if(print_time){
1365  curr = summary_table[j].time[i];
1366  }else{
1367  curr = summary_table[j].rate[i];
1368  }
1369 
1370  var += pow((mean - curr), 2);
1371  }
1372  var = var / (iterations);
1373  sd = sqrt(var);
1374  switch (i) {
1375  case 8: strcpy(access, "Tree creation :"); break;
1376  case 9: strcpy(access, "Tree removal :"); break;
1377  default: strcpy(access, "ERR"); break;
1378  }
1379  fprintf(out_logfile, " %s ", access);
1380  fprintf(out_logfile, "%14.3f ", max);
1381  fprintf(out_logfile, "%14.3f ", min);
1382  fprintf(out_logfile, "%14.3f ", mean);
1383  fprintf(out_logfile, "%14.3f\n", sd);
1384  fflush(out_logfile);
1385  sum = var = 0;
1386  }
1387 }
1388 
1389 /* Checks to see if the test setup is valid. If it isn't, fail. */
1390 void valid_tests() {
1391 
1392  if (((stone_wall_timer_seconds > 0) && (branch_factor > 1)) || ! barriers) {
1393  FAIL( "Error, stone wall timer does only work with a branch factor <= 1 (current is %d) and with barriers\n", branch_factor);
1394  }
1395 
1396  if (!create_only && !stat_only && !read_only && !remove_only) {
1398  VERBOSE(1,-1,"main: Setting create/stat/read/remove_only to True" );
1399  }
1400 
1401  VERBOSE(1,-1,"Entering valid_tests..." );
1402 
1403  /* if dirs_only and files_only were both left unset, set both now */
1404  if (!dirs_only && !files_only) {
1405  dirs_only = files_only = 1;
1406  }
1407 
1408  /* if shared file 'S' access, no directory tests */
1409  if (shared_file) {
1410  dirs_only = 0;
1411  }
1412 
1413  /* check for no barriers with shifting processes for different phases.
1414  that is, one may not specify both -B and -N as it will introduce
1415  race conditions that may cause errors stat'ing or deleting after
1416  creates.
1417  */
1418  if (( barriers == 0 ) && ( nstride != 0 ) && ( rank == 0 )) {
1419  FAIL( "Possible race conditions will occur: -B not compatible with -N");
1420  }
1421 
1422  /* check for collective_creates incompatibilities */
1423  if (shared_file && collective_creates && rank == 0) {
1424  FAIL("-c not compatible with -S");
1425  }
1426  if (path_count > 1 && collective_creates && rank == 0) {
1427  FAIL("-c not compatible with multiple test directories");
1428  }
1429  if (collective_creates && !barriers) {
1430  FAIL("-c not compatible with -B");
1431  }
1432 
1433  /* check for shared file incompatibilities */
1434  if (unique_dir_per_task && shared_file && rank == 0) {
1435  FAIL("-u not compatible with -S");
1436  }
1437 
1438  /* check multiple directory paths and strided option */
1439  if (path_count > 1 && nstride > 0) {
1440  FAIL("cannot have multiple directory paths with -N strides between neighbor tasks");
1441  }
1442 
1443  /* check for shared directory and multiple directories incompatibility */
1444  if (path_count > 1 && unique_dir_per_task != 1) {
1445  FAIL("shared directory mode is not compatible with multiple directory paths");
1446  }
1447 
1448  /* check if more directory paths than ranks */
1449  if (path_count > size) {
1450  FAIL("cannot have more directory paths than MPI tasks");
1451  }
1452 
1453  /* check depth */
1454  if (depth < 0) {
1455  FAIL("depth must be greater than or equal to zero");
1456  }
1457  /* check branch_factor */
1458  if (branch_factor < 1 && depth > 0) {
1459  FAIL("branch factor must be greater than or equal to zero");
1460  }
1461  /* check for valid number of items */
1462  if ((items > 0) && (items_per_dir > 0)) {
1463  if(unique_dir_per_task){
1464  FAIL("only specify the number of items or the number of items per directory");
1465  }else if( items % items_per_dir != 0){
1466  FAIL("items must be a multiple of items per directory");
1467  }else if( stone_wall_timer_seconds != 0){
1468  FAIL("items + items_per_dir can only be set without stonewalling");
1469  }
1470  }
1471  /* check for using mknod */
1472  if (write_bytes > 0 && make_node) {
1473  FAIL("-k not compatible with -w");
1474  }
1475 }
1476 
1477 void show_file_system_size(char *file_system) {
1478  char real_path[MAX_PATHLEN];
1479  char file_system_unit_str[MAX_PATHLEN] = "GiB";
1480  char inode_unit_str[MAX_PATHLEN] = "Mi";
1481  int64_t file_system_unit_val = 1024 * 1024 * 1024;
1482  int64_t inode_unit_val = 1024 * 1024;
1483  int64_t total_file_system_size,
1484  free_file_system_size,
1485  total_inodes,
1486  free_inodes;
1487  double total_file_system_size_hr,
1488  used_file_system_percentage,
1489  used_inode_percentage;
1490  ior_aiori_statfs_t stat_buf;
1491  int ret;
1492 
1493  VERBOSE(1,-1,"Entering show_file_system_size on %s", file_system );
1494 
1495  ret = backend->statfs (file_system, &stat_buf, &param);
1496  if (0 != ret) {
1497  FAIL("unable to stat file system %s", file_system);
1498  }
1499 
1500  total_file_system_size = stat_buf.f_blocks * stat_buf.f_bsize;
1501  free_file_system_size = stat_buf.f_bfree * stat_buf.f_bsize;
1502 
1503  used_file_system_percentage = (1 - ((double)free_file_system_size
1504  / (double)total_file_system_size)) * 100;
1505  total_file_system_size_hr = (double)total_file_system_size
1506  / (double)file_system_unit_val;
1507  if (total_file_system_size_hr > 1024) {
1508  total_file_system_size_hr = total_file_system_size_hr / 1024;
1509  strcpy(file_system_unit_str, "TiB");
1510  }
1511 
1512  /* inodes */
1513  total_inodes = stat_buf.f_files;
1514  free_inodes = stat_buf.f_ffree;
1515 
1516  used_inode_percentage = (1 - ((double)free_inodes/(double)total_inodes))
1517  * 100;
1518 
1519  if (realpath(file_system, real_path) == NULL) {
1520  WARN("unable to use realpath() on file system");
1521  }
1522 
1523 
1524  /* show results */
1525  VERBOSE(0,-1,"Path: %s", real_path);
1526  VERBOSE(0,-1,"FS: %.1f %s Used FS: %2.1f%% Inodes: %.1f %s Used Inodes: %2.1f%%\n",
1527  total_file_system_size_hr, file_system_unit_str, used_file_system_percentage,
1528  (double)total_inodes / (double)inode_unit_val, inode_unit_str, used_inode_percentage);
1529 
1530  return;
1531 }
1532 
1534 {
1535  char dirpath[MAX_PATHLEN] = {0};
1536  int i;
1537  int directoryFound = 0;
1538 
1539 
1540  VERBOSE(3,5,"Entering display_freespace on %s...", testdirpath );
1541 
1542  strcpy(dirpath, testdirpath);
1543 
1544  /* get directory for outfile */
1545  i = strlen(dirpath);
1546  while (i-- > 0) {
1547  if (dirpath[i] == '/') {
1548  dirpath[i] = '\0';
1549  directoryFound = 1;
1550  break;
1551  }
1552  }
1553 
1554  /* if no directory/, use '.' */
1555  if (directoryFound == 0) {
1556  strcpy(dirpath, ".");
1557  }
1558 
1559  if (param.api && strcasecmp(param.api, "DFS") == 0)
1560  return;
1561 
1562  VERBOSE(3,5,"Before show_file_system_size, dirpath is '%s'", dirpath );
1563  show_file_system_size(dirpath);
1564  VERBOSE(3,5, "After show_file_system_size, dirpath is '%s'\n", dirpath );
1565 
1566  return;
1567 }
1568 
1570  int currDepth, char* path, int dirNum, rank_progress_t * progress) {
1571 
1572  unsigned i;
1573  char dir[MAX_PATHLEN];
1574 
1575 
1576  VERBOSE(1,5,"Entering create_remove_directory_tree on %s, currDepth = %d...", path, currDepth );
1577 
1578  if (currDepth == 0) {
1579  sprintf(dir, "%s/%s.%d/", path, base_tree_name, dirNum);
1580 
1581  if (create) {
1582  VERBOSE(2,5,"Making directory '%s'", dir);
1583  if (-1 == backend->mkdir (dir, DIRMODE, &param)) {
1584  fprintf(out_logfile, "error could not create directory '%s'\n", dir);
1585  }
1586 #ifdef HAVE_LUSTRE_LUSTREAPI
1587  /* internal node for branching, can be non-striped for children */
1588  if (global_dir_layout && \
1589  llapi_dir_set_default_lmv_stripe(dir, -1, 0,
1590  LMV_HASH_TYPE_FNV_1A_64,
1591  NULL) == -1) {
1592  FAIL("Unable to reset to global default directory layout");
1593  }
1594 #endif /* HAVE_LUSTRE_LUSTREAPI */
1595  }
1596 
1597  create_remove_directory_tree(create, ++currDepth, dir, ++dirNum, progress);
1598 
1599  if (!create) {
1600  VERBOSE(2,5,"Remove directory '%s'", dir);
1601  if (-1 == backend->rmdir(dir, &param)) {
1602  FAIL("Unable to remove directory %s", dir);
1603  }
1604  }
1605  } else if (currDepth <= depth) {
1606 
1607  char temp_path[MAX_PATHLEN];
1608  strcpy(temp_path, path);
1609  int currDir = dirNum;
1610 
1611  for (i=0; i<branch_factor; i++) {
1612  sprintf(dir, "%s.%d/", base_tree_name, currDir);
1613  strcat(temp_path, dir);
1614 
1615  if (create) {
1616  VERBOSE(2,5,"Making directory '%s'", temp_path);
1617  if (-1 == backend->mkdir(temp_path, DIRMODE, &param)) {
1618  FAIL("Unable to create directory %s", temp_path);
1619  }
1620  }
1621 
1622  create_remove_directory_tree(create, ++currDepth,
1623  temp_path, (branch_factor*currDir)+1, progress);
1624  currDepth--;
1625 
1626  if (!create) {
1627  VERBOSE(2,5,"Remove directory '%s'", temp_path);
1628  if (-1 == backend->rmdir(temp_path, &param)) {
1629  FAIL("Unable to remove directory %s", temp_path);
1630  }
1631  }
1632 
1633  strcpy(temp_path, path);
1634  currDir++;
1635  }
1636  }
1637 }
1638 
1639 static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t * summary_table){
1640  rank_progress_t progress_o;
1641  memset(& progress_o, 0 , sizeof(progress_o));
1642  progress_o.start_time = GetTimeStamp();
1644  progress_o.items_per_dir = items_per_dir;
1645  rank_progress_t * progress = & progress_o;
1646 
1647  /* start and end times of directory tree create/remove */
1648  double startCreate, endCreate;
1649  int k;
1650 
1651  VERBOSE(1,-1,"main: * iteration %d *", j+1);
1652 
1653  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
1654  prep_testdir(j, dir_iter);
1655 
1656  VERBOSE(2,5,"main (for j loop): making testdir, '%s'", testdir );
1657  if ((rank < path_count) && backend->access(testdir, F_OK, &param) != 0) {
1658  if (backend->mkdir(testdir, DIRMODE, &param) != 0) {
1659  FAIL("Unable to create test directory %s", testdir);
1660  }
1661 #ifdef HAVE_LUSTRE_LUSTREAPI
1662  /* internal node for branching, can be non-striped for children */
1663  if (global_dir_layout && unique_dir_per_task && llapi_dir_set_default_lmv_stripe(testdir, -1, 0, LMV_HASH_TYPE_FNV_1A_64, NULL) == -1) {
1664  FAIL("Unable to reset to global default directory layout");
1665  }
1666 #endif /* HAVE_LUSTRE_LUSTREAPI */
1667  }
1668  }
1669 
1670  if (create_only) {
1671  /* create hierarchical directory structure */
1672  MPI_Barrier(testComm);
1673 
1674  startCreate = GetTimeStamp();
1675  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
1676  prep_testdir(j, dir_iter);
1677 
1678  if (unique_dir_per_task) {
1679  if (collective_creates && (rank == 0)) {
1680  /*
1681  * This is inside two loops, one of which already uses "i" and the other uses "j".
1682  * I don't know how this ever worked. I'm changing this loop to use "k".
1683  */
1684  for (k=0; k<size; k++) {
1685  sprintf(base_tree_name, "mdtest_tree.%d", k);
1686 
1687  VERBOSE(3,5,"main (create hierarchical directory loop-collective): Calling create_remove_directory_tree with '%s'", testdir );
1688  /*
1689  * Let's pass in the path to the directory we most recently made so that we can use
1690  * full paths in the other calls.
1691  */
1692  create_remove_directory_tree(1, 0, testdir, 0, progress);
1693  if(CHECK_STONE_WALL(progress)){
1694  size = k;
1695  break;
1696  }
1697  }
1698  } else if (!collective_creates) {
1699  VERBOSE(3,5,"main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with '%s'", testdir );
1700  /*
1701  * Let's pass in the path to the directory we most recently made so that we can use
1702  * full paths in the other calls.
1703  */
1704  create_remove_directory_tree(1, 0, testdir, 0, progress);
1705  }
1706  } else {
1707  if (rank == 0) {
1708  VERBOSE(3,5,"main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with '%s'", testdir );
1709 
1710  /*
1711  * Let's pass in the path to the directory we most recently made so that we can use
1712  * full paths in the other calls.
1713  */
1714  create_remove_directory_tree(1, 0 , testdir, 0, progress);
1715  }
1716  }
1717  }
1718  MPI_Barrier(testComm);
1719  endCreate = GetTimeStamp();
1720  summary_table->rate[8] =
1721  num_dirs_in_tree / (endCreate - startCreate);
1722  summary_table->time[8] = (endCreate - startCreate);
1723  summary_table->items[8] = num_dirs_in_tree;
1724  summary_table->stonewall_last_item[8] = num_dirs_in_tree;
1725  VERBOSE(1,-1,"V-1: main: Tree creation : %14.3f sec, %14.3f ops/sec", (endCreate - startCreate), summary_table->rate[8]);
1726  }
1727  sprintf(unique_mk_dir, "%s.0", base_tree_name);
1728  sprintf(unique_chdir_dir, "%s.0", base_tree_name);
1729  sprintf(unique_stat_dir, "%s.0", base_tree_name);
1730  sprintf(unique_read_dir, "%s.0", base_tree_name);
1731  sprintf(unique_rm_dir, "%s.0", base_tree_name);
1732  unique_rm_uni_dir[0] = 0;
1733 
1734  if (!unique_dir_per_task) {
1735  VERBOSE(3,-1,"V-3: main: Using unique_mk_dir, '%s'", unique_mk_dir );
1736  }
1737 
1738  if (rank < i) {
1739  if (!shared_file) {
1740  sprintf(mk_name, "mdtest.%d.", (rank+(0*nstride))%i);
1741  sprintf(stat_name, "mdtest.%d.", (rank+(1*nstride))%i);
1742  sprintf(read_name, "mdtest.%d.", (rank+(2*nstride))%i);
1743  sprintf(rm_name, "mdtest.%d.", (rank+(3*nstride))%i);
1744  }
1745  if (unique_dir_per_task) {
1746  VERBOSE(3,5,"i %d nstride %d", i, nstride);
1747  sprintf(unique_mk_dir, "mdtest_tree.%d.0", (rank+(0*nstride))%i);
1748  sprintf(unique_chdir_dir, "mdtest_tree.%d.0", (rank+(1*nstride))%i);
1749  sprintf(unique_stat_dir, "mdtest_tree.%d.0", (rank+(2*nstride))%i);
1750  sprintf(unique_read_dir, "mdtest_tree.%d.0", (rank+(3*nstride))%i);
1751  sprintf(unique_rm_dir, "mdtest_tree.%d.0", (rank+(4*nstride))%i);
1752  unique_rm_uni_dir[0] = 0;
1753  VERBOSE(5,5,"mk_dir %s chdir %s stat_dir %s read_dir %s rm_dir %s\n", unique_mk_dir,unique_chdir_dir,unique_stat_dir,unique_read_dir,unique_rm_dir);
1754  }
1755 
1756  VERBOSE(3,-1,"V-3: main: Copied unique_mk_dir, '%s', to topdir", unique_mk_dir );
1757 
1758  if (dirs_only && !shared_file) {
1759  if (pre_delay) {
1761  }
1762  directory_test(j, i, unique_mk_dir, progress);
1763  }
1764  if (files_only) {
1765  if (pre_delay) {
1767  }
1768  VERBOSE(3,5,"will file_test on %s", unique_mk_dir);
1769  file_test(j, i, unique_mk_dir, progress);
1770  }
1771  }
1772 
1773  /* remove directory structure */
1774  if (!unique_dir_per_task) {
1775  VERBOSE(3,-1,"main: Using testdir, '%s'", testdir );
1776  }
1777 
1778  MPI_Barrier(testComm);
1779  if (remove_only) {
1780  progress->items_start = 0;
1781  startCreate = GetTimeStamp();
1782  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
1783  prep_testdir(j, dir_iter);
1784  if (unique_dir_per_task) {
1785  if (collective_creates && (rank == 0)) {
1786  /*
1787  * This is inside two loops, one of which already uses "i" and the other uses "j".
1788  * I don't know how this ever worked. I'm changing this loop to use "k".
1789  */
1790  for (k=0; k<size; k++) {
1791  sprintf(base_tree_name, "mdtest_tree.%d", k);
1792 
1793  VERBOSE(3,-1,"main (remove hierarchical directory loop-collective): Calling create_remove_directory_tree with '%s'", testdir );
1794 
1795  /*
1796  * Let's pass in the path to the directory we most recently made so that we can use
1797  * full paths in the other calls.
1798  */
1799  create_remove_directory_tree(0, 0, testdir, 0, progress);
1800  if(CHECK_STONE_WALL(progress)){
1801  size = k;
1802  break;
1803  }
1804  }
1805  } else if (!collective_creates) {
1806  VERBOSE(3,-1,"main (remove hierarchical directory loop-!collective): Calling create_remove_directory_tree with '%s'", testdir );
1807 
1808  /*
1809  * Let's pass in the path to the directory we most recently made so that we can use
1810  * full paths in the other calls.
1811  */
1812  create_remove_directory_tree(0, 0, testdir, 0, progress);
1813  }
1814  } else {
1815  if (rank == 0) {
1816  VERBOSE(3,-1,"V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with '%s'", testdir );
1817 
1818  /*
1819  * Let's pass in the path to the directory we most recently made so that we can use
1820  * full paths in the other calls.
1821  */
1822  create_remove_directory_tree(0, 0 , testdir, 0, progress);
1823  }
1824  }
1825  }
1826 
1827  MPI_Barrier(testComm);
1828  endCreate = GetTimeStamp();
1829  summary_table->rate[9] = num_dirs_in_tree / (endCreate - startCreate);
1830  summary_table->time[9] = endCreate - startCreate;
1831  summary_table->items[9] = num_dirs_in_tree;
1832  summary_table->stonewall_last_item[8] = num_dirs_in_tree;
1833  VERBOSE(1,-1,"main Tree removal : %14.3f sec, %14.3f ops/sec", (endCreate - startCreate), summary_table->rate[9]);
1834  VERBOSE(2,-1,"main (at end of for j loop): Removing testdir of '%s'\n", testdir );
1835 
1836  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
1837  prep_testdir(j, dir_iter);
1838  if ((rank < path_count) && backend->access(testdir, F_OK, &param) == 0) {
1839  //if (( rank == 0 ) && access(testdir, F_OK) == 0) {
1840  if (backend->rmdir(testdir, &param) == -1) {
1841  FAIL("unable to remove directory %s", testdir);
1842  }
1843  }
1844  }
1845  } else {
1846  summary_table->rate[9] = 0;
1847  }
1848 }
1849 
1851  barriers = 1;
1852  branch_factor = 1;
1853  throttle = 1;
1855  create_only = 0;
1856  stat_only = 0;
1857  read_only = 0;
1858  verify_read = 0;
1859  verification_error = 0;
1860  remove_only = 0;
1861  leaf_only = 0;
1862  depth = 0;
1863  num_dirs_in_tree = 0;
1864  items_per_dir = 0;
1865  random_seed = 0;
1866  print_time = 0;
1867  print_rate_and_time = 0;
1868  shared_file = 0;
1869  files_only = 0;
1870  dirs_only = 0;
1871  pre_delay = 0;
1872  unique_dir_per_task = 0;
1874  items = 0;
1876  collective_creates = 0;
1877  write_bytes = 0;
1879  read_bytes = 0;
1880  sync_file = 0;
1881  call_sync = 0;
1882  path_count = 0;
1883  nstride = 0;
1884  make_node = 0;
1885 #ifdef HAVE_LUSTRE_LUSTREAPI
1886  global_dir_layout = 0;
1887 #endif /* HAVE_LUSTRE_LUSTREAPI */
1888 }
1889 
1890 mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * world_out) {
1891  testComm = world_com;
1892  out_logfile = world_out;
1893  mpi_comm_world = world_com;
1894 
1895  init_clock();
1896 
1897  mdtest_init_args();
1898  int i, j;
1899  int numNodes;
1900  int numTasksOnNode0 = 0;
1901  MPI_Group worldgroup, testgroup;
1902  struct {
1903  int first;
1904  int last;
1905  int stride;
1906  } range = {0, 0, 1};
1907  int first = 1;
1908  int last = 0;
1909  int stride = 1;
1910  int iterations = 1;
1911 
1912  verbose = 0;
1913  int no_barriers = 0;
1914  char * path = "./out";
1915  int randomize = 0;
1916  char APIs[1024];
1917  char APIs_legacy[1024];
1918  aiori_supported_apis(APIs, APIs_legacy, MDTEST);
1919  char apiStr[1024];
1920  sprintf(apiStr, "API for I/O [%s]", APIs);
1921 
1922  option_help options [] = {
1923  {'a', NULL, apiStr, OPTION_OPTIONAL_ARGUMENT, 's', & param.api},
1924  {'b', NULL, "branching factor of hierarchical directory structure", OPTION_OPTIONAL_ARGUMENT, 'd', & branch_factor},
1925  {'d', NULL, "the directory in which the tests will run", OPTION_OPTIONAL_ARGUMENT, 's', & path},
1926  {'B', NULL, "no barriers between phases", OPTION_OPTIONAL_ARGUMENT, 'd', & no_barriers},
1927  {'C', NULL, "only create files/dirs", OPTION_FLAG, 'd', & create_only},
1928  {'T', NULL, "only stat files/dirs", OPTION_FLAG, 'd', & stat_only},
1929  {'E', NULL, "only read files/dir", OPTION_FLAG, 'd', & read_only},
1930  {'r', NULL, "only remove files or directories left behind by previous runs", OPTION_FLAG, 'd', & remove_only},
1931  {'D', NULL, "perform test on directories only (no files)", OPTION_FLAG, 'd', & dirs_only},
1932  {'e', NULL, "bytes to read from each file", OPTION_OPTIONAL_ARGUMENT, 'l', & read_bytes},
1933  {'f', NULL, "first number of tasks on which the test will run", OPTION_OPTIONAL_ARGUMENT, 'd', & first},
1934  {'F', NULL, "perform test on files only (no directories)", OPTION_FLAG, 'd', & files_only},
1935 #ifdef HAVE_LUSTRE_LUSTREAPI
1936  {'g', NULL, "global default directory layout for test subdirectories (deletes inherited striping layout)", OPTION_FLAG, 'd', & global_dir_layout},
1937 #endif /* HAVE_LUSTRE_LUSTREAPI */
1938  {'i', NULL, "number of iterations the test will run", OPTION_OPTIONAL_ARGUMENT, 'd', & iterations},
1939  {'I', NULL, "number of items per directory in tree", OPTION_OPTIONAL_ARGUMENT, 'l', & items_per_dir},
1940  {'k', NULL, "use mknod to create file", OPTION_FLAG, 'd', & make_node},
1941  {'l', NULL, "last number of tasks on which the test will run", OPTION_OPTIONAL_ARGUMENT, 'd', & last},
1942  {'L', NULL, "files only at leaf level of tree", OPTION_FLAG, 'd', & leaf_only},
1943  {'n', NULL, "every process will creat/stat/read/remove # directories and files", OPTION_OPTIONAL_ARGUMENT, 'l', & items},
1944  {'N', NULL, "stride # between tasks for file/dir operation (local=0; set to 1 to avoid client cache)", OPTION_OPTIONAL_ARGUMENT, 'd', & nstride},
1945  {'p', NULL, "pre-iteration delay (in seconds)", OPTION_OPTIONAL_ARGUMENT, 'd', & pre_delay},
1946  {'P', NULL, "print rate AND time", OPTION_FLAG, 'd', & print_rate_and_time},
1947  {'R', NULL, "random access to files (only for stat)", OPTION_FLAG, 'd', & randomize},
1948  {0, "random-seed", "random seed for -R", OPTION_OPTIONAL_ARGUMENT, 'd', & random_seed},
1949  {'s', NULL, "stride between the number of tasks for each test", OPTION_OPTIONAL_ARGUMENT, 'd', & stride},
1950  {'S', NULL, "shared file access (file only, no directories)", OPTION_FLAG, 'd', & shared_file},
1951  {'c', NULL, "collective creates: task 0 does all creates", OPTION_FLAG, 'd', & collective_creates},
1952  {'t', NULL, "time unique working directory overhead", OPTION_FLAG, 'd', & time_unique_dir_overhead},
1953  {'u', NULL, "unique working directory for each task", OPTION_FLAG, 'd', & unique_dir_per_task},
1954  {'v', NULL, "verbosity (each instance of option increments by one)", OPTION_FLAG, 'd', & verbose},
1955  {'V', NULL, "verbosity value", OPTION_OPTIONAL_ARGUMENT, 'd', & verbose},
1956  {'w', NULL, "bytes to write to each file after it is created", OPTION_OPTIONAL_ARGUMENT, 'l', & write_bytes},
1957  {'W', NULL, "number in seconds; stonewall timer, write as many seconds and ensure all processes did the same number of operations (currently only stops during create phase)", OPTION_OPTIONAL_ARGUMENT, 'd', & stone_wall_timer_seconds},
1958  {'x', NULL, "StoneWallingStatusFile; contains the number of iterations of the creation phase, can be used to split phases across runs", OPTION_OPTIONAL_ARGUMENT, 's', & stoneWallingStatusFile},
1959  {'X', "verify-read", "Verify the data read", OPTION_FLAG, 'd', & verify_read},
1960  {'y', NULL, "sync file after writing", OPTION_FLAG, 'd', & sync_file},
1961  {'Y', NULL, "call the sync command after each phase (included in the timing; note it causes all IO to be flushed from your node)", OPTION_FLAG, 'd', & call_sync},
1962  {'z', NULL, "depth of hierarchical directory structure", OPTION_OPTIONAL_ARGUMENT, 'd', & depth},
1963  {'Z', NULL, "print time instead of rate", OPTION_FLAG, 'd', & print_time},
1964  LAST_OPTION
1965  };
1967  option_parse(argc, argv, global_options);
1968  updateParsedOptions(& param, global_options);
1969 
1970  free(global_options->modules);
1971  free(global_options);
1972  backend = param.backend;
1973 
1974  MPI_Comm_rank(testComm, &rank);
1975  MPI_Comm_size(testComm, &size);
1976 
1977  if (backend->initialize)
1978  backend->initialize();
1979 
1980  pid = getpid();
1981  uid = getuid();
1982 
1983  numNodes = GetNumNodes(testComm);
1984  numTasksOnNode0 = GetNumTasksOnNode0(testComm);
1985 
1986  char cmd_buffer[4096];
1987  strncpy(cmd_buffer, argv[0], 4096);
1988  for (i = 1; i < argc; i++) {
1989  snprintf(&cmd_buffer[strlen(cmd_buffer)], 4096-strlen(cmd_buffer), " '%s'", argv[i]);
1990  }
1991 
1992  VERBOSE(0,-1,"-- started at %s --\n", PrintTimestamp());
1993  VERBOSE(0,-1,"mdtest-%s was launched with %d total task(s) on %d node(s)", RELEASE_VERS, size, numNodes);
1994  VERBOSE(0,-1,"Command line used: %s", cmd_buffer);
1995 
1996  /* adjust special variables */
1997  barriers = ! no_barriers;
1998  if (path != NULL){
1999  parse_dirpath(path);
2000  }
2001  if( randomize > 0 ){
2002  if (random_seed == 0) {
2003  /* Ensure all procs have the same random number */
2004  random_seed = time(NULL);
2005  MPI_Barrier(testComm);
2006  MPI_Bcast(&random_seed, 1, MPI_INT, 0, testComm);
2007  }
2008  random_seed += rank;
2009  }
2010  if ((items > 0) && (items_per_dir > 0) && (! unique_dir_per_task)) {
2012  }else{
2013  directory_loops = 1;
2014  }
2015  valid_tests();
2016 
2017  // option_print_current(options);
2018  VERBOSE(1,-1, "api : %s", param.api);
2019  VERBOSE(1,-1, "barriers : %s", ( barriers ? "True" : "False" ));
2020  VERBOSE(1,-1, "collective_creates : %s", ( collective_creates ? "True" : "False" ));
2021  VERBOSE(1,-1, "create_only : %s", ( create_only ? "True" : "False" ));
2022  VERBOSE(1,-1, "dirpath(s):" );
2023  for ( i = 0; i < path_count; i++ ) {
2024  VERBOSE(1,-1, "\t%s", filenames[i] );
2025  }
2026  VERBOSE(1,-1, "dirs_only : %s", ( dirs_only ? "True" : "False" ));
2027  VERBOSE(1,-1, "read_bytes : "LLU"", read_bytes );
2028  VERBOSE(1,-1, "read_only : %s", ( read_only ? "True" : "False" ));
2029  VERBOSE(1,-1, "first : %d", first );
2030  VERBOSE(1,-1, "files_only : %s", ( files_only ? "True" : "False" ));
2031 #ifdef HAVE_LUSTRE_LUSTREAPI
2032  VERBOSE(1,-1, "global_dir_layout : %s", ( global_dir_layout ? "True" : "False" ));
2033 #endif /* HAVE_LUSTRE_LUSTREAPI */
2034  VERBOSE(1,-1, "iterations : %d", iterations );
2035  VERBOSE(1,-1, "items_per_dir : "LLU"", items_per_dir );
2036  VERBOSE(1,-1, "last : %d", last );
2037  VERBOSE(1,-1, "leaf_only : %s", ( leaf_only ? "True" : "False" ));
2038  VERBOSE(1,-1, "items : "LLU"", items );
2039  VERBOSE(1,-1, "nstride : %d", nstride );
2040  VERBOSE(1,-1, "pre_delay : %d", pre_delay );
2041  VERBOSE(1,-1, "remove_only : %s", ( leaf_only ? "True" : "False" ));
2042  VERBOSE(1,-1, "random_seed : %d", random_seed );
2043  VERBOSE(1,-1, "stride : %d", stride );
2044  VERBOSE(1,-1, "shared_file : %s", ( shared_file ? "True" : "False" ));
2045  VERBOSE(1,-1, "time_unique_dir_overhead: %s", ( time_unique_dir_overhead ? "True" : "False" ));
2046  VERBOSE(1,-1, "stone_wall_timer_seconds: %d", stone_wall_timer_seconds);
2047  VERBOSE(1,-1, "stat_only : %s", ( stat_only ? "True" : "False" ));
2048  VERBOSE(1,-1, "unique_dir_per_task : %s", ( unique_dir_per_task ? "True" : "False" ));
2049  VERBOSE(1,-1, "write_bytes : "LLU"", write_bytes );
2050  VERBOSE(1,-1, "sync_file : %s", ( sync_file ? "True" : "False" ));
2051  VERBOSE(1,-1, "call_sync : %s", ( call_sync ? "True" : "False" ));
2052  VERBOSE(1,-1, "depth : %d", depth );
2053  VERBOSE(1,-1, "make_node : %d", make_node );
2054 
2055  /* setup total number of items and number of items per dir */
2056  if (depth <= 0) {
2057  num_dirs_in_tree = 1;
2058  } else {
2059  if (branch_factor < 1) {
2060  num_dirs_in_tree = 1;
2061  } else if (branch_factor == 1) {
2062  num_dirs_in_tree = depth + 1;
2063  } else {
2064  num_dirs_in_tree = (pow(branch_factor, depth+1) - 1) / (branch_factor - 1);
2065  }
2066  }
2067  if (items_per_dir > 0) {
2068  if(items == 0){
2069  if (leaf_only) {
2070  items = items_per_dir * (uint64_t) pow(branch_factor, depth);
2071  } else {
2073  }
2074  }else{
2076  }
2077  } else {
2078  if (leaf_only) {
2079  if (branch_factor <= 1) {
2080  items_per_dir = items;
2081  } else {
2082  items_per_dir = (uint64_t) (items / pow(branch_factor, depth));
2083  items = items_per_dir * (uint64_t) pow(branch_factor, depth);
2084  }
2085  } else {
2088  }
2089  }
2090 
2091  /* initialize rand_array */
2092  if (random_seed > 0) {
2093  srand(random_seed);
2094 
2095  uint64_t s;
2096 
2097  rand_array = (uint64_t *) malloc( items * sizeof(*rand_array));
2098 
2099  for (s=0; s < items; s++) {
2100  rand_array[s] = s;
2101  }
2102 
2103  /* shuffle list randomly */
2104  uint64_t n = items;
2105  while (n>1) {
2106  n--;
2107 
2108  /*
2109  * Generate a random number in the range 0 .. n
2110  *
2111  * rand() returns a number from 0 .. RAND_MAX. Divide that
2112  * by RAND_MAX and you get a floating point number in the
2113  * range 0 .. 1. Multiply that by n and you get a number in
2114  * the range 0 .. n.
2115  */
2116  uint64_t k = ( uint64_t ) ((( double )rand() / ( double )RAND_MAX ) * ( double )n );
2117 
2118  /*
2119  * Now move the nth element to the kth (randomly chosen)
2120  * element, and the kth element to the nth element.
2121  */
2122 
2123  uint64_t tmp = rand_array[k];
2124  rand_array[k] = rand_array[n];
2125  rand_array[n] = tmp;
2126  }
2127  }
2128 
2129  /* allocate and initialize write buffer with # */
2130  if (write_bytes > 0) {
2131  int alloc_res = posix_memalign((void**)&write_buffer, sysconf(_SC_PAGESIZE), write_bytes);
2132  if (alloc_res) {
2133  FAIL("out of memory");
2134  }
2135  generate_memory_pattern(write_buffer, write_bytes);
2136  }
2137 
2138  /* setup directory path to work in */
2139  if (path_count == 0) { /* special case where no directory path provided with '-d' option */
2140  char *ret = getcwd(testdirpath, MAX_PATHLEN);
2141  if (ret == NULL) {
2142  FAIL("Unable to get current working directory on %s", testdirpath);
2143  }
2144  path_count = 1;
2145  } else {
2146  strcpy(testdirpath, filenames[rank%path_count]);
2147  }
2148 
2149  /* if directory does not exist, create it */
2150  if ((rank < path_count) && backend->access(testdirpath, F_OK, &param) != 0) {
2151  if (backend->mkdir(testdirpath, DIRMODE, &param) != 0) {
2152  FAIL("Unable to create test directory path %s", testdirpath);
2153  }
2154  }
2155 
2156  /* display disk usage */
2157  VERBOSE(3,-1,"main (before display_freespace): testdirpath is '%s'", testdirpath );
2158 
2159  if (rank == 0) display_freespace(testdirpath);
2160  int tasksBlockMapping = QueryNodeMapping(testComm, true);
2161 
2162  /* set the shift to mimic IOR and shift by procs per node */
2163  if (nstride > 0) {
2164  if ( numNodes > 1 && tasksBlockMapping ) {
2165  /* the user set the stride presumably to get the consumer tasks on a different node than the producer tasks
2166  however, if the mpirun scheduler placed the tasks by-slot (in a contiguous block) then we need to adjust the shift by ppn */
2167  nstride *= numTasksOnNode0;
2168  }
2169  VERBOSE(0,5,"Shifting ranks by %d for each phase.", nstride);
2170  }
2171 
2172  VERBOSE(3,-1,"main (after display_freespace): testdirpath is '%s'", testdirpath );
2173 
2174  if (rank == 0) {
2175  if (random_seed > 0) {
2176  VERBOSE(0,-1,"random seed: %d", random_seed);
2177  }
2178  }
2179 
2180  if (gethostname(hostname, MAX_PATHLEN) == -1) {
2181  perror("gethostname");
2182  MPI_Abort(testComm, 2);
2183  }
2184 
2185  if (last == 0) {
2186  first = size;
2187  last = size;
2188  }
2189 
2190  /* setup summary table for recording results */
2191  summary_table = (mdtest_results_t *) malloc(iterations * sizeof(mdtest_results_t));
2192  memset(summary_table, 0, iterations * sizeof(mdtest_results_t));
2193  for(int i=0; i < iterations; i++){
2194  for(int j=0; j < MDTEST_LAST_NUM; j++){
2195  summary_table[i].rate[j] = 0.0;
2196  summary_table[i].time[j] = 0.0;
2197  }
2198  }
2199 
2200  if (summary_table == NULL) {
2201  FAIL("out of memory");
2202  }
2203 
2204  if (unique_dir_per_task) {
2205  sprintf(base_tree_name, "mdtest_tree.%d", rank);
2206  } else {
2207  sprintf(base_tree_name, "mdtest_tree");
2208  }
2209 
2210  /* default use shared directory */
2211  strcpy(mk_name, "mdtest.shared.");
2212  strcpy(stat_name, "mdtest.shared.");
2213  strcpy(read_name, "mdtest.shared.");
2214  strcpy(rm_name, "mdtest.shared.");
2215 
2216  MPI_Comm_group(testComm, &worldgroup);
2217 
2218  /* Run the tests */
2219  for (i = first; i <= last && i <= size; i += stride) {
2220  range.last = i - 1;
2221  MPI_Group_range_incl(worldgroup, 1, (void *)&range, &testgroup);
2222  MPI_Comm_create(testComm, testgroup, &testComm);
2223  if (rank == 0) {
2224  uint64_t items_all = i * items;
2226  items_all *= num_dirs_in_tree_calc;
2227  }
2228  if (files_only && dirs_only) {
2229  VERBOSE(0,-1,"%d tasks, "LLU" files/directories", i, items_all);
2230  } else if (files_only) {
2231  if (!shared_file) {
2232  VERBOSE(0,-1,"%d tasks, "LLU" files", i, items_all);
2233  }
2234  else {
2235  VERBOSE(0,-1,"%d tasks, 1 file", i);
2236  }
2237  } else if (dirs_only) {
2238  VERBOSE(0,-1,"%d tasks, "LLU" directories", i, items_all);
2239  }
2240  }
2241  VERBOSE(1,-1,"");
2242  VERBOSE(1,-1," Operation Duration Rate");
2243  VERBOSE(1,-1," --------- -------- ----");
2244 
2245  for (j = 0; j < iterations; j++) {
2246  // keep track of the current status for stonewalling
2247  mdtest_iteration(i, j, testgroup, & summary_table[j]);
2248  }
2249  if (print_rate_and_time){
2250  summarize_results(iterations, 0);
2251  summarize_results(iterations, 1);
2252  }else{
2253  summarize_results(iterations, print_time);
2254  }
2255  if (i == 1 && stride > 1) {
2256  i = 0;
2257  }
2258  }
2259 
2260  if(verification_error){
2261  VERBOSE(0, -1, "\nERROR: verifying the data read! Take the performance values with care!\n");
2262  }
2263  VERBOSE(0,-1,"-- finished at %s --\n", PrintTimestamp());
2264 
2265  if (random_seed > 0) {
2266  free(rand_array);
2267  }
2268 
2269  if (backend->finalize)
2270  backend->finalize();
2271 
2272  return summary_table;
2273 }
option_module * modules
Definition: option.h:34
static int shared_file
Definition: mdtest.c:143
Definition: aiori.h:95
uint64_t stonewall_item_sum[MDTEST_LAST_NUM]
Definition: mdtest.h:32
static char testdir[MAX_PATHLEN]
Definition: mdtest.c:93
static int verification_error
Definition: mdtest.c:119
double rate[MDTEST_LAST_NUM]
Definition: mdtest.h:24
static char unique_rm_dir[MAX_PATHLEN]
Definition: mdtest.c:106
#define VERBOSE(root, any,...)
Definition: mdtest.c:195
uint64_t f_blocks
Definition: aiori.h:59
static char testdirpath[MAX_PATHLEN]
Definition: mdtest.c:94
#define FILEMODE
Definition: mdtest.c:83
void(* delete)(char *, IOR_param_t *)
Definition: aiori.h:76
static char unique_read_dir[MAX_PATHLEN]
Definition: mdtest.c:105
static char rm_name[MAX_PATHLEN]
Definition: mdtest.c:101
uint64_t f_bfree
Definition: aiori.h:60
int(* rmdir)(const char *path, IOR_param_t *param)
Definition: aiori.h:82
static int stone_wall_timer_seconds
Definition: mdtest.c:152
#define LAST_OPTION
Definition: option.h:37
int(* mknod)(char *)
Definition: aiori.h:71
static char stat_name[MAX_PATHLEN]
Definition: mdtest.c:99
static char unique_rm_uni_dir[MAX_PATHLEN]
Definition: mdtest.c:107
static char * verify_read_buffer
Definition: mdtest.c:110
static size_t read_bytes
Definition: mdtest.c:153
int filePerProc
Definition: ior.h:111
void VerboseMessage(int root_level, int any_level, int line, char *format,...)
Definition: mdtest.c:196
static int size
Definition: mdtest.c:91
static int barriers
Definition: mdtest.c:114
int option_parse(int argc, char **argv, options_all_t *opt_all)
Definition: option.c:376
static void create_file(const char *path, uint64_t itemNum)
Definition: mdtest.c:346
static const ior_aiori_t * backend
Definition: mdtest.c:168
void mdtest_init_args()
Definition: mdtest.c:1850
static int time_unique_dir_overhead
Definition: mdtest.c:148
int64_t ReadStoneWallingIterations(char *const filename)
Definition: utilities.c:795
mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *world_out)
Definition: mdtest.c:1890
static int nstride
Definition: mdtest.c:157
static int unique_dir_per_task
Definition: mdtest.c:147
uint64_t stonewall_item_min[MDTEST_LAST_NUM]
Definition: mdtest.h:31
void parse_dirpath(char *dirpath_arg)
Definition: mdtest.c:233
static uint64_t items
Definition: mdtest.c:136
static int collective_creates
Definition: mdtest.c:150
#define DIRMODE
Definition: mdtest.c:84
uint64_t items_start
Definition: mdtest.c:178
int QueryNodeMapping(MPI_Comm comm, int print_nodemap)
Definition: utilities.c:230
uint64_t f_ffree
Definition: aiori.h:63
void offset_timers(double *t, int tcount)
Definition: mdtest.c:220
void summarize_results(int iterations, int print_time)
Definition: mdtest.c:1237
static void prep_testdir(int j, int dir_iter)
Definition: mdtest.c:265
void create_remove_items(int currDepth, const int dirs, const int create, const int collective, const char *path, uint64_t dirNum, rank_progress_t *progress)
Definition: mdtest.c:474
unsigned int openFlags
Definition: ior.h:88
int fsyncPerWrite
Definition: ior.h:162
static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t *summary_table)
Definition: mdtest.c:1639
static unsigned branch_factor
Definition: mdtest.c:122
int(* access)(const char *path, int mode, IOR_param_t *param)
Definition: aiori.h:83
void mdtest_read(int random, int dirs, const long dir_iter, char *path)
Definition: mdtest.c:627
int stone_wall_timer_seconds
Definition: mdtest.c:176
void show_file_system_size(char *file_system)
Definition: mdtest.c:1477
#define WRITE
Definition: iordef.h:95
void(* close)(void *, IOR_param_t *)
Definition: aiori.h:75
double start_time
Definition: mdtest.c:174
static char * write_buffer
Definition: mdtest.c:108
static int directory_loops
Definition: mdtest.c:139
unsigned int mode
Definition: ior.h:87
#define READ
Definition: iordef.h:97
static size_t write_bytes
Definition: mdtest.c:151
void mdtest_stat(const int random, const int dirs, const long dir_iter, const char *path, rank_progress_t *progress)
Definition: mdtest.c:544
char * PrintTimestamp()
Definition: utilities.c:780
static int make_node
Definition: mdtest.c:158
void init_clock()
Definition: utilities.c:775
#define IOR_CREAT
Definition: aiori.h:38
void *(* open)(char *, IOR_param_t *)
Definition: aiori.h:72
static int read_only
Definition: mdtest.c:117
static char * stoneWallingStatusFile
Definition: mdtest.c:111
void collective_helper(const int dirs, const int create, const char *path, uint64_t itemNum, rank_progress_t *progress)
Definition: mdtest.c:435
void file_test(const int iteration, const int ntasks, const char *path, rank_progress_t *progress)
Definition: mdtest.c:1020
void(* finalize)(void)
Definition: aiori.h:86
double time[MDTEST_LAST_NUM]
Definition: mdtest.h:25
static int print_rate_and_time
Definition: mdtest.c:141
static int files_only
Definition: mdtest.c:144
static char base_tree_name[MAX_PATHLEN]
Definition: mdtest.c:95
static int leaf_only
Definition: mdtest.c:121
uint64_t f_files
Definition: aiori.h:62
MPI_Comm testComm
Definition: utilities.c:60
static option_help options[]
Definition: aiori-CEPHFS.c:54
static int print_time
Definition: mdtest.c:140
static mdtest_results_t * summary_table
Definition: mdtest.c:163
uint64_t f_bsize
Definition: aiori.h:58
uint64_t items[MDTEST_LAST_NUM]
Definition: mdtest.h:26
static int create_only
Definition: mdtest.c:115
void valid_tests()
Definition: mdtest.c:1390
static uint64_t num_dirs_in_tree
Definition: mdtest.c:130
static char unique_chdir_dir[MAX_PATHLEN]
Definition: mdtest.c:103
void collective_create_remove(const int create, const int dirs, const int ntasks, const char *path, rank_progress_t *progress)
Definition: mdtest.c:743
static uid_t uid
Definition: mdtest.c:165
void updateParsedOptions(IOR_param_t *options, options_all_t *global_options)
Definition: utilities.c:121
int GetNumNodes(MPI_Comm comm)
Definition: utilities.c:274
void(* initialize)(void)
Definition: aiori.h:85
#define FAIL(...)
Definition: utilities.h:42
void directory_test(const int iteration, const int ntasks, const char *path, rank_progress_t *progress)
Definition: mdtest.c:821
void create_remove_items_helper(const int dirs, const int create, const char *path, uint64_t itemNum, rank_progress_t *progress)
Definition: mdtest.c:409
options_all_t * airoi_create_all_module_options(option_help *global_options)
Definition: aiori.c:99
double GetTimeStamp(void)
Definition: utilities.c:726
int updateStoneWallIterations(int iteration, rank_progress_t *progress, double tstart)
Definition: mdtest.c:993
void display_freespace(char *testdirpath)
Definition: mdtest.c:1533
void create_remove_directory_tree(int create, int currDepth, char *path, int dirNum, rank_progress_t *progress)
Definition: mdtest.c:1569
void aiori_supported_apis(char *APIs, char *APIs_legacy, enum bench_type type)
Definition: aiori.c:118
static int sync_file
Definition: mdtest.c:154
uint64_t items_done
Definition: mdtest.c:179
static int dirs_only
Definition: mdtest.c:145
int(* mkdir)(const char *path, mode_t mode, IOR_param_t *param)
Definition: aiori.h:81
static IOR_param_t param
Definition: mdtest.c:170
void generate_memory_pattern(char *buffer, size_t bytes)
Definition: mdtest.c:214
static void remove_file(const char *path, uint64_t itemNum)
Definition: mdtest.c:331
void StoreStoneWallingIterations(char *const filename, int64_t count)
Definition: utilities.c:817
#define IOR_WRONLY
Definition: aiori.h:35
static char hostname[MAX_PATHLEN]
Definition: mdtest.c:97
static char unique_stat_dir[MAX_PATHLEN]
Definition: mdtest.c:104
static int path_count
Definition: mdtest.c:156
static char read_name[MAX_PATHLEN]
Definition: mdtest.c:100
#define RELEASE_VERS
Definition: mdtest.c:85
static options_all_t * global_options
Definition: parse_options.c:43
static uint64_t num_dirs_in_tree_calc
Definition: mdtest.c:138
static int random_seed
Definition: mdtest.c:142
static void create_remove_dirs(const char *path, bool create, uint64_t itemNum)
Definition: mdtest.c:308
void *(* create)(char *, IOR_param_t *)
Definition: aiori.h:70
long long int IOR_size_t
Definition: iordef.h:123
#define WARN(MSG)
Definition: iordef.h:144
uint64_t stonewall_last_item[MDTEST_LAST_NUM]
Definition: mdtest.h:30
static int pre_delay
Definition: mdtest.c:146
static char * read_buffer
Definition: mdtest.c:109
static int throttle
Definition: mdtest.c:149
static pid_t pid
Definition: mdtest.c:164
uint64_t items_per_dir
Definition: mdtest.c:181
static void phase_end()
Definition: mdtest.c:274
IOR_offset_t offset
Definition: ior.h:126
#define MAX_PATHLEN
Definition: utilities.h:33
static char mk_name[MAX_PATHLEN]
Definition: mdtest.c:98
static int remove_only
Definition: mdtest.c:120
double stonewall_time[MDTEST_LAST_NUM]
Definition: mdtest.h:29
const struct ior_aiori * backend
Definition: ior.h:85
int(* statfs)(const char *, ior_aiori_statfs_t *, IOR_param_t *param)
Definition: aiori.h:80
static char unique_mk_dir[MAX_PATHLEN]
Definition: mdtest.c:102
void DelaySecs(int delay)
Definition: utilities.c:832
#define TEST_DIR
Definition: mdtest.c:86
int verbose
Definition: utilities.c:59
MPI_Comm mpi_comm_world
Definition: utilities.c:61
int calc_allreduce_index(int iter, int rank, int op)
Definition: mdtest.c:1232
char * api
Definition: ior.h:90
static char ** filenames
Definition: mdtest.c:96
static uint64_t items_per_dir
Definition: mdtest.c:137
void(* sync)(IOR_param_t *)
Definition: aiori.h:90
FILE * out_logfile
Definition: utilities.c:62
static int call_sync
Definition: mdtest.c:155
void unique_dir_access(int opt, char *to)
Definition: mdtest.c:292
int rank
Definition: utilities.c:57
#define ITEM_COUNT
Definition: mdtest.c:87
int GetNumTasksOnNode0(MPI_Comm comm)
Definition: utilities.c:349
int(* stat)(const char *path, struct stat *buf, IOR_param_t *param)
Definition: aiori.h:84
#define CHECK_STONE_WALL(p)
Definition: mdtest.c:184
static int stat_only
Definition: mdtest.c:116
IOR_offset_t(* xfer)(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori.h:73
#define LLU
Definition: mdtest.c:89
static int depth
Definition: mdtest.c:123
static uint64_t * rand_array
Definition: mdtest.c:92
#define NULL
Definition: iordef.h:79
static int verify_read
Definition: mdtest.c:118