IOR
aiori-HDFS.c
Go to the documentation of this file.
1 /* -*- mode: c; indent-tabs-mode: t; -*-
2  * vim:noexpandtab:
3  *
4  * Editing with tabs allows different users to pick their own indentation
5  * appearance without changing the file.
6  */
7 
8 /*
9  * Copyright (c) 2009, Los Alamos National Security, LLC All rights reserved.
10  * Copyright 2009. Los Alamos National Security, LLC. This software was produced
11  * under U.S. Government contract DE-AC52-06NA25396 for Los Alamos National
12  * Laboratory (LANL), which is operated by Los Alamos National Security, LLC for
13  * the U.S. Department of Energy. The U.S. Government has rights to use,
14  * reproduce, and distribute this software. NEITHER THE GOVERNMENT NOR LOS
15  * ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR
16  * ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. If software is
17  * modified to produce derivative works, such modified software should be
18  * clearly marked, so as not to confuse it with the version available from
19  * LANL.
20  *
21  * Additionally, redistribution and use in source and binary forms, with or
22  * without modification, are permitted provided that the following conditions are
23  * met:
24  *
25  * • Redistributions of source code must retain the above copyright notice,
26  * this list of conditions and the following disclaimer.
27  *
28  * • Redistributions in binary form must reproduce the above copyright notice,
29  * this list of conditions and the following disclaimer in the documentation
30  * and/or other materials provided with the distribution.
31  *
32  * • Neither the name of Los Alamos National Security, LLC, Los Alamos National
33  * Laboratory, LANL, the U.S. Government, nor the names of its contributors may be
34  * used to endorse or promote products derived from this software without specific
35  * prior written permission.
36  *
37  * THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS
38  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
39  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
40  * ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR
41  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
42  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
43  * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
44  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
45  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
46  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
47  * OF SUCH DAMAGE.
48  */
49 
50 /******************************************************************************\
51 *
52 * Implement of abstract I/O interface for HDFS.
53 *
54 * HDFS has the added concept of a "File System Handle" which has to be
55 * connected before files are opened. We store this in the IOR_param_t
56 * object that is always passed to our functions. The thing that callers
57 * think of as the "fd" is an hdfsFile, (a pointer).
58 *
59 \******************************************************************************/
60 
61 #ifdef HAVE_CONFIG_H
62 # include "config.h"
63 #endif
64 
65 #include <stdio.h>
66 #include <stdlib.h>
67 
68 #ifdef __linux__
69 # include <sys/ioctl.h> /* necessary for: */
70 # define __USE_GNU /* O_DIRECT and */
71 # include <fcntl.h> /* IO operations */
72 # undef __USE_GNU
73 #endif /* __linux__ */
74 
75 #include <errno.h>
76 #include <fcntl.h> /* IO operations */
77 #include <sys/stat.h>
78 #include <assert.h>
79 /*
80 #ifdef HAVE_LUSTRE_LUSTRE_USER_H
81 #include <lustre/lustre_user.h>
82 #endif
83 */
84 
85 #include "ior.h"
86 #include "aiori.h"
87 #include "iordef.h"
88 
89 #ifndef open64 /* necessary for TRU64 -- */
90 # define open64 open /* unlikely, but may pose */
91 #endif /* not open64 */ /* conflicting prototypes */
92 
93 #ifndef lseek64 /* necessary for TRU64 -- */
94 # define lseek64 lseek /* unlikely, but may pose */
95 #endif /* not lseek64 */ /* conflicting prototypes */
96 
97 #ifndef O_BINARY /* Required on Windows */
98 # define O_BINARY 0
99 #endif
100 
101 #include "hdfs.h"
102 
103 /**************************** P R O T O T Y P E S *****************************/
104 static void *HDFS_Create(char *, IOR_param_t *);
105 static void *HDFS_Open(char *, IOR_param_t *);
106 static IOR_offset_t HDFS_Xfer(int, void *, IOR_size_t *,
108 static void HDFS_Close(void *, IOR_param_t *);
109 static void HDFS_Delete(char *, IOR_param_t *);
110 static void HDFS_SetVersion(IOR_param_t *);
111 static void HDFS_Fsync(void *, IOR_param_t *);
112 static IOR_offset_t HDFS_GetFileSize(IOR_param_t *, MPI_Comm, char *);
113 
114 /************************** D E C L A R A T I O N S ***************************/
115 
117  .name = "HDFS",
118  .name_legacy = NULL,
119  .create = HDFS_Create,
120  .open = HDFS_Open,
121  .xfer = HDFS_Xfer,
122  .close = HDFS_Close,
123  .delete = HDFS_Delete,
124  .set_version = HDFS_SetVersion,
125  .fsync = HDFS_Fsync,
126  .get_file_size = HDFS_GetFileSize,
127 };
128 
129 /***************************** F U N C T I O N S ******************************/
130 
131 /* This is identical to the one in aiori-POSIX.c Doesn't seem like
132  * it would be appropriate in utilities.c.
133  */
134 
136 {
137 /* note that TRU64 needs O_DIRECTIO, SunOS uses directio(),
138  and everyone else needs O_DIRECT */
139 #ifndef O_DIRECT
140 # ifndef O_DIRECTIO
141  WARN("cannot use O_DIRECT");
142 # define O_DIRECT 000000
143 # else /* O_DIRECTIO */
144 # define O_DIRECT O_DIRECTIO
145 # endif /* not O_DIRECTIO */
146 #endif /* not O_DIRECT */
147 
148  *fd |= O_DIRECT;
149 }
150 
151 
152 /*
153  * "Connect" to an HDFS file-system. HDFS requires this be done before and
154  * files are opened. It is easy for ior_aiori.open/create to assure that
155  * we connect, if we haven't already done so. However, there's not a
156  * simple way to assure that we disconnect after the last file-close. For
157  * now, we'll make a special call at the end of ior.c
158  *
159  * NOTE: It's okay to call this thing whenever you need to be sure the HDFS
160  * filesystem is connected.
161  */
162 static void hdfs_connect( IOR_param_t* param ) {
163  if (param->verbose >= VERBOSE_4) {
164  printf("-> hdfs_connect [nn:\"%s\", port:%d, user:%s]\n",
165  param->hdfs_name_node,
166  param->hdfs_name_node_port,
167  param->hdfs_user );
168  }
169 
170  if ( param->hdfs_fs ) {
171  if (param->verbose >= VERBOSE_4) {
172  printf("<- hdfs_connect [nothing to do]\n"); /* DEBUGGING */
173  }
174  return;
175  }
176 
177  /* initialize a builder, holding parameters for hdfsBuilderConnect() */
178  struct hdfsBuilder* builder = hdfsNewBuilder();
179  if ( ! builder )
180  ERR_SIMPLE("couldn't create an hdfsBuilder");
181 
182  hdfsBuilderSetForceNewInstance ( builder ); /* don't use cached instance */
183 
184  hdfsBuilderSetNameNode ( builder, param->hdfs_name_node );
185  hdfsBuilderSetNameNodePort( builder, param->hdfs_name_node_port );
186  hdfsBuilderSetUserName ( builder, param->hdfs_user );
187 
188  /* NOTE: hdfsBuilderConnect() frees the builder */
189  param->hdfs_fs = hdfsBuilderConnect( builder );
190  if ( ! param->hdfs_fs )
191  ERR_SIMPLE("hdsfsBuilderConnect failed");
192 
193  if (param->verbose >= VERBOSE_4) {
194  printf("<- hdfs_connect [success]\n");
195  }
196 }
197 
199  if (param->verbose >= VERBOSE_4) {
200  printf("-> hdfs_disconnect\n");
201  }
202  if ( param->hdfs_fs ) {
203  hdfsDisconnect( param->hdfs_fs );
204  param->hdfs_fs = NULL;
205  }
206  if (param->verbose >= VERBOSE_4) {
207  printf("<- hdfs_disconnect\n");
208  }
209 }
210 
211 
212 /*
213  * Create or open the file. Pass TRUE if creating and FALSE if opening an existing file.
214  * Return an hdfsFile.
215  */
216 
217 static void *HDFS_Create_Or_Open( char *testFileName, IOR_param_t *param, unsigned char createFile ) {
218  if (param->verbose >= VERBOSE_4) {
219  printf("-> HDFS_Create_Or_Open\n");
220  }
221 
222  hdfsFile hdfs_file = NULL;
223  int fd_oflags = 0, hdfs_return;
224 
225  /* initialize file-system handle, if needed */
226  hdfs_connect( param );
227 
228  /*
229  * Check for unsupported flags.
230  *
231  * If they want RDWR, we don't know if they're going to try to do both, so we
232  * can't default to either O_RDONLY or O_WRONLY. Thus, we error and exit.
233  *
234  * The other two, we just note that they are not supported and don't do them.
235  */
236 
237  if ( param->openFlags & IOR_RDWR ) {
238  ERR( "Opening or creating a file in RDWR is not implemented in HDFS" );
239  }
240 
241  if ( param->openFlags & IOR_EXCL ) {
242  fprintf( stdout, "Opening or creating a file in Exclusive mode is not implemented in HDFS\n" );
243  }
244 
245  if ( param->openFlags & IOR_APPEND ) {
246  fprintf( stdout, "Opening or creating a file for appending is not implemented in HDFS\n" );
247  }
248 
249  /*
250  * Setup the flags to be used.
251  */
252 
253  if ( createFile == TRUE ) {
254  fd_oflags = O_CREAT;
255  }
256 
257  if ( param->openFlags & IOR_WRONLY ) {
258  if ( !param->filePerProc ) {
259 
260  // in N-1 mode, only rank 0 truncates the file
261  if ( rank != 0 ) {
262  fd_oflags |= O_WRONLY;
263  } else {
264  fd_oflags |= O_TRUNC;
265  fd_oflags |= O_WRONLY;
266  }
267 
268  } else {
269  // in N-N mode, everyone does truncate
270  fd_oflags |= O_TRUNC;
271  fd_oflags |= O_WRONLY;
272  }
273 
274  } else {
275  fd_oflags |= O_RDONLY;
276  }
277 
278  /*
279  * Now see if O_DIRECT is needed.
280  */
281 
282  if ( param->useO_DIRECT == TRUE ) {
283  hdfs_set_o_direct_flag( &fd_oflags );
284  }
285 
286  /*
287  * For N-1 write, All other ranks wait for Rank 0 to open the file.
288  * this is bec 0 does truncate and rest don't
289  * it would be dangerous for all to do truncate as they might
290  * truncate each other's writes
291  */
292 
293  if (( param->openFlags & IOR_WRONLY ) &&
294  ( !param->filePerProc ) &&
295  ( rank != 0 )) {
296 
297  MPI_CHECK(MPI_Barrier(testComm), "barrier error");
298  }
299 
300  /*
301  * Now rank zero can open and truncate, if necessary.
302  */
303 
304  if (param->verbose >= VERBOSE_4) {
305  printf("\thdfsOpenFile(0x%llx, %s, 0%o, %d, %d, %d)\n",
306  param->hdfs_fs,
307  testFileName,
308  fd_oflags, /* shown in octal to compare w/ <bits/fcntl.h> */
309  param->transferSize,
310  param->hdfs_replicas,
311  param->hdfs_block_size);
312  }
313  hdfs_file = hdfsOpenFile( param->hdfs_fs,
314  testFileName,
315  fd_oflags,
316  param->transferSize,
317  param->hdfs_replicas,
318  param->hdfs_block_size);
319  if ( ! hdfs_file ) {
320  ERR( "Failed to open the file" );
321  }
322 
323  /*
324  * For N-1 write, Rank 0 waits for the other ranks to open the file after it has.
325  */
326 
327  if (( param->openFlags & IOR_WRONLY ) &&
328  ( !param->filePerProc ) &&
329  ( rank == 0 )) {
330 
331  MPI_CHECK(MPI_Barrier(testComm), "barrier error");
332  }
333 
334  if (param->verbose >= VERBOSE_4) {
335  printf("<- HDFS_Create_Or_Open\n");
336  }
337  return ((void *) hdfs_file );
338 }
339 
340 /*
341  * Create and open a file through the HDFS interface.
342  */
343 
344 static void *HDFS_Create( char *testFileName, IOR_param_t * param ) {
345  if (param->verbose >= VERBOSE_4) {
346  printf("-> HDFS_Create\n");
347  }
348 
349  if (param->verbose >= VERBOSE_4) {
350  printf("<- HDFS_Create\n");
351  }
352  return HDFS_Create_Or_Open( testFileName, param, TRUE );
353 }
354 
355 /*
356  * Open a file through the HDFS interface.
357  */
358 static void *HDFS_Open( char *testFileName, IOR_param_t * param ) {
359  if (param->verbose >= VERBOSE_4) {
360  printf("-> HDFS_Open\n");
361  }
362 
363  if ( param->openFlags & IOR_CREAT ) {
364  if (param->verbose >= VERBOSE_4) {
365  printf("<- HDFS_Open( ... TRUE)\n");
366  }
367  return HDFS_Create_Or_Open( testFileName, param, TRUE );
368  }
369  else {
370  if (param->verbose >= VERBOSE_4) {
371  printf("<- HDFS_Open( ... FALSE)\n");
372  }
373  return HDFS_Create_Or_Open( testFileName, param, FALSE );
374  }
375 }
376 
377 /*
378  * Write or read to file using the HDFS interface.
379  */
380 
381 static IOR_offset_t HDFS_Xfer(int access, void *file, IOR_size_t * buffer,
382  IOR_offset_t length, IOR_param_t * param) {
383  if (param->verbose >= VERBOSE_4) {
384  printf("-> HDFS_Xfer(acc:%d, file:0x%llx, buf:0x%llx, len:%llu, 0x%llx)\n",
385  access, file, buffer, length, param);
386  }
387 
388  int xferRetries = 0;
389  long long remaining = (long long)length;
390  char* ptr = (char *)buffer;
391  long long rc;
392  off_t offset = param->offset;
393  hdfsFS hdfs_fs = param->hdfs_fs; /* (void*) */
394  hdfsFile hdfs_file = (hdfsFile)file; /* (void*) */
395 
396 
397  while ( remaining > 0 ) {
398 
399  /* write/read file */
400  if (access == WRITE) { /* WRITE */
401  if (verbose >= VERBOSE_4) {
402  fprintf( stdout, "task %d writing to offset %lld\n",
403  rank,
404  param->offset + length - remaining);
405  }
406 
407  if (param->verbose >= VERBOSE_4) {
408  printf("\thdfsWrite( 0x%llx, 0x%llx, 0x%llx, %lld)\n",
409  hdfs_fs, hdfs_file, ptr, remaining ); /* DEBUGGING */
410  }
411  rc = hdfsWrite( hdfs_fs, hdfs_file, ptr, remaining );
412  if ( rc < 0 ) {
413  ERR( "hdfsWrite() failed" );
414  }
415 
416  offset += rc;
417 
418  if ( param->fsyncPerWrite == TRUE ) {
419  HDFS_Fsync( hdfs_file, param );
420  }
421  }
422  else { /* READ or CHECK */
423  if (verbose >= VERBOSE_4) {
424  fprintf( stdout, "task %d reading from offset %lld\n",
425  rank,
426  param->offset + length - remaining );
427  }
428 
429  if (param->verbose >= VERBOSE_4) {
430  printf("\thdfsRead( 0x%llx, 0x%llx, 0x%llx, %lld)\n",
431  hdfs_fs, hdfs_file, ptr, remaining ); /* DEBUGGING */
432  }
433  rc = hdfsRead( hdfs_fs, hdfs_file, ptr, remaining );
434 
435  if ( rc == 0 ) {
436  ERR( "hdfs_read() returned EOF prematurely" );
437  }
438 
439  if ( rc < 0 ) {
440  ERR( "hdfs_read() failed" );
441  }
442 
443  offset += rc;
444  }
445 
446 
447  if ( rc < remaining ) {
448  fprintf(stdout, "WARNING: Task %d, partial %s, %lld of %lld bytes at offset %lld\n",
449  rank,
450  access == WRITE ? "hdfsWrite()" : "hdfs_read()",
451  rc, remaining,
452  param->offset + length - remaining );
453 
454  if ( param->singleXferAttempt == TRUE ) {
455  MPI_CHECK( MPI_Abort( MPI_COMM_WORLD, -1 ), "barrier error" );
456  }
457 
458  if ( xferRetries > MAX_RETRY ) {
459  ERR( "too many retries -- aborting" );
460  }
461  }
462 
463  assert( rc >= 0 );
464  assert( rc <= remaining );
465  remaining -= rc;
466  ptr += rc;
467  xferRetries++;
468  }
469 
470  if (param->verbose >= VERBOSE_4) {
471  printf("<- HDFS_Xfer\n");
472  }
473  return ( length );
474 }
475 
476 /*
477  * Perform hdfs_sync().
478  */
479 
480 static void HDFS_Fsync( void *fd, IOR_param_t * param ) {
481  if (param->verbose >= VERBOSE_4) {
482  printf("-> HDFS_Fsync\n");
483  }
484  hdfsFS hdfs_fs = param->hdfs_fs; /* (void *) */
485  hdfsFile hdfs_file = (hdfsFile)fd; /* (void *) */
486 
487 #if 0
488  if (param->verbose >= VERBOSE_4) {
489  printf("\thdfsHSync(0x%llx, 0x%llx)\n", hdfs_fs, hdfs_file);
490  }
491  if ( hdfsHSync( hdfs_fs, hdfs_file ) != 0 ) {
492  EWARN( "hdfsHSync() failed" );
493  }
494 #elif 0
495  if (param->verbose >= VERBOSE_4) {
496  printf("\thdfsHFlush(0x%llx, 0x%llx)\n", hdfs_fs, hdfs_file);
497  }
498  if ( hdfsHFlush( hdfs_fs, hdfs_file ) != 0 ) {
499  EWARN( "hdfsHFlush() failed" );
500  }
501 #else
502  if (param->verbose >= VERBOSE_4) {
503  printf("\thdfsFlush(0x%llx, 0x%llx)\n", hdfs_fs, hdfs_file);
504  }
505  if ( hdfsFlush( hdfs_fs, hdfs_file ) != 0 ) {
506  EWARN( "hdfsFlush() failed" );
507  }
508 #endif
509 
510  if (param->verbose >= VERBOSE_4) {
511  printf("<- HDFS_Fsync\n");
512  }
513 }
514 
515 /*
516  * Close a file through the HDFS interface.
517  */
518 
519 static void HDFS_Close( void *fd, IOR_param_t * param ) {
520  if (param->verbose >= VERBOSE_4) {
521  printf("-> HDFS_Close\n");
522  }
523 
524  hdfsFS hdfs_fs = param->hdfs_fs; /* (void *) */
525  hdfsFile hdfs_file = (hdfsFile)fd; /* (void *) */
526 
527  int open_flags;
528 
529  if ( param->openFlags & IOR_WRONLY ) {
530  open_flags = O_CREAT | O_WRONLY;
531  } else {
532  open_flags = O_RDONLY;
533  }
534 
535  if ( hdfsCloseFile( hdfs_fs, hdfs_file ) != 0 ) {
536  ERR( "hdfsCloseFile() failed" );
537  }
538 
539  if (param->verbose >= VERBOSE_4) {
540  printf("<- HDFS_Close\n");
541  }
542 }
543 
544 /*
545  * Delete a file through the HDFS interface.
546  *
547  * NOTE: The signature for ior_aiori.delete doesn't include a parameter to
548  * select recursive deletes. We'll assume that that is never needed.
549  */
550 static void HDFS_Delete( char *testFileName, IOR_param_t * param ) {
551  if (param->verbose >= VERBOSE_4) {
552  printf("-> HDFS_Delete\n");
553  }
554 
555  char errmsg[256];
556 
557  /* initialize file-system handle, if needed */
558  hdfs_connect( param );
559 
560  if ( ! param->hdfs_fs )
561  ERR_SIMPLE( "Can't delete a file without an HDFS connection" );
562 
563  if ( hdfsDelete( param->hdfs_fs, testFileName, 0 ) != 0 ) {
564  sprintf(errmsg,
565  "[RANK %03d]: hdfsDelete() of file \"%s\" failed\n",
566  rank, testFileName);
567 
568  EWARN( errmsg );
569  }
570  if (param->verbose >= VERBOSE_4) {
571  printf("<- HDFS_Delete\n");
572  }
573 }
574 
575 /*
576  * Determine api version.
577  */
578 
580  if (param->verbose >= VERBOSE_4) {
581  printf("-> HDFS_SetVersion\n");
582  }
583 
584  strcpy( param->apiVersion, param->api );
585  if (param->verbose >= VERBOSE_4) {
586  printf("<- HDFS_SetVersion\n");
587  }
588 }
589 
590 /*
591  * Use hdfsGetPathInfo() to get info about file?
592  * Is there an fstat we can use on hdfs?
593  * Should we just use POSIX fstat?
594  */
595 
596 static IOR_offset_t
598  MPI_Comm testComm,
599  char * testFileName) {
600  if (param->verbose >= VERBOSE_4) {
601  printf("-> HDFS_GetFileSize(%s)\n", testFileName);
602  }
603 
604  IOR_offset_t aggFileSizeFromStat;
605  IOR_offset_t tmpMin, tmpMax, tmpSum;
606 
607  /* make sure file-system is connected */
608  hdfs_connect( param );
609 
610  /* file-info struct includes size in bytes */
611  if (param->verbose >= VERBOSE_4) {
612  printf("\thdfsGetPathInfo(%s) ...", testFileName);fflush(stdout);
613  }
614 
615  hdfsFileInfo* info = hdfsGetPathInfo( param->hdfs_fs, testFileName );
616  if ( ! info )
617  ERR_SIMPLE( "hdfsGetPathInfo() failed" );
618  if (param->verbose >= VERBOSE_4) {
619  printf("done.\n");fflush(stdout);
620  }
621 
622  aggFileSizeFromStat = info->mSize;
623 
624  if ( param->filePerProc == TRUE ) {
625  if (param->verbose >= VERBOSE_4) {
626  printf("\tall-reduce (1)\n");
627  }
628  MPI_CHECK(
629  MPI_Allreduce(
630  &aggFileSizeFromStat, &tmpSum, 1, MPI_LONG_LONG_INT, MPI_SUM, testComm ),
631  "cannot total data moved" );
632 
633  aggFileSizeFromStat = tmpSum;
634  }
635  else {
636  if (param->verbose >= VERBOSE_4) {
637  printf("\tall-reduce (2a)\n");
638  }
639  MPI_CHECK(
640  MPI_Allreduce(
641  &aggFileSizeFromStat, &tmpMin, 1, MPI_LONG_LONG_INT, MPI_MIN, testComm ),
642  "cannot total data moved" );
643 
644  if (param->verbose >= VERBOSE_4) {
645  printf("\tall-reduce (2b)\n");
646  }
647  MPI_CHECK(
648  MPI_Allreduce(
649  &aggFileSizeFromStat, &tmpMax, 1, MPI_LONG_LONG_INT, MPI_MAX, testComm ),
650  "cannot total data moved" );
651 
652  if ( tmpMin != tmpMax ) {
653  if ( rank == 0 ) {
654  WARN( "inconsistent file size by different tasks" );
655  }
656 
657  /* incorrect, but now consistent across tasks */
658  aggFileSizeFromStat = tmpMin;
659  }
660  }
661 
662  if (param->verbose >= VERBOSE_4) {
663  printf("<- HDFS_GetFileSize [%llu]\n", aggFileSizeFromStat);
664  }
665  return ( aggFileSizeFromStat );
666 }
char * hdfs_user
Definition: ior.h:176
#define ERR(MSG)
Definition: iordef.h:184
static void HDFS_Delete(char *, IOR_param_t *)
Definition: aiori-HDFS.c:550
int filePerProc
Definition: ior.h:111
CURLcode rc
Definition: aiori-S3.c:121
ior_aiori_t hdfs_aiori
Definition: aiori-HDFS.c:116
IOR_offset_t transferSize
Definition: ior.h:125
#define IOR_APPEND
Definition: aiori.h:37
char * apiVersion
Definition: ior.h:91
int hdfs_replicas
Definition: ior.h:180
unsigned int openFlags
Definition: ior.h:88
int fsyncPerWrite
Definition: ior.h:162
#define ERR_SIMPLE(MSG)
Definition: iordef.h:190
#define WRITE
Definition: iordef.h:95
#define EWARN(MSG)
Definition: iordef.h:169
static IOR_offset_t HDFS_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori-HDFS.c:381
#define IOR_CREAT
Definition: aiori.h:38
#define IOR_EXCL
Definition: aiori.h:40
hdfsFS hdfs_fs
Definition: ior.h:179
tPort hdfs_name_node_port
Definition: ior.h:178
static void * HDFS_Create_Or_Open(char *testFileName, IOR_param_t *param, unsigned char createFile)
Definition: aiori-HDFS.c:217
MPI_Comm testComm
Definition: utilities.c:60
#define O_DIRECT
static void HDFS_SetVersion(IOR_param_t *)
Definition: aiori-HDFS.c:579
void * hdfsFS
Definition: ior.h:29
int verbose
Definition: ior.h:144
#define MPI_CHECK(MPI_STATUS, MSG)
Definition: iordef.h:224
int singleXferAttempt
Definition: ior.h:161
Definition: ior.h:48
#define MAX_RETRY
Definition: iordef.h:110
void hdfs_set_o_direct_flag(int *fd)
Definition: aiori-HDFS.c:135
static void hdfs_connect(IOR_param_t *param)
Definition: aiori-HDFS.c:162
static IOR_param_t param
Definition: mdtest.c:170
static void * HDFS_Create(char *, IOR_param_t *)
Definition: aiori-HDFS.c:344
#define IOR_WRONLY
Definition: aiori.h:35
#define FALSE
Definition: iordef.h:71
long long int IOR_size_t
Definition: iordef.h:123
#define WARN(MSG)
Definition: iordef.h:144
int hdfs_block_size
Definition: ior.h:181
static void hdfs_disconnect(IOR_param_t *param)
Definition: aiori-HDFS.c:198
IOR_offset_t offset
Definition: ior.h:126
#define VERBOSE_4
Definition: iordef.h:105
static IOR_offset_t HDFS_GetFileSize(IOR_param_t *, MPI_Comm, char *)
Definition: aiori-HDFS.c:597
#define IOR_RDWR
Definition: aiori.h:36
static void * HDFS_Open(char *, IOR_param_t *)
Definition: aiori-HDFS.c:358
int verbose
Definition: utilities.c:59
char * api
Definition: ior.h:90
char * name
Definition: aiori.h:68
long long int IOR_offset_t
Definition: iordef.h:122
int rank
Definition: utilities.c:57
#define TRUE
Definition: iordef.h:75
static void HDFS_Fsync(void *, IOR_param_t *)
Definition: aiori-HDFS.c:480
const char * hdfs_name_node
Definition: ior.h:177
static void HDFS_Close(void *, IOR_param_t *)
Definition: aiori-HDFS.c:519
#define NULL
Definition: iordef.h:79