IOR
aiori-HDFS.c
Go to the documentation of this file.
1 /* -*- mode: c; indent-tabs-mode: t; -*-
2  * vim:noexpandtab:
3  *
4  * Editing with tabs allows different users to pick their own indentation
5  * appearance without changing the file.
6  */
7 
8 /*
9  * Copyright (c) 2009, Los Alamos National Security, LLC All rights reserved.
10  * Copyright 2009. Los Alamos National Security, LLC. This software was produced
11  * under U.S. Government contract DE-AC52-06NA25396 for Los Alamos National
12  * Laboratory (LANL), which is operated by Los Alamos National Security, LLC for
13  * the U.S. Department of Energy. The U.S. Government has rights to use,
14  * reproduce, and distribute this software. NEITHER THE GOVERNMENT NOR LOS
15  * ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR
16  * ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. If software is
17  * modified to produce derivative works, such modified software should be
18  * clearly marked, so as not to confuse it with the version available from
19  * LANL.
20  *
21  * Additionally, redistribution and use in source and binary forms, with or
22  * without modification, are permitted provided that the following conditions are
23  * met:
24  *
25  * • Redistributions of source code must retain the above copyright notice,
26  * this list of conditions and the following disclaimer.
27  *
28  * • Redistributions in binary form must reproduce the above copyright notice,
29  * this list of conditions and the following disclaimer in the documentation
30  * and/or other materials provided with the distribution.
31  *
32  * • Neither the name of Los Alamos National Security, LLC, Los Alamos National
33  * Laboratory, LANL, the U.S. Government, nor the names of its contributors may be
34  * used to endorse or promote products derived from this software without specific
35  * prior written permission.
36  *
37  * THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS
38  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
39  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
40  * ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR
41  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
42  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
43  * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
44  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
45  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
46  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
47  * OF SUCH DAMAGE.
48  */
49 
50 /******************************************************************************\
51 *
52 * Implement of abstract I/O interface for HDFS.
53 *
54 * HDFS has the added concept of a "File System Handle" which has to be
55 * connected before files are opened. We store this in the IOR_param_t
56 * object that is always passed to our functions. The thing that callers
57 * think of as the "fd" is an hdfsFile, (a pointer).
58 *
59 \******************************************************************************/
60 
61 #ifdef HAVE_CONFIG_H
62 # include "config.h"
63 #endif
64 
65 #include <stdio.h>
66 #include <stdlib.h>
67 
68 #ifdef __linux__
69 # include <sys/ioctl.h> /* necessary for: */
70 # define __USE_GNU /* O_DIRECT and */
71 # include <fcntl.h> /* IO operations */
72 # undef __USE_GNU
73 #endif /* __linux__ */
74 
75 #include <errno.h>
76 #include <fcntl.h> /* IO operations */
77 #include <sys/stat.h>
78 #include <assert.h>
79 /*
80 #ifdef HAVE_LUSTRE_USER
81 #include <lustre/lustre_user.h>
82 #endif
83 */
84 #include "ior.h"
85 #include "aiori.h"
86 #include "utilities.h"
87 
88 #ifndef open64 /* necessary for TRU64 -- */
89 # define open64 open /* unlikely, but may pose */
90 #endif /* not open64 */ /* conflicting prototypes */
91 
92 #ifndef lseek64 /* necessary for TRU64 -- */
93 # define lseek64 lseek /* unlikely, but may pose */
94 #endif /* not lseek64 */ /* conflicting prototypes */
95 
96 #ifndef O_BINARY /* Required on Windows */
97 # define O_BINARY 0
98 #endif
99 
100 #include "hdfs.h"
101 
102 /**************************** P R O T O T Y P E S *****************************/
103 static aiori_fd_t *HDFS_Create(char *testFileName, int flags, aiori_mod_opt_t * param);
104 static aiori_fd_t *HDFS_Open(char *testFileName, int flags, aiori_mod_opt_t * param);
105 static IOR_offset_t HDFS_Xfer(int access, aiori_fd_t *file, IOR_size_t * buffer,
107 static void HDFS_Close(aiori_fd_t *, aiori_mod_opt_t *);
108 static void HDFS_Delete(char *testFileName, aiori_mod_opt_t * param);
109 static void HDFS_Fsync(aiori_fd_t *, aiori_mod_opt_t *);
111 static void hdfs_xfer_hints(aiori_xfer_hint_t * params);
112 static option_help * HDFS_options(aiori_mod_opt_t ** init_backend_options, aiori_mod_opt_t * init_values);
113 static int HDFS_mkdir (const char *path, mode_t mode, aiori_mod_opt_t * options);
114 static int HDFS_rmdir (const char *path, aiori_mod_opt_t * options);
115 static int HDFS_access (const char *path, int mode, aiori_mod_opt_t * options);
116 static int HDFS_stat (const char *path, struct stat *buf, aiori_mod_opt_t * options);
117 static int HDFS_statfs (const char * path, ior_aiori_statfs_t * stat, aiori_mod_opt_t * options);
118 
120 
121 /************************** D E C L A R A T I O N S ***************************/
122 
124  .name = "HDFS",
125  .name_legacy = NULL,
126  .create = HDFS_Create,
127  .open = HDFS_Open,
128  .xfer = HDFS_Xfer,
129  .close = HDFS_Close,
130  .delete = HDFS_Delete,
131  .get_options = HDFS_options,
132  .get_version = aiori_get_version,
133  .xfer_hints = hdfs_xfer_hints,
134  .fsync = HDFS_Fsync,
135  .get_file_size = HDFS_GetFileSize,
136  .statfs = HDFS_statfs,
137  .mkdir = HDFS_mkdir,
138  .rmdir = HDFS_rmdir,
139  .access = HDFS_access,
140  .stat = HDFS_stat,
141  .enable_mdtest = true
142 };
143 
144 /***************************** F U N C T I O N S ******************************/
145 
147  hints = params;
148 }
149 
150 /************************** O P T I O N S *****************************/
151 typedef struct {
152  char * user;
153  char * name_node;
154  int replicas; /* n block replicas. (0 gets default) */
156  IOR_offset_t block_size; /* internal blk-size. (0 gets default) */
157  // runtime options
158  hdfsFS fs; /* file-system handle */
159  tPort name_node_port; /* (uint16_t) */
161 
162 static void hdfs_connect( hdfs_options_t* o );
163 
164 option_help * HDFS_options(aiori_mod_opt_t ** init_backend_options, aiori_mod_opt_t * init_values){
165  hdfs_options_t * o = malloc(sizeof(hdfs_options_t));
166 
167  if (init_values != NULL){
168  memcpy(o, init_values, sizeof(hdfs_options_t));
169  }else{
170  memset(o, 0, sizeof(hdfs_options_t));
171  char *hdfs_user;
172  hdfs_user = getenv("USER");
173  if (!hdfs_user){
174  hdfs_user = "";
175  }
176  o->user = strdup(hdfs_user);
177  o->name_node = "default";
178  }
179 
180  *init_backend_options = (aiori_mod_opt_t*) o;
181 
182  option_help h [] = {
183  {0, "hdfs.odirect", "Direct I/O Mode", OPTION_FLAG, 'd', & o->direct_io},
184  {0, "hdfs.user", "Username", OPTION_OPTIONAL_ARGUMENT, 's', & o->user},
185  {0, "hdfs.name_node", "Namenode", OPTION_OPTIONAL_ARGUMENT, 's', & o->name_node},
186  {0, "hdfs.replicas", "Number of replicas", OPTION_OPTIONAL_ARGUMENT, 'd', & o->replicas},
187  {0, "hdfs.block_size", "Blocksize", OPTION_OPTIONAL_ARGUMENT, 'l', & o->block_size},
189  };
190  option_help * help = malloc(sizeof(h));
191  memcpy(help, h, sizeof(h));
192  return help;
193 }
194 
195 
196 int HDFS_mkdir (const char *path, mode_t mode, aiori_mod_opt_t * options){
197  hdfs_options_t * o = (hdfs_options_t*) options;
198  hdfs_connect(o);
199  return hdfsCreateDirectory(o->fs, path);
200 }
201 
202 int HDFS_rmdir (const char *path, aiori_mod_opt_t * options){
203  hdfs_options_t * o = (hdfs_options_t*) options;
204  hdfs_connect(o);
205  return hdfsDelete(o->fs, path, 1);
206 }
207 
208 int HDFS_access (const char *path, int mode, aiori_mod_opt_t * options){
209  hdfs_options_t * o = (hdfs_options_t*) options;
210  hdfs_connect(o);
211  return hdfsExists(o->fs, path);
212 }
213 
214 int HDFS_stat (const char *path, struct stat *buf, aiori_mod_opt_t * options){
215  hdfsFileInfo * stat;
216  hdfs_options_t * o = (hdfs_options_t*) options;
217  hdfs_connect(o);
218  stat = hdfsGetPathInfo(o->fs, path);
219  if(stat == NULL){
220  return 1;
221  }
222  memset(buf, 0, sizeof(struct stat));
223  buf->st_atime = stat->mLastAccess;
224  buf->st_size = stat->mSize;
225  buf->st_mtime = stat->mLastMod;
226  buf->st_mode = stat->mPermissions;
227 
228  hdfsFreeFileInfo(stat, 1);
229  return 0;
230 }
231 
232 int HDFS_statfs (const char * path, ior_aiori_statfs_t * stat, aiori_mod_opt_t * options){
233  hdfs_options_t * o = (hdfs_options_t*) options;
234  hdfs_connect(o);
235 
236  stat->f_bsize = hdfsGetDefaultBlockSize(o->fs);
237  stat->f_blocks = hdfsGetCapacity(o->fs) / hdfsGetDefaultBlockSize(o->fs);
238  stat->f_bfree = stat->f_blocks - hdfsGetUsed(o->fs) / hdfsGetDefaultBlockSize(o->fs);
239  stat->f_bavail = 1;
240  stat->f_files = 1;
241  stat->f_ffree = 1;
242  return 0;
243 }
244 
245 /* This is identical to the one in aiori-POSIX.c Doesn't seem like
246  * it would be appropriate in utilities.c.
247  */
248 
250 {
251 /* note that TRU64 needs O_DIRECTIO, SunOS uses directio(),
252  and everyone else needs O_DIRECT */
253 #ifndef O_DIRECT
254 # ifndef O_DIRECTIO
255  WARN("cannot use O_DIRECT");
256 # define O_DIRECT 000000
257 # else /* O_DIRECTIO */
258 # define O_DIRECT O_DIRECTIO
259 # endif /* not O_DIRECTIO */
260 #endif /* not O_DIRECT */
261 
262  *fd |= O_DIRECT;
263 }
264 
265 
266 /*
267  * "Connect" to an HDFS file-system. HDFS requires this be done before and
268  * files are opened. It is easy for ior_aiori.open/create to assure that
269  * we connect, if we haven't already done so. However, there's not a
270  * simple way to assure that we disconnect after the last file-close. For
271  * now, we'll make a special call at the end of ior.c
272  *
273  * NOTE: It's okay to call this thing whenever you need to be sure the HDFS
274  * filesystem is connected.
275  */
277  if (verbose >= VERBOSE_4) {
278  printf("-> hdfs_connect [nn:\"%s\", port:%d, user:%s]\n",
279  o->name_node,
280  o->name_node_port,
281  o->user );
282  }
283 
284  if ( o->fs ) {
285  if (verbose >= VERBOSE_4) {
286  printf("<- hdfs_connect [nothing to do]\n"); /* DEBUGGING */
287  }
288  return;
289  }
290 
291  /* initialize a builder, holding parameters for hdfsBuilderConnect() */
292  struct hdfsBuilder* builder = hdfsNewBuilder();
293  if ( ! builder ){
294  ERR("couldn't create an hdfsBuilder");
295  }
296 
297  hdfsBuilderSetForceNewInstance ( builder ); /* don't use cached instance */
298 
299  hdfsBuilderSetNameNode ( builder, o->name_node );
300  hdfsBuilderSetNameNodePort( builder, o->name_node_port );
301  hdfsBuilderSetUserName ( builder, o->user );
302 
303  /* NOTE: hdfsBuilderConnect() frees the builder */
304  o->fs = hdfsBuilderConnect( builder );
305  if ( ! o->fs )
306  ERR("hdsfsBuilderConnect failed");
307 
308  if (verbose >= VERBOSE_4) {
309  printf("<- hdfs_connect [success]\n");
310  }
311 }
312 
314  if (verbose >= VERBOSE_4) {
315  printf("-> hdfs_disconnect\n");
316  }
317  if ( o->fs ) {
318  hdfsDisconnect( o->fs );
319  o->fs = NULL;
320  }
321  if (verbose >= VERBOSE_4) {
322  printf("<- hdfs_disconnect\n");
323  }
324 }
325 
326 
327 /*
328  * Create or open the file. Pass TRUE if creating and FALSE if opening an existing file.
329  * Return an hdfsFile.
330  */
331 
332 static void *HDFS_Create_Or_Open( char *testFileName, int flags, aiori_mod_opt_t *param, unsigned char createFile ) {
333  if (verbose >= VERBOSE_4) {
334  printf("-> HDFS_Create_Or_Open\n");
335  }
336  hdfs_options_t * o = (hdfs_options_t*) param;
337 
338  hdfsFile hdfs_file = NULL;
339  int fd_oflags = 0, hdfs_return;
340 
341  /* initialize file-system handle, if needed */
342  hdfs_connect( o );
343 
344  /*
345  * Check for unsupported flags.
346  *
347  * If they want RDWR, we don't know if they're going to try to do both, so we
348  * can't default to either O_RDONLY or O_WRONLY. Thus, we error and exit.
349  *
350  * The other two, we just note that they are not supported and don't do them.
351  */
352 
353  if ( flags & IOR_RDWR ) {
354  ERR( "Opening or creating a file in RDWR is not implemented in HDFS" );
355  }
356 
357  if ( flags & IOR_EXCL ) {
358  fprintf( stdout, "Opening or creating a file in Exclusive mode is not implemented in HDFS\n" );
359  }
360 
361  if ( flags & IOR_APPEND ) {
362  fprintf( stdout, "Opening or creating a file for appending is not implemented in HDFS\n" );
363  }
364 
365  /*
366  * Setup the flags to be used.
367  */
368 
369  if ( createFile == TRUE ) {
370  fd_oflags = O_CREAT;
371  }
372 
373  if ( flags & IOR_WRONLY ) {
374  if ( ! hints->filePerProc ) {
375 
376  // in N-1 mode, only rank 0 truncates the file
377  if ( rank != 0 ) {
378  fd_oflags |= O_WRONLY;
379  } else {
380  fd_oflags |= O_TRUNC;
381  fd_oflags |= O_WRONLY;
382  }
383 
384  } else {
385  // in N-N mode, everyone does truncate
386  fd_oflags |= O_TRUNC;
387  fd_oflags |= O_WRONLY;
388  }
389 
390  } else {
391  fd_oflags |= O_RDONLY;
392  }
393 
394  /*
395  * Now see if O_DIRECT is needed.
396  */
397 
398  if ( o->direct_io == TRUE ) {
399  hdfs_set_o_direct_flag( &fd_oflags );
400  }
401 
402  /*
403  * For N-1 write, All other ranks wait for Rank 0 to open the file.
404  * this is bec 0 does truncate and rest don't
405  * it would be dangerous for all to do truncate as they might
406  * truncate each other's writes
407  */
408 
409  if (( flags & IOR_WRONLY ) && ( ! hints->filePerProc ) && ( rank != 0 )) {
410  MPI_CHECK(MPI_Barrier(testComm), "barrier error");
411  }
412 
413  /*
414  * Now rank zero can open and truncate, if necessary.
415  */
416 
417  if (verbose >= VERBOSE_4) {
418  printf("\thdfsOpenFile(%p, %s, 0%o, %lld, %d, %lld)\n",
419  o->fs,
420  testFileName,
421  fd_oflags, /* shown in octal to compare w/ <bits/fcntl.h> */
422  hints->transferSize,
423  o->replicas,
424  o->block_size);
425  }
426  hdfs_file = hdfsOpenFile( o->fs, testFileName, fd_oflags, hints->transferSize, o->replicas, o->block_size);
427  if ( ! hdfs_file ) {
428  ERR( "Failed to open the file" );
429  }
430 
431  /*
432  * For N-1 write, Rank 0 waits for the other ranks to open the file after it has.
433  */
434 
435  if (( flags & IOR_WRONLY ) &&
436  ( !hints->filePerProc ) &&
437  ( rank == 0 )) {
438 
439  MPI_CHECK(MPI_Barrier(testComm), "barrier error");
440  }
441 
442  if (verbose >= VERBOSE_4) {
443  printf("<- HDFS_Create_Or_Open\n");
444  }
445  return ((void *) hdfs_file );
446 }
447 
448 /*
449  * Create and open a file through the HDFS interface.
450  */
451 
452 static aiori_fd_t *HDFS_Create(char *testFileName, int flags, aiori_mod_opt_t * param) {
453  if (verbose >= VERBOSE_4) {
454  printf("-> HDFS_Create\n");
455  }
456 
457  if (verbose >= VERBOSE_4) {
458  printf("<- HDFS_Create\n");
459  }
460  return HDFS_Create_Or_Open( testFileName, flags, param, TRUE );
461 }
462 
463 /*
464  * Open a file through the HDFS interface.
465  */
466 static aiori_fd_t *HDFS_Open(char *testFileName, int flags, aiori_mod_opt_t * param) {
467  if (verbose >= VERBOSE_4) {
468  printf("-> HDFS_Open\n");
469  }
470 
471  if ( flags & IOR_CREAT ) {
472  if (verbose >= VERBOSE_4) {
473  printf("<- HDFS_Open( ... TRUE)\n");
474  }
475  return HDFS_Create_Or_Open( testFileName, flags, param, TRUE );
476  }
477  else {
478  if (verbose >= VERBOSE_4) {
479  printf("<- HDFS_Open( ... FALSE)\n");
480  }
481  return HDFS_Create_Or_Open( testFileName, flags, param, FALSE );
482  }
483 }
484 
485 /*
486  * Write or read to file using the HDFS interface.
487  */
488 
489 static IOR_offset_t HDFS_Xfer(int access, aiori_fd_t *file, IOR_size_t * buffer,
490  IOR_offset_t length, IOR_offset_t offset, aiori_mod_opt_t * param) {
491  if (verbose >= VERBOSE_4) {
492  printf("-> HDFS_Xfer(acc:%d, file:%p, buf:%p, len:%llu, %p)\n",
493  access, file, buffer, length, param);
494  }
495  hdfs_options_t * o = (hdfs_options_t*) param;
496  int xferRetries = 0;
497  long long remaining = (long long)length;
498  char* ptr = (char *)buffer;
499  long long rc;
500  hdfsFS hdfs_fs = o->fs; /* (void*) */
501  hdfsFile hdfs_file = (hdfsFile)file; /* (void*) */
502 
503 
504  while ( remaining > 0 ) {
505 
506  /* write/read file */
507  if (access == WRITE) { /* WRITE */
508  if (verbose >= VERBOSE_4) {
509  fprintf( stdout, "task %d writing to offset %lld\n",
510  rank,
511  offset + length - remaining);
512  }
513 
514  if (verbose >= VERBOSE_4) {
515  printf("\thdfsWrite( %p, %p, %p, %lld)\n",
516  hdfs_fs, hdfs_file, ptr, remaining ); /* DEBUGGING */
517  }
518  rc = hdfsWrite( hdfs_fs, hdfs_file, ptr, remaining );
519  if ( rc < 0 ) {
520  ERR( "hdfsWrite() failed" );
521  }
522  offset += rc;
523 
524  if ( hints->fsyncPerWrite == TRUE ) {
525  HDFS_Fsync( file, param );
526  }
527  }
528  else { /* READ or CHECK */
529  if (verbose >= VERBOSE_4) {
530  fprintf( stdout, "task %d reading from offset %lld\n",
531  rank, offset + length - remaining );
532  }
533 
534  if (verbose >= VERBOSE_4) {
535  printf("\thdfsRead( %p, %p, %p, %lld)\n",
536  hdfs_fs, hdfs_file, ptr, remaining ); /* DEBUGGING */
537  }
538  rc = hdfsPread(hdfs_fs, hdfs_file, offset, ptr, remaining);
539  if ( rc == 0 ) {
540  ERR( "hdfs_read() returned EOF prematurely" );
541  }
542 
543  if ( rc < 0 ) {
544  ERR( "hdfs_read() failed" );
545  }
546 
547  offset += rc;
548  }
549 
550 
551  if ( rc < remaining ) {
552  fprintf(stdout, "WARNING: Task %d, partial %s, %lld of %lld bytes at offset %lld\n",
553  rank,
554  access == WRITE ? "hdfsWrite()" : "hdfs_read()",
555  rc, remaining,
556  offset + length - remaining );
557 
558  if ( hints->singleXferAttempt == TRUE ) {
559  MPI_CHECK( MPI_Abort( MPI_COMM_WORLD, -1 ), "barrier error" );
560  }
561 
562  if ( xferRetries > MAX_RETRY ) {
563  ERR( "too many retries -- aborting" );
564  }
565  }
566 
567  assert( rc >= 0 );
568  assert( rc <= remaining );
569  remaining -= rc;
570  ptr += rc;
571  xferRetries++;
572  }
573 
574  if(access == WRITE){
575  // flush user buffer, this makes the write visible to readers
576  // it is the expected semantics of read/writes
577  rc = hdfsHFlush(hdfs_fs, hdfs_file);
578  if(rc != 0){
579  WARN("Error during flush");
580  }
581  }
582 
583  if (verbose >= VERBOSE_4) {
584  printf("<- HDFS_Xfer\n");
585  }
586  return ( length );
587 }
588 
589 /*
590  * Perform hdfs_sync().
591  */
592 static void HDFS_Fsync(aiori_fd_t * fd, aiori_mod_opt_t * param) {
593  hdfs_options_t * o = (hdfs_options_t*) param;
594  hdfsFS hdfs_fs = o->fs; /* (void *) */
595  hdfsFile hdfs_file = (hdfsFile)fd; /* (void *) */
596 
597  if (verbose >= VERBOSE_4) {
598  printf("\thdfsFlush(%p, %p)\n", hdfs_fs, hdfs_file);
599  }
600  if ( hdfsHSync( hdfs_fs, hdfs_file ) != 0 ) {
601  // Hsync is implemented to flush out data with newer Hadoop versions
602  EWARN( "hdfsFlush() failed" );
603  }
604 }
605 
606 /*
607  * Close a file through the HDFS interface.
608  */
609 
610 static void HDFS_Close(aiori_fd_t * fd, aiori_mod_opt_t * param) {
611  if (verbose >= VERBOSE_4) {
612  printf("-> HDFS_Close\n");
613  }
614  hdfs_options_t * o = (hdfs_options_t*) param;
615 
616  hdfsFS hdfs_fs = o->fs; /* (void *) */
617  hdfsFile hdfs_file = (hdfsFile)fd; /* (void *) */
618 
619  if ( hdfsCloseFile( hdfs_fs, hdfs_file ) != 0 ) {
620  ERR( "hdfsCloseFile() failed" );
621  }
622 
623  if (verbose >= VERBOSE_4) {
624  printf("<- HDFS_Close\n");
625  }
626 }
627 
628 /*
629  * Delete a file through the HDFS interface.
630  *
631  * NOTE: The signature for ior_aiori.delete doesn't include a parameter to
632  * select recursive deletes. We'll assume that that is never needed.
633  */
634 static void HDFS_Delete( char *testFileName, aiori_mod_opt_t * param ) {
635  if (verbose >= VERBOSE_4) {
636  printf("-> HDFS_Delete\n");
637  }
638 
639  hdfs_options_t * o = (hdfs_options_t*) param;
640  char errmsg[256];
641 
642  /* initialize file-system handle, if needed */
643  hdfs_connect(o);
644 
645  if ( ! o->fs )
646  ERR( "Can't delete a file without an HDFS connection" );
647 
648  if ( hdfsDelete( o->fs, testFileName, 0 ) != 0 ) {
649  sprintf(errmsg, "[RANK %03d]: hdfsDelete() of file \"%s\" failed\n",
650  rank, testFileName);
651 
652  EWARN( errmsg );
653  }
654  if (verbose >= VERBOSE_4) {
655  printf("<- HDFS_Delete\n");
656  }
657 }
658 
659 /*
660  * Use hdfsGetPathInfo() to get info about file?
661  * Is there an fstat we can use on hdfs?
662  * Should we just use POSIX fstat?
663  */
664 
666  char * testFileName) {
667  if (verbose >= VERBOSE_4) {
668  printf("-> HDFS_GetFileSize(%s)\n", testFileName);
669  }
670  hdfs_options_t * o = (hdfs_options_t*) param;
671 
672  IOR_offset_t aggFileSizeFromStat;
673  IOR_offset_t tmpMin, tmpMax, tmpSum;
674 
675  /* make sure file-system is connected */
676  hdfs_connect( o );
677 
678  /* file-info struct includes size in bytes */
679  if (verbose >= VERBOSE_4) {
680  printf("\thdfsGetPathInfo(%s) ...", testFileName);
681  fflush(stdout);
682  }
683 
684  hdfsFileInfo* info = hdfsGetPathInfo( o->fs, testFileName );
685  if ( ! info )
686  ERR( "hdfsGetPathInfo() failed" );
687  if (verbose >= VERBOSE_4) {
688  printf("done.\n");fflush(stdout);
689  }
690 
691  aggFileSizeFromStat = info->mSize;
692 
693  if (verbose >= VERBOSE_4) {
694  printf("<- HDFS_GetFileSize [%llu]\n", aggFileSizeFromStat);
695  }
696  return ( aggFileSizeFromStat );
697 }
static void * HDFS_Create_Or_Open(char *testFileName, int flags, aiori_mod_opt_t *param, unsigned char createFile)
Definition: aiori-HDFS.c:332
static int HDFS_access(const char *path, int mode, aiori_mod_opt_t *options)
Definition: aiori-HDFS.c:208
static int HDFS_stat(const char *path, struct stat *buf, aiori_mod_opt_t *options)
Definition: aiori-HDFS.c:214
static aiori_fd_t * HDFS_Create(char *testFileName, int flags, aiori_mod_opt_t *param)
Definition: aiori-HDFS.c:452
uint64_t f_blocks
Definition: aiori.h:53
static int HDFS_rmdir(const char *path, aiori_mod_opt_t *options)
Definition: aiori-HDFS.c:202
static void HDFS_Delete(char *testFileName, aiori_mod_opt_t *param)
Definition: aiori-HDFS.c:634
static void HDFS_Fsync(aiori_fd_t *, aiori_mod_opt_t *)
Definition: aiori-HDFS.c:592
uint64_t f_bfree
Definition: aiori.h:54
uint16_t tPort
Definition: ior.h:28
#define LAST_OPTION
Definition: option.h:39
static aiori_xfer_hint_t * hints
Definition: aiori-HDFS.c:119
CURLcode rc
Definition: aiori-S3-4c.c:111
struct benchmark_options o
Definition: md-workbench.c:128
ior_aiori_t hdfs_aiori
Definition: aiori-HDFS.c:123
uint64_t f_ffree
Definition: aiori.h:57
#define IOR_APPEND
Definition: aiori.h:31
static IOR_offset_t HDFS_Xfer(int access, aiori_fd_t *file, IOR_size_t *buffer, IOR_offset_t length, IOR_offset_t offset, aiori_mod_opt_t *param)
Definition: aiori-HDFS.c:489
#define MPI_CHECK(MPI_STATUS, MSG)
Definition: aiori-debug.h:127
#define WRITE
Definition: iordef.h:86
tPort name_node_port
Definition: aiori-HDFS.c:159
#define IOR_CREAT
Definition: aiori.h:32
#define IOR_EXCL
Definition: aiori.h:34
char * aiori_get_version()
Definition: aiori.c:232
static int HDFS_statfs(const char *path, ior_aiori_statfs_t *stat, aiori_mod_opt_t *options)
Definition: aiori-HDFS.c:232
uint64_t f_files
Definition: aiori.h:56
MPI_Comm testComm
Definition: utilities.c:71
#define O_DIRECT
static option_help options[]
Definition: aiori-CEPHFS.c:54
char * name_node
Definition: aiori-HDFS.c:153
static void hdfs_xfer_hints(aiori_xfer_hint_t *params)
Definition: aiori-HDFS.c:146
uint64_t f_bsize
Definition: aiori.h:52
void * hdfsFS
Definition: ior.h:29
IOR_offset_t block_size
Definition: aiori-HDFS.c:156
#define WARN(MSG)
Definition: aiori-debug.h:32
static int HDFS_mkdir(const char *path, mode_t mode, aiori_mod_opt_t *options)
Definition: aiori-HDFS.c:196
int singleXferAttempt
Definition: aiori.h:75
Definition: ior.h:56
#define MAX_RETRY
Definition: iordef.h:101
void hdfs_set_o_direct_flag(int *fd)
Definition: aiori-HDFS.c:249
#define EWARN(MSG)
Definition: aiori-debug.h:59
IOR_offset_t transferSize
Definition: aiori.h:73
static option_help * HDFS_options(aiori_mod_opt_t **init_backend_options, aiori_mod_opt_t *init_values)
Definition: aiori-HDFS.c:164
static void hdfs_connect(hdfs_options_t *o)
Definition: aiori-HDFS.c:276
#define IOR_WRONLY
Definition: aiori.h:29
#define FALSE
Definition: iordef.h:62
static void HDFS_Close(aiori_fd_t *, aiori_mod_opt_t *)
Definition: aiori-HDFS.c:610
long long int IOR_size_t
Definition: iordef.h:110
uint64_t f_bavail
Definition: aiori.h:55
static aiori_fd_t * HDFS_Open(char *testFileName, int flags, aiori_mod_opt_t *param)
Definition: aiori-HDFS.c:466
int verbose
Definition: utilities.c:70
#define VERBOSE_4
Definition: iordef.h:96
static void hdfs_disconnect(hdfs_options_t *o)
Definition: aiori-HDFS.c:313
#define ERR(MSG)
Definition: aiori-debug.h:92
#define IOR_RDWR
Definition: aiori.h:30
int fsyncPerWrite
Definition: aiori.h:70
char * name
Definition: aiori.h:88
static IOR_offset_t HDFS_GetFileSize(aiori_mod_opt_t *, char *)
Definition: aiori-HDFS.c:665
int filePerProc
Definition: aiori.h:65
long long int IOR_offset_t
Definition: iordef.h:109
int rank
Definition: utilities.c:68
#define TRUE
Definition: iordef.h:66
#define NULL
Definition: iordef.h:70