IOR
Data Structures | Macros | Typedefs | Functions | Variables
aiori-S3-4c.c File Reference
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include <curl/curl.h>
#include <libxml/parser.h>
#include <libxml/tree.h>
#include "aws4c.h"
#include "aws4c_extra.h"
#include "ior.h"
#include "aiori.h"
#include "aiori-debug.h"
Include dependency graph for aiori-S3-4c.c:

Go to the source code of this file.

Data Structures

struct  s3_options_t
 

Macros

#define BUFF_SIZE   1024
 
#define IOR_CURL_INIT   0x01 /* curl top-level inits were performed once? */
 
#define IOR_CURL_NOCONTINUE   0x02
 
#define IOR_CURL_S3_EMC_EXT   0x04 /* allow EMC extensions to S3? */
 
#define MAX_UPLOAD_ID_SIZE   256 /* TODO don't know the actual value */
 
#define CURL_ERR(MSG, CURL_ERRNO, PARAM)
 
#define CURL_WARN(MSG, CURL_ERRNO)
 

Typedefs

typedef void CURL
 
typedef void IOBuf
 

Functions

static void S3_xfer_hints (aiori_xfer_hint_t *params)
 
static aiori_fd_tS3_Create (char *path, int iorflags, aiori_mod_opt_t *options)
 
static aiori_fd_tS3_Open (char *path, int flags, aiori_mod_opt_t *options)
 
static IOR_offset_t S3_Xfer (int access, aiori_fd_t *afd, IOR_size_t *buffer, IOR_offset_t length, IOR_offset_t offset, aiori_mod_opt_t *options)
 
static void S3_Close (aiori_fd_t *afd, aiori_mod_opt_t *options)
 
static aiori_fd_tEMC_Create (char *path, int iorflags, aiori_mod_opt_t *options)
 
static aiori_fd_tEMC_Open (char *path, int flags, aiori_mod_opt_t *options)
 
static IOR_offset_t EMC_Xfer (int access, aiori_fd_t *afd, IOR_size_t *buffer, IOR_offset_t length, IOR_offset_t offset, aiori_mod_opt_t *options)
 
static void EMC_Close (aiori_fd_t *afd, aiori_mod_opt_t *options)
 
static void S3_Delete (char *path, aiori_mod_opt_t *options)
 
static void S3_Fsync (aiori_fd_t *fd, aiori_mod_opt_t *options)
 
static IOR_offset_t S3_GetFileSize (aiori_mod_opt_t *options, char *testFileName)
 
static void S3_init (aiori_mod_opt_t *options)
 
static void S3_finalize (aiori_mod_opt_t *options)
 
static int S3_check_params (aiori_mod_opt_t *options)
 
static option_helpS3_options (aiori_mod_opt_t **init_backend_options, aiori_mod_opt_t *init_values)
 
static void s3_connect (s3_options_t *param)
 
static void s3_disconnect (s3_options_t *param)
 
void s3_MPU_reset (s3_options_t *param)
 
static aiori_fd_tS3_Create_Or_Open_internal (char *testFileName, int openFlags, s3_options_t *param, int multi_part_upload_p)
 
static IOR_offset_t S3_Xfer_internal (int access, aiori_fd_t *file, IOR_size_t *buffer, IOR_offset_t length, IOR_offset_t offset, s3_options_t *param, int multi_part_upload_p)
 
static void S3_Close_internal (aiori_fd_t *fd, s3_options_t *param, int multi_part_upload_p)
 
static void EMC_Delete (char *testFileName, aiori_mod_opt_t *options)
 

Variables

int rank
 
MPI_Comm testComm
 
const int ETAG_SIZE = 32
 
CURLcode rc
 
static aiori_xfer_hint_thints = NULL
 
ior_aiori_t s3_4c_aiori
 
ior_aiori_t s3_plus_aiori
 
ior_aiori_t s3_emc_aiori
 

Macro Definition Documentation

◆ BUFF_SIZE

#define BUFF_SIZE   1024

Definition at line 109 of file aiori-S3-4c.c.

Referenced by S3_Close_internal(), S3_Create_Or_Open_internal(), and S3_Xfer_internal().

◆ CURL_ERR

#define CURL_ERR (   MSG,
  CURL_ERRNO,
  PARAM 
)
Value:
do { \
fprintf(stdout, "ior ERROR: %s: %s (curl-errno=%d) (%s:%d)\n", \
MSG, curl_easy_strerror(CURL_ERRNO), CURL_ERRNO, \
__FILE__, __LINE__); \
fflush(stdout); \
MPI_Abort((PARAM)->testComm, -1); \
} while (0)
MPI_Comm testComm
Definition: utilities.c:71

Definition at line 286 of file aiori-S3-4c.c.

◆ CURL_WARN

#define CURL_WARN (   MSG,
  CURL_ERRNO 
)
Value:
do { \
fprintf(stdout, "ior WARNING: %s: %s (curl-errno=%d) (%s:%d)\n", \
MSG, curl_easy_strerror(CURL_ERRNO), CURL_ERRNO, \
__FILE__, __LINE__); \
fflush(stdout); \
} while (0)

Definition at line 296 of file aiori-S3-4c.c.

◆ IOR_CURL_INIT

#define IOR_CURL_INIT   0x01 /* curl top-level inits were performed once? */

Definition at line 116 of file aiori-S3-4c.c.

Referenced by s3_connect().

◆ IOR_CURL_NOCONTINUE

#define IOR_CURL_NOCONTINUE   0x02

Definition at line 117 of file aiori-S3-4c.c.

◆ IOR_CURL_S3_EMC_EXT

#define IOR_CURL_S3_EMC_EXT   0x04 /* allow EMC extensions to S3? */

Definition at line 118 of file aiori-S3-4c.c.

Referenced by s3_connect().

◆ MAX_UPLOAD_ID_SIZE

#define MAX_UPLOAD_ID_SIZE   256 /* TODO don't know the actual value */

Definition at line 120 of file aiori-S3-4c.c.

Referenced by S3_Create_Or_Open_internal().

Typedef Documentation

◆ CURL

typedef void CURL

Definition at line 127 of file aiori-S3-4c.c.

◆ IOBuf

typedef void IOBuf

Definition at line 128 of file aiori-S3-4c.c.

Function Documentation

◆ EMC_Close()

static void EMC_Close ( aiori_fd_t afd,
aiori_mod_opt_t options 
)
static

Definition at line 1299 of file aiori-S3-4c.c.

References FALSE, and S3_Close_internal().

Referenced by S3_xfer_hints().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ EMC_Create()

static aiori_fd_t * EMC_Create ( char *  path,
int  iorflags,
aiori_mod_opt_t options 
)
static

Definition at line 625 of file aiori-S3-4c.c.

References FALSE, and S3_Create_Or_Open_internal().

Referenced by S3_xfer_hints().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ EMC_Delete()

static void EMC_Delete ( char *  testFileName,
aiori_mod_opt_t options 
)
static

Definition at line 1339 of file aiori-S3-4c.c.

References s3_options_t::io_buf, and s3_connect().

Here is the call graph for this function:

◆ EMC_Open()

static aiori_fd_t * EMC_Open ( char *  path,
int  flags,
aiori_mod_opt_t options 
)
static

Definition at line 644 of file aiori-S3-4c.c.

References FALSE, and S3_Create_Or_Open_internal().

Referenced by S3_xfer_hints().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ EMC_Xfer()

static IOR_offset_t EMC_Xfer ( int  access,
aiori_fd_t afd,
IOR_size_t buffer,
IOR_offset_t  length,
IOR_offset_t  offset,
aiori_mod_opt_t options 
)
static

Definition at line 942 of file aiori-S3-4c.c.

References FALSE, and S3_Xfer_internal().

Referenced by S3_xfer_hints().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ S3_check_params()

static int S3_check_params ( aiori_mod_opt_t options)
static

Definition at line 268 of file aiori-S3-4c.c.

References aiori_xfer_hint_t::blockSize, ERR, aiori_xfer_hint_t::filePerProc, aiori_xfer_hint_t::segmentCount, and aiori_xfer_hint_t::transferSize.

Referenced by S3_xfer_hints().

Here is the caller graph for this function:

◆ S3_Close()

static void S3_Close ( aiori_fd_t afd,
aiori_mod_opt_t options 
)
static

Definition at line 1295 of file aiori-S3-4c.c.

References S3_Close_internal(), and TRUE.

Referenced by S3_xfer_hints().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ S3_Close_internal()

static void S3_Close_internal ( aiori_fd_t fd,
s3_options_t param,
int  multi_part_upload_p 
)
static

Definition at line 1019 of file aiori-S3-4c.c.

References BUFF_SIZE, ERR, ETAG_SIZE, s3_options_t::etags, aiori_xfer_hint_t::filePerProc, MPI_CHECK, NULL, aiori_xfer_hint_t::numTasks, rank, s3_MPU_reset(), aiori_xfer_hint_t::segmentCount, testComm, s3_options_t::UploadId, verbose, VERBOSE_3, VERBOSE_4, and s3_options_t::written.

Referenced by EMC_Close(), and S3_Close().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ s3_connect()

static void s3_connect ( s3_options_t param)
static

◆ S3_Create()

static aiori_fd_t * S3_Create ( char *  path,
int  iorflags,
aiori_mod_opt_t options 
)
static

Definition at line 614 of file aiori-S3-4c.c.

References S3_Create_Or_Open_internal(), and TRUE.

Referenced by S3_xfer_hints().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ S3_Create_Or_Open_internal()

static aiori_fd_t* S3_Create_Or_Open_internal ( char *  testFileName,
int  openFlags,
s3_options_t param,
int  multi_part_upload_p 
)
static

Definition at line 490 of file aiori-S3-4c.c.

References BUFF_SIZE, ERR, aiori_xfer_hint_t::filePerProc, s3_options_t::io_buf, IOR_CREAT, IOR_RDWR, IOR_TRUNC, IOR_WRONLY, MAX_UPLOAD_ID_SIZE, NULL, rank, s3_connect(), testComm, s3_options_t::UploadId, verbose, VERBOSE_3, and s3_options_t::written.

Referenced by EMC_Create(), EMC_Open(), S3_Create(), and S3_Open().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ S3_Delete()

static void S3_Delete ( char *  path,
aiori_mod_opt_t options 
)
static

Definition at line 1314 of file aiori-S3-4c.c.

References s3_options_t::io_buf, and s3_connect().

Referenced by S3_xfer_hints().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ s3_disconnect()

static void s3_disconnect ( s3_options_t param)
static

Definition at line 438 of file aiori-S3-4c.c.

◆ S3_finalize()

static void S3_finalize ( aiori_mod_opt_t options)
static

Definition at line 262 of file aiori-S3-4c.c.

Referenced by S3_xfer_hints().

Here is the caller graph for this function:

◆ S3_Fsync()

static void S3_Fsync ( aiori_fd_t fd,
aiori_mod_opt_t options 
)
static

Definition at line 987 of file aiori-S3-4c.c.

Referenced by S3_xfer_hints().

Here is the caller graph for this function:

◆ S3_GetFileSize()

static IOR_offset_t S3_GetFileSize ( aiori_mod_opt_t options,
char *  testFileName 
)
static

Definition at line 1373 of file aiori-S3-4c.c.

References s3_options_t::io_buf, rank, s3_connect(), and testComm.

Referenced by S3_xfer_hints().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ S3_init()

static void S3_init ( aiori_mod_opt_t options)
static

Definition at line 255 of file aiori-S3-4c.c.

Referenced by S3_xfer_hints().

Here is the caller graph for this function:

◆ s3_MPU_reset()

void s3_MPU_reset ( s3_options_t param)

Definition at line 454 of file aiori-S3-4c.c.

References s3_options_t::etags, s3_options_t::io_buf, and s3_options_t::part_number.

Referenced by S3_Close_internal().

Here is the caller graph for this function:

◆ S3_Open()

static aiori_fd_t * S3_Open ( char *  path,
int  flags,
aiori_mod_opt_t options 
)
static

Definition at line 636 of file aiori-S3-4c.c.

References S3_Create_Or_Open_internal(), and TRUE.

Referenced by S3_xfer_hints().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ S3_options()

static option_help * S3_options ( aiori_mod_opt_t **  init_backend_options,
aiori_mod_opt_t init_values 
)
static

Definition at line 232 of file aiori-S3-4c.c.

References s3_options_t::bucket_name, s3_options_t::host, LAST_OPTION, NULL, o, OPTION_OPTIONAL_ARGUMENT, and s3_options_t::user.

Referenced by S3_xfer_hints().

Here is the caller graph for this function:

◆ S3_Xfer()

static IOR_offset_t S3_Xfer ( int  access,
aiori_fd_t afd,
IOR_size_t buffer,
IOR_offset_t  length,
IOR_offset_t  offset,
aiori_mod_opt_t options 
)
static

Definition at line 930 of file aiori-S3-4c.c.

References S3_Xfer_internal(), and TRUE.

Referenced by S3_xfer_hints().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ S3_xfer_hints()

static void S3_xfer_hints ( aiori_xfer_hint_t params)
static

◆ S3_Xfer_internal()

static IOR_offset_t S3_Xfer_internal ( int  access,
aiori_fd_t file,
IOR_size_t buffer,
IOR_offset_t  length,
IOR_offset_t  offset,
s3_options_t param,
int  multi_part_upload_p 
)
static

Variable Documentation

◆ ETAG_SIZE

const int ETAG_SIZE = 32

Definition at line 110 of file aiori-S3-4c.c.

Referenced by S3_Close_internal(), and S3_Xfer_internal().

◆ hints

aiori_xfer_hint_t* hints = NULL
static

Definition at line 148 of file aiori-S3-4c.c.

◆ rank

int rank

◆ rc

CURLcode rc

◆ s3_4c_aiori

ior_aiori_t s3_4c_aiori
Initial value:
= {
.name = "S3-4c",
.name_legacy = NULL,
.create = S3_Create,
.open = S3_Open,
.xfer = S3_Xfer,
.xfer_hints = S3_xfer_hints,
.close = S3_Close,
.delete = S3_Delete,
.get_version = aiori_get_version,
.fsync = S3_Fsync,
.get_file_size = S3_GetFileSize,
.initialize = S3_init,
.finalize = S3_finalize,
.check_params = S3_check_params,
.get_options = S3_options,
.enable_mdtest = true
}
static void S3_xfer_hints(aiori_xfer_hint_t *params)
Definition: aiori-S3-4c.c:150
static IOR_offset_t S3_Xfer(int access, aiori_fd_t *afd, IOR_size_t *buffer, IOR_offset_t length, IOR_offset_t offset, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:930
static void S3_init(aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:255
static int S3_check_params(aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:268
static void S3_finalize(aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:262
static aiori_fd_t * S3_Open(char *path, int flags, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:636
char * aiori_get_version()
Definition: aiori.c:232
static void S3_Close(aiori_fd_t *afd, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:1295
static IOR_offset_t S3_GetFileSize(aiori_mod_opt_t *options, char *testFileName)
Definition: aiori-S3-4c.c:1373
static option_help * S3_options(aiori_mod_opt_t **init_backend_options, aiori_mod_opt_t *init_values)
Definition: aiori-S3-4c.c:232
static void S3_Fsync(aiori_fd_t *fd, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:987
static void S3_Delete(char *path, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:1314
static aiori_fd_t * S3_Create(char *path, int iorflags, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:614
#define NULL
Definition: iordef.h:70

Definition at line 178 of file aiori-S3-4c.c.

◆ s3_emc_aiori

ior_aiori_t s3_emc_aiori
Initial value:
= {
.name = "S3_EMC",
.create = EMC_Create,
.open = EMC_Open,
.xfer = EMC_Xfer,
.close = EMC_Close,
.delete = S3_Delete,
.get_version = aiori_get_version,
.fsync = S3_Fsync,
.get_file_size = S3_GetFileSize,
.initialize = S3_init,
.finalize = S3_finalize
}
static void S3_init(aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:255
static void EMC_Close(aiori_fd_t *afd, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:1299
static void S3_finalize(aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:262
char * aiori_get_version()
Definition: aiori.c:232
static IOR_offset_t EMC_Xfer(int access, aiori_fd_t *afd, IOR_size_t *buffer, IOR_offset_t length, IOR_offset_t offset, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:942
static IOR_offset_t S3_GetFileSize(aiori_mod_opt_t *options, char *testFileName)
Definition: aiori-S3-4c.c:1373
static void S3_Fsync(aiori_fd_t *fd, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:987
static aiori_fd_t * EMC_Create(char *path, int iorflags, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:625
static void S3_Delete(char *path, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:1314
static aiori_fd_t * EMC_Open(char *path, int flags, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:644

Definition at line 217 of file aiori-S3-4c.c.

◆ s3_plus_aiori

ior_aiori_t s3_plus_aiori
Initial value:
= {
.name = "S3_plus",
.create = S3_Create,
.open = S3_Open,
.xfer = S3_Xfer,
.close = S3_Close,
.delete = S3_Delete,
.get_version = aiori_get_version,
.fsync = S3_Fsync,
.get_file_size = S3_GetFileSize,
.initialize = S3_init,
.finalize = S3_finalize
}
static IOR_offset_t S3_Xfer(int access, aiori_fd_t *afd, IOR_size_t *buffer, IOR_offset_t length, IOR_offset_t offset, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:930
static void S3_init(aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:255
static void S3_finalize(aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:262
static aiori_fd_t * S3_Open(char *path, int flags, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:636
char * aiori_get_version()
Definition: aiori.c:232
static void S3_Close(aiori_fd_t *afd, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:1295
static IOR_offset_t S3_GetFileSize(aiori_mod_opt_t *options, char *testFileName)
Definition: aiori-S3-4c.c:1373
static void S3_Fsync(aiori_fd_t *fd, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:987
static void S3_Delete(char *path, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:1314
static aiori_fd_t * S3_Create(char *path, int iorflags, aiori_mod_opt_t *options)
Definition: aiori-S3-4c.c:614

Definition at line 200 of file aiori-S3-4c.c.

◆ testComm

MPI_Comm testComm