IOR
Macros | Typedefs | Functions | Variables
aiori-S3.c File Reference
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include "ior.h"
#include "aiori.h"
#include "iordef.h"
#include <curl/curl.h>
#include <libxml/parser.h>
#include <libxml/tree.h>
#include "aws4c.h"
#include "aws4c_extra.h"
Include dependency graph for aiori-S3.c:

Go to the source code of this file.

Macros

#define BUFF_SIZE   1024
 
#define IOR_CURL_INIT   0x01 /* curl top-level inits were perfomed once? */
 
#define IOR_CURL_NOCONTINUE   0x02
 
#define IOR_CURL_S3_EMC_EXT   0x04 /* allow EMC extensions to S3? */
 
#define CURL_ERR(MSG, CURL_ERRNO, PARAM)
 
#define CURL_WARN(MSG, CURL_ERRNO)
 

Typedefs

typedef void CURL
 
typedef void IOBuf
 

Functions

static void * S3_Create (char *, IOR_param_t *)
 
static void * S3_Open (char *, IOR_param_t *)
 
static IOR_offset_t S3_Xfer (int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
 
static void S3_Close (void *, IOR_param_t *)
 
static void * EMC_Create (char *, IOR_param_t *)
 
static void * EMC_Open (char *, IOR_param_t *)
 
static IOR_offset_t EMC_Xfer (int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
 
static void EMC_Close (void *, IOR_param_t *)
 
static void S3_Delete (char *, IOR_param_t *)
 
static void S3_Fsync (void *, IOR_param_t *)
 
static IOR_offset_t S3_GetFileSize (IOR_param_t *, MPI_Comm, char *)
 
static void S3_init ()
 
static void S3_finalize ()
 
static int S3_check_params (IOR_param_t *)
 
static void s3_connect (IOR_param_t *param)
 
static void s3_disconnect (IOR_param_t *param)
 
void s3_MPU_reset (IOR_param_t *param)
 
static void * S3_Create_Or_Open_internal (char *testFileName, IOR_param_t *param, unsigned char createFile, int multi_part_upload_p)
 
static IOR_offset_t S3_Xfer_internal (int access, void *file, IOR_size_t *buffer, IOR_offset_t length, IOR_param_t *param, int multi_part_upload_p)
 
static void S3_Close_internal (void *fd, IOR_param_t *param, int multi_part_upload_p)
 
static void EMC_Delete (char *testFileName, IOR_param_t *param)
 

Variables

static char buff [BUFF_SIZE]
 
const int ETAG_SIZE = 32
 
CURLcode rc
 
const char * bucket_name = "ior"
 
IOBufio_buf
 
IOBufetags
 
ior_aiori_t s3_aiori
 
ior_aiori_t s3_plus_aiori
 
ior_aiori_t s3_emc_aiori
 

Macro Definition Documentation

◆ BUFF_SIZE

#define BUFF_SIZE   1024

◆ CURL_ERR

#define CURL_ERR (   MSG,
  CURL_ERRNO,
  PARAM 
)
Value:
do { \
fprintf(stdout, "ior ERROR: %s: %s (curl-errno=%d) (%s:%d)\n", \
MSG, curl_easy_strerror(CURL_ERRNO), CURL_ERRNO, \
__FILE__, __LINE__); \
fflush(stdout); \
MPI_Abort((PARAM)->testComm, -1); \
} while (0)
MPI_Comm testComm
Definition: utilities.c:60

Definition at line 251 of file aiori-S3.c.

◆ CURL_WARN

#define CURL_WARN (   MSG,
  CURL_ERRNO 
)
Value:
do { \
fprintf(stdout, "ior WARNING: %s: %s (curl-errno=%d) (%s:%d)\n", \
MSG, curl_easy_strerror(CURL_ERRNO), CURL_ERRNO, \
__FILE__, __LINE__); \
fflush(stdout); \
} while (0)

Definition at line 261 of file aiori-S3.c.

◆ IOR_CURL_INIT

#define IOR_CURL_INIT   0x01 /* curl top-level inits were perfomed once? */

Definition at line 129 of file aiori-S3.c.

Referenced by s3_connect().

◆ IOR_CURL_NOCONTINUE

#define IOR_CURL_NOCONTINUE   0x02

Definition at line 130 of file aiori-S3.c.

◆ IOR_CURL_S3_EMC_EXT

#define IOR_CURL_S3_EMC_EXT   0x04 /* allow EMC extensions to S3? */

Definition at line 131 of file aiori-S3.c.

Referenced by s3_connect().

Typedef Documentation

◆ CURL

typedef void CURL

Definition at line 137 of file aiori-S3.c.

◆ IOBuf

typedef void IOBuf

Definition at line 138 of file aiori-S3.c.

Function Documentation

◆ EMC_Close()

static void EMC_Close ( void *  fd,
IOR_param_t param 
)
static

Definition at line 1336 of file aiori-S3.c.

References FALSE, and S3_Close_internal().

Here is the call graph for this function:

◆ EMC_Create()

static void * EMC_Create ( char *  testFileName,
IOR_param_t param 
)
static

Definition at line 602 of file aiori-S3.c.

References FALSE, S3_Create_Or_Open_internal(), TRUE, IOR_param_t::verbose, and VERBOSE_2.

Here is the call graph for this function:

◆ EMC_Delete()

static void EMC_Delete ( char *  testFileName,
IOR_param_t param 
)
static

Definition at line 1383 of file aiori-S3.c.

References s3_connect(), IOR_param_t::verbose, and VERBOSE_2.

Here is the call graph for this function:

◆ EMC_Open()

static void * EMC_Open ( char *  testFileName,
IOR_param_t param 
)
static

Definition at line 640 of file aiori-S3.c.

References FALSE, IOR_CREAT, IOR_param_t::openFlags, S3_Create_Or_Open_internal(), TRUE, IOR_param_t::verbose, and VERBOSE_2.

Here is the call graph for this function:

◆ EMC_Xfer()

static IOR_offset_t EMC_Xfer ( int  access,
void *  file,
IOR_size_t buffer,
IOR_offset_t  length,
IOR_param_t param 
)
static

Definition at line 951 of file aiori-S3.c.

References FALSE, and S3_Xfer_internal().

Here is the call graph for this function:

◆ S3_check_params()

static int S3_check_params ( IOR_param_t test)
static

◆ S3_Close()

static void S3_Close ( void *  fd,
IOR_param_t param 
)
static

Definition at line 1330 of file aiori-S3.c.

References S3_Close_internal(), and TRUE.

Here is the call graph for this function:

◆ S3_Close_internal()

static void S3_Close_internal ( void *  fd,
IOR_param_t param,
int  multi_part_upload_p 
)
static

Definition at line 1035 of file aiori-S3.c.

References buff, BUFF_SIZE, ERR_SIMPLE, ETAG_SIZE, IOR_param_t::filePerProc, MPI_CHECK, NULL, IOR_param_t::numTasks, IOR_param_t::open, rank, s3_MPU_reset(), IOR_param_t::segmentCount, IOR_param_t::testComm, IOR_param_t::UploadId, IOR_param_t::verbose, VERBOSE_2, VERBOSE_3, VERBOSE_4, and WRITE.

Referenced by EMC_Close(), and S3_Close().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ s3_connect()

static void s3_connect ( IOR_param_t param)
static

Definition at line 295 of file aiori-S3.c.

References bucket_name, buff, BUFF_SIZE, IOR_CURL_INIT, IOR_CURL_S3_EMC_EXT, MPI_CHECK, rank, IOR_param_t::testComm, IOR_param_t::verbose, and VERBOSE_2.

Referenced by EMC_Delete(), S3_Create_Or_Open_internal(), S3_Delete(), and S3_GetFileSize().

Here is the caller graph for this function:

◆ S3_Create()

static void * S3_Create ( char *  testFileName,
IOR_param_t param 
)
static

Definition at line 590 of file aiori-S3.c.

References S3_Create_Or_Open_internal(), TRUE, IOR_param_t::verbose, and VERBOSE_2.

Here is the call graph for this function:

◆ S3_Create_Or_Open_internal()

static void* S3_Create_Or_Open_internal ( char *  testFileName,
IOR_param_t param,
unsigned char  createFile,
int  multi_part_upload_p 
)
static

Definition at line 458 of file aiori-S3.c.

References buff, BUFF_SIZE, ERR_SIMPLE, IOR_param_t::filePerProc, IOR_EXCL, IOR_TRUNC, NULL, IOR_param_t::open, IOR_param_t::openFlags, rank, s3_connect(), IOR_param_t::testComm, TRUE, IOR_param_t::UploadId, verbose, IOR_param_t::verbose, VERBOSE_2, VERBOSE_3, and WRITE.

Referenced by EMC_Create(), EMC_Open(), S3_Create(), and S3_Open().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ S3_Delete()

static void S3_Delete ( char *  testFileName,
IOR_param_t param 
)
static

Definition at line 1354 of file aiori-S3.c.

References s3_connect(), IOR_param_t::verbose, and VERBOSE_2.

Here is the call graph for this function:

◆ s3_disconnect()

static void s3_disconnect ( IOR_param_t param)
static

Definition at line 402 of file aiori-S3.c.

References IOR_param_t::verbose, and VERBOSE_2.

◆ S3_finalize()

static void S3_finalize ( )
static

Definition at line 228 of file aiori-S3.c.

◆ S3_Fsync()

static void S3_Fsync ( void *  fd,
IOR_param_t param 
)
static

Definition at line 997 of file aiori-S3.c.

References IOR_param_t::verbose, and VERBOSE_2.

◆ S3_GetFileSize()

static IOR_offset_t S3_GetFileSize ( IOR_param_t param,
MPI_Comm  testComm,
char *  testFileName 
)
static

Definition at line 1425 of file aiori-S3.c.

References IOR_param_t::filePerProc, MPI_CHECK, rank, s3_connect(), IOR_param_t::testComm, TRUE, IOR_param_t::verbose, VERBOSE_2, and WARN.

Here is the call graph for this function:

◆ S3_init()

static void S3_init ( )
static

Definition at line 221 of file aiori-S3.c.

◆ s3_MPU_reset()

void s3_MPU_reset ( IOR_param_t param)

Definition at line 420 of file aiori-S3.c.

References IOR_param_t::part_number.

Referenced by S3_Close_internal().

Here is the caller graph for this function:

◆ S3_Open()

static void * S3_Open ( char *  testFileName,
IOR_param_t param 
)
static

Definition at line 620 of file aiori-S3.c.

References FALSE, IOR_CREAT, IOR_param_t::openFlags, S3_Create_Or_Open_internal(), TRUE, IOR_param_t::verbose, and VERBOSE_2.

Here is the call graph for this function:

◆ S3_Xfer()

static IOR_offset_t S3_Xfer ( int  access,
void *  file,
IOR_size_t buffer,
IOR_offset_t  length,
IOR_param_t param 
)
static

Definition at line 942 of file aiori-S3.c.

References S3_Xfer_internal(), and TRUE.

Here is the call graph for this function:

◆ S3_Xfer_internal()

static IOR_offset_t S3_Xfer_internal ( int  access,
void *  file,
IOR_size_t buffer,
IOR_offset_t  length,
IOR_param_t param,
int  multi_part_upload_p 
)
static

Variable Documentation

◆ bucket_name

const char* bucket_name = "ior"

Definition at line 124 of file aiori-S3.c.

Referenced by s3_connect().

◆ buff

char buff[BUFF_SIZE]
static

◆ ETAG_SIZE

const int ETAG_SIZE = 32

Definition at line 119 of file aiori-S3.c.

Referenced by S3_Close_internal(), and S3_Xfer_internal().

◆ etags

IOBuf* etags

Definition at line 142 of file aiori-S3.c.

◆ io_buf

IOBuf* io_buf

Definition at line 141 of file aiori-S3.c.

◆ rc

CURLcode rc

◆ s3_aiori

ior_aiori_t s3_aiori
Initial value:
= {
.name = "S3",
.name_legacy = NULL,
.create = S3_Create,
.open = S3_Open,
.xfer = S3_Xfer,
.close = S3_Close,
.delete = S3_Delete,
.get_version = aiori_get_version,
.fsync = S3_Fsync,
.get_file_size = S3_GetFileSize,
.initialize = S3_init,
.finalize = S3_finalize,
.check_params = S3_check_params
}
static void S3_Fsync(void *, IOR_param_t *)
Definition: aiori-S3.c:997
static void S3_finalize()
Definition: aiori-S3.c:228
static void S3_Close(void *, IOR_param_t *)
Definition: aiori-S3.c:1330
static IOR_offset_t S3_GetFileSize(IOR_param_t *, MPI_Comm, char *)
Definition: aiori-S3.c:1425
char * aiori_get_version()
Definition: aiori.c:200
static void S3_init()
Definition: aiori-S3.c:221
static int S3_check_params(IOR_param_t *)
Definition: aiori-S3.c:234
static void * S3_Open(char *, IOR_param_t *)
Definition: aiori-S3.c:620
static void * S3_Create(char *, IOR_param_t *)
Definition: aiori-S3.c:590
static IOR_offset_t S3_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori-S3.c:942
static void S3_Delete(char *, IOR_param_t *)
Definition: aiori-S3.c:1354
#define NULL
Definition: iordef.h:79

Definition at line 170 of file aiori-S3.c.

◆ s3_emc_aiori

ior_aiori_t s3_emc_aiori
Initial value:
= {
.name = "S3_EMC",
.create = EMC_Create,
.open = EMC_Open,
.xfer = EMC_Xfer,
.close = EMC_Close,
.delete = S3_Delete,
.set_version = S3_SetVersion,
.fsync = S3_Fsync,
.get_file_size = S3_GetFileSize,
.initialize = S3_init,
.finalize = S3_finalize
}
static void S3_Fsync(void *, IOR_param_t *)
Definition: aiori-S3.c:997
static void EMC_Close(void *, IOR_param_t *)
Definition: aiori-S3.c:1336
static IOR_offset_t EMC_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori-S3.c:951
static void * EMC_Create(char *, IOR_param_t *)
Definition: aiori-S3.c:602
static void S3_finalize()
Definition: aiori-S3.c:228
static IOR_offset_t S3_GetFileSize(IOR_param_t *, MPI_Comm, char *)
Definition: aiori-S3.c:1425
static void * EMC_Open(char *, IOR_param_t *)
Definition: aiori-S3.c:640
static void S3_init()
Definition: aiori-S3.c:221
static void S3_Delete(char *, IOR_param_t *)
Definition: aiori-S3.c:1354

Definition at line 206 of file aiori-S3.c.

◆ s3_plus_aiori

ior_aiori_t s3_plus_aiori
Initial value:
= {
.name = "S3_plus",
.create = S3_Create,
.open = S3_Open,
.xfer = S3_Xfer,
.close = S3_Close,
.delete = S3_Delete,
.set_version = S3_SetVersion,
.fsync = S3_Fsync,
.get_file_size = S3_GetFileSize,
.initialize = S3_init,
.finalize = S3_finalize
}
static void S3_Fsync(void *, IOR_param_t *)
Definition: aiori-S3.c:997
static void S3_finalize()
Definition: aiori-S3.c:228
static void S3_Close(void *, IOR_param_t *)
Definition: aiori-S3.c:1330
static IOR_offset_t S3_GetFileSize(IOR_param_t *, MPI_Comm, char *)
Definition: aiori-S3.c:1425
static void S3_init()
Definition: aiori-S3.c:221
static void * S3_Open(char *, IOR_param_t *)
Definition: aiori-S3.c:620
static void * S3_Create(char *, IOR_param_t *)
Definition: aiori-S3.c:590
static IOR_offset_t S3_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori-S3.c:942
static void S3_Delete(char *, IOR_param_t *)
Definition: aiori-S3.c:1354

Definition at line 189 of file aiori-S3.c.