94 #include <curl/curl.h> 96 #include <libxml/parser.h> 97 #include <libxml/tree.h> 100 #include "aws4c_extra.h" 109 #define BUFF_SIZE 1024 116 # define IOR_CURL_INIT 0x01 117 # define IOR_CURL_NOCONTINUE 0x02 118 # define IOR_CURL_S3_EMC_EXT 0x04 120 #define MAX_UPLOAD_ID_SIZE 256 123 #ifdef USE_S3_4C_AIORI 124 # include <curl/curl.h> 194 .enable_mdtest =
true 234 if (init_values !=
NULL){
250 memcpy(help, h,
sizeof(h));
259 AWS4C_CHECK( aws_init() );
269 if(! hints)
return 0;
277 if (Nto1 && (s != 1) && (b != t)) {
278 ERR(
"N:1 (strided) requires xfer-size == block-size");
286 #define CURL_ERR(MSG, CURL_ERRNO, PARAM) \ 288 fprintf(stdout, "ior ERROR: %s: %s (curl-errno=%d) (%s:%d)\n", \ 289 MSG, curl_easy_strerror(CURL_ERRNO), CURL_ERRNO, \ 290 __FILE__, __LINE__); \ 292 MPI_Abort((PARAM)->testComm, -1); \ 296 #define CURL_WARN(MSG, CURL_ERRNO) \ 298 fprintf(stdout, "ior WARNING: %s: %s (curl-errno=%d) (%s:%d)\n", \ 299 MSG, curl_easy_strerror(CURL_ERRNO), CURL_ERRNO, \ 300 __FILE__, __LINE__); \ 357 aws_read_config(param->
user);
358 aws_reuse_connections(1);
363 param->
io_buf = aws_iobuf_new();
364 aws_iobuf_growth_size(param->
io_buf, 1024*1024*1);
366 param->
etags = aws_iobuf_new();
367 aws_iobuf_growth_size(param->
etags, 1024*1024*8);
404 s3_set_host(param->
host);
410 AWS4C_CHECK( s3_head(param->
io_buf,
"") );
411 if ( param->
io_buf->code == 404 ) {
412 printf(
" bucket '%s' doesn't exist\n", param->
bucket_name);
414 AWS4C_CHECK( s3_put(param->
io_buf,
"") );
415 AWS4C_CHECK_OK( param->
io_buf );
416 printf(
"created bucket '%s'\n", param->
bucket_name);
419 AWS4C_CHECK_OK( param->
io_buf );
455 aws_iobuf_reset(param->
io_buf);
456 aws_iobuf_reset(param->
etags);
491 unsigned char createFile = openFlags &
IOR_CREAT;
511 int n_to_1 = ! n_to_n;
515 if (! multi_part_upload_p)
519 else if (createFile) {
530 if (multi_part_upload_p) {
536 if ( n_to_n || (
rank == 0) ) {
540 aws_iobuf_reset(param->
io_buf);
541 AWS4C_CHECK( s3_put(param->
io_buf, testFileName) );
542 AWS4C_CHECK_OK( param->
io_buf );
546 snprintf(buff,
BUFF_SIZE,
"%s?uploads", testFileName);
547 IOBuf* response = aws_iobuf_new();
548 AWS4C_CHECK( s3_post2(param->
io_buf, buff,
NULL, response) );
549 AWS4C_CHECK_OK( param->
io_buf );
552 aws_iobuf_realloc(response);
553 xmlDocPtr doc = xmlReadMemory(response->first->buf,
554 response->first->len,
557 ERR(
"Rank0 Failed to find POST response\n");
560 xmlNode* root_element = xmlDocGetRootElement(doc);
561 const char* upload_id = find_element_named(root_element, (
char*)
"UploadId");
563 ERR(
"couldn't find 'UploadId' in returned XML\n");
568 const size_t upload_id_len = strlen(upload_id);
575 memcpy(param->
UploadId, upload_id, upload_id_len);
580 aws_iobuf_free(response);
597 fprintf( stdout,
"rank %d resetting\n",
601 aws_iobuf_reset(param->
io_buf);
602 AWS4C_CHECK( s3_put(param->
io_buf, testFileName) );
603 AWS4C_CHECK_OK( param->
io_buf );
732 int multi_part_upload_p ) {
738 char* fname = (
char*)file;
739 size_t remaining = (size_t)length;
740 char* data_ptr = (
char *)buffer;
744 int n_to_1 = (! n_to_n);
748 if (access ==
WRITE) {
757 if (multi_part_upload_p) {
804 "%s?partNumber=%zd&uploadId=%s",
805 fname, part_number, param->
UploadId);
817 aws_iobuf_reset(param->
io_buf);
818 aws_iobuf_append_static(param->
io_buf, data_ptr, remaining);
819 AWS4C_CHECK( s3_put(param->
io_buf, buff) );
820 AWS4C_CHECK_OK( param->
io_buf );
842 fprintf(stderr,
"Rank %d: ERROR: expected ETag to be %d hex digits\n",
851 aws_iobuf_append(param->
etags,
853 strlen(param->
io_buf->eTag) -2);
860 aws_iobuf_reset(param->
io_buf);
868 s3_set_byte_range(-1,-1);
870 s3_set_byte_range(offset, remaining);
876 aws_iobuf_reset(param->
io_buf);
877 aws_iobuf_append_static(param->
io_buf, data_ptr, remaining);
878 AWS4C_CHECK ( s3_put(param->
io_buf, (
char*) file) );
879 AWS4C_CHECK_OK( param->
io_buf );
882 aws_iobuf_reset(param->
io_buf);
887 WARN(
"S3 doesn't support 'fsync'" );
901 s3_set_byte_range(offset, remaining);
908 aws_iobuf_reset(param->
io_buf);
909 aws_iobuf_extend_static(param->
io_buf, data_ptr, remaining);
910 AWS4C_CHECK( s3_get(param->
io_buf, (
char*) file) );
911 if (param->
io_buf->code != 206) {
914 "Unexpected result (%d, '%s')",
920 aws_iobuf_reset(param->
io_buf);
1021 char* fname = (
char*)fd;
1025 int n_to_1 = (! n_to_n);
1031 if (multi_part_upload_p) {
1034 size_t etag_data_size = param->
etags->write_count;
1035 size_t etags_per_rank = etag_data_size /
ETAG_SIZE;
1044 MPI_Datatype mpi_size_t;
1045 if (
sizeof(
size_t) ==
sizeof(int))
1046 mpi_size_t = MPI_INT;
1047 else if (
sizeof(
size_t) ==
sizeof(long))
1048 mpi_size_t = MPI_LONG;
1050 mpi_size_t = MPI_LONG_LONG;
1053 size_t etag_count_max = 0;
1054 MPI_Allreduce(&etags_per_rank, &etag_count_max,
1056 if (etags_per_rank != etag_count_max) {
1057 printf(
"Rank %d: etag count mismatch: max:%zd, mine:%zd\n",
1058 rank, etag_count_max, etags_per_rank);
1063 aws_iobuf_realloc(param->
etags);
1064 char* etag_data = param->
etags->first->buf;
1072 char* etag_vec = (
char*)malloc((hints->
numTasks * etag_data_size) +1);
1074 fprintf(stderr,
"rank 0 failed to malloc %zd bytes\n",
1078 MPI_Gather(etag_data, etag_data_size, MPI_BYTE,
1079 etag_vec, etag_data_size, MPI_BYTE, 0,
testComm);
1084 printf(
"rank 0: gathered %zd etags from all ranks:\n", etags_per_rank);
1086 for (rnk=0; rnk < hints->
numTasks; ++rnk) {
1087 printf(
"\t[%d]: '", rnk);
1090 for (ii=0; ii < etag_data_size; ++ii)
1091 printf(
"%c", etag_ptr[ii]);
1094 etag_ptr += etag_data_size;
1145 size_t start_multiplier;
1150 j_max = etags_per_rank;
1151 start_multiplier = etag_data_size;
1155 i_max = etags_per_rank;
1158 stride = etag_data_size;
1162 xml = aws_iobuf_new();
1163 aws_iobuf_growth_size(xml, 1024 * 8);
1166 aws_iobuf_append_str(xml,
"<CompleteMultipartUpload>\n");
1169 for (i=0; i<i_max; ++i) {
1171 etag_ptr=etag_vec + (i * start_multiplier);
1173 for (j=0; j<j_max; ++j) {
1183 " <PartNumber>%d</PartNumber>\n" 1184 " <ETag>%s</ETag>\n" 1188 aws_iobuf_append_str(xml, buff);
1196 aws_iobuf_append_str(xml,
"</CompleteMultipartUpload>\n");
1198 MPI_Gather(etag_data, etag_data_size, MPI_BYTE,
1203 xml = aws_iobuf_new();
1204 aws_iobuf_growth_size(xml, 1024 * 8);
1207 aws_iobuf_append_str(xml,
"<CompleteMultipartUpload>\n");
1214 for (i=0; i<etags_per_rank; ++i) {
1221 "Read of ETag %d had length %d (not %d)\n",
1231 " <PartNumber>%d</PartNumber>\n" 1232 " <ETag>%s</ETag>\n" 1236 aws_iobuf_append_str(xml, buff);
1242 aws_iobuf_append_str(xml,
"</CompleteMultipartUpload>\n");
1246 if (n_to_n || (
rank == 0)) {
1250 debug_iobuf(xml, 1, 1);
1257 AWS4C_CHECK ( s3_post(xml, buff) );
1258 AWS4C_CHECK_OK( xml );
1260 aws_iobuf_free(xml);
1287 aws_reset_connection();
1326 AWS4C_CHECK( s3_delete(param->
io_buf, testFileName) );
1329 aws_iobuf_reset(param->
io_buf);
1330 AWS4C_CHECK ( s3_put(param->
io_buf, testFileName) );
1333 AWS4C_CHECK_OK( param->
io_buf );
1352 AWS4C_CHECK( s3_delete(param->
io_buf, testFileName) );
1355 aws_iobuf_reset(param->
io_buf);
1356 AWS4C_CHECK ( s3_put(param->
io_buf, testFileName) );
1359 AWS4C_CHECK_OK( param->
io_buf );
1387 AWS4C_CHECK( s3_head(param->
io_buf, testFileName) );
1388 if ( ! AWS4C_OK(param->
io_buf) ) {
1389 fprintf(stderr,
"rank %d: couldn't stat '%s': %s\n",
1393 aggFileSizeFromStat = param->
io_buf->contentLen;
1395 return ( aggFileSizeFromStat );
static void S3_xfer_hints(aiori_xfer_hint_t *params)
static IOR_offset_t S3_Xfer(int access, aiori_fd_t *afd, IOR_size_t *buffer, IOR_offset_t length, IOR_offset_t offset, aiori_mod_opt_t *options)
IOR_offset_t segmentCount
static void S3_init(aiori_mod_opt_t *options)
static void EMC_Close(aiori_fd_t *afd, aiori_mod_opt_t *options)
struct benchmark_options o
static int S3_check_params(aiori_mod_opt_t *options)
static void EMC_Delete(char *testFileName, aiori_mod_opt_t *options)
static void S3_finalize(aiori_mod_opt_t *options)
static IOR_offset_t S3_Xfer_internal(int access, aiori_fd_t *file, IOR_size_t *buffer, IOR_offset_t length, IOR_offset_t offset, s3_options_t *param, int multi_part_upload_p)
#define MPI_CHECK(MPI_STATUS, MSG)
static void s3_connect(s3_options_t *param)
static aiori_fd_t * S3_Open(char *path, int flags, aiori_mod_opt_t *options)
char * aiori_get_version()
static option_help options[]
static IOR_offset_t EMC_Xfer(int access, aiori_fd_t *afd, IOR_size_t *buffer, IOR_offset_t length, IOR_offset_t offset, aiori_mod_opt_t *options)
void s3_MPU_reset(s3_options_t *param)
static void S3_Close(aiori_fd_t *afd, aiori_mod_opt_t *options)
static aiori_fd_t * S3_Create_Or_Open_internal(char *testFileName, int openFlags, s3_options_t *param, int multi_part_upload_p)
IOR_offset_t transferSize
static IOR_offset_t S3_GetFileSize(aiori_mod_opt_t *options, char *testFileName)
static option_help * S3_options(aiori_mod_opt_t **init_backend_options, aiori_mod_opt_t *init_values)
static void s3_disconnect(s3_options_t *param)
static void S3_Fsync(aiori_fd_t *fd, aiori_mod_opt_t *options)
char UploadId[MAX_UPLOAD_ID_SIZE]
#define IOR_CURL_S3_EMC_EXT
static aiori_fd_t * EMC_Create(char *path, int iorflags, aiori_mod_opt_t *options)
static void S3_Delete(char *path, aiori_mod_opt_t *options)
static aiori_xfer_hint_t * hints
static void S3_Close_internal(aiori_fd_t *fd, s3_options_t *param, int multi_part_upload_p)
#define MAX_UPLOAD_ID_SIZE
static aiori_fd_t * S3_Create(char *path, int iorflags, aiori_mod_opt_t *options)
ior_aiori_t s3_plus_aiori
long long int IOR_offset_t
static aiori_fd_t * EMC_Open(char *path, int flags, aiori_mod_opt_t *options)