IOR
aiori-DAOS.c
Go to the documentation of this file.
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  */
4 /*
5  * Copyright (C) 2018-2020 Intel Corporation
6  * See the file COPYRIGHT for a complete copyright notice and license.
7  */
8 
9 /*
10  * This file implements the abstract I/O interface for DAOS Array API.
11  */
12 
13 #define _BSD_SOURCE
14 
15 #ifdef HAVE_CONFIG_H
16 #include "config.h"
17 #endif
18 
19 #include <stdint.h>
20 #include <assert.h>
21 #include <unistd.h>
22 #include <strings.h>
23 #include <sys/types.h>
24 #include <libgen.h>
25 #include <stdbool.h>
26 
27 #include <gurt/common.h>
28 #include <daos.h>
29 
30 #include "ior.h"
31 #include "aiori.h"
32 #include "iordef.h"
33 
34 /************************** O P T I O N S *****************************/
35 struct daos_options{
36  char *pool;
37  char *svcl;
38  char *group;
39  char *cont;
41  int destroy;
42  char *oclass;
43 };
44 
45 static struct daos_options o = {
46  .pool = NULL,
47  .svcl = NULL,
48  .group = NULL,
49  .cont = NULL,
50  .chunk_size = 1048576,
51  .destroy = 0,
52  .oclass = NULL,
53 };
54 
55 static option_help options [] = {
56  {0, "daos.pool", "pool uuid", OPTION_OPTIONAL_ARGUMENT, 's', &o.pool},
57  {0, "daos.svcl", "pool SVCL", OPTION_OPTIONAL_ARGUMENT, 's', &o.svcl},
58  {0, "daos.group", "server group", OPTION_OPTIONAL_ARGUMENT, 's', &o.group},
59  {0, "daos.cont", "container uuid", OPTION_OPTIONAL_ARGUMENT, 's', &o.cont},
60  {0, "daos.chunk_size", "chunk size", OPTION_OPTIONAL_ARGUMENT, 'd', &o.chunk_size},
61  {0, "daos.destroy", "Destroy Container", OPTION_FLAG, 'd', &o.destroy},
62  {0, "daos.oclass", "object class", OPTION_OPTIONAL_ARGUMENT, 's', &o.oclass},
64 };
65 
66 /**************************** P R O T O T Y P E S *****************************/
67 
68 static void DAOS_Init();
69 static void DAOS_Fini();
70 static void *DAOS_Create(char *, IOR_param_t *);
71 static void *DAOS_Open(char *, IOR_param_t *);
72 static int DAOS_Access(const char *, int, IOR_param_t *);
73 static IOR_offset_t DAOS_Xfer(int, void *, IOR_size_t *,
75 static void DAOS_Close(void *, IOR_param_t *);
76 static void DAOS_Delete(char *, IOR_param_t *);
77 static char* DAOS_GetVersion();
78 static void DAOS_Fsync(void *, IOR_param_t *);
79 static IOR_offset_t DAOS_GetFileSize(IOR_param_t *, MPI_Comm, char *);
80 static option_help * DAOS_options();
81 
82 /************************** D E C L A R A T I O N S ***************************/
83 
85  .name = "DAOS",
86  .create = DAOS_Create,
87  .open = DAOS_Open,
88  .access = DAOS_Access,
89  .xfer = DAOS_Xfer,
90  .close = DAOS_Close,
91  .delete = DAOS_Delete,
92  .get_version = DAOS_GetVersion,
93  .fsync = DAOS_Fsync,
94  .get_file_size = DAOS_GetFileSize,
95  .initialize = DAOS_Init,
96  .finalize = DAOS_Fini,
97  .get_options = DAOS_options,
98  .statfs = aiori_posix_statfs,
99  .mkdir = aiori_posix_mkdir,
100  .rmdir = aiori_posix_rmdir,
101  .stat = aiori_posix_stat,
102 };
103 
104 #define IOR_DAOS_MUR_SEED 0xDEAD10CC
105 
110 };
111 
112 static daos_handle_t poh;
113 static daos_handle_t coh;
114 static daos_handle_t aoh;
115 static daos_oclass_id_t objectClass = OC_SX;
116 static bool daos_initialized = false;
117 
118 /***************************** F U N C T I O N S ******************************/
119 
120 /* For DAOS methods. */
121 #define DCHECK(rc, format, ...) \
122 do { \
123  int _rc = (rc); \
124  \
125  if (_rc < 0) { \
126  fprintf(stderr, "ior ERROR (%s:%d): %d: %d: " \
127  format"\n", __FILE__, __LINE__, rank, _rc, \
128  ##__VA_ARGS__); \
129  fflush(stdout); \
130  MPI_Abort(MPI_COMM_WORLD, -1); \
131  } \
132 } while (0)
133 
134 #define INFO(level, format, ...) \
135 do { \
136  if (verbose >= level) \
137  printf("[%d] "format"\n", rank, ##__VA_ARGS__); \
138 } while (0)
139 
140 /* For generic errors like invalid command line options. */
141 #define GERR(format, ...) \
142 do { \
143  fprintf(stderr, format"\n", ##__VA_ARGS__); \
144  MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); \
145 } while (0)
146 
147 /* Distribute process 0's pool or container handle to others. */
148 static void
149 HandleDistribute(daos_handle_t *handle, enum handleType type)
150 {
151  d_iov_t global;
152  int rc;
153 
154  global.iov_buf = NULL;
155  global.iov_buf_len = 0;
156  global.iov_len = 0;
157 
158  if (rank == 0) {
159  /* Get the global handle size. */
160  if (type == POOL_HANDLE)
161  rc = daos_pool_local2global(*handle, &global);
162  else if (type == CONT_HANDLE)
163  rc = daos_cont_local2global(*handle, &global);
164  else
165  rc = daos_array_local2global(*handle, &global);
166  DCHECK(rc, "Failed to get global handle size");
167  }
168 
169  MPI_CHECK(MPI_Bcast(&global.iov_buf_len, 1, MPI_UINT64_T, 0,
170  MPI_COMM_WORLD),
171  "Failed to bcast global handle buffer size");
172 
173  global.iov_len = global.iov_buf_len;
174  global.iov_buf = malloc(global.iov_buf_len);
175  if (global.iov_buf == NULL)
176  ERR("Failed to allocate global handle buffer");
177 
178  if (rank == 0) {
179  if (type == POOL_HANDLE)
180  rc = daos_pool_local2global(*handle, &global);
181  else if (type == CONT_HANDLE)
182  rc = daos_cont_local2global(*handle, &global);
183  else
184  rc = daos_array_local2global(*handle, &global);
185  DCHECK(rc, "Failed to create global handle");
186  }
187 
188  MPI_CHECK(MPI_Bcast(global.iov_buf, global.iov_buf_len, MPI_BYTE, 0,
189  MPI_COMM_WORLD),
190  "Failed to bcast global pool handle");
191 
192  if (rank != 0) {
193  if (type == POOL_HANDLE)
194  rc = daos_pool_global2local(global, handle);
195  else if (type == CONT_HANDLE)
196  rc = daos_cont_global2local(poh, global, handle);
197  else
198  rc = daos_array_global2local(coh, global, 0, handle);
199  DCHECK(rc, "Failed to get local handle");
200  }
201 
202  free(global.iov_buf);
203 }
204 
205 static option_help *
207 {
208  return options;
209 }
210 
211 static void
213 {
214  int rc;
215 
216  if (daos_initialized)
217  return;
218 
219  if (o.pool == NULL || o.svcl == NULL || o.cont == NULL)
220  return;
221 
222  if (o.oclass) {
223  objectClass = daos_oclass_name2id(o.oclass);
224  if (objectClass == OC_UNKNOWN)
225  GERR("Invalid DAOS Object class %s\n", o.oclass);
226  }
227 
228  rc = daos_init();
229  if (rc)
230  DCHECK(rc, "Failed to initialize daos");
231 
232  if (rank == 0) {
233  uuid_t uuid;
234  d_rank_list_t *svcl = NULL;
235  static daos_pool_info_t po_info;
236  static daos_cont_info_t co_info;
237 
238  INFO(VERBOSE_1, "Connecting to pool %s", o.pool);
239 
240  rc = uuid_parse(o.pool, uuid);
241  DCHECK(rc, "Failed to parse 'pool': %s", o.pool);
242 
243  svcl = daos_rank_list_parse(o.svcl, ":");
244  if (svcl == NULL)
245  ERR("Failed to allocate svcl");
246 
247  rc = daos_pool_connect(uuid, o.group, svcl, DAOS_PC_RW,
248  &poh, &po_info, NULL);
249  d_rank_list_free(svcl);
250  DCHECK(rc, "Failed to connect to pool %s", o.pool);
251 
252  INFO(VERBOSE_1, "Create/Open Container %s", o.cont);
253 
254  uuid_clear(uuid);
255  rc = uuid_parse(o.cont, uuid);
256  DCHECK(rc, "Failed to parse 'cont': %s", o.cont);
257 
258  rc = daos_cont_open(poh, uuid, DAOS_COO_RW, &coh, &co_info,
259  NULL);
260  /* If NOEXIST we create it */
261  if (rc == -DER_NONEXIST) {
262  INFO(VERBOSE_2, "Creating DAOS Container...\n");
263  rc = daos_cont_create(poh, uuid, NULL, NULL);
264  if (rc == 0)
265  rc = daos_cont_open(poh, uuid, DAOS_COO_RW,
266  &coh, &co_info, NULL);
267  }
268  DCHECK(rc, "Failed to create container");
269  }
270 
273  aoh.cookie = 0;
274 
275  daos_initialized = true;
276 }
277 
278 static void
280 {
281  int rc;
282 
283  if (!daos_initialized)
284  return;
285 
286  MPI_Barrier(MPI_COMM_WORLD);
287  rc = daos_cont_close(coh, NULL);
288  if (rc) {
289  DCHECK(rc, "Failed to close container %s (%d)", o.cont, rc);
290  MPI_Abort(MPI_COMM_WORLD, -1);
291  }
292  MPI_Barrier(MPI_COMM_WORLD);
293 
294  if (o.destroy) {
295  if (rank == 0) {
296  uuid_t uuid;
297  double t1, t2;
298 
299  INFO(VERBOSE_1, "Destroying DAOS Container %s", o.cont);
300  uuid_parse(o.cont, uuid);
301  t1 = MPI_Wtime();
302  rc = daos_cont_destroy(poh, uuid, 1, NULL);
303  t2 = MPI_Wtime();
304  if (rc == 0)
305  INFO(VERBOSE_1, "Container Destroy time = %f secs", t2-t1);
306  }
307 
308  MPI_Bcast(&rc, 1, MPI_INT, 0, MPI_COMM_WORLD);
309  if (rc) {
310  if (rank == 0)
311  DCHECK(rc, "Failed to destroy container %s (%d)", o.cont, rc);
312  MPI_Abort(MPI_COMM_WORLD, -1);
313  }
314  }
315 
316  if (rank == 0)
317  INFO(VERBOSE_1, "Disconnecting from DAOS POOL..");
318 
319  rc = daos_pool_disconnect(poh, NULL);
320  DCHECK(rc, "Failed to disconnect from pool %s", o.pool);
321 
322  MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD), "barrier error");
323  if (rank == 0)
324  INFO(VERBOSE_1, "Finalizing DAOS..");
325 
326  rc = daos_fini();
327  DCHECK(rc, "Failed to finalize daos");
328 
329  daos_initialized = false;
330 }
331 
332 static void
333 gen_oid(const char *name, daos_obj_id_t *oid)
334 {
335 
336  oid->lo = d_hash_murmur64(name, strlen(name), IOR_DAOS_MUR_SEED);
337  oid->hi = 0;
338 
339  daos_array_generate_id(oid, objectClass, true, 0);
340 }
341 
342 static void *
343 DAOS_Create(char *testFileName, IOR_param_t *param)
344 {
345  daos_obj_id_t oid;
346  int rc;
347 
349  gen_oid(testFileName, &oid);
350 
352  if (param->filePerProc || rank == 0) {
353  rc = daos_array_create(coh, oid, DAOS_TX_NONE, 1, o.chunk_size,
354  &aoh, NULL);
355  DCHECK(rc, "Failed to create array object\n");
356  }
357 
359  if (!param->filePerProc)
361 
362  return &aoh;
363 }
364 
365 static int
366 DAOS_Access(const char *testFileName, int mode, IOR_param_t * param)
367 {
368  daos_obj_id_t oid;
369  daos_size_t cell_size, chunk_size;
370  int rc;
371 
373  gen_oid(testFileName, &oid);
374 
375  rc = daos_array_open(coh, oid, DAOS_TX_NONE, DAOS_OO_RO,
376  &cell_size, &chunk_size, &aoh, NULL);
377  if (rc)
378  return rc;
379 
380  if (cell_size != 1)
381  GERR("Invalid DAOS Array object.\n");
382 
383  rc = daos_array_close(aoh, NULL);
384  aoh.cookie = 0;
385  return rc;
386 }
387 
388 static void *
389 DAOS_Open(char *testFileName, IOR_param_t *param)
390 {
391  daos_obj_id_t oid;
392 
394  gen_oid(testFileName, &oid);
395 
397  if (param->filePerProc || rank == 0) {
398  daos_size_t cell_size, chunk_size;
399  int rc;
400 
401  rc = daos_array_open(coh, oid, DAOS_TX_NONE, DAOS_OO_RW,
402  &cell_size, &chunk_size, &aoh, NULL);
403  DCHECK(rc, "Failed to create array object\n");
404 
405  if (cell_size != 1)
406  GERR("Invalid DAOS Array object.\n");
407  }
408 
410  if (!param->filePerProc)
412 
413  return &aoh;
414 }
415 
416 static IOR_offset_t
417 DAOS_Xfer(int access, void *file, IOR_size_t *buffer,
418  IOR_offset_t length, IOR_param_t *param)
419 {
420  daos_array_iod_t iod;
421  daos_range_t rg;
422  d_sg_list_t sgl;
423  d_iov_t iov;
424  int rc;
425 
427  iod.arr_nr = 1;
428  rg.rg_len = length;
429  rg.rg_idx = param->offset;
430  iod.arr_rgs = &rg;
431 
433  sgl.sg_nr = 1;
434  d_iov_set(&iov, buffer, length);
435  sgl.sg_iovs = &iov;
436 
437  if (access == WRITE) {
438  rc = daos_array_write(aoh, DAOS_TX_NONE, &iod, &sgl, NULL);
439  DCHECK(rc, "daos_array_write() failed (%d).", rc);
440  } else {
441  rc = daos_array_read(aoh, DAOS_TX_NONE, &iod, &sgl, NULL);
442  DCHECK(rc, "daos_array_read() failed (%d).", rc);
443  }
444 
445  return length;
446 }
447 
448 static void
450 {
451  int rc;
452 
453  if (!daos_initialized)
454  GERR("DAOS is not initialized!");
455 
456  rc = daos_array_close(aoh, NULL);
457  DCHECK(rc, "daos_array_close() failed (%d).", rc);
458 
459  aoh.cookie = 0;
460 }
461 
462 static void
463 DAOS_Delete(char *testFileName, IOR_param_t *param)
464 {
465  daos_obj_id_t oid;
466  daos_size_t cell_size, chunk_size;
467  int rc;
468 
469  if (!daos_initialized)
470  GERR("DAOS is not initialized!");
471 
473  gen_oid(testFileName, &oid);
474 
476  rc = daos_array_open(coh, oid, DAOS_TX_NONE, DAOS_OO_RW,
477  &cell_size, &chunk_size, &aoh, NULL);
478  DCHECK(rc, "daos_array_open() failed (%d).", rc);
479 
480  if (cell_size != 1)
481  GERR("Invalid DAOS Array object.\n");
482 
483  rc = daos_array_destroy(aoh, DAOS_TX_NONE, NULL);
484  DCHECK(rc, "daos_array_destroy() failed (%d).", rc);
485 
486  rc = daos_array_close(aoh, NULL);
487  DCHECK(rc, "daos_array_close() failed (%d).", rc);
488  aoh.cookie = 0;
489 }
490 
491 static char *
493 {
494  static char ver[1024] = {};
495 
496  sprintf(ver, "%s", "DAOS");
497  return ver;
498 }
499 
500 static void
502 {
503  return;
504 }
505 
506 static IOR_offset_t
507 DAOS_GetFileSize(IOR_param_t *param, MPI_Comm testComm, char *testFileName)
508 {
509  daos_obj_id_t oid;
510  daos_size_t size;
511  int rc;
512 
513  if (!daos_initialized)
514  GERR("DAOS is not initialized!");
515 
517  gen_oid(testFileName, &oid);
518 
520  if (param->filePerProc || rank == 0) {
521  daos_size_t cell_size, chunk_size;
522 
523  rc = daos_array_open(coh, oid, DAOS_TX_NONE, DAOS_OO_RO,
524  &cell_size, &chunk_size, &aoh, NULL);
525  DCHECK(rc, "daos_array_open() failed (%d).", rc);
526 
527  if (cell_size != 1)
528  GERR("Invalid DAOS Array object.\n");
529 
530  rc = daos_array_get_size(aoh, DAOS_TX_NONE, &size, NULL);
531  DCHECK(rc, "daos_array_get_size() failed (%d).", rc);
532 
533  rc = daos_array_close(aoh, NULL);
534  DCHECK(rc, "daos_array_close() failed (%d).", rc);
535  aoh.cookie = 0;
536  }
537 
538  if (!param->filePerProc)
539  MPI_Bcast(&size, 1, MPI_LONG, 0, MPI_COMM_WORLD);
540 
541  return size;
542 }
static daos_handle_t poh
Definition: aiori-DAOS.c:112
#define ERR(MSG)
Definition: iordef.h:184
static void gen_oid(const char *name, daos_obj_id_t *oid)
Definition: aiori-DAOS.c:333
static void DAOS_Fini()
Definition: aiori-DAOS.c:279
#define LAST_OPTION
Definition: option.h:37
ior_aiori_t daos_aiori
Definition: aiori-DAOS.c:84
int filePerProc
Definition: ior.h:111
static int size
Definition: mdtest.c:91
static daos_handle_t coh
Definition: aiori-DAOS.c:113
static void DAOS_Fsync(void *, IOR_param_t *)
Definition: aiori-DAOS.c:501
CURLcode rc
Definition: aiori-S3.c:121
static void * DAOS_Open(char *, IOR_param_t *)
Definition: aiori-DAOS.c:389
static IOR_offset_t DAOS_GetFileSize(IOR_param_t *, MPI_Comm, char *)
Definition: aiori-DAOS.c:507
char * svcl
Definition: aiori-DAOS.c:37
char * group
Definition: aiori-DAOS.c:38
int aiori_posix_rmdir(const char *path, IOR_param_t *param)
Definition: aiori.c:185
int chunk_size
Definition: aiori-DAOS.c:40
static daos_oclass_id_t objectClass
Definition: aiori-DAOS.c:115
int aiori_posix_mkdir(const char *path, mode_t mode, IOR_param_t *param)
Definition: aiori.c:180
static IOR_offset_t DAOS_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori-DAOS.c:417
static void * DAOS_Create(char *, IOR_param_t *)
Definition: aiori-DAOS.c:343
#define WRITE
Definition: iordef.h:95
int aiori_posix_statfs(const char *path, ior_aiori_statfs_t *stat_buf, IOR_param_t *param)
Definition: aiori.c:155
char * cont
Definition: aiori-DAOS.c:39
static void DAOS_Close(void *, IOR_param_t *)
Definition: aiori-DAOS.c:449
static void DAOS_Delete(char *, IOR_param_t *)
Definition: aiori-DAOS.c:463
#define IOR_DAOS_MUR_SEED
Definition: aiori-DAOS.c:104
static char * DAOS_GetVersion()
Definition: aiori-DAOS.c:492
MPI_Comm testComm
Definition: utilities.c:60
static option_help options[]
Definition: aiori-DAOS.c:55
#define MPI_CHECK(MPI_STATUS, MSG)
Definition: iordef.h:224
static option_help * DAOS_options()
Definition: aiori-DAOS.c:206
static struct daos_options o
Definition: aiori-DAOS.c:45
static IOR_param_t param
Definition: mdtest.c:170
static void DAOS_Init()
Definition: aiori-DAOS.c:212
handleType
Definition: aiori-DAOS.c:106
static bool daos_initialized
Definition: aiori-DAOS.c:116
#define GERR(format,...)
Definition: aiori-DAOS.c:141
long long int IOR_size_t
Definition: iordef.h:123
#define INFO(level, format,...)
Definition: aiori-DAOS.c:134
#define VERBOSE_2
Definition: iordef.h:103
IOR_offset_t offset
Definition: ior.h:126
static void HandleDistribute(daos_handle_t *handle, enum handleType type)
Definition: aiori-DAOS.c:149
int aiori_posix_stat(const char *path, struct stat *buf, IOR_param_t *param)
Definition: aiori.c:195
#define VERBOSE_1
Definition: iordef.h:102
static int DAOS_Access(const char *, int, IOR_param_t *)
Definition: aiori-DAOS.c:366
static daos_handle_t aoh
Definition: aiori-DAOS.c:114
char * name
Definition: aiori.h:68
long long int IOR_offset_t
Definition: iordef.h:122
int rank
Definition: utilities.c:57
char * oclass
Definition: aiori-DAOS.c:42
#define DCHECK(rc, format,...)
Definition: aiori-DAOS.c:121
#define NULL
Definition: iordef.h:79
char * pool
Definition: aiori-DAOS.c:36