IOR
aiori-RADOS.c
Go to the documentation of this file.
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  */
4 /******************************************************************************\
5 * *
6 * (C) 2015 The University of Chicago *
7 * *
8 * See COPYRIGHT in top-level directory. *
9 * *
10 ********************************************************************************
11 *
12 * Implement abstract I/O interface for RADOS.
13 *
14 \******************************************************************************/
15 
16 #ifdef HAVE_CONFIG_H
17 # include "config.h"
18 #endif
19 
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <sys/stat.h>
23 #include <rados/librados.h>
24 
25 #include "ior.h"
26 #include "iordef.h"
27 #include "aiori.h"
28 #include "utilities.h"
29 
30 /************************** O P T I O N S *****************************/
32  char * user;
33  char * conf;
34  char * pool;
35 };
36 
37 static struct rados_options o = {
38  .user = NULL,
39  .conf = NULL,
40  .pool = NULL,
41 };
42 
43 static option_help options [] = {
44  {0, "rados.user", "Username for the RADOS cluster", OPTION_REQUIRED_ARGUMENT, 's', & o.user},
45  {0, "rados.conf", "Config file for the RADOS cluster", OPTION_REQUIRED_ARGUMENT, 's', & o.conf},
46  {0, "rados.pool", "RADOS pool to use for I/O", OPTION_REQUIRED_ARGUMENT, 's', & o.pool},
48 };
49 
50 
51 /**************************** P R O T O T Y P E S *****************************/
52 static void *RADOS_Create(char *, IOR_param_t *);
53 static void *RADOS_Open(char *, IOR_param_t *);
54 static IOR_offset_t RADOS_Xfer(int, void *, IOR_size_t *,
56 static void RADOS_Close(void *, IOR_param_t *);
57 static void RADOS_Delete(char *, IOR_param_t *);
58 static void RADOS_Fsync(void *, IOR_param_t *);
59 static IOR_offset_t RADOS_GetFileSize(IOR_param_t *, MPI_Comm, char *);
60 static int RADOS_StatFS(const char *, ior_aiori_statfs_t *, IOR_param_t *);
61 static int RADOS_MkDir(const char *, mode_t, IOR_param_t *);
62 static int RADOS_RmDir(const char *, IOR_param_t *);
63 static int RADOS_Access(const char *, int, IOR_param_t *);
64 static int RADOS_Stat(const char *, struct stat *, IOR_param_t *);
65 static option_help * RADOS_options();
66 
67 /************************** D E C L A R A T I O N S ***************************/
69  .name = "RADOS",
70  .name_legacy = NULL,
71  .create = RADOS_Create,
72  .open = RADOS_Open,
73  .xfer = RADOS_Xfer,
74  .close = RADOS_Close,
75  .delete = RADOS_Delete,
76  .get_version = aiori_get_version,
77  .fsync = RADOS_Fsync,
78  .get_file_size = RADOS_GetFileSize,
79  .statfs = RADOS_StatFS,
80  .mkdir = RADOS_MkDir,
81  .rmdir = RADOS_RmDir,
82  .access = RADOS_Access,
83  .stat = RADOS_Stat,
84  .get_options = RADOS_options,
85 };
86 
87 #define RADOS_ERR(__err_str, __ret) do { \
88  errno = -__ret; \
89  ERR(__err_str); \
90 } while(0)
91 
92 /***************************** F U N C T I O N S ******************************/
94  return options;
95 }
96 
97 static void RADOS_Cluster_Init(IOR_param_t * param)
98 {
99  int ret;
100 
101  /* create RADOS cluster handle */
102  ret = rados_create(&param->rados_cluster, o.user);
103  if (ret)
104  RADOS_ERR("unable to create RADOS cluster handle", ret);
105 
106  /* set the handle using the Ceph config */
107  ret = rados_conf_read_file(param->rados_cluster, o.conf);
108  if (ret)
109  RADOS_ERR("unable to read RADOS config file", ret);
110 
111  /* connect to the RADOS cluster */
112  ret = rados_connect(param->rados_cluster);
113  if (ret)
114  RADOS_ERR("unable to connect to the RADOS cluster", ret);
115 
116  /* create an io context for the pool we are operating on */
117  ret = rados_ioctx_create(param->rados_cluster, o.pool, &param->rados_ioctx);
118  if (ret)
119  RADOS_ERR("unable to create an I/O context for the RADOS cluster", ret);
120 
121  return;
122 }
123 
125 {
126  /* ioctx destroy */
127  rados_ioctx_destroy(param->rados_ioctx);
128 
129  /* shutdown */
130  rados_shutdown(param->rados_cluster);
131 }
132 
133 static void *RADOS_Create_Or_Open(char *testFileName, IOR_param_t * param, int create_flag)
134 {
135  int ret;
136  char *oid;
137 
138  RADOS_Cluster_Init(param);
139 
140  oid = strdup(testFileName);
141  if (!oid)
142  ERR("unable to allocate RADOS oid");
143 
144  if (create_flag)
145  {
146  rados_write_op_t create_op;
147  int rados_create_flag;
148 
149  if (param->openFlags & IOR_EXCL)
150  rados_create_flag = LIBRADOS_CREATE_EXCLUSIVE;
151  else
152  rados_create_flag = LIBRADOS_CREATE_IDEMPOTENT;
153 
154  /* create a RADOS "write op" for creating the object */
155  create_op = rados_create_write_op();
156  rados_write_op_create(create_op, rados_create_flag, NULL);
157  ret = rados_write_op_operate(create_op, param->rados_ioctx, oid,
158  NULL, 0);
159  rados_release_write_op(create_op);
160  if (ret)
161  RADOS_ERR("unable to create RADOS object", ret);
162  }
163  else
164  {
165  /* XXX actually, we should probably assert oid existence here? */
166  }
167 
168  return (void *)oid;
169 }
170 
171 static void *RADOS_Create(char *testFileName, IOR_param_t * param)
172 {
173  return RADOS_Create_Or_Open(testFileName, param, TRUE);
174 }
175 
176 static void *RADOS_Open(char *testFileName, IOR_param_t * param)
177 {
178  if (param->openFlags & IOR_CREAT)
179  return RADOS_Create_Or_Open(testFileName, param, TRUE);
180  else
181  return RADOS_Create_Or_Open(testFileName, param, FALSE);
182 }
183 
184 static IOR_offset_t RADOS_Xfer(int access, void *fd, IOR_size_t * buffer,
185  IOR_offset_t length, IOR_param_t * param)
186 {
187  int ret;
188  char *oid = (char *)fd;
189 
190  if (access == WRITE)
191  {
192  rados_write_op_t write_op;
193 
194  write_op = rados_create_write_op();
195  rados_write_op_write(write_op, (const char *)buffer,
196  length, param->offset);
197  ret = rados_write_op_operate(write_op, param->rados_ioctx,
198  oid, NULL, 0);
199  rados_release_write_op(write_op);
200  if (ret)
201  RADOS_ERR("unable to write RADOS object", ret);
202  }
203  else /* READ */
204  {
205  int read_ret;
206  size_t bytes_read;
207  rados_read_op_t read_op;
208 
209  read_op = rados_create_read_op();
210  rados_read_op_read(read_op, param->offset, length, (char *)buffer,
211  &bytes_read, &read_ret);
212  ret = rados_read_op_operate(read_op, param->rados_ioctx, oid, 0);
213  rados_release_read_op(read_op);
214  if (ret || read_ret || ((IOR_offset_t)bytes_read != length))
215  RADOS_ERR("unable to read RADOS object", ret);
216  }
217 
218  return length;
219 }
220 
221 static void RADOS_Fsync(void *fd, IOR_param_t * param)
222 {
223  return;
224 }
225 
226 static void RADOS_Close(void *fd, IOR_param_t * param)
227 {
228  char *oid = (char *)fd;
229 
230  /* object does not need to be "closed", but we should tear the cluster down */
231  RADOS_Cluster_Finalize(param);
232  free(oid);
233 
234  return;
235 }
236 
237 static void RADOS_Delete(char *testFileName, IOR_param_t * param)
238 {
239  int ret;
240  char *oid = testFileName;
241  rados_write_op_t remove_op;
242 
243  /* we have to reestablish cluster connection here... */
244  RADOS_Cluster_Init(param);
245 
246  /* remove the object */
247  remove_op = rados_create_write_op();
248  rados_write_op_remove(remove_op);
249  ret = rados_write_op_operate(remove_op, param->rados_ioctx,
250  oid, NULL, 0);
251  rados_release_write_op(remove_op);
252  if (ret)
253  RADOS_ERR("unable to remove RADOS object", ret);
254 
255  RADOS_Cluster_Finalize(param);
256 
257  return;
258 }
259 
261  char *testFileName)
262 {
263  int ret;
264  char *oid = testFileName;
265  rados_read_op_t stat_op;
266  uint64_t oid_size;
267  int stat_ret;
268  IOR_offset_t aggSizeFromStat, tmpMin, tmpMax, tmpSum;
269 
270  /* we have to reestablish cluster connection here... */
271  RADOS_Cluster_Init(test);
272 
273  /* stat the object */
274  stat_op = rados_create_read_op();
275  rados_read_op_stat(stat_op, &oid_size, NULL, &stat_ret);
276  ret = rados_read_op_operate(stat_op, test->rados_ioctx, oid, 0);
277  rados_release_read_op(stat_op);
278  if (ret || stat_ret)
279  RADOS_ERR("unable to stat RADOS object", stat_ret);
280  aggSizeFromStat = oid_size;
281 
282  if (test->filePerProc == TRUE)
283  {
284  MPI_CHECK(MPI_Allreduce(&aggSizeFromStat, &tmpSum, 1,
285  MPI_LONG_LONG_INT, MPI_SUM, testComm),
286  "cannot total data moved");
287  aggSizeFromStat = tmpSum;
288  }
289  else
290  {
291  MPI_CHECK(MPI_Allreduce(&aggSizeFromStat, &tmpMin, 1,
292  MPI_LONG_LONG_INT, MPI_MIN, testComm),
293  "cannot total data moved");
294  MPI_CHECK(MPI_Allreduce(&aggSizeFromStat, &tmpMax, 1,
295  MPI_LONG_LONG_INT, MPI_MAX, testComm),
296  "cannot total data moved");
297  if (tmpMin != tmpMax)
298  {
299  if (rank == 0)
300  WARN("inconsistent file size by different tasks");
301 
302  /* incorrect, but now consistent across tasks */
303  aggSizeFromStat = tmpMin;
304  }
305  }
306 
308 
309  return aggSizeFromStat;
310 }
311 
312 static int RADOS_StatFS(const char *oid, ior_aiori_statfs_t *stat_buf,
313  IOR_param_t *param)
314 {
315  WARN("statfs not supported in RADOS backend!");
316  return -1;
317 }
318 
319 static int RADOS_MkDir(const char *oid, mode_t mode, IOR_param_t *param)
320 {
321  WARN("mkdir not supported in RADOS backend!");
322  return -1;
323 }
324 
325 static int RADOS_RmDir(const char *oid, IOR_param_t *param)
326 {
327  WARN("rmdir not supported in RADOS backend!");
328  return -1;
329 }
330 
331 static int RADOS_Access(const char *oid, int mode, IOR_param_t *param)
332 {
333  rados_read_op_t read_op;
334  int ret;
335  int prval;
336  uint64_t oid_size;
337 
338  /* we have to reestablish cluster connection here... */
339  RADOS_Cluster_Init(param);
340 
341  /* use read_op stat to check for oid existence */
342  read_op = rados_create_read_op();
343  rados_read_op_stat(read_op, &oid_size, NULL, &prval);
344  ret = rados_read_op_operate(read_op, param->rados_ioctx, oid, 0);
345  rados_release_read_op(read_op);
346 
347  RADOS_Cluster_Finalize(param);
348 
349  if (ret | prval)
350  return -1;
351  else
352  return 0;
353 }
354 
355 static int RADOS_Stat(const char *oid, struct stat *buf, IOR_param_t *param)
356 {
357  WARN("stat not supported in RADOS backend!");
358  return -1;
359 }
static IOR_offset_t RADOS_GetFileSize(IOR_param_t *, MPI_Comm, char *)
Definition: aiori-RADOS.c:260
#define LAST_OPTION
Definition: option.h:39
static int RADOS_RmDir(const char *, IOR_param_t *)
Definition: aiori-RADOS.c:325
int filePerProc
Definition: ior.h:125
#define RADOS_ERR(__err_str, __ret)
Definition: aiori-RADOS.c:87
static void RADOS_Close(void *, IOR_param_t *)
Definition: aiori-RADOS.c:226
static void * RADOS_Create(char *, IOR_param_t *)
Definition: aiori-RADOS.c:171
static void RADOS_Fsync(void *, IOR_param_t *)
Definition: aiori-RADOS.c:221
static void * RADOS_Create_Or_Open(char *testFileName, IOR_param_t *param, int create_flag)
Definition: aiori-RADOS.c:133
ior_aiori_t rados_aiori
Definition: aiori-RADOS.c:68
#define MPI_CHECK(MPI_STATUS, MSG)
Definition: aiori-debug.h:127
#define WRITE
Definition: iordef.h:86
static void * RADOS_Open(char *, IOR_param_t *)
Definition: aiori-RADOS.c:176
static void RADOS_Delete(char *, IOR_param_t *)
Definition: aiori-RADOS.c:237
#define IOR_CREAT
Definition: aiori.h:32
#define IOR_EXCL
Definition: aiori.h:34
char * aiori_get_version()
Definition: aiori.c:232
MPI_Comm testComm
Definition: utilities.c:71
#define WARN(MSG)
Definition: aiori-debug.h:32
static int RADOS_Access(const char *, int, IOR_param_t *)
Definition: aiori-RADOS.c:331
static struct rados_options o
Definition: aiori-RADOS.c:37
static int RADOS_Stat(const char *, struct stat *, IOR_param_t *)
Definition: aiori-RADOS.c:355
#define FALSE
Definition: iordef.h:62
long long int IOR_size_t
Definition: iordef.h:110
rados_t rados_cluster
Definition: ior.h:176
static void RADOS_Cluster_Init(IOR_param_t *param)
Definition: aiori-RADOS.c:97
static int RADOS_StatFS(const char *, ior_aiori_statfs_t *, IOR_param_t *)
Definition: aiori-RADOS.c:312
static IOR_offset_t RADOS_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori-RADOS.c:184
static option_help * RADOS_options()
Definition: aiori-RADOS.c:93
#define ERR(MSG)
Definition: aiori-debug.h:92
static int RADOS_MkDir(const char *, mode_t, IOR_param_t *)
Definition: aiori-RADOS.c:319
char * name
Definition: aiori.h:88
rados_ioctx_t rados_ioctx
Definition: ior.h:177
long long int IOR_offset_t
Definition: iordef.h:109
int rank
Definition: utilities.c:68
#define TRUE
Definition: iordef.h:66
static void RADOS_Cluster_Finalize(IOR_param_t *param)
Definition: aiori-RADOS.c:124
static option_help options[]
Definition: aiori-RADOS.c:43
#define NULL
Definition: iordef.h:70