1 /* check-sources:disable-copyright-check */
2 /*
3 * simple buffered example which use ID capable REST services (emulate blocks
4 * with UKS)
5 */
6
7 #include <droplet.h>
8 #include <droplet/utils.h>
9 #include <droplet/uks/uks.h>
10 #include <droplet/dbuf.h>
11 #include <droplet/scal/gc.h>
12 #include <assert.h>
13
my_dpl_uks_bn2hex(const BIGNUM * id)14 char* my_dpl_uks_bn2hex(const BIGNUM* id)
15 {
16 char* id_str = NULL;
17
18 id_str = malloc(DPL_UKS_BCH_LEN + 1);
19 if (!id_str) return NULL;
20
21 if (dpl_uks_bn2hex(id, id_str) != DPL_SUCCESS) {
22 free(id_str);
23 return NULL;
24 }
25
26 return id_str;
27 }
28
main(int argc,char ** argv)29 int main(int argc, char** argv)
30 {
31 int ret;
32 dpl_ctx_t* ctx;
33 dpl_dict_t* metadata = NULL;
34 char* data_buf = NULL;
35 char* data_ptr;
36 size_t data_len;
37 char* data_buf_returned = NULL;
38 u_int data_len_returned;
39 dpl_dict_t* metadata_returned = NULL;
40 dpl_dict_t* metadata2_returned = NULL;
41 dpl_dict_var_t* metadatum = NULL;
42 dpl_option_t option;
43 BIGNUM* id = NULL;
44 char* main_id = NULL;
45 dpl_conn_t* conn = NULL;
46 int block_size, remain_size, offset;
47 int connection_close;
48 dpl_dict_t* headers_returned;
49 uint64_t file_identifier;
50 int first;
51 dpl_dbuf_t* dbuf = NULL;
52 char* gc_id = NULL;
53
54 if (1 != argc) {
55 fprintf(stderr, "usage: idtestbuffered3\n");
56 ret = 1;
57 goto end;
58 }
59
60 ret = dpl_init(); // init droplet library
61 if (DPL_SUCCESS != ret) {
62 fprintf(stderr, "dpl_init failed\n");
63 ret = 1;
64 goto end;
65 }
66
67 // open default profile
68 ctx = dpl_ctx_new(NULL, // droplet directory, default: "~/.droplet"
69 NULL); // droplet profile, default: "default"
70 if (NULL == ctx) {
71 fprintf(stderr, "dpl_ctx_new failed\n");
72 ret = 1;
73 goto free_dpl;
74 }
75
76 // ctx->trace_level = ~0;
77 // ctx->trace_buffers = 1;
78
79 data_len = 10000;
80 data_buf = malloc(data_len);
81 if (NULL == data_buf) {
82 fprintf(stderr, "alloc data failed\n");
83 ret = 1;
84 goto free_all;
85 }
86
87 memset(data_buf, 'z', data_len);
88
89 metadata = dpl_dict_new(13);
90 if (NULL == metadata) {
91 fprintf(stderr, "dpl_dict_new failed\n");
92 ret = 1;
93 goto free_all;
94 }
95
96 ret = dpl_dict_add(metadata, "foo", "bar", 0);
97 if (DPL_SUCCESS != ret) {
98 fprintf(stderr, "dpl_dict_add failed\n");
99 ret = 1;
100 goto free_all;
101 }
102
103 ret = dpl_dict_add(metadata, "foo2", "qux", 0);
104 if (DPL_SUCCESS != ret) {
105 fprintf(stderr, "dpl_dict_add failed\n");
106 ret = 1;
107 goto free_all;
108 }
109
110 /**/
111
112 fprintf(stderr, "setting object+MD\n");
113
114 file_identifier = dpl_rand_u64();
115
116 id = BN_new();
117 if (NULL == id) {
118 perror("BN_new");
119 ret = 1;
120 goto free_all;
121 }
122
123 ret = dpl_uks_gen_key(id,
124 file_identifier, // file id
125 0, // volume id
126 0xc0, // customer tag
127 0); // block id
128 if (DPL_SUCCESS != ret) {
129 fprintf(stderr, "dpl_uks_gen_key failed: %s (%d)\n", dpl_status_str(ret),
130 ret);
131 ret = 1;
132 goto free_all;
133 }
134
135 ret = dpl_uks_set_class(id,
136 2); // means 3 replicas
137 if (DPL_SUCCESS != ret) {
138 fprintf(stderr, "dpl_uks_set_class failed: %s (%d)\n", dpl_status_str(ret),
139 ret);
140 ret = 1;
141 goto free_all;
142 }
143
144 main_id = my_dpl_uks_bn2hex(id);
145 if (NULL == main_id) {
146 perror("my_dpl_uks_bn2hex");
147 ret = 1;
148 goto free_all;
149 }
150
151 first = 1;
152 block_size = 1000;
153 remain_size = data_len;
154 offset = 0;
155 while (remain_size > 0) {
156 int size = MIN(block_size, remain_size);
157 int block_nb = offset / block_size;
158 char* block_id = NULL;
159
160 ret = dpl_uks_gen_key(id,
161 file_identifier, // file id
162 0, // volume id
163 0xc0, // customer tag
164 block_nb); // block nb
165 if (DPL_SUCCESS != ret) {
166 fprintf(stderr, "dpl_uks_gen_key failed: %s (%d)\n", dpl_status_str(ret),
167 ret);
168 ret = 1;
169 goto free_all;
170 }
171
172 ret = dpl_uks_set_class(id,
173 2); // means 3 replicas
174 if (DPL_SUCCESS != ret) {
175 fprintf(stderr, "dpl_uks_set_class failed: %s (%d)\n",
176 dpl_status_str(ret), ret);
177 ret = 1;
178 goto free_all;
179 }
180
181 block_id = my_dpl_uks_bn2hex(id);
182 if (NULL == block_id) {
183 perror("my_dpl_uks_bn2hex");
184 ret = 1;
185 goto free_all;
186 }
187
188 ret = dpl_put_id(ctx, // the context
189 NULL, // no bucket
190 block_id, // the id
191 NULL, // no option
192 DPL_FTYPE_REG, // regular object
193 NULL, // no condition
194 NULL, // no range
195 first ? metadata : NULL, // the metadata
196 NULL, // no sysmd
197 data_buf + offset, // object body
198 size); // object length
199 if (DPL_SUCCESS != ret) {
200 fprintf(stderr, "dpl_put_id %s failed: %s (%d)\n", block_id,
201 dpl_status_str(ret), ret);
202 ret = 1;
203 goto free_all;
204 }
205
206 first = 0;
207 offset += size;
208 remain_size -= size;
209
210 free(block_id);
211 }
212
213 /**/
214
215 fprintf(stderr, "getting object+MD\n");
216
217 // we known file size in advance
218 data_buf_returned = malloc(data_len);
219 if (NULL == data_buf_returned) {
220 perror("malloc");
221 exit(1);
222 }
223 data_len_returned = data_len;
224
225 first = 1;
226 remain_size = data_len;
227 offset = 0;
228 while (remain_size > 0) {
229 int size = MIN(block_size, remain_size);
230 int block_nb = offset / block_size;
231 char* block_id = NULL;
232 char* bufp;
233 int len;
234
235 ret = dpl_uks_gen_key(id,
236 file_identifier, // file id
237 0, // volume id
238 0xc0, // customer tag
239 block_nb); // block nb
240 if (DPL_SUCCESS != ret) {
241 fprintf(stderr, "dpl_uks_gen_key failed: %s (%d)\n", dpl_status_str(ret),
242 ret);
243 ret = 1;
244 goto free_all;
245 }
246
247 ret = dpl_uks_set_class(id,
248 2); // means 3 replicas
249 if (DPL_SUCCESS != ret) {
250 fprintf(stderr, "dpl_uks_set_class failed: %s (%d)\n",
251 dpl_status_str(ret), ret);
252 ret = 1;
253 goto free_all;
254 }
255
256 block_id = my_dpl_uks_bn2hex(id);
257 if (NULL == block_id) {
258 perror("my_dpl_uks_bn2hex");
259 ret = 1;
260 goto free_all;
261 }
262
263 bufp = data_buf_returned + offset;
264 len = size;
265
266 option.mask = DPL_OPTION_NOALLOC; // we already allocated buffer
267
268 ret = dpl_get_id(ctx, // the context
269 NULL, // no bucket
270 block_id, // the key
271 &option, // options
272 DPL_FTYPE_REG, // object type
273 NULL, // no condition
274 NULL, // no range
275 &bufp, // data object
276 &len, // data object length
277 first ? &metadata_returned : NULL, // metadata
278 NULL); // sysmd
279 if (DPL_SUCCESS != ret) {
280 fprintf(stderr, "dpl_get_id %s failed: %s (%d)\n", block_id,
281 dpl_status_str(ret), ret);
282 ret = 1;
283 goto free_all;
284 }
285
286 if (len != size) {
287 fprintf(stderr, "short read %d != %d\n", data_len_returned, size);
288 ret = 1;
289 goto free_all;
290 }
291
292 first = 0;
293 offset += size;
294 remain_size -= size;
295
296 free(block_id);
297 }
298
299 fprintf(stderr, "checking object\n");
300
301 if (data_len != data_len_returned) {
302 fprintf(stderr, "data lengths mismatch\n");
303 ret = 1;
304 goto free_all;
305 }
306
307 if (0 != memcmp(data_buf, data_buf_returned, data_len)) {
308 fprintf(stderr, "data content mismatch\n");
309 ret = 1;
310 goto free_all;
311 }
312
313 fprintf(stderr, "checking metadata\n");
314
315 metadatum = dpl_dict_get(metadata_returned, "foo");
316 if (NULL == metadatum) {
317 fprintf(stderr, "missing metadatum\n");
318 ret = 1;
319 goto free_all;
320 }
321
322 assert(metadatum->val->type == DPL_VALUE_STRING);
323 if (strcmp(dpl_sbuf_get_str(metadatum->val->string), "bar")) {
324 fprintf(stderr, "bad value in metadatum\n");
325 ret = 1;
326 goto free_all;
327 }
328
329 metadatum = dpl_dict_get(metadata_returned, "foo2");
330 if (NULL == metadatum) {
331 fprintf(stderr, "missing metadatum\n");
332 ret = 1;
333 goto free_all;
334 }
335
336 assert(metadatum->val->type == DPL_VALUE_STRING);
337 if (strcmp(dpl_sbuf_get_str(metadatum->val->string), "qux")) {
338 fprintf(stderr, "bad value in metadatum\n");
339 ret = 1;
340 goto free_all;
341 }
342
343 /**/
344
345 fprintf(stderr, "setting MD only\n");
346
347 ret = dpl_dict_add(metadata, "foo", "bar2", 0);
348 if (DPL_SUCCESS != ret) {
349 fprintf(stderr, "error updating metadatum: %s (%d)\n", dpl_status_str(ret),
350 ret);
351 ret = 1;
352 goto free_all;
353 }
354
355 ret = dpl_copy_id(
356 ctx, // the context
357 NULL, // no src bucket
358 main_id, // the key
359 NULL, // no dst bucket
360 main_id, // the same key
361 NULL, // no option
362 DPL_FTYPE_REG, // object type
363 DPL_COPY_DIRECTIVE_METADATA_REPLACE, // tell server to replace metadata
364 metadata, // the updated metadata
365 NULL, // no sysmd
366 NULL); // no condition
367 if (DPL_SUCCESS != ret) {
368 fprintf(stderr, "error updating metadata: %s (%d)\n", dpl_status_str(ret),
369 ret);
370 ret = 1;
371 goto free_all;
372 }
373
374 /**/
375
376 fprintf(stderr, "getting MD only\n");
377
378 ret = dpl_head_id(ctx, // the context
379 NULL, // no bucket,
380 main_id, // the key
381 NULL, // option
382 DPL_FTYPE_UNDEF, // no matter the file type
383 NULL, // no condition,
384 &metadata2_returned,
385 NULL); // no sysmd
386 if (DPL_SUCCESS != ret) {
387 fprintf(stderr, "error getting metadata: %s (%d)\n", dpl_status_str(ret),
388 ret);
389 ret = 1;
390 goto free_all;
391 }
392
393 fprintf(stderr, "checking metadata\n");
394
395 metadatum = dpl_dict_get(metadata2_returned, "foo");
396 if (NULL == metadatum) {
397 fprintf(stderr, "missing metadatum\n");
398 ret = 1;
399 goto free_all;
400 }
401
402 assert(metadatum->val->type == DPL_VALUE_STRING);
403 if (strcmp(dpl_sbuf_get_str(metadatum->val->string), "bar2")) {
404 fprintf(stderr, "bad value in metadatum\n");
405 ret = 1;
406 goto free_all;
407 }
408
409 metadatum = dpl_dict_get(metadata2_returned, "foo2");
410 if (NULL == metadatum) {
411 fprintf(stderr, "missing metadatum\n");
412 ret = 1;
413 goto free_all;
414 }
415
416 assert(metadatum->val->type == DPL_VALUE_STRING);
417 if (strcmp(dpl_sbuf_get_str(metadatum->val->string), "qux")) {
418 fprintf(stderr, "bad value in metadatum\n");
419 ret = 1;
420 goto free_all;
421 }
422
423 /**/
424
425 fprintf(stderr, "delete object+MD\n");
426
427 ret = dpl_scal_gc_index_init(&dbuf);
428 if (DPL_SUCCESS != ret) {
429 fprintf(stderr, "gc init failed\n");
430 ret = 1;
431 goto free_all;
432 }
433
434 remain_size = data_len;
435 offset = 0;
436 while (remain_size > 0) {
437 int size = MIN(block_size, remain_size);
438 int block_nb = offset / block_size;
439
440 ret = dpl_uks_gen_key(id,
441 file_identifier, // file id
442 0, // volume id
443 0xc0, // customer tag
444 block_nb); // block nb
445 if (DPL_SUCCESS != ret) {
446 fprintf(stderr, "dpl_uks_gen_key failed: %s (%d)\n", dpl_status_str(ret),
447 ret);
448 ret = 1;
449 goto free_all;
450 }
451
452 ret = dpl_uks_set_class(id,
453 2); // means 3 replicas
454 if (DPL_SUCCESS != ret) {
455 fprintf(stderr, "dpl_uks_set_class failed: %s (%d)\n",
456 dpl_status_str(ret), ret);
457 ret = 1;
458 goto free_all;
459 }
460
461 ret = dpl_scal_gc_index_serialize(
462 id,
463 offset, // offset of block (not mandatory)
464 block_size, // size of block (not mandatory)
465 dbuf);
466 if (DPL_SUCCESS != ret) {
467 fprintf(stderr, "error marking object for deletion: %s (%d)\n",
468 dpl_status_str(ret), ret);
469 ret = 1;
470 goto free_all;
471 }
472
473 offset += size;
474 remain_size -= size;
475 }
476
477 // generate GC chunk
478 dpl_scal_gc_gen_key(id, 2);
479
480 gc_id = my_dpl_uks_bn2hex(id);
481 if (NULL == gc_id) {
482 perror("my_dpl_uks_bn2hex");
483 ret = 1;
484 goto free_all;
485 }
486
487 // push GC chunk
488 ret = dpl_put_id(ctx, // the context
489 NULL, // no bucket
490 gc_id, // the id
491 NULL, // no option
492 DPL_FTYPE_REG, // regular object
493 NULL, // no condition
494 NULL, // no range
495 NULL, // no metadata
496 NULL, // no sysmd
497 dbuf->data, // object body
498 dpl_dbuf_length(dbuf)); // object length
499 if (DPL_SUCCESS != ret) {
500 fprintf(stderr, "dpl_put_id %s failed: %s (%d)\n", gc_id,
501 dpl_status_str(ret), ret);
502 ret = 1;
503 goto free_all;
504 }
505
506 ret = 0;
507
508 free_all:
509
510 if (NULL != gc_id) free(gc_id);
511
512 if (NULL != main_id) free(main_id);
513
514 if (NULL != id) BN_free(id);
515
516 if (NULL != metadata2_returned) dpl_dict_free(metadata2_returned);
517
518 if (NULL != metadata_returned) dpl_dict_free(metadata_returned);
519
520 if (NULL != data_buf_returned) free(data_buf_returned);
521
522 if (NULL != metadata) dpl_dict_free(metadata);
523
524 if (NULL != data_buf) free(data_buf);
525
526 if (NULL != dbuf) dpl_dbuf_free(dbuf);
527
528 dpl_ctx_free(ctx); // free context
529
530 free_dpl:
531 dpl_free(); // free droplet library
532
533 end:
534 return ret;
535 }
536