1f27aaf4bSChristian Brunner /* 2f27aaf4bSChristian Brunner * QEMU Block driver for RADOS (Ceph) 3f27aaf4bSChristian Brunner * 4*ad32e9c0SJosh Durgin * Copyright (C) 2010-2011 Christian Brunner <chb@muc.de>, 5*ad32e9c0SJosh Durgin * Josh Durgin <josh.durgin@dreamhost.com> 6f27aaf4bSChristian Brunner * 7f27aaf4bSChristian Brunner * This work is licensed under the terms of the GNU GPL, version 2. See 8f27aaf4bSChristian Brunner * the COPYING file in the top-level directory. 9f27aaf4bSChristian Brunner * 10f27aaf4bSChristian Brunner */ 11f27aaf4bSChristian Brunner 12*ad32e9c0SJosh Durgin #include <inttypes.h> 13*ad32e9c0SJosh Durgin 14f27aaf4bSChristian Brunner #include "qemu-common.h" 15f27aaf4bSChristian Brunner #include "qemu-error.h" 16f27aaf4bSChristian Brunner 17f27aaf4bSChristian Brunner #include "block_int.h" 18f27aaf4bSChristian Brunner 19*ad32e9c0SJosh Durgin #include <rbd/librbd.h> 20f27aaf4bSChristian Brunner 21f27aaf4bSChristian Brunner 22f27aaf4bSChristian Brunner 23f27aaf4bSChristian Brunner /* 24f27aaf4bSChristian Brunner * When specifying the image filename use: 25f27aaf4bSChristian Brunner * 26f27aaf4bSChristian Brunner * rbd:poolname/devicename 27f27aaf4bSChristian Brunner * 28f27aaf4bSChristian Brunner * poolname must be the name of an existing rados pool 29f27aaf4bSChristian Brunner * 30f27aaf4bSChristian Brunner * devicename is the basename for all objects used to 31f27aaf4bSChristian Brunner * emulate the raw device. 32f27aaf4bSChristian Brunner * 33f27aaf4bSChristian Brunner * Metadata information (image size, ...) is stored in an 34f27aaf4bSChristian Brunner * object with the name "devicename.rbd". 35f27aaf4bSChristian Brunner * 36f27aaf4bSChristian Brunner * The raw device is split into 4MB sized objects by default. 37f27aaf4bSChristian Brunner * The sequencenumber is encoded in a 12 byte long hex-string, 38f27aaf4bSChristian Brunner * and is attached to the devicename, separated by a dot. 39f27aaf4bSChristian Brunner * e.g. "devicename.1234567890ab" 40f27aaf4bSChristian Brunner * 41f27aaf4bSChristian Brunner */ 42f27aaf4bSChristian Brunner 43f27aaf4bSChristian Brunner #define OBJ_MAX_SIZE (1UL << OBJ_DEFAULT_OBJ_ORDER) 44f27aaf4bSChristian Brunner 45*ad32e9c0SJosh Durgin #define RBD_MAX_CONF_NAME_SIZE 128 46*ad32e9c0SJosh Durgin #define RBD_MAX_CONF_VAL_SIZE 512 47*ad32e9c0SJosh Durgin #define RBD_MAX_CONF_SIZE 1024 48*ad32e9c0SJosh Durgin #define RBD_MAX_POOL_NAME_SIZE 128 49*ad32e9c0SJosh Durgin #define RBD_MAX_SNAP_NAME_SIZE 128 50*ad32e9c0SJosh Durgin #define RBD_MAX_SNAPS 100 51*ad32e9c0SJosh Durgin 52f27aaf4bSChristian Brunner typedef struct RBDAIOCB { 53f27aaf4bSChristian Brunner BlockDriverAIOCB common; 54f27aaf4bSChristian Brunner QEMUBH *bh; 55f27aaf4bSChristian Brunner int ret; 56f27aaf4bSChristian Brunner QEMUIOVector *qiov; 57f27aaf4bSChristian Brunner char *bounce; 58f27aaf4bSChristian Brunner int write; 59f27aaf4bSChristian Brunner int64_t sector_num; 60f27aaf4bSChristian Brunner int error; 61f27aaf4bSChristian Brunner struct BDRVRBDState *s; 62f27aaf4bSChristian Brunner int cancelled; 63f27aaf4bSChristian Brunner } RBDAIOCB; 64f27aaf4bSChristian Brunner 65f27aaf4bSChristian Brunner typedef struct RADOSCB { 66f27aaf4bSChristian Brunner int rcbid; 67f27aaf4bSChristian Brunner RBDAIOCB *acb; 68f27aaf4bSChristian Brunner struct BDRVRBDState *s; 69f27aaf4bSChristian Brunner int done; 70*ad32e9c0SJosh Durgin int64_t size; 71f27aaf4bSChristian Brunner char *buf; 72f27aaf4bSChristian Brunner int ret; 73f27aaf4bSChristian Brunner } RADOSCB; 74f27aaf4bSChristian Brunner 75f27aaf4bSChristian Brunner #define RBD_FD_READ 0 76f27aaf4bSChristian Brunner #define RBD_FD_WRITE 1 77f27aaf4bSChristian Brunner 78f27aaf4bSChristian Brunner typedef struct BDRVRBDState { 79f27aaf4bSChristian Brunner int fds[2]; 80*ad32e9c0SJosh Durgin rados_t cluster; 81*ad32e9c0SJosh Durgin rados_ioctx_t io_ctx; 82*ad32e9c0SJosh Durgin rbd_image_t image; 83*ad32e9c0SJosh Durgin char name[RBD_MAX_IMAGE_NAME_SIZE]; 84f27aaf4bSChristian Brunner int qemu_aio_count; 85*ad32e9c0SJosh Durgin char *snap; 86f27aaf4bSChristian Brunner int event_reader_pos; 87f27aaf4bSChristian Brunner RADOSCB *event_rcb; 88f27aaf4bSChristian Brunner } BDRVRBDState; 89f27aaf4bSChristian Brunner 90f27aaf4bSChristian Brunner static void rbd_aio_bh_cb(void *opaque); 91f27aaf4bSChristian Brunner 92*ad32e9c0SJosh Durgin static int qemu_rbd_next_tok(char *dst, int dst_len, 93f27aaf4bSChristian Brunner char *src, char delim, 94f27aaf4bSChristian Brunner const char *name, 95f27aaf4bSChristian Brunner char **p) 96f27aaf4bSChristian Brunner { 97f27aaf4bSChristian Brunner int l; 98f27aaf4bSChristian Brunner char *end; 99f27aaf4bSChristian Brunner 100f27aaf4bSChristian Brunner *p = NULL; 101f27aaf4bSChristian Brunner 102f27aaf4bSChristian Brunner if (delim != '\0') { 103f27aaf4bSChristian Brunner end = strchr(src, delim); 104f27aaf4bSChristian Brunner if (end) { 105f27aaf4bSChristian Brunner *p = end + 1; 106f27aaf4bSChristian Brunner *end = '\0'; 107f27aaf4bSChristian Brunner } 108f27aaf4bSChristian Brunner } 109f27aaf4bSChristian Brunner l = strlen(src); 110f27aaf4bSChristian Brunner if (l >= dst_len) { 111f27aaf4bSChristian Brunner error_report("%s too long", name); 112f27aaf4bSChristian Brunner return -EINVAL; 113f27aaf4bSChristian Brunner } else if (l == 0) { 114f27aaf4bSChristian Brunner error_report("%s too short", name); 115f27aaf4bSChristian Brunner return -EINVAL; 116f27aaf4bSChristian Brunner } 117f27aaf4bSChristian Brunner 118f27aaf4bSChristian Brunner pstrcpy(dst, dst_len, src); 119f27aaf4bSChristian Brunner 120f27aaf4bSChristian Brunner return 0; 121f27aaf4bSChristian Brunner } 122f27aaf4bSChristian Brunner 123*ad32e9c0SJosh Durgin static int qemu_rbd_parsename(const char *filename, 124f27aaf4bSChristian Brunner char *pool, int pool_len, 125f27aaf4bSChristian Brunner char *snap, int snap_len, 126f27aaf4bSChristian Brunner char *name, int name_len) 127f27aaf4bSChristian Brunner { 128f27aaf4bSChristian Brunner const char *start; 129f27aaf4bSChristian Brunner char *p, *buf; 130f27aaf4bSChristian Brunner int ret; 131f27aaf4bSChristian Brunner 132f27aaf4bSChristian Brunner if (!strstart(filename, "rbd:", &start)) { 133f27aaf4bSChristian Brunner return -EINVAL; 134f27aaf4bSChristian Brunner } 135f27aaf4bSChristian Brunner 136f27aaf4bSChristian Brunner buf = qemu_strdup(start); 137f27aaf4bSChristian Brunner p = buf; 138f27aaf4bSChristian Brunner 139*ad32e9c0SJosh Durgin ret = qemu_rbd_next_tok(pool, pool_len, p, '/', "pool name", &p); 140f27aaf4bSChristian Brunner if (ret < 0 || !p) { 141f27aaf4bSChristian Brunner ret = -EINVAL; 142f27aaf4bSChristian Brunner goto done; 143f27aaf4bSChristian Brunner } 144*ad32e9c0SJosh Durgin ret = qemu_rbd_next_tok(name, name_len, p, '@', "object name", &p); 145f27aaf4bSChristian Brunner if (ret < 0) { 146f27aaf4bSChristian Brunner goto done; 147f27aaf4bSChristian Brunner } 148f27aaf4bSChristian Brunner if (!p) { 149f27aaf4bSChristian Brunner *snap = '\0'; 150f27aaf4bSChristian Brunner goto done; 151f27aaf4bSChristian Brunner } 152f27aaf4bSChristian Brunner 153*ad32e9c0SJosh Durgin ret = qemu_rbd_next_tok(snap, snap_len, p, '\0', "snap name", &p); 154f27aaf4bSChristian Brunner 155f27aaf4bSChristian Brunner done: 156f27aaf4bSChristian Brunner qemu_free(buf); 157f27aaf4bSChristian Brunner return ret; 158f27aaf4bSChristian Brunner } 159f27aaf4bSChristian Brunner 160*ad32e9c0SJosh Durgin static int qemu_rbd_create(const char *filename, QEMUOptionParameter *options) 161f27aaf4bSChristian Brunner { 162f27aaf4bSChristian Brunner int64_t bytes = 0; 163f27aaf4bSChristian Brunner int64_t objsize; 164*ad32e9c0SJosh Durgin int obj_order = 0; 165*ad32e9c0SJosh Durgin char pool[RBD_MAX_POOL_NAME_SIZE]; 166*ad32e9c0SJosh Durgin char name[RBD_MAX_IMAGE_NAME_SIZE]; 167*ad32e9c0SJosh Durgin char snap_buf[RBD_MAX_SNAP_NAME_SIZE]; 168f27aaf4bSChristian Brunner char *snap = NULL; 169*ad32e9c0SJosh Durgin rados_t cluster; 170*ad32e9c0SJosh Durgin rados_ioctx_t io_ctx; 171f27aaf4bSChristian Brunner int ret; 172f27aaf4bSChristian Brunner 173*ad32e9c0SJosh Durgin if (qemu_rbd_parsename(filename, pool, sizeof(pool), 174f27aaf4bSChristian Brunner snap_buf, sizeof(snap_buf), 175f27aaf4bSChristian Brunner name, sizeof(name)) < 0) { 176f27aaf4bSChristian Brunner return -EINVAL; 177f27aaf4bSChristian Brunner } 178f27aaf4bSChristian Brunner if (snap_buf[0] != '\0') { 179f27aaf4bSChristian Brunner snap = snap_buf; 180f27aaf4bSChristian Brunner } 181f27aaf4bSChristian Brunner 182f27aaf4bSChristian Brunner /* Read out options */ 183f27aaf4bSChristian Brunner while (options && options->name) { 184f27aaf4bSChristian Brunner if (!strcmp(options->name, BLOCK_OPT_SIZE)) { 185f27aaf4bSChristian Brunner bytes = options->value.n; 186f27aaf4bSChristian Brunner } else if (!strcmp(options->name, BLOCK_OPT_CLUSTER_SIZE)) { 187f27aaf4bSChristian Brunner if (options->value.n) { 188f27aaf4bSChristian Brunner objsize = options->value.n; 189f27aaf4bSChristian Brunner if ((objsize - 1) & objsize) { /* not a power of 2? */ 190f27aaf4bSChristian Brunner error_report("obj size needs to be power of 2"); 191f27aaf4bSChristian Brunner return -EINVAL; 192f27aaf4bSChristian Brunner } 193f27aaf4bSChristian Brunner if (objsize < 4096) { 194f27aaf4bSChristian Brunner error_report("obj size too small"); 195f27aaf4bSChristian Brunner return -EINVAL; 196f27aaf4bSChristian Brunner } 197f27aaf4bSChristian Brunner obj_order = ffs(objsize) - 1; 198f27aaf4bSChristian Brunner } 199f27aaf4bSChristian Brunner } 200f27aaf4bSChristian Brunner options++; 201f27aaf4bSChristian Brunner } 202f27aaf4bSChristian Brunner 203*ad32e9c0SJosh Durgin if (rados_create(&cluster, NULL) < 0) { 204f27aaf4bSChristian Brunner error_report("error initializing"); 205f27aaf4bSChristian Brunner return -EIO; 206f27aaf4bSChristian Brunner } 207f27aaf4bSChristian Brunner 208*ad32e9c0SJosh Durgin if (rados_conf_read_file(cluster, NULL) < 0) { 209*ad32e9c0SJosh Durgin error_report("error reading config file"); 210*ad32e9c0SJosh Durgin rados_shutdown(cluster); 211*ad32e9c0SJosh Durgin return -EIO; 212*ad32e9c0SJosh Durgin } 213*ad32e9c0SJosh Durgin 214*ad32e9c0SJosh Durgin if (rados_connect(cluster) < 0) { 215*ad32e9c0SJosh Durgin error_report("error connecting"); 216*ad32e9c0SJosh Durgin rados_shutdown(cluster); 217*ad32e9c0SJosh Durgin return -EIO; 218*ad32e9c0SJosh Durgin } 219*ad32e9c0SJosh Durgin 220*ad32e9c0SJosh Durgin if (rados_ioctx_create(cluster, pool, &io_ctx) < 0) { 221f27aaf4bSChristian Brunner error_report("error opening pool %s", pool); 222*ad32e9c0SJosh Durgin rados_shutdown(cluster); 223f27aaf4bSChristian Brunner return -EIO; 224f27aaf4bSChristian Brunner } 225f27aaf4bSChristian Brunner 226*ad32e9c0SJosh Durgin ret = rbd_create(io_ctx, name, bytes, &obj_order); 227*ad32e9c0SJosh Durgin rados_ioctx_destroy(io_ctx); 228*ad32e9c0SJosh Durgin rados_shutdown(cluster); 229f27aaf4bSChristian Brunner 230f27aaf4bSChristian Brunner return ret; 231f27aaf4bSChristian Brunner } 232f27aaf4bSChristian Brunner 233f27aaf4bSChristian Brunner /* 234*ad32e9c0SJosh Durgin * This aio completion is being called from qemu_rbd_aio_event_reader() 235*ad32e9c0SJosh Durgin * and runs in qemu context. It schedules a bh, but just in case the aio 236f27aaf4bSChristian Brunner * was not cancelled before. 237f27aaf4bSChristian Brunner */ 238*ad32e9c0SJosh Durgin static void qemu_rbd_complete_aio(RADOSCB *rcb) 239f27aaf4bSChristian Brunner { 240f27aaf4bSChristian Brunner RBDAIOCB *acb = rcb->acb; 241f27aaf4bSChristian Brunner int64_t r; 242f27aaf4bSChristian Brunner 243f27aaf4bSChristian Brunner if (acb->cancelled) { 244f27aaf4bSChristian Brunner qemu_vfree(acb->bounce); 245f27aaf4bSChristian Brunner qemu_aio_release(acb); 246f27aaf4bSChristian Brunner goto done; 247f27aaf4bSChristian Brunner } 248f27aaf4bSChristian Brunner 249f27aaf4bSChristian Brunner r = rcb->ret; 250f27aaf4bSChristian Brunner 251f27aaf4bSChristian Brunner if (acb->write) { 252f27aaf4bSChristian Brunner if (r < 0) { 253f27aaf4bSChristian Brunner acb->ret = r; 254f27aaf4bSChristian Brunner acb->error = 1; 255f27aaf4bSChristian Brunner } else if (!acb->error) { 256*ad32e9c0SJosh Durgin acb->ret = rcb->size; 257f27aaf4bSChristian Brunner } 258f27aaf4bSChristian Brunner } else { 259*ad32e9c0SJosh Durgin if (r < 0) { 260*ad32e9c0SJosh Durgin memset(rcb->buf, 0, rcb->size); 261f27aaf4bSChristian Brunner acb->ret = r; 262f27aaf4bSChristian Brunner acb->error = 1; 263*ad32e9c0SJosh Durgin } else if (r < rcb->size) { 264*ad32e9c0SJosh Durgin memset(rcb->buf + r, 0, rcb->size - r); 265f27aaf4bSChristian Brunner if (!acb->error) { 266*ad32e9c0SJosh Durgin acb->ret = rcb->size; 267f27aaf4bSChristian Brunner } 268f27aaf4bSChristian Brunner } else if (!acb->error) { 269*ad32e9c0SJosh Durgin acb->ret = r; 270f27aaf4bSChristian Brunner } 271f27aaf4bSChristian Brunner } 272f27aaf4bSChristian Brunner /* Note that acb->bh can be NULL in case where the aio was cancelled */ 273f27aaf4bSChristian Brunner acb->bh = qemu_bh_new(rbd_aio_bh_cb, acb); 274f27aaf4bSChristian Brunner qemu_bh_schedule(acb->bh); 275f27aaf4bSChristian Brunner done: 276f27aaf4bSChristian Brunner qemu_free(rcb); 277f27aaf4bSChristian Brunner } 278f27aaf4bSChristian Brunner 279f27aaf4bSChristian Brunner /* 280f27aaf4bSChristian Brunner * aio fd read handler. It runs in the qemu context and calls the 281f27aaf4bSChristian Brunner * completion handling of completed rados aio operations. 282f27aaf4bSChristian Brunner */ 283*ad32e9c0SJosh Durgin static void qemu_rbd_aio_event_reader(void *opaque) 284f27aaf4bSChristian Brunner { 285f27aaf4bSChristian Brunner BDRVRBDState *s = opaque; 286f27aaf4bSChristian Brunner 287f27aaf4bSChristian Brunner ssize_t ret; 288f27aaf4bSChristian Brunner 289f27aaf4bSChristian Brunner do { 290f27aaf4bSChristian Brunner char *p = (char *)&s->event_rcb; 291f27aaf4bSChristian Brunner 292f27aaf4bSChristian Brunner /* now read the rcb pointer that was sent from a non qemu thread */ 293f27aaf4bSChristian Brunner if ((ret = read(s->fds[RBD_FD_READ], p + s->event_reader_pos, 294f27aaf4bSChristian Brunner sizeof(s->event_rcb) - s->event_reader_pos)) > 0) { 295f27aaf4bSChristian Brunner if (ret > 0) { 296f27aaf4bSChristian Brunner s->event_reader_pos += ret; 297f27aaf4bSChristian Brunner if (s->event_reader_pos == sizeof(s->event_rcb)) { 298f27aaf4bSChristian Brunner s->event_reader_pos = 0; 299*ad32e9c0SJosh Durgin qemu_rbd_complete_aio(s->event_rcb); 300f27aaf4bSChristian Brunner s->qemu_aio_count--; 301f27aaf4bSChristian Brunner } 302f27aaf4bSChristian Brunner } 303f27aaf4bSChristian Brunner } 304f27aaf4bSChristian Brunner } while (ret < 0 && errno == EINTR); 305f27aaf4bSChristian Brunner } 306f27aaf4bSChristian Brunner 307*ad32e9c0SJosh Durgin static int qemu_rbd_aio_flush_cb(void *opaque) 308f27aaf4bSChristian Brunner { 309f27aaf4bSChristian Brunner BDRVRBDState *s = opaque; 310f27aaf4bSChristian Brunner 311f27aaf4bSChristian Brunner return (s->qemu_aio_count > 0); 312f27aaf4bSChristian Brunner } 313f27aaf4bSChristian Brunner 314*ad32e9c0SJosh Durgin static int qemu_rbd_open(BlockDriverState *bs, const char *filename, int flags) 315f27aaf4bSChristian Brunner { 316f27aaf4bSChristian Brunner BDRVRBDState *s = bs->opaque; 317*ad32e9c0SJosh Durgin char pool[RBD_MAX_POOL_NAME_SIZE]; 318*ad32e9c0SJosh Durgin char snap_buf[RBD_MAX_SNAP_NAME_SIZE]; 319f27aaf4bSChristian Brunner int r; 320f27aaf4bSChristian Brunner 321*ad32e9c0SJosh Durgin if (qemu_rbd_parsename(filename, pool, sizeof(pool), 322f27aaf4bSChristian Brunner snap_buf, sizeof(snap_buf), 323f27aaf4bSChristian Brunner s->name, sizeof(s->name)) < 0) { 324f27aaf4bSChristian Brunner return -EINVAL; 325f27aaf4bSChristian Brunner } 326*ad32e9c0SJosh Durgin s->snap = NULL; 327f27aaf4bSChristian Brunner if (snap_buf[0] != '\0') { 328*ad32e9c0SJosh Durgin s->snap = qemu_strdup(snap_buf); 329f27aaf4bSChristian Brunner } 330f27aaf4bSChristian Brunner 331*ad32e9c0SJosh Durgin r = rados_create(&s->cluster, NULL); 332*ad32e9c0SJosh Durgin if (r < 0) { 333f27aaf4bSChristian Brunner error_report("error initializing"); 334f27aaf4bSChristian Brunner return r; 335f27aaf4bSChristian Brunner } 336f27aaf4bSChristian Brunner 337*ad32e9c0SJosh Durgin r = rados_conf_read_file(s->cluster, NULL); 338f27aaf4bSChristian Brunner if (r < 0) { 339*ad32e9c0SJosh Durgin error_report("error reading config file"); 340*ad32e9c0SJosh Durgin rados_shutdown(s->cluster); 341*ad32e9c0SJosh Durgin return r; 342f27aaf4bSChristian Brunner } 343f27aaf4bSChristian Brunner 344*ad32e9c0SJosh Durgin r = rados_connect(s->cluster); 345*ad32e9c0SJosh Durgin if (r < 0) { 346*ad32e9c0SJosh Durgin error_report("error connecting"); 347*ad32e9c0SJosh Durgin rados_shutdown(s->cluster); 348*ad32e9c0SJosh Durgin return r; 349*ad32e9c0SJosh Durgin } 350*ad32e9c0SJosh Durgin 351*ad32e9c0SJosh Durgin r = rados_ioctx_create(s->cluster, pool, &s->io_ctx); 352*ad32e9c0SJosh Durgin if (r < 0) { 353*ad32e9c0SJosh Durgin error_report("error opening pool %s", pool); 354*ad32e9c0SJosh Durgin rados_shutdown(s->cluster); 355*ad32e9c0SJosh Durgin return r; 356*ad32e9c0SJosh Durgin } 357*ad32e9c0SJosh Durgin 358*ad32e9c0SJosh Durgin r = rbd_open(s->io_ctx, s->name, &s->image, s->snap); 359*ad32e9c0SJosh Durgin if (r < 0) { 360*ad32e9c0SJosh Durgin error_report("error reading header from %s", s->name); 361*ad32e9c0SJosh Durgin rados_ioctx_destroy(s->io_ctx); 362*ad32e9c0SJosh Durgin rados_shutdown(s->cluster); 363*ad32e9c0SJosh Durgin return r; 364*ad32e9c0SJosh Durgin } 365*ad32e9c0SJosh Durgin 366*ad32e9c0SJosh Durgin bs->read_only = (s->snap != NULL); 367f27aaf4bSChristian Brunner 368f27aaf4bSChristian Brunner s->event_reader_pos = 0; 369f27aaf4bSChristian Brunner r = qemu_pipe(s->fds); 370f27aaf4bSChristian Brunner if (r < 0) { 371f27aaf4bSChristian Brunner error_report("error opening eventfd"); 372f27aaf4bSChristian Brunner goto failed; 373f27aaf4bSChristian Brunner } 374f27aaf4bSChristian Brunner fcntl(s->fds[0], F_SETFL, O_NONBLOCK); 375f27aaf4bSChristian Brunner fcntl(s->fds[1], F_SETFL, O_NONBLOCK); 376*ad32e9c0SJosh Durgin qemu_aio_set_fd_handler(s->fds[RBD_FD_READ], qemu_rbd_aio_event_reader, 377*ad32e9c0SJosh Durgin NULL, qemu_rbd_aio_flush_cb, NULL, s); 378f27aaf4bSChristian Brunner 379f27aaf4bSChristian Brunner 380f27aaf4bSChristian Brunner return 0; 381f27aaf4bSChristian Brunner 382f27aaf4bSChristian Brunner failed: 383*ad32e9c0SJosh Durgin rbd_close(s->image); 384*ad32e9c0SJosh Durgin rados_ioctx_destroy(s->io_ctx); 385*ad32e9c0SJosh Durgin rados_shutdown(s->cluster); 386f27aaf4bSChristian Brunner return r; 387f27aaf4bSChristian Brunner } 388f27aaf4bSChristian Brunner 389*ad32e9c0SJosh Durgin static void qemu_rbd_close(BlockDriverState *bs) 390f27aaf4bSChristian Brunner { 391f27aaf4bSChristian Brunner BDRVRBDState *s = bs->opaque; 392f27aaf4bSChristian Brunner 393f27aaf4bSChristian Brunner close(s->fds[0]); 394f27aaf4bSChristian Brunner close(s->fds[1]); 395f27aaf4bSChristian Brunner qemu_aio_set_fd_handler(s->fds[RBD_FD_READ], NULL , NULL, NULL, NULL, 396f27aaf4bSChristian Brunner NULL); 397f27aaf4bSChristian Brunner 398*ad32e9c0SJosh Durgin rbd_close(s->image); 399*ad32e9c0SJosh Durgin rados_ioctx_destroy(s->io_ctx); 400*ad32e9c0SJosh Durgin qemu_free(s->snap); 401*ad32e9c0SJosh Durgin rados_shutdown(s->cluster); 402f27aaf4bSChristian Brunner } 403f27aaf4bSChristian Brunner 404f27aaf4bSChristian Brunner /* 405f27aaf4bSChristian Brunner * Cancel aio. Since we don't reference acb in a non qemu threads, 406f27aaf4bSChristian Brunner * it is safe to access it here. 407f27aaf4bSChristian Brunner */ 408*ad32e9c0SJosh Durgin static void qemu_rbd_aio_cancel(BlockDriverAIOCB *blockacb) 409f27aaf4bSChristian Brunner { 410f27aaf4bSChristian Brunner RBDAIOCB *acb = (RBDAIOCB *) blockacb; 411f27aaf4bSChristian Brunner acb->cancelled = 1; 412f27aaf4bSChristian Brunner } 413f27aaf4bSChristian Brunner 414f27aaf4bSChristian Brunner static AIOPool rbd_aio_pool = { 415f27aaf4bSChristian Brunner .aiocb_size = sizeof(RBDAIOCB), 416*ad32e9c0SJosh Durgin .cancel = qemu_rbd_aio_cancel, 417f27aaf4bSChristian Brunner }; 418f27aaf4bSChristian Brunner 419*ad32e9c0SJosh Durgin static int qemu_rbd_send_pipe(BDRVRBDState *s, RADOSCB *rcb) 420f27aaf4bSChristian Brunner { 421*ad32e9c0SJosh Durgin int ret = 0; 422f27aaf4bSChristian Brunner while (1) { 423f27aaf4bSChristian Brunner fd_set wfd; 424*ad32e9c0SJosh Durgin int fd = s->fds[RBD_FD_WRITE]; 425f27aaf4bSChristian Brunner 426*ad32e9c0SJosh Durgin /* send the op pointer to the qemu thread that is responsible 427*ad32e9c0SJosh Durgin for the aio/op completion. Must do it in a qemu thread context */ 428f27aaf4bSChristian Brunner ret = write(fd, (void *)&rcb, sizeof(rcb)); 429f27aaf4bSChristian Brunner if (ret >= 0) { 430f27aaf4bSChristian Brunner break; 431f27aaf4bSChristian Brunner } 432f27aaf4bSChristian Brunner if (errno == EINTR) { 433f27aaf4bSChristian Brunner continue; 434f27aaf4bSChristian Brunner } 435f27aaf4bSChristian Brunner if (errno != EAGAIN) { 436f27aaf4bSChristian Brunner break; 437f27aaf4bSChristian Brunner } 438f27aaf4bSChristian Brunner 439f27aaf4bSChristian Brunner FD_ZERO(&wfd); 440f27aaf4bSChristian Brunner FD_SET(fd, &wfd); 441f27aaf4bSChristian Brunner do { 442f27aaf4bSChristian Brunner ret = select(fd + 1, NULL, &wfd, NULL, NULL); 443f27aaf4bSChristian Brunner } while (ret < 0 && errno == EINTR); 444f27aaf4bSChristian Brunner } 445f27aaf4bSChristian Brunner 446*ad32e9c0SJosh Durgin return ret; 447*ad32e9c0SJosh Durgin } 448*ad32e9c0SJosh Durgin 449*ad32e9c0SJosh Durgin /* 450*ad32e9c0SJosh Durgin * This is the callback function for rbd_aio_read and _write 451*ad32e9c0SJosh Durgin * 452*ad32e9c0SJosh Durgin * Note: this function is being called from a non qemu thread so 453*ad32e9c0SJosh Durgin * we need to be careful about what we do here. Generally we only 454*ad32e9c0SJosh Durgin * write to the block notification pipe, and do the rest of the 455*ad32e9c0SJosh Durgin * io completion handling from qemu_rbd_aio_event_reader() which 456*ad32e9c0SJosh Durgin * runs in a qemu context. 457*ad32e9c0SJosh Durgin */ 458*ad32e9c0SJosh Durgin static void rbd_finish_aiocb(rbd_completion_t c, RADOSCB *rcb) 459*ad32e9c0SJosh Durgin { 460*ad32e9c0SJosh Durgin int ret; 461*ad32e9c0SJosh Durgin rcb->ret = rbd_aio_get_return_value(c); 462*ad32e9c0SJosh Durgin rbd_aio_release(c); 463*ad32e9c0SJosh Durgin ret = qemu_rbd_send_pipe(rcb->s, rcb); 464f27aaf4bSChristian Brunner if (ret < 0) { 465*ad32e9c0SJosh Durgin error_report("failed writing to acb->s->fds"); 466f27aaf4bSChristian Brunner qemu_free(rcb); 467f27aaf4bSChristian Brunner } 468f27aaf4bSChristian Brunner } 469f27aaf4bSChristian Brunner 470*ad32e9c0SJosh Durgin /* Callback when all queued rbd_aio requests are complete */ 471f27aaf4bSChristian Brunner 472f27aaf4bSChristian Brunner static void rbd_aio_bh_cb(void *opaque) 473f27aaf4bSChristian Brunner { 474f27aaf4bSChristian Brunner RBDAIOCB *acb = opaque; 475f27aaf4bSChristian Brunner 476f27aaf4bSChristian Brunner if (!acb->write) { 477f27aaf4bSChristian Brunner qemu_iovec_from_buffer(acb->qiov, acb->bounce, acb->qiov->size); 478f27aaf4bSChristian Brunner } 479f27aaf4bSChristian Brunner qemu_vfree(acb->bounce); 480f27aaf4bSChristian Brunner acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret)); 481f27aaf4bSChristian Brunner qemu_bh_delete(acb->bh); 482f27aaf4bSChristian Brunner acb->bh = NULL; 483f27aaf4bSChristian Brunner 484f27aaf4bSChristian Brunner qemu_aio_release(acb); 485f27aaf4bSChristian Brunner } 486f27aaf4bSChristian Brunner 487f27aaf4bSChristian Brunner static BlockDriverAIOCB *rbd_aio_rw_vector(BlockDriverState *bs, 488f27aaf4bSChristian Brunner int64_t sector_num, 489f27aaf4bSChristian Brunner QEMUIOVector *qiov, 490f27aaf4bSChristian Brunner int nb_sectors, 491f27aaf4bSChristian Brunner BlockDriverCompletionFunc *cb, 492f27aaf4bSChristian Brunner void *opaque, int write) 493f27aaf4bSChristian Brunner { 494f27aaf4bSChristian Brunner RBDAIOCB *acb; 495f27aaf4bSChristian Brunner RADOSCB *rcb; 496*ad32e9c0SJosh Durgin rbd_completion_t c; 497f27aaf4bSChristian Brunner int64_t off, size; 498f27aaf4bSChristian Brunner char *buf; 499f27aaf4bSChristian Brunner 500f27aaf4bSChristian Brunner BDRVRBDState *s = bs->opaque; 501f27aaf4bSChristian Brunner 502f27aaf4bSChristian Brunner acb = qemu_aio_get(&rbd_aio_pool, bs, cb, opaque); 503f27aaf4bSChristian Brunner acb->write = write; 504f27aaf4bSChristian Brunner acb->qiov = qiov; 505f27aaf4bSChristian Brunner acb->bounce = qemu_blockalign(bs, qiov->size); 506f27aaf4bSChristian Brunner acb->ret = 0; 507f27aaf4bSChristian Brunner acb->error = 0; 508f27aaf4bSChristian Brunner acb->s = s; 509f27aaf4bSChristian Brunner acb->cancelled = 0; 510f27aaf4bSChristian Brunner acb->bh = NULL; 511f27aaf4bSChristian Brunner 512f27aaf4bSChristian Brunner if (write) { 513f27aaf4bSChristian Brunner qemu_iovec_to_buffer(acb->qiov, acb->bounce); 514f27aaf4bSChristian Brunner } 515f27aaf4bSChristian Brunner 516f27aaf4bSChristian Brunner buf = acb->bounce; 517f27aaf4bSChristian Brunner 518f27aaf4bSChristian Brunner off = sector_num * BDRV_SECTOR_SIZE; 519f27aaf4bSChristian Brunner size = nb_sectors * BDRV_SECTOR_SIZE; 520f27aaf4bSChristian Brunner 521*ad32e9c0SJosh Durgin s->qemu_aio_count++; /* All the RADOSCB */ 522f27aaf4bSChristian Brunner 523f27aaf4bSChristian Brunner rcb = qemu_malloc(sizeof(RADOSCB)); 524f27aaf4bSChristian Brunner rcb->done = 0; 525f27aaf4bSChristian Brunner rcb->acb = acb; 526f27aaf4bSChristian Brunner rcb->buf = buf; 527f27aaf4bSChristian Brunner rcb->s = acb->s; 528*ad32e9c0SJosh Durgin rcb->size = size; 529f27aaf4bSChristian Brunner 530f27aaf4bSChristian Brunner if (write) { 531*ad32e9c0SJosh Durgin rbd_aio_create_completion(rcb, (rbd_callback_t) rbd_finish_aiocb, &c); 532*ad32e9c0SJosh Durgin rbd_aio_write(s->image, off, size, buf, c); 533f27aaf4bSChristian Brunner } else { 534*ad32e9c0SJosh Durgin rbd_aio_create_completion(rcb, (rbd_callback_t) rbd_finish_aiocb, &c); 535*ad32e9c0SJosh Durgin rbd_aio_read(s->image, off, size, buf, c); 536f27aaf4bSChristian Brunner } 537f27aaf4bSChristian Brunner 538f27aaf4bSChristian Brunner return &acb->common; 539f27aaf4bSChristian Brunner } 540f27aaf4bSChristian Brunner 541*ad32e9c0SJosh Durgin static BlockDriverAIOCB *qemu_rbd_aio_readv(BlockDriverState *bs, 542*ad32e9c0SJosh Durgin int64_t sector_num, 543*ad32e9c0SJosh Durgin QEMUIOVector *qiov, 544f27aaf4bSChristian Brunner int nb_sectors, 545f27aaf4bSChristian Brunner BlockDriverCompletionFunc *cb, 546f27aaf4bSChristian Brunner void *opaque) 547f27aaf4bSChristian Brunner { 548f27aaf4bSChristian Brunner return rbd_aio_rw_vector(bs, sector_num, qiov, nb_sectors, cb, opaque, 0); 549f27aaf4bSChristian Brunner } 550f27aaf4bSChristian Brunner 551*ad32e9c0SJosh Durgin static BlockDriverAIOCB *qemu_rbd_aio_writev(BlockDriverState *bs, 552*ad32e9c0SJosh Durgin int64_t sector_num, 553*ad32e9c0SJosh Durgin QEMUIOVector *qiov, 554f27aaf4bSChristian Brunner int nb_sectors, 555f27aaf4bSChristian Brunner BlockDriverCompletionFunc *cb, 556f27aaf4bSChristian Brunner void *opaque) 557f27aaf4bSChristian Brunner { 558f27aaf4bSChristian Brunner return rbd_aio_rw_vector(bs, sector_num, qiov, nb_sectors, cb, opaque, 1); 559f27aaf4bSChristian Brunner } 560f27aaf4bSChristian Brunner 561*ad32e9c0SJosh Durgin static int qemu_rbd_getinfo(BlockDriverState *bs, BlockDriverInfo *bdi) 562f27aaf4bSChristian Brunner { 563f27aaf4bSChristian Brunner BDRVRBDState *s = bs->opaque; 564*ad32e9c0SJosh Durgin rbd_image_info_t info; 565*ad32e9c0SJosh Durgin int r; 566*ad32e9c0SJosh Durgin 567*ad32e9c0SJosh Durgin r = rbd_stat(s->image, &info, sizeof(info)); 568*ad32e9c0SJosh Durgin if (r < 0) { 569*ad32e9c0SJosh Durgin return r; 570*ad32e9c0SJosh Durgin } 571*ad32e9c0SJosh Durgin 572*ad32e9c0SJosh Durgin bdi->cluster_size = info.obj_size; 573f27aaf4bSChristian Brunner return 0; 574f27aaf4bSChristian Brunner } 575f27aaf4bSChristian Brunner 576*ad32e9c0SJosh Durgin static int64_t qemu_rbd_getlength(BlockDriverState *bs) 577f27aaf4bSChristian Brunner { 578f27aaf4bSChristian Brunner BDRVRBDState *s = bs->opaque; 579*ad32e9c0SJosh Durgin rbd_image_info_t info; 580*ad32e9c0SJosh Durgin int r; 581f27aaf4bSChristian Brunner 582*ad32e9c0SJosh Durgin r = rbd_stat(s->image, &info, sizeof(info)); 583*ad32e9c0SJosh Durgin if (r < 0) { 584*ad32e9c0SJosh Durgin return r; 585f27aaf4bSChristian Brunner } 586f27aaf4bSChristian Brunner 587*ad32e9c0SJosh Durgin return info.size; 588*ad32e9c0SJosh Durgin } 589*ad32e9c0SJosh Durgin 590*ad32e9c0SJosh Durgin static int qemu_rbd_snap_create(BlockDriverState *bs, 591*ad32e9c0SJosh Durgin QEMUSnapshotInfo *sn_info) 592f27aaf4bSChristian Brunner { 593f27aaf4bSChristian Brunner BDRVRBDState *s = bs->opaque; 594f27aaf4bSChristian Brunner int r; 595f27aaf4bSChristian Brunner 596f27aaf4bSChristian Brunner if (sn_info->name[0] == '\0') { 597f27aaf4bSChristian Brunner return -EINVAL; /* we need a name for rbd snapshots */ 598f27aaf4bSChristian Brunner } 599f27aaf4bSChristian Brunner 600f27aaf4bSChristian Brunner /* 601f27aaf4bSChristian Brunner * rbd snapshots are using the name as the user controlled unique identifier 602f27aaf4bSChristian Brunner * we can't use the rbd snapid for that purpose, as it can't be set 603f27aaf4bSChristian Brunner */ 604f27aaf4bSChristian Brunner if (sn_info->id_str[0] != '\0' && 605f27aaf4bSChristian Brunner strcmp(sn_info->id_str, sn_info->name) != 0) { 606f27aaf4bSChristian Brunner return -EINVAL; 607f27aaf4bSChristian Brunner } 608f27aaf4bSChristian Brunner 609f27aaf4bSChristian Brunner if (strlen(sn_info->name) >= sizeof(sn_info->id_str)) { 610f27aaf4bSChristian Brunner return -ERANGE; 611f27aaf4bSChristian Brunner } 612f27aaf4bSChristian Brunner 613*ad32e9c0SJosh Durgin r = rbd_snap_create(s->image, sn_info->name); 614f27aaf4bSChristian Brunner if (r < 0) { 615*ad32e9c0SJosh Durgin error_report("failed to create snap: %s", strerror(-r)); 616f27aaf4bSChristian Brunner return r; 617f27aaf4bSChristian Brunner } 618f27aaf4bSChristian Brunner 619f27aaf4bSChristian Brunner return 0; 620f27aaf4bSChristian Brunner } 621f27aaf4bSChristian Brunner 622*ad32e9c0SJosh Durgin static int qemu_rbd_snap_list(BlockDriverState *bs, 623*ad32e9c0SJosh Durgin QEMUSnapshotInfo **psn_tab) 624f27aaf4bSChristian Brunner { 625f27aaf4bSChristian Brunner BDRVRBDState *s = bs->opaque; 626f27aaf4bSChristian Brunner QEMUSnapshotInfo *sn_info, *sn_tab = NULL; 627*ad32e9c0SJosh Durgin int i, snap_count; 628*ad32e9c0SJosh Durgin rbd_snap_info_t *snaps; 629*ad32e9c0SJosh Durgin int max_snaps = RBD_MAX_SNAPS; 630f27aaf4bSChristian Brunner 631*ad32e9c0SJosh Durgin do { 632*ad32e9c0SJosh Durgin snaps = qemu_malloc(sizeof(*snaps) * max_snaps); 633*ad32e9c0SJosh Durgin snap_count = rbd_snap_list(s->image, snaps, &max_snaps); 634*ad32e9c0SJosh Durgin if (snap_count < 0) { 635*ad32e9c0SJosh Durgin qemu_free(snaps); 636f27aaf4bSChristian Brunner } 637*ad32e9c0SJosh Durgin } while (snap_count == -ERANGE); 638f27aaf4bSChristian Brunner 639*ad32e9c0SJosh Durgin if (snap_count <= 0) { 640*ad32e9c0SJosh Durgin return snap_count; 641f27aaf4bSChristian Brunner } 642f27aaf4bSChristian Brunner 643f27aaf4bSChristian Brunner sn_tab = qemu_mallocz(snap_count * sizeof(QEMUSnapshotInfo)); 644f27aaf4bSChristian Brunner 645*ad32e9c0SJosh Durgin for (i = 0; i < snap_count; i++) { 646*ad32e9c0SJosh Durgin const char *snap_name = snaps[i].name; 647f27aaf4bSChristian Brunner 648f27aaf4bSChristian Brunner sn_info = sn_tab + i; 649f27aaf4bSChristian Brunner pstrcpy(sn_info->id_str, sizeof(sn_info->id_str), snap_name); 650f27aaf4bSChristian Brunner pstrcpy(sn_info->name, sizeof(sn_info->name), snap_name); 651f27aaf4bSChristian Brunner 652*ad32e9c0SJosh Durgin sn_info->vm_state_size = snaps[i].size; 653f27aaf4bSChristian Brunner sn_info->date_sec = 0; 654f27aaf4bSChristian Brunner sn_info->date_nsec = 0; 655f27aaf4bSChristian Brunner sn_info->vm_clock_nsec = 0; 656f27aaf4bSChristian Brunner } 657*ad32e9c0SJosh Durgin rbd_snap_list_end(snaps); 658*ad32e9c0SJosh Durgin 659f27aaf4bSChristian Brunner *psn_tab = sn_tab; 660f27aaf4bSChristian Brunner return snap_count; 661f27aaf4bSChristian Brunner } 662f27aaf4bSChristian Brunner 663*ad32e9c0SJosh Durgin static QEMUOptionParameter qemu_rbd_create_options[] = { 664f27aaf4bSChristian Brunner { 665f27aaf4bSChristian Brunner .name = BLOCK_OPT_SIZE, 666f27aaf4bSChristian Brunner .type = OPT_SIZE, 667f27aaf4bSChristian Brunner .help = "Virtual disk size" 668f27aaf4bSChristian Brunner }, 669f27aaf4bSChristian Brunner { 670f27aaf4bSChristian Brunner .name = BLOCK_OPT_CLUSTER_SIZE, 671f27aaf4bSChristian Brunner .type = OPT_SIZE, 672f27aaf4bSChristian Brunner .help = "RBD object size" 673f27aaf4bSChristian Brunner }, 674f27aaf4bSChristian Brunner {NULL} 675f27aaf4bSChristian Brunner }; 676f27aaf4bSChristian Brunner 677f27aaf4bSChristian Brunner static BlockDriver bdrv_rbd = { 678f27aaf4bSChristian Brunner .format_name = "rbd", 679f27aaf4bSChristian Brunner .instance_size = sizeof(BDRVRBDState), 680*ad32e9c0SJosh Durgin .bdrv_file_open = qemu_rbd_open, 681*ad32e9c0SJosh Durgin .bdrv_close = qemu_rbd_close, 682*ad32e9c0SJosh Durgin .bdrv_create = qemu_rbd_create, 683*ad32e9c0SJosh Durgin .bdrv_get_info = qemu_rbd_getinfo, 684*ad32e9c0SJosh Durgin .create_options = qemu_rbd_create_options, 685*ad32e9c0SJosh Durgin .bdrv_getlength = qemu_rbd_getlength, 686f27aaf4bSChristian Brunner .protocol_name = "rbd", 687f27aaf4bSChristian Brunner 688*ad32e9c0SJosh Durgin .bdrv_aio_readv = qemu_rbd_aio_readv, 689*ad32e9c0SJosh Durgin .bdrv_aio_writev = qemu_rbd_aio_writev, 690f27aaf4bSChristian Brunner 691*ad32e9c0SJosh Durgin .bdrv_snapshot_create = qemu_rbd_snap_create, 692*ad32e9c0SJosh Durgin .bdrv_snapshot_list = qemu_rbd_snap_list, 693f27aaf4bSChristian Brunner }; 694f27aaf4bSChristian Brunner 695f27aaf4bSChristian Brunner static void bdrv_rbd_init(void) 696f27aaf4bSChristian Brunner { 697f27aaf4bSChristian Brunner bdrv_register(&bdrv_rbd); 698f27aaf4bSChristian Brunner } 699f27aaf4bSChristian Brunner 700f27aaf4bSChristian Brunner block_init(bdrv_rbd_init); 701