1f27aaf4bSChristian Brunner /* 2f27aaf4bSChristian Brunner * QEMU Block driver for RADOS (Ceph) 3f27aaf4bSChristian Brunner * 4ad32e9c0SJosh Durgin * Copyright (C) 2010-2011 Christian Brunner <chb@muc.de>, 5ad32e9c0SJosh Durgin * Josh Durgin <josh.durgin@dreamhost.com> 6f27aaf4bSChristian Brunner * 7f27aaf4bSChristian Brunner * This work is licensed under the terms of the GNU GPL, version 2. See 8f27aaf4bSChristian Brunner * the COPYING file in the top-level directory. 9f27aaf4bSChristian Brunner * 106b620ca3SPaolo Bonzini * Contributions after 2012-01-13 are licensed under the terms of the 116b620ca3SPaolo Bonzini * GNU GPL, version 2 or (at your option) any later version. 12f27aaf4bSChristian Brunner */ 13f27aaf4bSChristian Brunner 14ad32e9c0SJosh Durgin #include <inttypes.h> 15ad32e9c0SJosh Durgin 16f27aaf4bSChristian Brunner #include "qemu-common.h" 171de7afc9SPaolo Bonzini #include "qemu/error-report.h" 18737e150eSPaolo Bonzini #include "block/block_int.h" 19f27aaf4bSChristian Brunner 20ad32e9c0SJosh Durgin #include <rbd/librbd.h> 21f27aaf4bSChristian Brunner 22f27aaf4bSChristian Brunner /* 23f27aaf4bSChristian Brunner * When specifying the image filename use: 24f27aaf4bSChristian Brunner * 25fab5cf59SJosh Durgin * rbd:poolname/devicename[@snapshotname][:option1=value1[:option2=value2...]] 26f27aaf4bSChristian Brunner * 279e1fbcdeSSage Weil * poolname must be the name of an existing rados pool. 28f27aaf4bSChristian Brunner * 299e1fbcdeSSage Weil * devicename is the name of the rbd image. 30f27aaf4bSChristian Brunner * 319e1fbcdeSSage Weil * Each option given is used to configure rados, and may be any valid 329e1fbcdeSSage Weil * Ceph option, "id", or "conf". 33fab5cf59SJosh Durgin * 349e1fbcdeSSage Weil * The "id" option indicates what user we should authenticate as to 359e1fbcdeSSage Weil * the Ceph cluster. If it is excluded we will use the Ceph default 369e1fbcdeSSage Weil * (normally 'admin'). 37f27aaf4bSChristian Brunner * 389e1fbcdeSSage Weil * The "conf" option specifies a Ceph configuration file to read. If 399e1fbcdeSSage Weil * it is not specified, we will read from the default Ceph locations 409e1fbcdeSSage Weil * (e.g., /etc/ceph/ceph.conf). To avoid reading _any_ configuration 419e1fbcdeSSage Weil * file, specify conf=/dev/null. 42f27aaf4bSChristian Brunner * 439e1fbcdeSSage Weil * Configuration values containing :, @, or = can be escaped with a 449e1fbcdeSSage Weil * leading "\". 45f27aaf4bSChristian Brunner */ 46f27aaf4bSChristian Brunner 47787f3133SJosh Durgin /* rbd_aio_discard added in 0.1.2 */ 48787f3133SJosh Durgin #if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(0, 1, 2) 49787f3133SJosh Durgin #define LIBRBD_SUPPORTS_DISCARD 50787f3133SJosh Durgin #else 51787f3133SJosh Durgin #undef LIBRBD_SUPPORTS_DISCARD 52787f3133SJosh Durgin #endif 53787f3133SJosh Durgin 54f27aaf4bSChristian Brunner #define OBJ_MAX_SIZE (1UL << OBJ_DEFAULT_OBJ_ORDER) 55f27aaf4bSChristian Brunner 56ad32e9c0SJosh Durgin #define RBD_MAX_CONF_NAME_SIZE 128 57ad32e9c0SJosh Durgin #define RBD_MAX_CONF_VAL_SIZE 512 58ad32e9c0SJosh Durgin #define RBD_MAX_CONF_SIZE 1024 59ad32e9c0SJosh Durgin #define RBD_MAX_POOL_NAME_SIZE 128 60ad32e9c0SJosh Durgin #define RBD_MAX_SNAP_NAME_SIZE 128 61ad32e9c0SJosh Durgin #define RBD_MAX_SNAPS 100 62ad32e9c0SJosh Durgin 63787f3133SJosh Durgin typedef enum { 64787f3133SJosh Durgin RBD_AIO_READ, 65787f3133SJosh Durgin RBD_AIO_WRITE, 66dc7588c1SJosh Durgin RBD_AIO_DISCARD, 67dc7588c1SJosh Durgin RBD_AIO_FLUSH 68787f3133SJosh Durgin } RBDAIOCmd; 69787f3133SJosh Durgin 70f27aaf4bSChristian Brunner typedef struct RBDAIOCB { 71f27aaf4bSChristian Brunner BlockDriverAIOCB common; 72f27aaf4bSChristian Brunner QEMUBH *bh; 7308448d51SStefan Priebe int64_t ret; 74f27aaf4bSChristian Brunner QEMUIOVector *qiov; 75f27aaf4bSChristian Brunner char *bounce; 76787f3133SJosh Durgin RBDAIOCmd cmd; 77f27aaf4bSChristian Brunner int64_t sector_num; 78f27aaf4bSChristian Brunner int error; 79f27aaf4bSChristian Brunner struct BDRVRBDState *s; 80f27aaf4bSChristian Brunner int cancelled; 81473c7f02SStefan Priebe int status; 82f27aaf4bSChristian Brunner } RBDAIOCB; 83f27aaf4bSChristian Brunner 84f27aaf4bSChristian Brunner typedef struct RADOSCB { 85f27aaf4bSChristian Brunner int rcbid; 86f27aaf4bSChristian Brunner RBDAIOCB *acb; 87f27aaf4bSChristian Brunner struct BDRVRBDState *s; 88f27aaf4bSChristian Brunner int done; 89ad32e9c0SJosh Durgin int64_t size; 90f27aaf4bSChristian Brunner char *buf; 9108448d51SStefan Priebe int64_t ret; 92f27aaf4bSChristian Brunner } RADOSCB; 93f27aaf4bSChristian Brunner 94f27aaf4bSChristian Brunner #define RBD_FD_READ 0 95f27aaf4bSChristian Brunner #define RBD_FD_WRITE 1 96f27aaf4bSChristian Brunner 97f27aaf4bSChristian Brunner typedef struct BDRVRBDState { 98f27aaf4bSChristian Brunner int fds[2]; 99ad32e9c0SJosh Durgin rados_t cluster; 100ad32e9c0SJosh Durgin rados_ioctx_t io_ctx; 101ad32e9c0SJosh Durgin rbd_image_t image; 102ad32e9c0SJosh Durgin char name[RBD_MAX_IMAGE_NAME_SIZE]; 103f27aaf4bSChristian Brunner int qemu_aio_count; 104ad32e9c0SJosh Durgin char *snap; 105f27aaf4bSChristian Brunner int event_reader_pos; 106f27aaf4bSChristian Brunner RADOSCB *event_rcb; 107f27aaf4bSChristian Brunner } BDRVRBDState; 108f27aaf4bSChristian Brunner 109f27aaf4bSChristian Brunner static void rbd_aio_bh_cb(void *opaque); 110f27aaf4bSChristian Brunner 111ad32e9c0SJosh Durgin static int qemu_rbd_next_tok(char *dst, int dst_len, 112f27aaf4bSChristian Brunner char *src, char delim, 113f27aaf4bSChristian Brunner const char *name, 114f27aaf4bSChristian Brunner char **p) 115f27aaf4bSChristian Brunner { 116f27aaf4bSChristian Brunner int l; 117f27aaf4bSChristian Brunner char *end; 118f27aaf4bSChristian Brunner 119f27aaf4bSChristian Brunner *p = NULL; 120f27aaf4bSChristian Brunner 121f27aaf4bSChristian Brunner if (delim != '\0') { 12216a06b24SSage Weil for (end = src; *end; ++end) { 12316a06b24SSage Weil if (*end == delim) { 12416a06b24SSage Weil break; 12516a06b24SSage Weil } 12616a06b24SSage Weil if (*end == '\\' && end[1] != '\0') { 12716a06b24SSage Weil end++; 12816a06b24SSage Weil } 12916a06b24SSage Weil } 13016a06b24SSage Weil if (*end == delim) { 131f27aaf4bSChristian Brunner *p = end + 1; 132f27aaf4bSChristian Brunner *end = '\0'; 133f27aaf4bSChristian Brunner } 134f27aaf4bSChristian Brunner } 135f27aaf4bSChristian Brunner l = strlen(src); 136f27aaf4bSChristian Brunner if (l >= dst_len) { 137f27aaf4bSChristian Brunner error_report("%s too long", name); 138f27aaf4bSChristian Brunner return -EINVAL; 139f27aaf4bSChristian Brunner } else if (l == 0) { 140f27aaf4bSChristian Brunner error_report("%s too short", name); 141f27aaf4bSChristian Brunner return -EINVAL; 142f27aaf4bSChristian Brunner } 143f27aaf4bSChristian Brunner 144f27aaf4bSChristian Brunner pstrcpy(dst, dst_len, src); 145f27aaf4bSChristian Brunner 146f27aaf4bSChristian Brunner return 0; 147f27aaf4bSChristian Brunner } 148f27aaf4bSChristian Brunner 14916a06b24SSage Weil static void qemu_rbd_unescape(char *src) 15016a06b24SSage Weil { 15116a06b24SSage Weil char *p; 15216a06b24SSage Weil 15316a06b24SSage Weil for (p = src; *src; ++src, ++p) { 15416a06b24SSage Weil if (*src == '\\' && src[1] != '\0') { 15516a06b24SSage Weil src++; 15616a06b24SSage Weil } 15716a06b24SSage Weil *p = *src; 15816a06b24SSage Weil } 15916a06b24SSage Weil *p = '\0'; 16016a06b24SSage Weil } 16116a06b24SSage Weil 162ad32e9c0SJosh Durgin static int qemu_rbd_parsename(const char *filename, 163f27aaf4bSChristian Brunner char *pool, int pool_len, 164f27aaf4bSChristian Brunner char *snap, int snap_len, 165fab5cf59SJosh Durgin char *name, int name_len, 166fab5cf59SJosh Durgin char *conf, int conf_len) 167f27aaf4bSChristian Brunner { 168f27aaf4bSChristian Brunner const char *start; 169f27aaf4bSChristian Brunner char *p, *buf; 170f27aaf4bSChristian Brunner int ret; 171f27aaf4bSChristian Brunner 172f27aaf4bSChristian Brunner if (!strstart(filename, "rbd:", &start)) { 173f27aaf4bSChristian Brunner return -EINVAL; 174f27aaf4bSChristian Brunner } 175f27aaf4bSChristian Brunner 1767267c094SAnthony Liguori buf = g_strdup(start); 177f27aaf4bSChristian Brunner p = buf; 178fab5cf59SJosh Durgin *snap = '\0'; 179fab5cf59SJosh Durgin *conf = '\0'; 180f27aaf4bSChristian Brunner 181ad32e9c0SJosh Durgin ret = qemu_rbd_next_tok(pool, pool_len, p, '/', "pool name", &p); 182f27aaf4bSChristian Brunner if (ret < 0 || !p) { 183f27aaf4bSChristian Brunner ret = -EINVAL; 184f27aaf4bSChristian Brunner goto done; 185f27aaf4bSChristian Brunner } 18616a06b24SSage Weil qemu_rbd_unescape(pool); 187fab5cf59SJosh Durgin 188fab5cf59SJosh Durgin if (strchr(p, '@')) { 189ad32e9c0SJosh Durgin ret = qemu_rbd_next_tok(name, name_len, p, '@', "object name", &p); 190f27aaf4bSChristian Brunner if (ret < 0) { 191f27aaf4bSChristian Brunner goto done; 192f27aaf4bSChristian Brunner } 193fab5cf59SJosh Durgin ret = qemu_rbd_next_tok(snap, snap_len, p, ':', "snap name", &p); 19416a06b24SSage Weil qemu_rbd_unescape(snap); 195fab5cf59SJosh Durgin } else { 196fab5cf59SJosh Durgin ret = qemu_rbd_next_tok(name, name_len, p, ':', "object name", &p); 197fab5cf59SJosh Durgin } 19816a06b24SSage Weil qemu_rbd_unescape(name); 199fab5cf59SJosh Durgin if (ret < 0 || !p) { 200f27aaf4bSChristian Brunner goto done; 201f27aaf4bSChristian Brunner } 202f27aaf4bSChristian Brunner 203fab5cf59SJosh Durgin ret = qemu_rbd_next_tok(conf, conf_len, p, '\0', "configuration", &p); 204f27aaf4bSChristian Brunner 205f27aaf4bSChristian Brunner done: 2067267c094SAnthony Liguori g_free(buf); 207f27aaf4bSChristian Brunner return ret; 208f27aaf4bSChristian Brunner } 209f27aaf4bSChristian Brunner 2107c7e9df0SSage Weil static char *qemu_rbd_parse_clientname(const char *conf, char *clientname) 2117c7e9df0SSage Weil { 2127c7e9df0SSage Weil const char *p = conf; 2137c7e9df0SSage Weil 2147c7e9df0SSage Weil while (*p) { 2157c7e9df0SSage Weil int len; 2167c7e9df0SSage Weil const char *end = strchr(p, ':'); 2177c7e9df0SSage Weil 2187c7e9df0SSage Weil if (end) { 2197c7e9df0SSage Weil len = end - p; 2207c7e9df0SSage Weil } else { 2217c7e9df0SSage Weil len = strlen(p); 2227c7e9df0SSage Weil } 2237c7e9df0SSage Weil 2247c7e9df0SSage Weil if (strncmp(p, "id=", 3) == 0) { 2257c7e9df0SSage Weil len -= 3; 2267c7e9df0SSage Weil strncpy(clientname, p + 3, len); 2277c7e9df0SSage Weil clientname[len] = '\0'; 2287c7e9df0SSage Weil return clientname; 2297c7e9df0SSage Weil } 2307c7e9df0SSage Weil if (end == NULL) { 2317c7e9df0SSage Weil break; 2327c7e9df0SSage Weil } 2337c7e9df0SSage Weil p = end + 1; 2347c7e9df0SSage Weil } 2357c7e9df0SSage Weil return NULL; 2367c7e9df0SSage Weil } 2377c7e9df0SSage Weil 238fab5cf59SJosh Durgin static int qemu_rbd_set_conf(rados_t cluster, const char *conf) 239fab5cf59SJosh Durgin { 240fab5cf59SJosh Durgin char *p, *buf; 241fab5cf59SJosh Durgin char name[RBD_MAX_CONF_NAME_SIZE]; 242fab5cf59SJosh Durgin char value[RBD_MAX_CONF_VAL_SIZE]; 243fab5cf59SJosh Durgin int ret = 0; 244fab5cf59SJosh Durgin 2457267c094SAnthony Liguori buf = g_strdup(conf); 246fab5cf59SJosh Durgin p = buf; 247fab5cf59SJosh Durgin 248fab5cf59SJosh Durgin while (p) { 249fab5cf59SJosh Durgin ret = qemu_rbd_next_tok(name, sizeof(name), p, 250fab5cf59SJosh Durgin '=', "conf option name", &p); 251fab5cf59SJosh Durgin if (ret < 0) { 252fab5cf59SJosh Durgin break; 253fab5cf59SJosh Durgin } 25416a06b24SSage Weil qemu_rbd_unescape(name); 255fab5cf59SJosh Durgin 256fab5cf59SJosh Durgin if (!p) { 257fab5cf59SJosh Durgin error_report("conf option %s has no value", name); 258fab5cf59SJosh Durgin ret = -EINVAL; 259fab5cf59SJosh Durgin break; 260fab5cf59SJosh Durgin } 261fab5cf59SJosh Durgin 262fab5cf59SJosh Durgin ret = qemu_rbd_next_tok(value, sizeof(value), p, 263fab5cf59SJosh Durgin ':', "conf option value", &p); 264fab5cf59SJosh Durgin if (ret < 0) { 265fab5cf59SJosh Durgin break; 266fab5cf59SJosh Durgin } 26716a06b24SSage Weil qemu_rbd_unescape(value); 268fab5cf59SJosh Durgin 2697c7e9df0SSage Weil if (strcmp(name, "conf") == 0) { 2707c7e9df0SSage Weil ret = rados_conf_read_file(cluster, value); 2717c7e9df0SSage Weil if (ret < 0) { 2727c7e9df0SSage Weil error_report("error reading conf file %s", value); 2737c7e9df0SSage Weil break; 2747c7e9df0SSage Weil } 2757c7e9df0SSage Weil } else if (strcmp(name, "id") == 0) { 2767c7e9df0SSage Weil /* ignore, this is parsed by qemu_rbd_parse_clientname() */ 2777c7e9df0SSage Weil } else { 278fab5cf59SJosh Durgin ret = rados_conf_set(cluster, name, value); 279fab5cf59SJosh Durgin if (ret < 0) { 280fab5cf59SJosh Durgin error_report("invalid conf option %s", name); 281fab5cf59SJosh Durgin ret = -EINVAL; 282fab5cf59SJosh Durgin break; 283fab5cf59SJosh Durgin } 284fab5cf59SJosh Durgin } 285fab5cf59SJosh Durgin } 286fab5cf59SJosh Durgin 2877267c094SAnthony Liguori g_free(buf); 288fab5cf59SJosh Durgin return ret; 289fab5cf59SJosh Durgin } 290fab5cf59SJosh Durgin 291ad32e9c0SJosh Durgin static int qemu_rbd_create(const char *filename, QEMUOptionParameter *options) 292f27aaf4bSChristian Brunner { 293f27aaf4bSChristian Brunner int64_t bytes = 0; 294f27aaf4bSChristian Brunner int64_t objsize; 295ad32e9c0SJosh Durgin int obj_order = 0; 296ad32e9c0SJosh Durgin char pool[RBD_MAX_POOL_NAME_SIZE]; 297ad32e9c0SJosh Durgin char name[RBD_MAX_IMAGE_NAME_SIZE]; 298ad32e9c0SJosh Durgin char snap_buf[RBD_MAX_SNAP_NAME_SIZE]; 299fab5cf59SJosh Durgin char conf[RBD_MAX_CONF_SIZE]; 3007c7e9df0SSage Weil char clientname_buf[RBD_MAX_CONF_SIZE]; 3017c7e9df0SSage Weil char *clientname; 302ad32e9c0SJosh Durgin rados_t cluster; 303ad32e9c0SJosh Durgin rados_ioctx_t io_ctx; 304f27aaf4bSChristian Brunner int ret; 305f27aaf4bSChristian Brunner 306ad32e9c0SJosh Durgin if (qemu_rbd_parsename(filename, pool, sizeof(pool), 307f27aaf4bSChristian Brunner snap_buf, sizeof(snap_buf), 308fab5cf59SJosh Durgin name, sizeof(name), 309fab5cf59SJosh Durgin conf, sizeof(conf)) < 0) { 310f27aaf4bSChristian Brunner return -EINVAL; 311f27aaf4bSChristian Brunner } 312f27aaf4bSChristian Brunner 313f27aaf4bSChristian Brunner /* Read out options */ 314f27aaf4bSChristian Brunner while (options && options->name) { 315f27aaf4bSChristian Brunner if (!strcmp(options->name, BLOCK_OPT_SIZE)) { 316f27aaf4bSChristian Brunner bytes = options->value.n; 317f27aaf4bSChristian Brunner } else if (!strcmp(options->name, BLOCK_OPT_CLUSTER_SIZE)) { 318f27aaf4bSChristian Brunner if (options->value.n) { 319f27aaf4bSChristian Brunner objsize = options->value.n; 320f27aaf4bSChristian Brunner if ((objsize - 1) & objsize) { /* not a power of 2? */ 321f27aaf4bSChristian Brunner error_report("obj size needs to be power of 2"); 322f27aaf4bSChristian Brunner return -EINVAL; 323f27aaf4bSChristian Brunner } 324f27aaf4bSChristian Brunner if (objsize < 4096) { 325f27aaf4bSChristian Brunner error_report("obj size too small"); 326f27aaf4bSChristian Brunner return -EINVAL; 327f27aaf4bSChristian Brunner } 328f27aaf4bSChristian Brunner obj_order = ffs(objsize) - 1; 329f27aaf4bSChristian Brunner } 330f27aaf4bSChristian Brunner } 331f27aaf4bSChristian Brunner options++; 332f27aaf4bSChristian Brunner } 333f27aaf4bSChristian Brunner 3347c7e9df0SSage Weil clientname = qemu_rbd_parse_clientname(conf, clientname_buf); 3357c7e9df0SSage Weil if (rados_create(&cluster, clientname) < 0) { 336f27aaf4bSChristian Brunner error_report("error initializing"); 337f27aaf4bSChristian Brunner return -EIO; 338f27aaf4bSChristian Brunner } 339f27aaf4bSChristian Brunner 340fab5cf59SJosh Durgin if (strstr(conf, "conf=") == NULL) { 341f9fe18ecSSage Weil /* try default location, but ignore failure */ 342f9fe18ecSSage Weil rados_conf_read_file(cluster, NULL); 343fab5cf59SJosh Durgin } 344fab5cf59SJosh Durgin 345fab5cf59SJosh Durgin if (conf[0] != '\0' && 346fab5cf59SJosh Durgin qemu_rbd_set_conf(cluster, conf) < 0) { 347fab5cf59SJosh Durgin error_report("error setting config options"); 348fab5cf59SJosh Durgin rados_shutdown(cluster); 349fab5cf59SJosh Durgin return -EIO; 350fab5cf59SJosh Durgin } 351ad32e9c0SJosh Durgin 352ad32e9c0SJosh Durgin if (rados_connect(cluster) < 0) { 353ad32e9c0SJosh Durgin error_report("error connecting"); 354ad32e9c0SJosh Durgin rados_shutdown(cluster); 355ad32e9c0SJosh Durgin return -EIO; 356ad32e9c0SJosh Durgin } 357ad32e9c0SJosh Durgin 358ad32e9c0SJosh Durgin if (rados_ioctx_create(cluster, pool, &io_ctx) < 0) { 359f27aaf4bSChristian Brunner error_report("error opening pool %s", pool); 360ad32e9c0SJosh Durgin rados_shutdown(cluster); 361f27aaf4bSChristian Brunner return -EIO; 362f27aaf4bSChristian Brunner } 363f27aaf4bSChristian Brunner 364ad32e9c0SJosh Durgin ret = rbd_create(io_ctx, name, bytes, &obj_order); 365ad32e9c0SJosh Durgin rados_ioctx_destroy(io_ctx); 366ad32e9c0SJosh Durgin rados_shutdown(cluster); 367f27aaf4bSChristian Brunner 368f27aaf4bSChristian Brunner return ret; 369f27aaf4bSChristian Brunner } 370f27aaf4bSChristian Brunner 371f27aaf4bSChristian Brunner /* 372ad32e9c0SJosh Durgin * This aio completion is being called from qemu_rbd_aio_event_reader() 373ad32e9c0SJosh Durgin * and runs in qemu context. It schedules a bh, but just in case the aio 374f27aaf4bSChristian Brunner * was not cancelled before. 375f27aaf4bSChristian Brunner */ 376ad32e9c0SJosh Durgin static void qemu_rbd_complete_aio(RADOSCB *rcb) 377f27aaf4bSChristian Brunner { 378f27aaf4bSChristian Brunner RBDAIOCB *acb = rcb->acb; 379f27aaf4bSChristian Brunner int64_t r; 380f27aaf4bSChristian Brunner 381f27aaf4bSChristian Brunner r = rcb->ret; 382f27aaf4bSChristian Brunner 383dc7588c1SJosh Durgin if (acb->cmd != RBD_AIO_READ) { 384f27aaf4bSChristian Brunner if (r < 0) { 385f27aaf4bSChristian Brunner acb->ret = r; 386f27aaf4bSChristian Brunner acb->error = 1; 387f27aaf4bSChristian Brunner } else if (!acb->error) { 388ad32e9c0SJosh Durgin acb->ret = rcb->size; 389f27aaf4bSChristian Brunner } 390f27aaf4bSChristian Brunner } else { 391ad32e9c0SJosh Durgin if (r < 0) { 392ad32e9c0SJosh Durgin memset(rcb->buf, 0, rcb->size); 393f27aaf4bSChristian Brunner acb->ret = r; 394f27aaf4bSChristian Brunner acb->error = 1; 395ad32e9c0SJosh Durgin } else if (r < rcb->size) { 396ad32e9c0SJosh Durgin memset(rcb->buf + r, 0, rcb->size - r); 397f27aaf4bSChristian Brunner if (!acb->error) { 398ad32e9c0SJosh Durgin acb->ret = rcb->size; 399f27aaf4bSChristian Brunner } 400f27aaf4bSChristian Brunner } else if (!acb->error) { 401ad32e9c0SJosh Durgin acb->ret = r; 402f27aaf4bSChristian Brunner } 403f27aaf4bSChristian Brunner } 404f27aaf4bSChristian Brunner /* Note that acb->bh can be NULL in case where the aio was cancelled */ 405f27aaf4bSChristian Brunner acb->bh = qemu_bh_new(rbd_aio_bh_cb, acb); 406f27aaf4bSChristian Brunner qemu_bh_schedule(acb->bh); 4077267c094SAnthony Liguori g_free(rcb); 408f27aaf4bSChristian Brunner } 409f27aaf4bSChristian Brunner 410f27aaf4bSChristian Brunner /* 411f27aaf4bSChristian Brunner * aio fd read handler. It runs in the qemu context and calls the 412f27aaf4bSChristian Brunner * completion handling of completed rados aio operations. 413f27aaf4bSChristian Brunner */ 414ad32e9c0SJosh Durgin static void qemu_rbd_aio_event_reader(void *opaque) 415f27aaf4bSChristian Brunner { 416f27aaf4bSChristian Brunner BDRVRBDState *s = opaque; 417f27aaf4bSChristian Brunner 418f27aaf4bSChristian Brunner ssize_t ret; 419f27aaf4bSChristian Brunner 420f27aaf4bSChristian Brunner do { 421f27aaf4bSChristian Brunner char *p = (char *)&s->event_rcb; 422f27aaf4bSChristian Brunner 423f27aaf4bSChristian Brunner /* now read the rcb pointer that was sent from a non qemu thread */ 424dfe80b07SSage Weil ret = read(s->fds[RBD_FD_READ], p + s->event_reader_pos, 425dfe80b07SSage Weil sizeof(s->event_rcb) - s->event_reader_pos); 426f27aaf4bSChristian Brunner if (ret > 0) { 427f27aaf4bSChristian Brunner s->event_reader_pos += ret; 428f27aaf4bSChristian Brunner if (s->event_reader_pos == sizeof(s->event_rcb)) { 429f27aaf4bSChristian Brunner s->event_reader_pos = 0; 430ad32e9c0SJosh Durgin qemu_rbd_complete_aio(s->event_rcb); 431f27aaf4bSChristian Brunner s->qemu_aio_count--; 432f27aaf4bSChristian Brunner } 433f27aaf4bSChristian Brunner } 434f27aaf4bSChristian Brunner } while (ret < 0 && errno == EINTR); 435f27aaf4bSChristian Brunner } 436f27aaf4bSChristian Brunner 437ad32e9c0SJosh Durgin static int qemu_rbd_aio_flush_cb(void *opaque) 438f27aaf4bSChristian Brunner { 439f27aaf4bSChristian Brunner BDRVRBDState *s = opaque; 440f27aaf4bSChristian Brunner 441f27aaf4bSChristian Brunner return (s->qemu_aio_count > 0); 442f27aaf4bSChristian Brunner } 443f27aaf4bSChristian Brunner 444a9ccedc3SKevin Wolf /* TODO Convert to fine grained options */ 445a9ccedc3SKevin Wolf static QemuOptsList runtime_opts = { 446a9ccedc3SKevin Wolf .name = "rbd", 447a9ccedc3SKevin Wolf .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head), 448a9ccedc3SKevin Wolf .desc = { 449a9ccedc3SKevin Wolf { 450a9ccedc3SKevin Wolf .name = "filename", 451a9ccedc3SKevin Wolf .type = QEMU_OPT_STRING, 452a9ccedc3SKevin Wolf .help = "Specification of the rbd image", 453a9ccedc3SKevin Wolf }, 454a9ccedc3SKevin Wolf { /* end of list */ } 455a9ccedc3SKevin Wolf }, 456a9ccedc3SKevin Wolf }; 457a9ccedc3SKevin Wolf 45856d1b4d2SKevin Wolf static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags) 459f27aaf4bSChristian Brunner { 460f27aaf4bSChristian Brunner BDRVRBDState *s = bs->opaque; 461ad32e9c0SJosh Durgin char pool[RBD_MAX_POOL_NAME_SIZE]; 462ad32e9c0SJosh Durgin char snap_buf[RBD_MAX_SNAP_NAME_SIZE]; 463fab5cf59SJosh Durgin char conf[RBD_MAX_CONF_SIZE]; 4647c7e9df0SSage Weil char clientname_buf[RBD_MAX_CONF_SIZE]; 4657c7e9df0SSage Weil char *clientname; 466a9ccedc3SKevin Wolf QemuOpts *opts; 467a9ccedc3SKevin Wolf Error *local_err = NULL; 468a9ccedc3SKevin Wolf const char *filename; 469f27aaf4bSChristian Brunner int r; 470f27aaf4bSChristian Brunner 471a9ccedc3SKevin Wolf opts = qemu_opts_create_nofail(&runtime_opts); 472a9ccedc3SKevin Wolf qemu_opts_absorb_qdict(opts, options, &local_err); 473a9ccedc3SKevin Wolf if (error_is_set(&local_err)) { 474a9ccedc3SKevin Wolf qerror_report_err(local_err); 475a9ccedc3SKevin Wolf error_free(local_err); 476a9ccedc3SKevin Wolf qemu_opts_del(opts); 477a9ccedc3SKevin Wolf return -EINVAL; 478a9ccedc3SKevin Wolf } 479a9ccedc3SKevin Wolf 480a9ccedc3SKevin Wolf filename = qemu_opt_get(opts, "filename"); 481a9ccedc3SKevin Wolf 482ad32e9c0SJosh Durgin if (qemu_rbd_parsename(filename, pool, sizeof(pool), 483f27aaf4bSChristian Brunner snap_buf, sizeof(snap_buf), 484fab5cf59SJosh Durgin s->name, sizeof(s->name), 485fab5cf59SJosh Durgin conf, sizeof(conf)) < 0) { 486*c3ca988dSKevin Wolf r = -EINVAL; 487*c3ca988dSKevin Wolf goto failed_opts; 488f27aaf4bSChristian Brunner } 489f27aaf4bSChristian Brunner 4907c7e9df0SSage Weil clientname = qemu_rbd_parse_clientname(conf, clientname_buf); 4917c7e9df0SSage Weil r = rados_create(&s->cluster, clientname); 492ad32e9c0SJosh Durgin if (r < 0) { 493f27aaf4bSChristian Brunner error_report("error initializing"); 494*c3ca988dSKevin Wolf goto failed_opts; 495f27aaf4bSChristian Brunner } 496f27aaf4bSChristian Brunner 497eb93d5d9SSage Weil s->snap = NULL; 498eb93d5d9SSage Weil if (snap_buf[0] != '\0') { 499eb93d5d9SSage Weil s->snap = g_strdup(snap_buf); 500eb93d5d9SSage Weil } 501eb93d5d9SSage Weil 502b11f38fcSJosh Durgin /* 503b11f38fcSJosh Durgin * Fallback to more conservative semantics if setting cache 504b11f38fcSJosh Durgin * options fails. Ignore errors from setting rbd_cache because the 505b11f38fcSJosh Durgin * only possible error is that the option does not exist, and 506b11f38fcSJosh Durgin * librbd defaults to no caching. If write through caching cannot 507b11f38fcSJosh Durgin * be set up, fall back to no caching. 508b11f38fcSJosh Durgin */ 509b11f38fcSJosh Durgin if (flags & BDRV_O_NOCACHE) { 510b11f38fcSJosh Durgin rados_conf_set(s->cluster, "rbd_cache", "false"); 511b11f38fcSJosh Durgin } else { 512b11f38fcSJosh Durgin rados_conf_set(s->cluster, "rbd_cache", "true"); 513b11f38fcSJosh Durgin } 514b11f38fcSJosh Durgin 515fab5cf59SJosh Durgin if (strstr(conf, "conf=") == NULL) { 516f9fe18ecSSage Weil /* try default location, but ignore failure */ 517f9fe18ecSSage Weil rados_conf_read_file(s->cluster, NULL); 518fab5cf59SJosh Durgin } 519fab5cf59SJosh Durgin 520fab5cf59SJosh Durgin if (conf[0] != '\0') { 521fab5cf59SJosh Durgin r = qemu_rbd_set_conf(s->cluster, conf); 522fab5cf59SJosh Durgin if (r < 0) { 523fab5cf59SJosh Durgin error_report("error setting config options"); 524eb93d5d9SSage Weil goto failed_shutdown; 525fab5cf59SJosh Durgin } 526fab5cf59SJosh Durgin } 527f27aaf4bSChristian Brunner 528ad32e9c0SJosh Durgin r = rados_connect(s->cluster); 529ad32e9c0SJosh Durgin if (r < 0) { 530ad32e9c0SJosh Durgin error_report("error connecting"); 531eb93d5d9SSage Weil goto failed_shutdown; 532ad32e9c0SJosh Durgin } 533ad32e9c0SJosh Durgin 534ad32e9c0SJosh Durgin r = rados_ioctx_create(s->cluster, pool, &s->io_ctx); 535ad32e9c0SJosh Durgin if (r < 0) { 536ad32e9c0SJosh Durgin error_report("error opening pool %s", pool); 537eb93d5d9SSage Weil goto failed_shutdown; 538ad32e9c0SJosh Durgin } 539ad32e9c0SJosh Durgin 540ad32e9c0SJosh Durgin r = rbd_open(s->io_ctx, s->name, &s->image, s->snap); 541ad32e9c0SJosh Durgin if (r < 0) { 542ad32e9c0SJosh Durgin error_report("error reading header from %s", s->name); 543eb93d5d9SSage Weil goto failed_open; 544ad32e9c0SJosh Durgin } 545ad32e9c0SJosh Durgin 546ad32e9c0SJosh Durgin bs->read_only = (s->snap != NULL); 547f27aaf4bSChristian Brunner 548f27aaf4bSChristian Brunner s->event_reader_pos = 0; 549f27aaf4bSChristian Brunner r = qemu_pipe(s->fds); 550f27aaf4bSChristian Brunner if (r < 0) { 551f27aaf4bSChristian Brunner error_report("error opening eventfd"); 552f27aaf4bSChristian Brunner goto failed; 553f27aaf4bSChristian Brunner } 554f27aaf4bSChristian Brunner fcntl(s->fds[0], F_SETFL, O_NONBLOCK); 555f27aaf4bSChristian Brunner fcntl(s->fds[1], F_SETFL, O_NONBLOCK); 556ad32e9c0SJosh Durgin qemu_aio_set_fd_handler(s->fds[RBD_FD_READ], qemu_rbd_aio_event_reader, 557bafbd6a1SPaolo Bonzini NULL, qemu_rbd_aio_flush_cb, s); 558f27aaf4bSChristian Brunner 559f27aaf4bSChristian Brunner 560*c3ca988dSKevin Wolf qemu_opts_del(opts); 561f27aaf4bSChristian Brunner return 0; 562f27aaf4bSChristian Brunner 563f27aaf4bSChristian Brunner failed: 564ad32e9c0SJosh Durgin rbd_close(s->image); 565eb93d5d9SSage Weil failed_open: 566ad32e9c0SJosh Durgin rados_ioctx_destroy(s->io_ctx); 567eb93d5d9SSage Weil failed_shutdown: 568ad32e9c0SJosh Durgin rados_shutdown(s->cluster); 569eb93d5d9SSage Weil g_free(s->snap); 570*c3ca988dSKevin Wolf failed_opts: 571*c3ca988dSKevin Wolf qemu_opts_del(opts); 572f27aaf4bSChristian Brunner return r; 573f27aaf4bSChristian Brunner } 574f27aaf4bSChristian Brunner 575ad32e9c0SJosh Durgin static void qemu_rbd_close(BlockDriverState *bs) 576f27aaf4bSChristian Brunner { 577f27aaf4bSChristian Brunner BDRVRBDState *s = bs->opaque; 578f27aaf4bSChristian Brunner 579f27aaf4bSChristian Brunner close(s->fds[0]); 580f27aaf4bSChristian Brunner close(s->fds[1]); 581bafbd6a1SPaolo Bonzini qemu_aio_set_fd_handler(s->fds[RBD_FD_READ], NULL, NULL, NULL, NULL); 582f27aaf4bSChristian Brunner 583ad32e9c0SJosh Durgin rbd_close(s->image); 584ad32e9c0SJosh Durgin rados_ioctx_destroy(s->io_ctx); 5857267c094SAnthony Liguori g_free(s->snap); 586ad32e9c0SJosh Durgin rados_shutdown(s->cluster); 587f27aaf4bSChristian Brunner } 588f27aaf4bSChristian Brunner 589f27aaf4bSChristian Brunner /* 590f27aaf4bSChristian Brunner * Cancel aio. Since we don't reference acb in a non qemu threads, 591f27aaf4bSChristian Brunner * it is safe to access it here. 592f27aaf4bSChristian Brunner */ 593ad32e9c0SJosh Durgin static void qemu_rbd_aio_cancel(BlockDriverAIOCB *blockacb) 594f27aaf4bSChristian Brunner { 595f27aaf4bSChristian Brunner RBDAIOCB *acb = (RBDAIOCB *) blockacb; 596f27aaf4bSChristian Brunner acb->cancelled = 1; 597473c7f02SStefan Priebe 598473c7f02SStefan Priebe while (acb->status == -EINPROGRESS) { 599473c7f02SStefan Priebe qemu_aio_wait(); 600473c7f02SStefan Priebe } 601473c7f02SStefan Priebe 602473c7f02SStefan Priebe qemu_aio_release(acb); 603f27aaf4bSChristian Brunner } 604f27aaf4bSChristian Brunner 605d7331bedSStefan Hajnoczi static const AIOCBInfo rbd_aiocb_info = { 606f27aaf4bSChristian Brunner .aiocb_size = sizeof(RBDAIOCB), 607ad32e9c0SJosh Durgin .cancel = qemu_rbd_aio_cancel, 608f27aaf4bSChristian Brunner }; 609f27aaf4bSChristian Brunner 610ad32e9c0SJosh Durgin static int qemu_rbd_send_pipe(BDRVRBDState *s, RADOSCB *rcb) 611f27aaf4bSChristian Brunner { 612ad32e9c0SJosh Durgin int ret = 0; 613f27aaf4bSChristian Brunner while (1) { 614f27aaf4bSChristian Brunner fd_set wfd; 615ad32e9c0SJosh Durgin int fd = s->fds[RBD_FD_WRITE]; 616f27aaf4bSChristian Brunner 617ad32e9c0SJosh Durgin /* send the op pointer to the qemu thread that is responsible 618ad32e9c0SJosh Durgin for the aio/op completion. Must do it in a qemu thread context */ 619f27aaf4bSChristian Brunner ret = write(fd, (void *)&rcb, sizeof(rcb)); 620f27aaf4bSChristian Brunner if (ret >= 0) { 621f27aaf4bSChristian Brunner break; 622f27aaf4bSChristian Brunner } 623f27aaf4bSChristian Brunner if (errno == EINTR) { 624f27aaf4bSChristian Brunner continue; 625f27aaf4bSChristian Brunner } 626f27aaf4bSChristian Brunner if (errno != EAGAIN) { 627f27aaf4bSChristian Brunner break; 628f27aaf4bSChristian Brunner } 629f27aaf4bSChristian Brunner 630f27aaf4bSChristian Brunner FD_ZERO(&wfd); 631f27aaf4bSChristian Brunner FD_SET(fd, &wfd); 632f27aaf4bSChristian Brunner do { 633f27aaf4bSChristian Brunner ret = select(fd + 1, NULL, &wfd, NULL, NULL); 634f27aaf4bSChristian Brunner } while (ret < 0 && errno == EINTR); 635f27aaf4bSChristian Brunner } 636f27aaf4bSChristian Brunner 637ad32e9c0SJosh Durgin return ret; 638ad32e9c0SJosh Durgin } 639ad32e9c0SJosh Durgin 640ad32e9c0SJosh Durgin /* 641ad32e9c0SJosh Durgin * This is the callback function for rbd_aio_read and _write 642ad32e9c0SJosh Durgin * 643ad32e9c0SJosh Durgin * Note: this function is being called from a non qemu thread so 644ad32e9c0SJosh Durgin * we need to be careful about what we do here. Generally we only 645ad32e9c0SJosh Durgin * write to the block notification pipe, and do the rest of the 646ad32e9c0SJosh Durgin * io completion handling from qemu_rbd_aio_event_reader() which 647ad32e9c0SJosh Durgin * runs in a qemu context. 648ad32e9c0SJosh Durgin */ 649ad32e9c0SJosh Durgin static void rbd_finish_aiocb(rbd_completion_t c, RADOSCB *rcb) 650ad32e9c0SJosh Durgin { 651ad32e9c0SJosh Durgin int ret; 652ad32e9c0SJosh Durgin rcb->ret = rbd_aio_get_return_value(c); 653ad32e9c0SJosh Durgin rbd_aio_release(c); 654ad32e9c0SJosh Durgin ret = qemu_rbd_send_pipe(rcb->s, rcb); 655f27aaf4bSChristian Brunner if (ret < 0) { 656ad32e9c0SJosh Durgin error_report("failed writing to acb->s->fds"); 6577267c094SAnthony Liguori g_free(rcb); 658f27aaf4bSChristian Brunner } 659f27aaf4bSChristian Brunner } 660f27aaf4bSChristian Brunner 661ad32e9c0SJosh Durgin /* Callback when all queued rbd_aio requests are complete */ 662f27aaf4bSChristian Brunner 663f27aaf4bSChristian Brunner static void rbd_aio_bh_cb(void *opaque) 664f27aaf4bSChristian Brunner { 665f27aaf4bSChristian Brunner RBDAIOCB *acb = opaque; 666f27aaf4bSChristian Brunner 667787f3133SJosh Durgin if (acb->cmd == RBD_AIO_READ) { 66803396148SMichael Tokarev qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size); 669f27aaf4bSChristian Brunner } 670f27aaf4bSChristian Brunner qemu_vfree(acb->bounce); 671f27aaf4bSChristian Brunner acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret)); 672f27aaf4bSChristian Brunner qemu_bh_delete(acb->bh); 673f27aaf4bSChristian Brunner acb->bh = NULL; 674473c7f02SStefan Priebe acb->status = 0; 675f27aaf4bSChristian Brunner 676473c7f02SStefan Priebe if (!acb->cancelled) { 677f27aaf4bSChristian Brunner qemu_aio_release(acb); 678f27aaf4bSChristian Brunner } 679473c7f02SStefan Priebe } 680f27aaf4bSChristian Brunner 681787f3133SJosh Durgin static int rbd_aio_discard_wrapper(rbd_image_t image, 682787f3133SJosh Durgin uint64_t off, 683787f3133SJosh Durgin uint64_t len, 684787f3133SJosh Durgin rbd_completion_t comp) 685787f3133SJosh Durgin { 686787f3133SJosh Durgin #ifdef LIBRBD_SUPPORTS_DISCARD 687787f3133SJosh Durgin return rbd_aio_discard(image, off, len, comp); 688787f3133SJosh Durgin #else 689787f3133SJosh Durgin return -ENOTSUP; 690787f3133SJosh Durgin #endif 691787f3133SJosh Durgin } 692787f3133SJosh Durgin 693dc7588c1SJosh Durgin static int rbd_aio_flush_wrapper(rbd_image_t image, 694dc7588c1SJosh Durgin rbd_completion_t comp) 695dc7588c1SJosh Durgin { 696dc7588c1SJosh Durgin #ifdef LIBRBD_SUPPORTS_AIO_FLUSH 697dc7588c1SJosh Durgin return rbd_aio_flush(image, comp); 698dc7588c1SJosh Durgin #else 699dc7588c1SJosh Durgin return -ENOTSUP; 700dc7588c1SJosh Durgin #endif 701dc7588c1SJosh Durgin } 702dc7588c1SJosh Durgin 703787f3133SJosh Durgin static BlockDriverAIOCB *rbd_start_aio(BlockDriverState *bs, 704f27aaf4bSChristian Brunner int64_t sector_num, 705f27aaf4bSChristian Brunner QEMUIOVector *qiov, 706f27aaf4bSChristian Brunner int nb_sectors, 707f27aaf4bSChristian Brunner BlockDriverCompletionFunc *cb, 708787f3133SJosh Durgin void *opaque, 709787f3133SJosh Durgin RBDAIOCmd cmd) 710f27aaf4bSChristian Brunner { 711f27aaf4bSChristian Brunner RBDAIOCB *acb; 712f27aaf4bSChristian Brunner RADOSCB *rcb; 713ad32e9c0SJosh Durgin rbd_completion_t c; 714f27aaf4bSChristian Brunner int64_t off, size; 715f27aaf4bSChristian Brunner char *buf; 71651a13528SJosh Durgin int r; 717f27aaf4bSChristian Brunner 718f27aaf4bSChristian Brunner BDRVRBDState *s = bs->opaque; 719f27aaf4bSChristian Brunner 720d7331bedSStefan Hajnoczi acb = qemu_aio_get(&rbd_aiocb_info, bs, cb, opaque); 721787f3133SJosh Durgin acb->cmd = cmd; 722f27aaf4bSChristian Brunner acb->qiov = qiov; 723dc7588c1SJosh Durgin if (cmd == RBD_AIO_DISCARD || cmd == RBD_AIO_FLUSH) { 724787f3133SJosh Durgin acb->bounce = NULL; 725787f3133SJosh Durgin } else { 726f27aaf4bSChristian Brunner acb->bounce = qemu_blockalign(bs, qiov->size); 727787f3133SJosh Durgin } 728f27aaf4bSChristian Brunner acb->ret = 0; 729f27aaf4bSChristian Brunner acb->error = 0; 730f27aaf4bSChristian Brunner acb->s = s; 731f27aaf4bSChristian Brunner acb->cancelled = 0; 732f27aaf4bSChristian Brunner acb->bh = NULL; 733473c7f02SStefan Priebe acb->status = -EINPROGRESS; 734f27aaf4bSChristian Brunner 735787f3133SJosh Durgin if (cmd == RBD_AIO_WRITE) { 736d5e6b161SMichael Tokarev qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size); 737f27aaf4bSChristian Brunner } 738f27aaf4bSChristian Brunner 739f27aaf4bSChristian Brunner buf = acb->bounce; 740f27aaf4bSChristian Brunner 741f27aaf4bSChristian Brunner off = sector_num * BDRV_SECTOR_SIZE; 742f27aaf4bSChristian Brunner size = nb_sectors * BDRV_SECTOR_SIZE; 743f27aaf4bSChristian Brunner 744ad32e9c0SJosh Durgin s->qemu_aio_count++; /* All the RADOSCB */ 745f27aaf4bSChristian Brunner 7467267c094SAnthony Liguori rcb = g_malloc(sizeof(RADOSCB)); 747f27aaf4bSChristian Brunner rcb->done = 0; 748f27aaf4bSChristian Brunner rcb->acb = acb; 749f27aaf4bSChristian Brunner rcb->buf = buf; 750f27aaf4bSChristian Brunner rcb->s = acb->s; 751ad32e9c0SJosh Durgin rcb->size = size; 75251a13528SJosh Durgin r = rbd_aio_create_completion(rcb, (rbd_callback_t) rbd_finish_aiocb, &c); 75351a13528SJosh Durgin if (r < 0) { 75451a13528SJosh Durgin goto failed; 75551a13528SJosh Durgin } 756f27aaf4bSChristian Brunner 757787f3133SJosh Durgin switch (cmd) { 758787f3133SJosh Durgin case RBD_AIO_WRITE: 75951a13528SJosh Durgin r = rbd_aio_write(s->image, off, size, buf, c); 760787f3133SJosh Durgin break; 761787f3133SJosh Durgin case RBD_AIO_READ: 76251a13528SJosh Durgin r = rbd_aio_read(s->image, off, size, buf, c); 763787f3133SJosh Durgin break; 764787f3133SJosh Durgin case RBD_AIO_DISCARD: 765787f3133SJosh Durgin r = rbd_aio_discard_wrapper(s->image, off, size, c); 766787f3133SJosh Durgin break; 767dc7588c1SJosh Durgin case RBD_AIO_FLUSH: 768dc7588c1SJosh Durgin r = rbd_aio_flush_wrapper(s->image, c); 769dc7588c1SJosh Durgin break; 770787f3133SJosh Durgin default: 771787f3133SJosh Durgin r = -EINVAL; 77251a13528SJosh Durgin } 77351a13528SJosh Durgin 77451a13528SJosh Durgin if (r < 0) { 77551a13528SJosh Durgin goto failed; 776f27aaf4bSChristian Brunner } 777f27aaf4bSChristian Brunner 778f27aaf4bSChristian Brunner return &acb->common; 77951a13528SJosh Durgin 78051a13528SJosh Durgin failed: 7817267c094SAnthony Liguori g_free(rcb); 78251a13528SJosh Durgin s->qemu_aio_count--; 78351a13528SJosh Durgin qemu_aio_release(acb); 78451a13528SJosh Durgin return NULL; 785f27aaf4bSChristian Brunner } 786f27aaf4bSChristian Brunner 787ad32e9c0SJosh Durgin static BlockDriverAIOCB *qemu_rbd_aio_readv(BlockDriverState *bs, 788ad32e9c0SJosh Durgin int64_t sector_num, 789ad32e9c0SJosh Durgin QEMUIOVector *qiov, 790f27aaf4bSChristian Brunner int nb_sectors, 791f27aaf4bSChristian Brunner BlockDriverCompletionFunc *cb, 792f27aaf4bSChristian Brunner void *opaque) 793f27aaf4bSChristian Brunner { 794787f3133SJosh Durgin return rbd_start_aio(bs, sector_num, qiov, nb_sectors, cb, opaque, 795787f3133SJosh Durgin RBD_AIO_READ); 796f27aaf4bSChristian Brunner } 797f27aaf4bSChristian Brunner 798ad32e9c0SJosh Durgin static BlockDriverAIOCB *qemu_rbd_aio_writev(BlockDriverState *bs, 799ad32e9c0SJosh Durgin int64_t sector_num, 800ad32e9c0SJosh Durgin QEMUIOVector *qiov, 801f27aaf4bSChristian Brunner int nb_sectors, 802f27aaf4bSChristian Brunner BlockDriverCompletionFunc *cb, 803f27aaf4bSChristian Brunner void *opaque) 804f27aaf4bSChristian Brunner { 805787f3133SJosh Durgin return rbd_start_aio(bs, sector_num, qiov, nb_sectors, cb, opaque, 806787f3133SJosh Durgin RBD_AIO_WRITE); 807f27aaf4bSChristian Brunner } 808f27aaf4bSChristian Brunner 809dc7588c1SJosh Durgin #ifdef LIBRBD_SUPPORTS_AIO_FLUSH 810dc7588c1SJosh Durgin static BlockDriverAIOCB *qemu_rbd_aio_flush(BlockDriverState *bs, 811dc7588c1SJosh Durgin BlockDriverCompletionFunc *cb, 812dc7588c1SJosh Durgin void *opaque) 813dc7588c1SJosh Durgin { 814dc7588c1SJosh Durgin return rbd_start_aio(bs, 0, NULL, 0, cb, opaque, RBD_AIO_FLUSH); 815dc7588c1SJosh Durgin } 816dc7588c1SJosh Durgin 817dc7588c1SJosh Durgin #else 818dc7588c1SJosh Durgin 8198b94ff85SPaolo Bonzini static int qemu_rbd_co_flush(BlockDriverState *bs) 8207a3f5fe9SSage Weil { 8217a3f5fe9SSage Weil #if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(0, 1, 1) 8227a3f5fe9SSage Weil /* rbd_flush added in 0.1.1 */ 8237a3f5fe9SSage Weil BDRVRBDState *s = bs->opaque; 8247a3f5fe9SSage Weil return rbd_flush(s->image); 8257a3f5fe9SSage Weil #else 8267a3f5fe9SSage Weil return 0; 8277a3f5fe9SSage Weil #endif 8287a3f5fe9SSage Weil } 829dc7588c1SJosh Durgin #endif 8307a3f5fe9SSage Weil 831ad32e9c0SJosh Durgin static int qemu_rbd_getinfo(BlockDriverState *bs, BlockDriverInfo *bdi) 832f27aaf4bSChristian Brunner { 833f27aaf4bSChristian Brunner BDRVRBDState *s = bs->opaque; 834ad32e9c0SJosh Durgin rbd_image_info_t info; 835ad32e9c0SJosh Durgin int r; 836ad32e9c0SJosh Durgin 837ad32e9c0SJosh Durgin r = rbd_stat(s->image, &info, sizeof(info)); 838ad32e9c0SJosh Durgin if (r < 0) { 839ad32e9c0SJosh Durgin return r; 840ad32e9c0SJosh Durgin } 841ad32e9c0SJosh Durgin 842ad32e9c0SJosh Durgin bdi->cluster_size = info.obj_size; 843f27aaf4bSChristian Brunner return 0; 844f27aaf4bSChristian Brunner } 845f27aaf4bSChristian Brunner 846ad32e9c0SJosh Durgin static int64_t qemu_rbd_getlength(BlockDriverState *bs) 847f27aaf4bSChristian Brunner { 848f27aaf4bSChristian Brunner BDRVRBDState *s = bs->opaque; 849ad32e9c0SJosh Durgin rbd_image_info_t info; 850ad32e9c0SJosh Durgin int r; 851f27aaf4bSChristian Brunner 852ad32e9c0SJosh Durgin r = rbd_stat(s->image, &info, sizeof(info)); 853ad32e9c0SJosh Durgin if (r < 0) { 854ad32e9c0SJosh Durgin return r; 855f27aaf4bSChristian Brunner } 856f27aaf4bSChristian Brunner 857ad32e9c0SJosh Durgin return info.size; 858ad32e9c0SJosh Durgin } 859ad32e9c0SJosh Durgin 86030cdc48cSJosh Durgin static int qemu_rbd_truncate(BlockDriverState *bs, int64_t offset) 86130cdc48cSJosh Durgin { 86230cdc48cSJosh Durgin BDRVRBDState *s = bs->opaque; 86330cdc48cSJosh Durgin int r; 86430cdc48cSJosh Durgin 86530cdc48cSJosh Durgin r = rbd_resize(s->image, offset); 86630cdc48cSJosh Durgin if (r < 0) { 86730cdc48cSJosh Durgin return r; 86830cdc48cSJosh Durgin } 86930cdc48cSJosh Durgin 87030cdc48cSJosh Durgin return 0; 87130cdc48cSJosh Durgin } 87230cdc48cSJosh Durgin 873ad32e9c0SJosh Durgin static int qemu_rbd_snap_create(BlockDriverState *bs, 874ad32e9c0SJosh Durgin QEMUSnapshotInfo *sn_info) 875f27aaf4bSChristian Brunner { 876f27aaf4bSChristian Brunner BDRVRBDState *s = bs->opaque; 877f27aaf4bSChristian Brunner int r; 878f27aaf4bSChristian Brunner 879f27aaf4bSChristian Brunner if (sn_info->name[0] == '\0') { 880f27aaf4bSChristian Brunner return -EINVAL; /* we need a name for rbd snapshots */ 881f27aaf4bSChristian Brunner } 882f27aaf4bSChristian Brunner 883f27aaf4bSChristian Brunner /* 884f27aaf4bSChristian Brunner * rbd snapshots are using the name as the user controlled unique identifier 885f27aaf4bSChristian Brunner * we can't use the rbd snapid for that purpose, as it can't be set 886f27aaf4bSChristian Brunner */ 887f27aaf4bSChristian Brunner if (sn_info->id_str[0] != '\0' && 888f27aaf4bSChristian Brunner strcmp(sn_info->id_str, sn_info->name) != 0) { 889f27aaf4bSChristian Brunner return -EINVAL; 890f27aaf4bSChristian Brunner } 891f27aaf4bSChristian Brunner 892f27aaf4bSChristian Brunner if (strlen(sn_info->name) >= sizeof(sn_info->id_str)) { 893f27aaf4bSChristian Brunner return -ERANGE; 894f27aaf4bSChristian Brunner } 895f27aaf4bSChristian Brunner 896ad32e9c0SJosh Durgin r = rbd_snap_create(s->image, sn_info->name); 897f27aaf4bSChristian Brunner if (r < 0) { 898ad32e9c0SJosh Durgin error_report("failed to create snap: %s", strerror(-r)); 899f27aaf4bSChristian Brunner return r; 900f27aaf4bSChristian Brunner } 901f27aaf4bSChristian Brunner 902f27aaf4bSChristian Brunner return 0; 903f27aaf4bSChristian Brunner } 904f27aaf4bSChristian Brunner 905bd603247SGregory Farnum static int qemu_rbd_snap_remove(BlockDriverState *bs, 906bd603247SGregory Farnum const char *snapshot_name) 907bd603247SGregory Farnum { 908bd603247SGregory Farnum BDRVRBDState *s = bs->opaque; 909bd603247SGregory Farnum int r; 910bd603247SGregory Farnum 911bd603247SGregory Farnum r = rbd_snap_remove(s->image, snapshot_name); 912bd603247SGregory Farnum return r; 913bd603247SGregory Farnum } 914bd603247SGregory Farnum 915bd603247SGregory Farnum static int qemu_rbd_snap_rollback(BlockDriverState *bs, 916bd603247SGregory Farnum const char *snapshot_name) 917bd603247SGregory Farnum { 918bd603247SGregory Farnum BDRVRBDState *s = bs->opaque; 919bd603247SGregory Farnum int r; 920bd603247SGregory Farnum 921bd603247SGregory Farnum r = rbd_snap_rollback(s->image, snapshot_name); 922bd603247SGregory Farnum return r; 923bd603247SGregory Farnum } 924bd603247SGregory Farnum 925ad32e9c0SJosh Durgin static int qemu_rbd_snap_list(BlockDriverState *bs, 926ad32e9c0SJosh Durgin QEMUSnapshotInfo **psn_tab) 927f27aaf4bSChristian Brunner { 928f27aaf4bSChristian Brunner BDRVRBDState *s = bs->opaque; 929f27aaf4bSChristian Brunner QEMUSnapshotInfo *sn_info, *sn_tab = NULL; 930ad32e9c0SJosh Durgin int i, snap_count; 931ad32e9c0SJosh Durgin rbd_snap_info_t *snaps; 932ad32e9c0SJosh Durgin int max_snaps = RBD_MAX_SNAPS; 933f27aaf4bSChristian Brunner 934ad32e9c0SJosh Durgin do { 9357267c094SAnthony Liguori snaps = g_malloc(sizeof(*snaps) * max_snaps); 936ad32e9c0SJosh Durgin snap_count = rbd_snap_list(s->image, snaps, &max_snaps); 937ad32e9c0SJosh Durgin if (snap_count < 0) { 9387267c094SAnthony Liguori g_free(snaps); 939f27aaf4bSChristian Brunner } 940ad32e9c0SJosh Durgin } while (snap_count == -ERANGE); 941f27aaf4bSChristian Brunner 942ad32e9c0SJosh Durgin if (snap_count <= 0) { 943b9c53290SJosh Durgin goto done; 944f27aaf4bSChristian Brunner } 945f27aaf4bSChristian Brunner 9467267c094SAnthony Liguori sn_tab = g_malloc0(snap_count * sizeof(QEMUSnapshotInfo)); 947f27aaf4bSChristian Brunner 948ad32e9c0SJosh Durgin for (i = 0; i < snap_count; i++) { 949ad32e9c0SJosh Durgin const char *snap_name = snaps[i].name; 950f27aaf4bSChristian Brunner 951f27aaf4bSChristian Brunner sn_info = sn_tab + i; 952f27aaf4bSChristian Brunner pstrcpy(sn_info->id_str, sizeof(sn_info->id_str), snap_name); 953f27aaf4bSChristian Brunner pstrcpy(sn_info->name, sizeof(sn_info->name), snap_name); 954f27aaf4bSChristian Brunner 955ad32e9c0SJosh Durgin sn_info->vm_state_size = snaps[i].size; 956f27aaf4bSChristian Brunner sn_info->date_sec = 0; 957f27aaf4bSChristian Brunner sn_info->date_nsec = 0; 958f27aaf4bSChristian Brunner sn_info->vm_clock_nsec = 0; 959f27aaf4bSChristian Brunner } 960ad32e9c0SJosh Durgin rbd_snap_list_end(snaps); 961ad32e9c0SJosh Durgin 962b9c53290SJosh Durgin done: 963f27aaf4bSChristian Brunner *psn_tab = sn_tab; 964f27aaf4bSChristian Brunner return snap_count; 965f27aaf4bSChristian Brunner } 966f27aaf4bSChristian Brunner 967787f3133SJosh Durgin #ifdef LIBRBD_SUPPORTS_DISCARD 968787f3133SJosh Durgin static BlockDriverAIOCB* qemu_rbd_aio_discard(BlockDriverState *bs, 969787f3133SJosh Durgin int64_t sector_num, 970787f3133SJosh Durgin int nb_sectors, 971787f3133SJosh Durgin BlockDriverCompletionFunc *cb, 972787f3133SJosh Durgin void *opaque) 973787f3133SJosh Durgin { 974787f3133SJosh Durgin return rbd_start_aio(bs, sector_num, NULL, nb_sectors, cb, opaque, 975787f3133SJosh Durgin RBD_AIO_DISCARD); 976787f3133SJosh Durgin } 977787f3133SJosh Durgin #endif 978787f3133SJosh Durgin 979ad32e9c0SJosh Durgin static QEMUOptionParameter qemu_rbd_create_options[] = { 980f27aaf4bSChristian Brunner { 981f27aaf4bSChristian Brunner .name = BLOCK_OPT_SIZE, 982f27aaf4bSChristian Brunner .type = OPT_SIZE, 983f27aaf4bSChristian Brunner .help = "Virtual disk size" 984f27aaf4bSChristian Brunner }, 985f27aaf4bSChristian Brunner { 986f27aaf4bSChristian Brunner .name = BLOCK_OPT_CLUSTER_SIZE, 987f27aaf4bSChristian Brunner .type = OPT_SIZE, 988f27aaf4bSChristian Brunner .help = "RBD object size" 989f27aaf4bSChristian Brunner }, 990f27aaf4bSChristian Brunner {NULL} 991f27aaf4bSChristian Brunner }; 992f27aaf4bSChristian Brunner 993f27aaf4bSChristian Brunner static BlockDriver bdrv_rbd = { 994f27aaf4bSChristian Brunner .format_name = "rbd", 995f27aaf4bSChristian Brunner .instance_size = sizeof(BDRVRBDState), 996ad32e9c0SJosh Durgin .bdrv_file_open = qemu_rbd_open, 997ad32e9c0SJosh Durgin .bdrv_close = qemu_rbd_close, 998ad32e9c0SJosh Durgin .bdrv_create = qemu_rbd_create, 999ad32e9c0SJosh Durgin .bdrv_get_info = qemu_rbd_getinfo, 1000ad32e9c0SJosh Durgin .create_options = qemu_rbd_create_options, 1001ad32e9c0SJosh Durgin .bdrv_getlength = qemu_rbd_getlength, 100230cdc48cSJosh Durgin .bdrv_truncate = qemu_rbd_truncate, 1003f27aaf4bSChristian Brunner .protocol_name = "rbd", 1004f27aaf4bSChristian Brunner 1005ad32e9c0SJosh Durgin .bdrv_aio_readv = qemu_rbd_aio_readv, 1006ad32e9c0SJosh Durgin .bdrv_aio_writev = qemu_rbd_aio_writev, 1007dc7588c1SJosh Durgin 1008dc7588c1SJosh Durgin #ifdef LIBRBD_SUPPORTS_AIO_FLUSH 1009dc7588c1SJosh Durgin .bdrv_aio_flush = qemu_rbd_aio_flush, 1010dc7588c1SJosh Durgin #else 1011c68b89acSKevin Wolf .bdrv_co_flush_to_disk = qemu_rbd_co_flush, 1012dc7588c1SJosh Durgin #endif 1013f27aaf4bSChristian Brunner 1014787f3133SJosh Durgin #ifdef LIBRBD_SUPPORTS_DISCARD 1015787f3133SJosh Durgin .bdrv_aio_discard = qemu_rbd_aio_discard, 1016787f3133SJosh Durgin #endif 1017787f3133SJosh Durgin 1018ad32e9c0SJosh Durgin .bdrv_snapshot_create = qemu_rbd_snap_create, 1019bd603247SGregory Farnum .bdrv_snapshot_delete = qemu_rbd_snap_remove, 1020ad32e9c0SJosh Durgin .bdrv_snapshot_list = qemu_rbd_snap_list, 1021bd603247SGregory Farnum .bdrv_snapshot_goto = qemu_rbd_snap_rollback, 1022f27aaf4bSChristian Brunner }; 1023f27aaf4bSChristian Brunner 1024f27aaf4bSChristian Brunner static void bdrv_rbd_init(void) 1025f27aaf4bSChristian Brunner { 1026f27aaf4bSChristian Brunner bdrv_register(&bdrv_rbd); 1027f27aaf4bSChristian Brunner } 1028f27aaf4bSChristian Brunner 1029f27aaf4bSChristian Brunner block_init(bdrv_rbd_init); 1030