1 /*
2  * Virtio Block Device
3  *
4  * Copyright IBM, Corp. 2007
5  *
6  * Authors:
7  *  Anthony Liguori   <aliguori@us.ibm.com>
8  *
9  * This work is licensed under the terms of the GNU GPL, version 2.  See
10  * the COPYING file in the top-level directory.
11  *
12  */
13 
14 #include "qemu/osdep.h"
15 #include "qapi/error.h"
16 #include "qemu-common.h"
17 #include "qemu/iov.h"
18 #include "qemu/error-report.h"
19 #include "trace.h"
20 #include "hw/block/block.h"
21 #include "sysemu/blockdev.h"
22 #include "hw/virtio/virtio-blk.h"
23 #include "dataplane/virtio-blk.h"
24 #include "scsi/constants.h"
25 #ifdef __linux__
26 # include <scsi/sg.h>
27 #endif
28 #include "hw/virtio/virtio-bus.h"
29 #include "hw/virtio/virtio-access.h"
30 
virtio_blk_init_request(VirtIOBlock * s,VirtQueue * vq,VirtIOBlockReq * req)31 static void virtio_blk_init_request(VirtIOBlock *s, VirtQueue *vq,
32                                     VirtIOBlockReq *req)
33 {
34     req->dev = s;
35     req->vq = vq;
36     req->qiov.size = 0;
37     req->in_len = 0;
38     req->next = NULL;
39     req->mr_next = NULL;
40 }
41 
virtio_blk_free_request(VirtIOBlockReq * req)42 static void virtio_blk_free_request(VirtIOBlockReq *req)
43 {
44     g_free(req);
45 }
46 
virtio_blk_req_complete(VirtIOBlockReq * req,unsigned char status)47 static void virtio_blk_req_complete(VirtIOBlockReq *req, unsigned char status)
48 {
49     VirtIOBlock *s = req->dev;
50     VirtIODevice *vdev = VIRTIO_DEVICE(s);
51 
52     trace_virtio_blk_req_complete(vdev, req, status);
53 
54     stb_p(&req->in->status, status);
55     virtqueue_push(req->vq, &req->elem, req->in_len);
56     if (s->dataplane_started && !s->dataplane_disabled) {
57         virtio_blk_data_plane_notify(s->dataplane, req->vq);
58     } else {
59         virtio_notify(vdev, req->vq);
60     }
61 }
62 
virtio_blk_handle_rw_error(VirtIOBlockReq * req,int error,bool is_read)63 static int virtio_blk_handle_rw_error(VirtIOBlockReq *req, int error,
64     bool is_read)
65 {
66     BlockErrorAction action = blk_get_error_action(req->dev->blk,
67                                                    is_read, error);
68     VirtIOBlock *s = req->dev;
69 
70     if (action == BLOCK_ERROR_ACTION_STOP) {
71         /* Break the link as the next request is going to be parsed from the
72          * ring again. Otherwise we may end up doing a double completion! */
73         req->mr_next = NULL;
74         req->next = s->rq;
75         s->rq = req;
76     } else if (action == BLOCK_ERROR_ACTION_REPORT) {
77         virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR);
78         block_acct_failed(blk_get_stats(s->blk), &req->acct);
79         virtio_blk_free_request(req);
80     }
81 
82     blk_error_action(s->blk, action, is_read, error);
83     return action != BLOCK_ERROR_ACTION_IGNORE;
84 }
85 
virtio_blk_rw_complete(void * opaque,int ret)86 static void virtio_blk_rw_complete(void *opaque, int ret)
87 {
88     VirtIOBlockReq *next = opaque;
89     VirtIOBlock *s = next->dev;
90     VirtIODevice *vdev = VIRTIO_DEVICE(s);
91 
92     aio_context_acquire(blk_get_aio_context(s->conf.conf.blk));
93     while (next) {
94         VirtIOBlockReq *req = next;
95         next = req->mr_next;
96         trace_virtio_blk_rw_complete(vdev, req, ret);
97 
98         if (req->qiov.nalloc != -1) {
99             /* If nalloc is != 1 req->qiov is a local copy of the original
100              * external iovec. It was allocated in submit_merged_requests
101              * to be able to merge requests. */
102             qemu_iovec_destroy(&req->qiov);
103         }
104 
105         if (ret) {
106             int p = virtio_ldl_p(VIRTIO_DEVICE(req->dev), &req->out.type);
107             bool is_read = !(p & VIRTIO_BLK_T_OUT);
108             /* Note that memory may be dirtied on read failure.  If the
109              * virtio request is not completed here, as is the case for
110              * BLOCK_ERROR_ACTION_STOP, the memory may not be copied
111              * correctly during live migration.  While this is ugly,
112              * it is acceptable because the device is free to write to
113              * the memory until the request is completed (which will
114              * happen on the other side of the migration).
115              */
116             if (virtio_blk_handle_rw_error(req, -ret, is_read)) {
117                 continue;
118             }
119         }
120 
121         virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
122         block_acct_done(blk_get_stats(req->dev->blk), &req->acct);
123         virtio_blk_free_request(req);
124     }
125     aio_context_release(blk_get_aio_context(s->conf.conf.blk));
126 }
127 
virtio_blk_flush_complete(void * opaque,int ret)128 static void virtio_blk_flush_complete(void *opaque, int ret)
129 {
130     VirtIOBlockReq *req = opaque;
131     VirtIOBlock *s = req->dev;
132 
133     aio_context_acquire(blk_get_aio_context(s->conf.conf.blk));
134     if (ret) {
135         if (virtio_blk_handle_rw_error(req, -ret, 0)) {
136             goto out;
137         }
138     }
139 
140     virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
141     block_acct_done(blk_get_stats(req->dev->blk), &req->acct);
142     virtio_blk_free_request(req);
143 
144 out:
145     aio_context_release(blk_get_aio_context(s->conf.conf.blk));
146 }
147 
148 #ifdef __linux__
149 
150 typedef struct {
151     VirtIOBlockReq *req;
152     struct sg_io_hdr hdr;
153 } VirtIOBlockIoctlReq;
154 
virtio_blk_ioctl_complete(void * opaque,int status)155 static void virtio_blk_ioctl_complete(void *opaque, int status)
156 {
157     VirtIOBlockIoctlReq *ioctl_req = opaque;
158     VirtIOBlockReq *req = ioctl_req->req;
159     VirtIOBlock *s = req->dev;
160     VirtIODevice *vdev = VIRTIO_DEVICE(s);
161     struct virtio_scsi_inhdr *scsi;
162     struct sg_io_hdr *hdr;
163 
164     scsi = (void *)req->elem.in_sg[req->elem.in_num - 2].iov_base;
165 
166     if (status) {
167         status = VIRTIO_BLK_S_UNSUPP;
168         virtio_stl_p(vdev, &scsi->errors, 255);
169         goto out;
170     }
171 
172     hdr = &ioctl_req->hdr;
173     /*
174      * From SCSI-Generic-HOWTO: "Some lower level drivers (e.g. ide-scsi)
175      * clear the masked_status field [hence status gets cleared too, see
176      * block/scsi_ioctl.c] even when a CHECK_CONDITION or COMMAND_TERMINATED
177      * status has occurred.  However they do set DRIVER_SENSE in driver_status
178      * field. Also a (sb_len_wr > 0) indicates there is a sense buffer.
179      */
180     if (hdr->status == 0 && hdr->sb_len_wr > 0) {
181         hdr->status = CHECK_CONDITION;
182     }
183 
184     virtio_stl_p(vdev, &scsi->errors,
185                  hdr->status | (hdr->msg_status << 8) |
186                  (hdr->host_status << 16) | (hdr->driver_status << 24));
187     virtio_stl_p(vdev, &scsi->residual, hdr->resid);
188     virtio_stl_p(vdev, &scsi->sense_len, hdr->sb_len_wr);
189     virtio_stl_p(vdev, &scsi->data_len, hdr->dxfer_len);
190 
191 out:
192     aio_context_acquire(blk_get_aio_context(s->conf.conf.blk));
193     virtio_blk_req_complete(req, status);
194     virtio_blk_free_request(req);
195     aio_context_release(blk_get_aio_context(s->conf.conf.blk));
196     g_free(ioctl_req);
197 }
198 
199 #endif
200 
virtio_blk_get_request(VirtIOBlock * s,VirtQueue * vq)201 static VirtIOBlockReq *virtio_blk_get_request(VirtIOBlock *s, VirtQueue *vq)
202 {
203     VirtIOBlockReq *req = virtqueue_pop(vq, sizeof(VirtIOBlockReq));
204 
205     if (req) {
206         virtio_blk_init_request(s, vq, req);
207     }
208     return req;
209 }
210 
virtio_blk_handle_scsi_req(VirtIOBlockReq * req)211 static int virtio_blk_handle_scsi_req(VirtIOBlockReq *req)
212 {
213     int status = VIRTIO_BLK_S_OK;
214     struct virtio_scsi_inhdr *scsi = NULL;
215     VirtIODevice *vdev = VIRTIO_DEVICE(req->dev);
216     VirtQueueElement *elem = &req->elem;
217     VirtIOBlock *blk = req->dev;
218 
219 #ifdef __linux__
220     int i;
221     VirtIOBlockIoctlReq *ioctl_req;
222     BlockAIOCB *acb;
223 #endif
224 
225     /*
226      * We require at least one output segment each for the virtio_blk_outhdr
227      * and the SCSI command block.
228      *
229      * We also at least require the virtio_blk_inhdr, the virtio_scsi_inhdr
230      * and the sense buffer pointer in the input segments.
231      */
232     if (elem->out_num < 2 || elem->in_num < 3) {
233         status = VIRTIO_BLK_S_IOERR;
234         goto fail;
235     }
236 
237     /*
238      * The scsi inhdr is placed in the second-to-last input segment, just
239      * before the regular inhdr.
240      */
241     scsi = (void *)elem->in_sg[elem->in_num - 2].iov_base;
242 
243     if (!blk->conf.scsi) {
244         status = VIRTIO_BLK_S_UNSUPP;
245         goto fail;
246     }
247 
248     /*
249      * No support for bidirection commands yet.
250      */
251     if (elem->out_num > 2 && elem->in_num > 3) {
252         status = VIRTIO_BLK_S_UNSUPP;
253         goto fail;
254     }
255 
256 #ifdef __linux__
257     ioctl_req = g_new0(VirtIOBlockIoctlReq, 1);
258     ioctl_req->req = req;
259     ioctl_req->hdr.interface_id = 'S';
260     ioctl_req->hdr.cmd_len = elem->out_sg[1].iov_len;
261     ioctl_req->hdr.cmdp = elem->out_sg[1].iov_base;
262     ioctl_req->hdr.dxfer_len = 0;
263 
264     if (elem->out_num > 2) {
265         /*
266          * If there are more than the minimally required 2 output segments
267          * there is write payload starting from the third iovec.
268          */
269         ioctl_req->hdr.dxfer_direction = SG_DXFER_TO_DEV;
270         ioctl_req->hdr.iovec_count = elem->out_num - 2;
271 
272         for (i = 0; i < ioctl_req->hdr.iovec_count; i++) {
273             ioctl_req->hdr.dxfer_len += elem->out_sg[i + 2].iov_len;
274         }
275 
276         ioctl_req->hdr.dxferp = elem->out_sg + 2;
277 
278     } else if (elem->in_num > 3) {
279         /*
280          * If we have more than 3 input segments the guest wants to actually
281          * read data.
282          */
283         ioctl_req->hdr.dxfer_direction = SG_DXFER_FROM_DEV;
284         ioctl_req->hdr.iovec_count = elem->in_num - 3;
285         for (i = 0; i < ioctl_req->hdr.iovec_count; i++) {
286             ioctl_req->hdr.dxfer_len += elem->in_sg[i].iov_len;
287         }
288 
289         ioctl_req->hdr.dxferp = elem->in_sg;
290     } else {
291         /*
292          * Some SCSI commands don't actually transfer any data.
293          */
294         ioctl_req->hdr.dxfer_direction = SG_DXFER_NONE;
295     }
296 
297     ioctl_req->hdr.sbp = elem->in_sg[elem->in_num - 3].iov_base;
298     ioctl_req->hdr.mx_sb_len = elem->in_sg[elem->in_num - 3].iov_len;
299 
300     acb = blk_aio_ioctl(blk->blk, SG_IO, &ioctl_req->hdr,
301                         virtio_blk_ioctl_complete, ioctl_req);
302     if (!acb) {
303         g_free(ioctl_req);
304         status = VIRTIO_BLK_S_UNSUPP;
305         goto fail;
306     }
307     return -EINPROGRESS;
308 #else
309     abort();
310 #endif
311 
312 fail:
313     /* Just put anything nonzero so that the ioctl fails in the guest.  */
314     if (scsi) {
315         virtio_stl_p(vdev, &scsi->errors, 255);
316     }
317     return status;
318 }
319 
virtio_blk_handle_scsi(VirtIOBlockReq * req)320 static void virtio_blk_handle_scsi(VirtIOBlockReq *req)
321 {
322     int status;
323 
324     status = virtio_blk_handle_scsi_req(req);
325     if (status != -EINPROGRESS) {
326         virtio_blk_req_complete(req, status);
327         virtio_blk_free_request(req);
328     }
329 }
330 
submit_requests(BlockBackend * blk,MultiReqBuffer * mrb,int start,int num_reqs,int niov)331 static inline void submit_requests(BlockBackend *blk, MultiReqBuffer *mrb,
332                                    int start, int num_reqs, int niov)
333 {
334     QEMUIOVector *qiov = &mrb->reqs[start]->qiov;
335     int64_t sector_num = mrb->reqs[start]->sector_num;
336     bool is_write = mrb->is_write;
337 
338     if (num_reqs > 1) {
339         int i;
340         struct iovec *tmp_iov = qiov->iov;
341         int tmp_niov = qiov->niov;
342 
343         /* mrb->reqs[start]->qiov was initialized from external so we can't
344          * modify it here. We need to initialize it locally and then add the
345          * external iovecs. */
346         qemu_iovec_init(qiov, niov);
347 
348         for (i = 0; i < tmp_niov; i++) {
349             qemu_iovec_add(qiov, tmp_iov[i].iov_base, tmp_iov[i].iov_len);
350         }
351 
352         for (i = start + 1; i < start + num_reqs; i++) {
353             qemu_iovec_concat(qiov, &mrb->reqs[i]->qiov, 0,
354                               mrb->reqs[i]->qiov.size);
355             mrb->reqs[i - 1]->mr_next = mrb->reqs[i];
356         }
357 
358         trace_virtio_blk_submit_multireq(VIRTIO_DEVICE(mrb->reqs[start]->dev),
359                                          mrb, start, num_reqs,
360                                          sector_num << BDRV_SECTOR_BITS,
361                                          qiov->size, is_write);
362         block_acct_merge_done(blk_get_stats(blk),
363                               is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ,
364                               num_reqs - 1);
365     }
366 
367     if (is_write) {
368         blk_aio_pwritev(blk, sector_num << BDRV_SECTOR_BITS, qiov, 0,
369                         virtio_blk_rw_complete, mrb->reqs[start]);
370     } else {
371         blk_aio_preadv(blk, sector_num << BDRV_SECTOR_BITS, qiov, 0,
372                        virtio_blk_rw_complete, mrb->reqs[start]);
373     }
374 }
375 
multireq_compare(const void * a,const void * b)376 static int multireq_compare(const void *a, const void *b)
377 {
378     const VirtIOBlockReq *req1 = *(VirtIOBlockReq **)a,
379                          *req2 = *(VirtIOBlockReq **)b;
380 
381     /*
382      * Note that we can't simply subtract sector_num1 from sector_num2
383      * here as that could overflow the return value.
384      */
385     if (req1->sector_num > req2->sector_num) {
386         return 1;
387     } else if (req1->sector_num < req2->sector_num) {
388         return -1;
389     } else {
390         return 0;
391     }
392 }
393 
virtio_blk_submit_multireq(BlockBackend * blk,MultiReqBuffer * mrb)394 static void virtio_blk_submit_multireq(BlockBackend *blk, MultiReqBuffer *mrb)
395 {
396     int i = 0, start = 0, num_reqs = 0, niov = 0, nb_sectors = 0;
397     uint32_t max_transfer;
398     int64_t sector_num = 0;
399 
400     if (mrb->num_reqs == 1) {
401         submit_requests(blk, mrb, 0, 1, -1);
402         mrb->num_reqs = 0;
403         return;
404     }
405 
406     max_transfer = blk_get_max_transfer(mrb->reqs[0]->dev->blk);
407 
408     qsort(mrb->reqs, mrb->num_reqs, sizeof(*mrb->reqs),
409           &multireq_compare);
410 
411     for (i = 0; i < mrb->num_reqs; i++) {
412         VirtIOBlockReq *req = mrb->reqs[i];
413         if (num_reqs > 0) {
414             /*
415              * NOTE: We cannot merge the requests in below situations:
416              * 1. requests are not sequential
417              * 2. merge would exceed maximum number of IOVs
418              * 3. merge would exceed maximum transfer length of backend device
419              */
420             if (sector_num + nb_sectors != req->sector_num ||
421                 niov > blk_get_max_iov(blk) - req->qiov.niov ||
422                 req->qiov.size > max_transfer ||
423                 nb_sectors > (max_transfer -
424                               req->qiov.size) / BDRV_SECTOR_SIZE) {
425                 submit_requests(blk, mrb, start, num_reqs, niov);
426                 num_reqs = 0;
427             }
428         }
429 
430         if (num_reqs == 0) {
431             sector_num = req->sector_num;
432             nb_sectors = niov = 0;
433             start = i;
434         }
435 
436         nb_sectors += req->qiov.size / BDRV_SECTOR_SIZE;
437         niov += req->qiov.niov;
438         num_reqs++;
439     }
440 
441     submit_requests(blk, mrb, start, num_reqs, niov);
442     mrb->num_reqs = 0;
443 }
444 
virtio_blk_handle_flush(VirtIOBlockReq * req,MultiReqBuffer * mrb)445 static void virtio_blk_handle_flush(VirtIOBlockReq *req, MultiReqBuffer *mrb)
446 {
447     block_acct_start(blk_get_stats(req->dev->blk), &req->acct, 0,
448                      BLOCK_ACCT_FLUSH);
449 
450     /*
451      * Make sure all outstanding writes are posted to the backing device.
452      */
453     if (mrb->is_write && mrb->num_reqs > 0) {
454         virtio_blk_submit_multireq(req->dev->blk, mrb);
455     }
456     blk_aio_flush(req->dev->blk, virtio_blk_flush_complete, req);
457 }
458 
virtio_blk_sect_range_ok(VirtIOBlock * dev,uint64_t sector,size_t size)459 static bool virtio_blk_sect_range_ok(VirtIOBlock *dev,
460                                      uint64_t sector, size_t size)
461 {
462     uint64_t nb_sectors = size >> BDRV_SECTOR_BITS;
463     uint64_t total_sectors;
464 
465     if (nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
466         return false;
467     }
468     if (sector & dev->sector_mask) {
469         return false;
470     }
471     if (size % dev->conf.conf.logical_block_size) {
472         return false;
473     }
474     blk_get_geometry(dev->blk, &total_sectors);
475     if (sector > total_sectors || nb_sectors > total_sectors - sector) {
476         return false;
477     }
478     return true;
479 }
480 
virtio_blk_handle_request(VirtIOBlockReq * req,MultiReqBuffer * mrb)481 static int virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb)
482 {
483     uint32_t type;
484     struct iovec *in_iov = req->elem.in_sg;
485     struct iovec *iov = req->elem.out_sg;
486     unsigned in_num = req->elem.in_num;
487     unsigned out_num = req->elem.out_num;
488     VirtIOBlock *s = req->dev;
489     VirtIODevice *vdev = VIRTIO_DEVICE(s);
490 
491     if (req->elem.out_num < 1 || req->elem.in_num < 1) {
492         virtio_error(vdev, "virtio-blk missing headers");
493         return -1;
494     }
495 
496     if (unlikely(iov_to_buf(iov, out_num, 0, &req->out,
497                             sizeof(req->out)) != sizeof(req->out))) {
498         virtio_error(vdev, "virtio-blk request outhdr too short");
499         return -1;
500     }
501 
502     iov_discard_front(&iov, &out_num, sizeof(req->out));
503 
504     if (in_iov[in_num - 1].iov_len < sizeof(struct virtio_blk_inhdr)) {
505         virtio_error(vdev, "virtio-blk request inhdr too short");
506         return -1;
507     }
508 
509     /* We always touch the last byte, so just see how big in_iov is.  */
510     req->in_len = iov_size(in_iov, in_num);
511     req->in = (void *)in_iov[in_num - 1].iov_base
512               + in_iov[in_num - 1].iov_len
513               - sizeof(struct virtio_blk_inhdr);
514     iov_discard_back(in_iov, &in_num, sizeof(struct virtio_blk_inhdr));
515 
516     type = virtio_ldl_p(VIRTIO_DEVICE(req->dev), &req->out.type);
517 
518     /* VIRTIO_BLK_T_OUT defines the command direction. VIRTIO_BLK_T_BARRIER
519      * is an optional flag. Although a guest should not send this flag if
520      * not negotiated we ignored it in the past. So keep ignoring it. */
521     switch (type & ~(VIRTIO_BLK_T_OUT | VIRTIO_BLK_T_BARRIER)) {
522     case VIRTIO_BLK_T_IN:
523     {
524         bool is_write = type & VIRTIO_BLK_T_OUT;
525         req->sector_num = virtio_ldq_p(VIRTIO_DEVICE(req->dev),
526                                        &req->out.sector);
527 
528         if (is_write) {
529             qemu_iovec_init_external(&req->qiov, iov, out_num);
530             trace_virtio_blk_handle_write(vdev, req, req->sector_num,
531                                           req->qiov.size / BDRV_SECTOR_SIZE);
532         } else {
533             qemu_iovec_init_external(&req->qiov, in_iov, in_num);
534             trace_virtio_blk_handle_read(vdev, req, req->sector_num,
535                                          req->qiov.size / BDRV_SECTOR_SIZE);
536         }
537 
538         if (!virtio_blk_sect_range_ok(req->dev, req->sector_num,
539                                       req->qiov.size)) {
540             virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR);
541             block_acct_invalid(blk_get_stats(req->dev->blk),
542                                is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ);
543             virtio_blk_free_request(req);
544             return 0;
545         }
546 
547         block_acct_start(blk_get_stats(req->dev->blk),
548                          &req->acct, req->qiov.size,
549                          is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ);
550 
551         /* merge would exceed maximum number of requests or IO direction
552          * changes */
553         if (mrb->num_reqs > 0 && (mrb->num_reqs == VIRTIO_BLK_MAX_MERGE_REQS ||
554                                   is_write != mrb->is_write ||
555                                   !req->dev->conf.request_merging)) {
556             virtio_blk_submit_multireq(req->dev->blk, mrb);
557         }
558 
559         assert(mrb->num_reqs < VIRTIO_BLK_MAX_MERGE_REQS);
560         mrb->reqs[mrb->num_reqs++] = req;
561         mrb->is_write = is_write;
562         break;
563     }
564     case VIRTIO_BLK_T_FLUSH:
565         virtio_blk_handle_flush(req, mrb);
566         break;
567     case VIRTIO_BLK_T_SCSI_CMD:
568         virtio_blk_handle_scsi(req);
569         break;
570     case VIRTIO_BLK_T_GET_ID:
571     {
572         VirtIOBlock *s = req->dev;
573 
574         /*
575          * NB: per existing s/n string convention the string is
576          * terminated by '\0' only when shorter than buffer.
577          */
578         const char *serial = s->conf.serial ? s->conf.serial : "";
579         size_t size = MIN(strlen(serial) + 1,
580                           MIN(iov_size(in_iov, in_num),
581                               VIRTIO_BLK_ID_BYTES));
582         iov_from_buf(in_iov, in_num, 0, serial, size);
583         virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
584         virtio_blk_free_request(req);
585         break;
586     }
587     default:
588         virtio_blk_req_complete(req, VIRTIO_BLK_S_UNSUPP);
589         virtio_blk_free_request(req);
590     }
591     return 0;
592 }
593 
virtio_blk_handle_vq(VirtIOBlock * s,VirtQueue * vq)594 bool virtio_blk_handle_vq(VirtIOBlock *s, VirtQueue *vq)
595 {
596     VirtIOBlockReq *req;
597     MultiReqBuffer mrb = {};
598     bool progress = false;
599 
600     aio_context_acquire(blk_get_aio_context(s->blk));
601     blk_io_plug(s->blk);
602 
603     do {
604         virtio_queue_set_notification(vq, 0);
605 
606         while ((req = virtio_blk_get_request(s, vq))) {
607             progress = true;
608             if (virtio_blk_handle_request(req, &mrb)) {
609                 virtqueue_detach_element(req->vq, &req->elem, 0);
610                 virtio_blk_free_request(req);
611                 break;
612             }
613         }
614 
615         virtio_queue_set_notification(vq, 1);
616     } while (!virtio_queue_empty(vq));
617 
618     if (mrb.num_reqs) {
619         virtio_blk_submit_multireq(s->blk, &mrb);
620     }
621 
622     blk_io_unplug(s->blk);
623     aio_context_release(blk_get_aio_context(s->blk));
624     return progress;
625 }
626 
virtio_blk_handle_output_do(VirtIOBlock * s,VirtQueue * vq)627 static void virtio_blk_handle_output_do(VirtIOBlock *s, VirtQueue *vq)
628 {
629     virtio_blk_handle_vq(s, vq);
630 }
631 
virtio_blk_handle_output(VirtIODevice * vdev,VirtQueue * vq)632 static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
633 {
634     VirtIOBlock *s = (VirtIOBlock *)vdev;
635 
636     if (s->dataplane) {
637         /* Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
638          * dataplane here instead of waiting for .set_status().
639          */
640         virtio_device_start_ioeventfd(vdev);
641         if (!s->dataplane_disabled) {
642             return;
643         }
644     }
645     virtio_blk_handle_output_do(s, vq);
646 }
647 
virtio_blk_dma_restart_bh(void * opaque)648 static void virtio_blk_dma_restart_bh(void *opaque)
649 {
650     VirtIOBlock *s = opaque;
651     VirtIOBlockReq *req = s->rq;
652     MultiReqBuffer mrb = {};
653 
654     qemu_bh_delete(s->bh);
655     s->bh = NULL;
656 
657     s->rq = NULL;
658 
659     aio_context_acquire(blk_get_aio_context(s->conf.conf.blk));
660     while (req) {
661         VirtIOBlockReq *next = req->next;
662         if (virtio_blk_handle_request(req, &mrb)) {
663             /* Device is now broken and won't do any processing until it gets
664              * reset. Already queued requests will be lost: let's purge them.
665              */
666             while (req) {
667                 next = req->next;
668                 virtqueue_detach_element(req->vq, &req->elem, 0);
669                 virtio_blk_free_request(req);
670                 req = next;
671             }
672             break;
673         }
674         req = next;
675     }
676 
677     if (mrb.num_reqs) {
678         virtio_blk_submit_multireq(s->blk, &mrb);
679     }
680     aio_context_release(blk_get_aio_context(s->conf.conf.blk));
681 }
682 
virtio_blk_dma_restart_cb(void * opaque,int running,RunState state)683 static void virtio_blk_dma_restart_cb(void *opaque, int running,
684                                       RunState state)
685 {
686     VirtIOBlock *s = opaque;
687 
688     if (!running) {
689         return;
690     }
691 
692     if (!s->bh) {
693         s->bh = aio_bh_new(blk_get_aio_context(s->conf.conf.blk),
694                            virtio_blk_dma_restart_bh, s);
695         qemu_bh_schedule(s->bh);
696     }
697 }
698 
virtio_blk_reset(VirtIODevice * vdev)699 static void virtio_blk_reset(VirtIODevice *vdev)
700 {
701     VirtIOBlock *s = VIRTIO_BLK(vdev);
702     AioContext *ctx;
703     VirtIOBlockReq *req;
704 
705     ctx = blk_get_aio_context(s->blk);
706     aio_context_acquire(ctx);
707     blk_drain(s->blk);
708 
709     /* We drop queued requests after blk_drain() because blk_drain() itself can
710      * produce them. */
711     while (s->rq) {
712         req = s->rq;
713         s->rq = req->next;
714         virtqueue_detach_element(req->vq, &req->elem, 0);
715         virtio_blk_free_request(req);
716     }
717 
718     aio_context_release(ctx);
719 
720     assert(!s->dataplane_started);
721     blk_set_enable_write_cache(s->blk, s->original_wce);
722 }
723 
724 /* coalesce internal state, copy to pci i/o region 0
725  */
virtio_blk_update_config(VirtIODevice * vdev,uint8_t * config)726 static void virtio_blk_update_config(VirtIODevice *vdev, uint8_t *config)
727 {
728     VirtIOBlock *s = VIRTIO_BLK(vdev);
729     BlockConf *conf = &s->conf.conf;
730     struct virtio_blk_config blkcfg;
731     uint64_t capacity;
732     int64_t length;
733     int blk_size = conf->logical_block_size;
734 
735     blk_get_geometry(s->blk, &capacity);
736     memset(&blkcfg, 0, sizeof(blkcfg));
737     virtio_stq_p(vdev, &blkcfg.capacity, capacity);
738     virtio_stl_p(vdev, &blkcfg.seg_max, 128 - 2);
739     virtio_stw_p(vdev, &blkcfg.geometry.cylinders, conf->cyls);
740     virtio_stl_p(vdev, &blkcfg.blk_size, blk_size);
741     virtio_stw_p(vdev, &blkcfg.min_io_size, conf->min_io_size / blk_size);
742     virtio_stw_p(vdev, &blkcfg.opt_io_size, conf->opt_io_size / blk_size);
743     blkcfg.geometry.heads = conf->heads;
744     /*
745      * We must ensure that the block device capacity is a multiple of
746      * the logical block size. If that is not the case, let's use
747      * sector_mask to adopt the geometry to have a correct picture.
748      * For those devices where the capacity is ok for the given geometry
749      * we don't touch the sector value of the geometry, since some devices
750      * (like s390 dasd) need a specific value. Here the capacity is already
751      * cyls*heads*secs*blk_size and the sector value is not block size
752      * divided by 512 - instead it is the amount of blk_size blocks
753      * per track (cylinder).
754      */
755     length = blk_getlength(s->blk);
756     if (length > 0 && length / conf->heads / conf->secs % blk_size) {
757         blkcfg.geometry.sectors = conf->secs & ~s->sector_mask;
758     } else {
759         blkcfg.geometry.sectors = conf->secs;
760     }
761     blkcfg.size_max = 0;
762     blkcfg.physical_block_exp = get_physical_block_exp(conf);
763     blkcfg.alignment_offset = 0;
764     blkcfg.wce = blk_enable_write_cache(s->blk);
765     virtio_stw_p(vdev, &blkcfg.num_queues, s->conf.num_queues);
766     memcpy(config, &blkcfg, sizeof(struct virtio_blk_config));
767 }
768 
virtio_blk_set_config(VirtIODevice * vdev,const uint8_t * config)769 static void virtio_blk_set_config(VirtIODevice *vdev, const uint8_t *config)
770 {
771     VirtIOBlock *s = VIRTIO_BLK(vdev);
772     struct virtio_blk_config blkcfg;
773 
774     memcpy(&blkcfg, config, sizeof(blkcfg));
775 
776     aio_context_acquire(blk_get_aio_context(s->blk));
777     blk_set_enable_write_cache(s->blk, blkcfg.wce != 0);
778     aio_context_release(blk_get_aio_context(s->blk));
779 }
780 
virtio_blk_get_features(VirtIODevice * vdev,uint64_t features,Error ** errp)781 static uint64_t virtio_blk_get_features(VirtIODevice *vdev, uint64_t features,
782                                         Error **errp)
783 {
784     VirtIOBlock *s = VIRTIO_BLK(vdev);
785 
786     virtio_add_feature(&features, VIRTIO_BLK_F_SEG_MAX);
787     virtio_add_feature(&features, VIRTIO_BLK_F_GEOMETRY);
788     virtio_add_feature(&features, VIRTIO_BLK_F_TOPOLOGY);
789     virtio_add_feature(&features, VIRTIO_BLK_F_BLK_SIZE);
790     if (virtio_has_feature(features, VIRTIO_F_VERSION_1)) {
791         if (s->conf.scsi) {
792             error_setg(errp, "Please set scsi=off for virtio-blk devices in order to use virtio 1.0");
793             return 0;
794         }
795     } else {
796         virtio_clear_feature(&features, VIRTIO_F_ANY_LAYOUT);
797         virtio_add_feature(&features, VIRTIO_BLK_F_SCSI);
798     }
799 
800     if (s->conf.config_wce) {
801         virtio_add_feature(&features, VIRTIO_BLK_F_CONFIG_WCE);
802     }
803     if (blk_enable_write_cache(s->blk)) {
804         virtio_add_feature(&features, VIRTIO_BLK_F_WCE);
805     }
806     if (blk_is_read_only(s->blk)) {
807         virtio_add_feature(&features, VIRTIO_BLK_F_RO);
808     }
809     if (s->conf.num_queues > 1) {
810         virtio_add_feature(&features, VIRTIO_BLK_F_MQ);
811     }
812 
813     return features;
814 }
815 
virtio_blk_set_status(VirtIODevice * vdev,uint8_t status)816 static void virtio_blk_set_status(VirtIODevice *vdev, uint8_t status)
817 {
818     VirtIOBlock *s = VIRTIO_BLK(vdev);
819 
820     if (!(status & (VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK))) {
821         assert(!s->dataplane_started);
822     }
823 
824     if (!(status & VIRTIO_CONFIG_S_DRIVER_OK)) {
825         return;
826     }
827 
828     /* A guest that supports VIRTIO_BLK_F_CONFIG_WCE must be able to send
829      * cache flushes.  Thus, the "auto writethrough" behavior is never
830      * necessary for guests that support the VIRTIO_BLK_F_CONFIG_WCE feature.
831      * Leaving it enabled would break the following sequence:
832      *
833      *     Guest started with "-drive cache=writethrough"
834      *     Guest sets status to 0
835      *     Guest sets DRIVER bit in status field
836      *     Guest reads host features (WCE=0, CONFIG_WCE=1)
837      *     Guest writes guest features (WCE=0, CONFIG_WCE=1)
838      *     Guest writes 1 to the WCE configuration field (writeback mode)
839      *     Guest sets DRIVER_OK bit in status field
840      *
841      * s->blk would erroneously be placed in writethrough mode.
842      */
843     if (!virtio_vdev_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE)) {
844         aio_context_acquire(blk_get_aio_context(s->blk));
845         blk_set_enable_write_cache(s->blk,
846                                    virtio_vdev_has_feature(vdev,
847                                                            VIRTIO_BLK_F_WCE));
848         aio_context_release(blk_get_aio_context(s->blk));
849     }
850 }
851 
virtio_blk_save_device(VirtIODevice * vdev,QEMUFile * f)852 static void virtio_blk_save_device(VirtIODevice *vdev, QEMUFile *f)
853 {
854     VirtIOBlock *s = VIRTIO_BLK(vdev);
855     VirtIOBlockReq *req = s->rq;
856 
857     while (req) {
858         qemu_put_sbyte(f, 1);
859 
860         if (s->conf.num_queues > 1) {
861             qemu_put_be32(f, virtio_get_queue_index(req->vq));
862         }
863 
864         qemu_put_virtqueue_element(f, &req->elem);
865         req = req->next;
866     }
867     qemu_put_sbyte(f, 0);
868 }
869 
virtio_blk_load_device(VirtIODevice * vdev,QEMUFile * f,int version_id)870 static int virtio_blk_load_device(VirtIODevice *vdev, QEMUFile *f,
871                                   int version_id)
872 {
873     VirtIOBlock *s = VIRTIO_BLK(vdev);
874 
875     while (qemu_get_sbyte(f)) {
876         unsigned nvqs = s->conf.num_queues;
877         unsigned vq_idx = 0;
878         VirtIOBlockReq *req;
879 
880         if (nvqs > 1) {
881             vq_idx = qemu_get_be32(f);
882 
883             if (vq_idx >= nvqs) {
884                 error_report("Invalid virtqueue index in request list: %#x",
885                              vq_idx);
886                 return -EINVAL;
887             }
888         }
889 
890         req = qemu_get_virtqueue_element(vdev, f, sizeof(VirtIOBlockReq));
891         virtio_blk_init_request(s, virtio_get_queue(vdev, vq_idx), req);
892         req->next = s->rq;
893         s->rq = req;
894     }
895 
896     return 0;
897 }
898 
virtio_blk_resize(void * opaque)899 static void virtio_blk_resize(void *opaque)
900 {
901     VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
902 
903     virtio_notify_config(vdev);
904 }
905 
906 static const BlockDevOps virtio_block_ops = {
907     .resize_cb = virtio_blk_resize,
908 };
909 
virtio_blk_device_realize(DeviceState * dev,Error ** errp)910 static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
911 {
912     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
913     VirtIOBlock *s = VIRTIO_BLK(dev);
914     VirtIOBlkConf *conf = &s->conf;
915     Error *err = NULL;
916     unsigned i;
917 
918     if (!conf->conf.blk) {
919         error_setg(errp, "drive property not set");
920         return;
921     }
922     if (!blk_is_inserted(conf->conf.blk)) {
923         error_setg(errp, "Device needs media, but drive is empty");
924         return;
925     }
926     if (!conf->num_queues) {
927         error_setg(errp, "num-queues property must be larger than 0");
928         return;
929     }
930     if (!is_power_of_2(conf->queue_size) ||
931         conf->queue_size > VIRTQUEUE_MAX_SIZE) {
932         error_setg(errp, "invalid queue-size property (%" PRIu16 "), "
933                    "must be a power of 2 (max %d)",
934                    conf->queue_size, VIRTQUEUE_MAX_SIZE);
935         return;
936     }
937 
938     if (!blkconf_apply_backend_options(&conf->conf,
939                                        blk_is_read_only(conf->conf.blk), true,
940                                        errp)) {
941         return;
942     }
943     s->original_wce = blk_enable_write_cache(conf->conf.blk);
944     if (!blkconf_geometry(&conf->conf, NULL, 65535, 255, 255, errp)) {
945         return;
946     }
947 
948     blkconf_blocksizes(&conf->conf);
949 
950     if (conf->conf.logical_block_size >
951         conf->conf.physical_block_size) {
952         error_setg(errp,
953                    "logical_block_size > physical_block_size not supported");
954         return;
955     }
956 
957     virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK,
958                 sizeof(struct virtio_blk_config));
959 
960     s->blk = conf->conf.blk;
961     s->rq = NULL;
962     s->sector_mask = (s->conf.conf.logical_block_size / BDRV_SECTOR_SIZE) - 1;
963 
964     for (i = 0; i < conf->num_queues; i++) {
965         virtio_add_queue(vdev, conf->queue_size, virtio_blk_handle_output);
966     }
967     virtio_blk_data_plane_create(vdev, conf, &s->dataplane, &err);
968     if (err != NULL) {
969         error_propagate(errp, err);
970         virtio_cleanup(vdev);
971         return;
972     }
973 
974     s->change = qemu_add_vm_change_state_handler(virtio_blk_dma_restart_cb, s);
975     blk_set_dev_ops(s->blk, &virtio_block_ops, s);
976     blk_set_guest_block_size(s->blk, s->conf.conf.logical_block_size);
977 
978     blk_iostatus_enable(s->blk);
979 }
980 
virtio_blk_device_unrealize(DeviceState * dev,Error ** errp)981 static void virtio_blk_device_unrealize(DeviceState *dev, Error **errp)
982 {
983     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
984     VirtIOBlock *s = VIRTIO_BLK(dev);
985 
986     virtio_blk_data_plane_destroy(s->dataplane);
987     s->dataplane = NULL;
988     qemu_del_vm_change_state_handler(s->change);
989     blockdev_mark_auto_del(s->blk);
990     virtio_cleanup(vdev);
991 }
992 
virtio_blk_instance_init(Object * obj)993 static void virtio_blk_instance_init(Object *obj)
994 {
995     VirtIOBlock *s = VIRTIO_BLK(obj);
996 
997     device_add_bootindex_property(obj, &s->conf.conf.bootindex,
998                                   "bootindex", "/disk@0,0",
999                                   DEVICE(obj), NULL);
1000 }
1001 
1002 static const VMStateDescription vmstate_virtio_blk = {
1003     .name = "virtio-blk",
1004     .minimum_version_id = 2,
1005     .version_id = 2,
1006     .fields = (VMStateField[]) {
1007         VMSTATE_VIRTIO_DEVICE,
1008         VMSTATE_END_OF_LIST()
1009     },
1010 };
1011 
1012 static Property virtio_blk_properties[] = {
1013     DEFINE_BLOCK_PROPERTIES(VirtIOBlock, conf.conf),
1014     DEFINE_BLOCK_ERROR_PROPERTIES(VirtIOBlock, conf.conf),
1015     DEFINE_BLOCK_CHS_PROPERTIES(VirtIOBlock, conf.conf),
1016     DEFINE_PROP_STRING("serial", VirtIOBlock, conf.serial),
1017     DEFINE_PROP_BIT("config-wce", VirtIOBlock, conf.config_wce, 0, true),
1018 #ifdef __linux__
1019     DEFINE_PROP_BIT("scsi", VirtIOBlock, conf.scsi, 0, false),
1020 #endif
1021     DEFINE_PROP_BIT("request-merging", VirtIOBlock, conf.request_merging, 0,
1022                     true),
1023     DEFINE_PROP_UINT16("num-queues", VirtIOBlock, conf.num_queues, 1),
1024     DEFINE_PROP_UINT16("queue-size", VirtIOBlock, conf.queue_size, 128),
1025     DEFINE_PROP_LINK("iothread", VirtIOBlock, conf.iothread, TYPE_IOTHREAD,
1026                      IOThread *),
1027     DEFINE_PROP_END_OF_LIST(),
1028 };
1029 
virtio_blk_class_init(ObjectClass * klass,void * data)1030 static void virtio_blk_class_init(ObjectClass *klass, void *data)
1031 {
1032     DeviceClass *dc = DEVICE_CLASS(klass);
1033     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
1034 
1035     dc->props = virtio_blk_properties;
1036     dc->vmsd = &vmstate_virtio_blk;
1037     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
1038     vdc->realize = virtio_blk_device_realize;
1039     vdc->unrealize = virtio_blk_device_unrealize;
1040     vdc->get_config = virtio_blk_update_config;
1041     vdc->set_config = virtio_blk_set_config;
1042     vdc->get_features = virtio_blk_get_features;
1043     vdc->set_status = virtio_blk_set_status;
1044     vdc->reset = virtio_blk_reset;
1045     vdc->save = virtio_blk_save_device;
1046     vdc->load = virtio_blk_load_device;
1047     vdc->start_ioeventfd = virtio_blk_data_plane_start;
1048     vdc->stop_ioeventfd = virtio_blk_data_plane_stop;
1049 }
1050 
1051 static const TypeInfo virtio_blk_info = {
1052     .name = TYPE_VIRTIO_BLK,
1053     .parent = TYPE_VIRTIO_DEVICE,
1054     .instance_size = sizeof(VirtIOBlock),
1055     .instance_init = virtio_blk_instance_init,
1056     .class_init = virtio_blk_class_init,
1057 };
1058 
virtio_register_types(void)1059 static void virtio_register_types(void)
1060 {
1061     type_register_static(&virtio_blk_info);
1062 }
1063 
1064 type_init(virtio_register_types)
1065