1 /*
2 * Copyright (c) 2013-2014 Douglas Gilbert.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <iostream>
30 #include <vector>
31 #include <system_error>
32 #include <thread>
33 #include <mutex>
34 #include <chrono>
35
36 #include <unistd.h>
37 #include <fcntl.h>
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <string.h>
41 #include <errno.h>
42 #include <ctype.h>
43 #include <sys/ioctl.h>
44 #include <sys/types.h>
45 #include <sys/stat.h>
46 #include "sg_lib.h"
47 #include "sg_pt.h"
48
49 static const char * version_str = "1.05 20140828";
50 static const char * util_name = "sg_tst_excl3";
51
52 /* This is a test program for checking O_EXCL on open() works. It uses
53 * multiple threads and can be run as multiple processes and attempts
54 * to "break" O_EXCL. The strategy is to open a device O_EXCL|O_NONBLOCK
55 * and do a double increment on a LB then close it from a single thread.
56 * the remaining threads open that device O_NONBLOCK and do a read and
57 * note of the number is odd. Assuming the count starts as an even
58 * (typically 0) then it should remain even. Odd instances
59 * are counted and reported at the end of the program, after all threads
60 * have completed.
61 *
62 * This is C++ code with some things from C++11 (e.g. threads) and was
63 * only just able to compile (when some things were reverted) with gcc/g++
64 * version 4.7.3 found in Ubuntu 13.04 . C++11 "feature complete" support
65 * was not available until g++ version 4.8.1 and that is found in Fedora
66 * 19 and Ubuntu 13.10 .
67 *
68 * The build uses various object files from the <sg3_utils>/lib directory
69 * which is assumed to be a sibling of this examples directory. Those
70 * object files in the lib directory can be built with:
71 * cd <sg3_utils> ; ./configure ; cd lib; make
72 * Then to build sg_tst_excl3 concatenate the next 3 lines:
73 * g++ -Wall -std=c++11 -pthread -I ../include ../lib/sg_lib.o
74 * ../lib/sg_lib_data.o ../lib/sg_pt_linux.o -o sg_tst_excl3
75 * sg_tst_excl3.cpp
76 * Alternatively use 'make -f Makefile.cplus sg_tst_excl3'
77 *
78 * BEWARE: this utility modifies a logical block (default LBA 1000) on the
79 * given device.
80 *
81 * Test breaks sg driver in lk 3.10.4 but works with proposed fix so should
82 * work soon thereafter. It works on standard block driver (e.g. /dev/sdc)
83 * in lk 3.10.4 (most of the time). It fails on bsg driver in lk 3.10.4
84 * because it ignores the O_EXCL flag (and that is unlikely to change in
85 * the short term).
86 *
87 */
88
89 using namespace std;
90 using namespace std::chrono;
91
92 #define DEF_NUM_PER_THREAD 200
93 #define DEF_NUM_THREADS 4
94 #define DEF_WAIT_MS 0 /* 0: yield; -1: don't wait; -2: sleep(0) */
95
96 #define DEF_LBA 1000
97
98 #define EBUFF_SZ 256
99
100
101 static mutex odd_count_mutex;
102 static mutex console_mutex;
103 static unsigned int odd_count;
104 static unsigned int ebusy_count;
105
106
107 static void
usage(void)108 usage(void)
109 {
110 printf("Usage: %s [-b] [-f] [-h] [-l <lba>] [-n <n_per_thr>]\n"
111 " [-R] [-t <num_thrs>] [-V] [-w <wait_ms>] "
112 "[-x]\n"
113 " <disk_device>\n", util_name);
114 printf(" where\n");
115 printf(" -b block on open (def: O_NONBLOCK)\n");
116 printf(" -f force: any SCSI disk (def: only "
117 "scsi_debug)\n");
118 printf(" WARNING: <lba> written to\n");
119 printf(" -h print this usage message then exit\n");
120 printf(" -l <lba> logical block to increment (def: %u)\n",
121 DEF_LBA);
122 printf(" -n <n_per_thr> number of loops per thread "
123 "(def: %d)\n", DEF_NUM_PER_THREAD);
124 printf(" -R all readers; so first thread (id=0) "
125 "just reads\n");
126 printf(" -t <num_thrs> number of threads (def: %d)\n",
127 DEF_NUM_THREADS);
128 printf(" -V print version number then exit\n");
129 printf(" -w <wait_ms> >0: sleep_for(<wait_ms>); =0: "
130 "yield(); -1: no\n"
131 " wait; -2: sleep(0) (def: %d)\n",
132 DEF_WAIT_MS);
133 printf(" -x don't use O_EXCL on first thread "
134 "(def: use\n"
135 " O_EXCL on first thread)\n\n");
136 printf("Test O_EXCL open flag with pass-through drivers. First thread "
137 "(id=0) does\nopen/close cycle with the O_EXCL flag then does a "
138 "double increment on\nlba (using its first 4 bytes). Remaining "
139 "theads read (without\nO_EXCL flag on open) and check the "
140 "value is even.\n");
141 }
142
143 /* Assumed a lock (mutex) held when pt_err() is called */
144 static int
pt_err(int res)145 pt_err(int res)
146 {
147 if (res < 0)
148 fprintf(stderr, " pass through os error: %s\n", safe_strerror(-res));
149 else if (SCSI_PT_DO_BAD_PARAMS == res)
150 fprintf(stderr, " bad pass through setup\n");
151 else if (SCSI_PT_DO_TIMEOUT == res)
152 fprintf(stderr, " pass through timeout\n");
153 else
154 fprintf(stderr, " do_scsi_pt error=%d\n", res);
155 return -1;
156 }
157
158 /* Assumed a lock (mutex) held when pt_cat_no_good() is called */
159 static int
pt_cat_no_good(int cat,struct sg_pt_base * ptp,const unsigned char * sbp)160 pt_cat_no_good(int cat, struct sg_pt_base * ptp, const unsigned char * sbp)
161 {
162 int slen;
163 char b[256];
164 const int bl = (int)sizeof(b);
165
166 switch (cat) {
167 case SCSI_PT_RESULT_STATUS: /* other than GOOD and CHECK CONDITION */
168 sg_get_scsi_status_str(get_scsi_pt_status_response(ptp), bl, b);
169 fprintf(stderr, " scsi status: %s\n", b);
170 break;
171 case SCSI_PT_RESULT_SENSE:
172 slen = get_scsi_pt_sense_len(ptp);
173 sg_get_sense_str("", sbp, slen, 1, bl, b);
174 fprintf(stderr, "%s", b);
175 break;
176 case SCSI_PT_RESULT_TRANSPORT_ERR:
177 get_scsi_pt_transport_err_str(ptp, bl, b);
178 fprintf(stderr, " transport: %s", b);
179 break;
180 case SCSI_PT_RESULT_OS_ERR:
181 get_scsi_pt_os_err_str(ptp, bl, b);
182 fprintf(stderr, " os: %s", b);
183 break;
184 default:
185 fprintf(stderr, " unknown pt result category (%d)\n", cat);
186 break;
187 }
188 return -1;
189 }
190
191 #define READ16_REPLY_LEN 512
192 #define READ16_CMD_LEN 16
193 #define WRITE16_REPLY_LEN 512
194 #define WRITE16_CMD_LEN 16
195
196 /* Opens dev_name and spins if busy (i.e. gets EBUSY), sleeping for
197 * wait_ms milliseconds if wait_ms is positive. Reads lba and treats the
198 * first 4 bytes as an int (SCSI endian), increments it and writes it back.
199 * Repeats so that happens twice. Then closes dev_name. If an error occurs
200 * returns -1 else returns 0 if first int read is even otherwise returns 1. */
201 static int
do_rd_inc_wr_twice(const char * dev_name,int read_only,unsigned int lba,int block,int excl,int wait_ms,unsigned int & ebusys)202 do_rd_inc_wr_twice(const char * dev_name, int read_only, unsigned int lba,
203 int block, int excl, int wait_ms, unsigned int & ebusys)
204 {
205 int k, sg_fd, res, cat;
206 int odd = 0;
207 unsigned int u = 0;
208 struct sg_pt_base * ptp = NULL;
209 unsigned char r16CmdBlk [READ16_CMD_LEN] =
210 {0x88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0};
211 unsigned char w16CmdBlk [WRITE16_CMD_LEN] =
212 {0x8a, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0};
213 unsigned char sense_buffer[64];
214 unsigned char lb[READ16_REPLY_LEN];
215 char ebuff[EBUFF_SZ];
216 int open_flags = O_RDWR;
217
218 r16CmdBlk[6] = w16CmdBlk[6] = (lba >> 24) & 0xff;
219 r16CmdBlk[7] = w16CmdBlk[7] = (lba >> 16) & 0xff;
220 r16CmdBlk[8] = w16CmdBlk[8] = (lba >> 8) & 0xff;
221 r16CmdBlk[9] = w16CmdBlk[9] = lba & 0xff;
222 if (! block)
223 open_flags |= O_NONBLOCK;
224 if (excl)
225 open_flags |= O_EXCL;
226
227 while (((sg_fd = scsi_pt_open_flags(dev_name, open_flags, 0)) < 0) &&
228 (-EBUSY == sg_fd)) {
229 ++ebusys;
230 if (wait_ms > 0)
231 this_thread::sleep_for(milliseconds{wait_ms});
232 else if (0 == wait_ms)
233 this_thread::yield(); // thread yield
234 else if (-2 == wait_ms)
235 sleep(0); // process yield ??
236 }
237 if (sg_fd < 0) {
238 snprintf(ebuff, EBUFF_SZ,
239 "do_rd_inc_wr_twice: error opening file: %s", dev_name);
240 {
241 lock_guard<mutex> lg(console_mutex);
242
243 perror(ebuff);
244 }
245 return -1;
246 }
247
248 ptp = construct_scsi_pt_obj();
249 for (k = 0; k < 2; ++k) {
250 /* Prepare READ_16 command */
251 clear_scsi_pt_obj(ptp);
252 set_scsi_pt_cdb(ptp, r16CmdBlk, sizeof(r16CmdBlk));
253 set_scsi_pt_sense(ptp, sense_buffer, sizeof(sense_buffer));
254 set_scsi_pt_data_in(ptp, lb, READ16_REPLY_LEN);
255 res = do_scsi_pt(ptp, sg_fd, 20 /* secs timeout */, 1);
256 if (res) {
257 {
258 lock_guard<mutex> lg(console_mutex);
259
260 fprintf(stderr, "READ_16 do_scsi_pt() submission error\n");
261 res = pt_err(res);
262 }
263 goto err;
264 }
265 cat = get_scsi_pt_result_category(ptp);
266 if (SCSI_PT_RESULT_GOOD != cat) {
267 {
268 lock_guard<mutex> lg(console_mutex);
269
270 fprintf(stderr, "READ_16 do_scsi_pt() category problem\n");
271 res = pt_cat_no_good(cat, ptp, sense_buffer);
272 }
273 goto err;
274 }
275
276 u = (lb[0] << 24) + (lb[1] << 16) + (lb[2] << 8) + lb[3];
277 // Assuming u starts test as even (probably 0), expect it to stay even
278 if (0 == k)
279 odd = (1 == (u % 2));
280
281 if (wait_ms > 0) /* allow daylight for bad things ... */
282 this_thread::sleep_for(milliseconds{wait_ms});
283 else if (0 == wait_ms)
284 this_thread::yield(); // thread yield
285 else if (-2 == wait_ms)
286 sleep(0); // process yield ??
287
288 if (read_only)
289 break;
290 ++u;
291 lb[0] = (u >> 24) & 0xff;
292 lb[1] = (u >> 16) & 0xff;
293 lb[2] = (u >> 8) & 0xff;
294 lb[3] = u & 0xff;
295
296 /* Prepare WRITE_16 command */
297 clear_scsi_pt_obj(ptp);
298 set_scsi_pt_cdb(ptp, w16CmdBlk, sizeof(w16CmdBlk));
299 set_scsi_pt_sense(ptp, sense_buffer, sizeof(sense_buffer));
300 set_scsi_pt_data_out(ptp, lb, WRITE16_REPLY_LEN);
301 res = do_scsi_pt(ptp, sg_fd, 20 /* secs timeout */, 1);
302 if (res) {
303 {
304 lock_guard<mutex> lg(console_mutex);
305
306 fprintf(stderr, "WRITE_16 do_scsi_pt() submission error\n");
307 res = pt_err(res);
308 }
309 goto err;
310 }
311 cat = get_scsi_pt_result_category(ptp);
312 if (SCSI_PT_RESULT_GOOD != cat) {
313 {
314 lock_guard<mutex> lg(console_mutex);
315
316 fprintf(stderr, "WRITE_16 do_scsi_pt() category problem\n");
317 res = pt_cat_no_good(cat, ptp, sense_buffer);
318 }
319 goto err;
320 }
321 }
322 err:
323 if (ptp)
324 destruct_scsi_pt_obj(ptp);
325 scsi_pt_close_device(sg_fd);
326 return odd;
327 }
328
329
330 #define INQ_REPLY_LEN 96
331 #define INQ_CMD_LEN 6
332
333 /* Send INQUIRY and fetches response. If okay puts PRODUCT ID field
334 * in b (up to m_blen bytes). Does not use O_EXCL flag. Returns 0 on success,
335 * else -1 . */
336 static int
do_inquiry_prod_id(const char * dev_name,int block,int wait_ms,unsigned int & ebusys,char * b,int b_mlen)337 do_inquiry_prod_id(const char * dev_name, int block, int wait_ms,
338 unsigned int & ebusys, char * b, int b_mlen)
339 {
340 int sg_fd, res, cat;
341 struct sg_pt_base * ptp = NULL;
342 unsigned char inqCmdBlk [INQ_CMD_LEN] =
343 {0x12, 0, 0, 0, INQ_REPLY_LEN, 0};
344 unsigned char inqBuff[INQ_REPLY_LEN];
345 unsigned char sense_buffer[64];
346 char ebuff[EBUFF_SZ];
347 int open_flags = O_RDWR; /* since O_EXCL | O_RDONLY gives EPERM */
348
349 if (! block)
350 open_flags |= O_NONBLOCK;
351 while (((sg_fd = scsi_pt_open_flags(dev_name, open_flags, 0)) < 0) &&
352 (-EBUSY == sg_fd)) {
353 ++ebusys;
354 if (wait_ms > 0)
355 this_thread::sleep_for(milliseconds{wait_ms});
356 else if (0 == wait_ms)
357 this_thread::yield(); // thread yield
358 else if (-2 == wait_ms)
359 sleep(0); // process yield ??
360 }
361 if (sg_fd < 0) {
362 snprintf(ebuff, EBUFF_SZ,
363 "do_inquiry_prod_id: error opening file: %s", dev_name);
364 perror(ebuff);
365 return -1;
366 }
367 /* Prepare INQUIRY command */
368 ptp = construct_scsi_pt_obj();
369 clear_scsi_pt_obj(ptp);
370 set_scsi_pt_cdb(ptp, inqCmdBlk, sizeof(inqCmdBlk));
371 set_scsi_pt_sense(ptp, sense_buffer, sizeof(sense_buffer));
372 set_scsi_pt_data_in(ptp, inqBuff, INQ_REPLY_LEN);
373 res = do_scsi_pt(ptp, sg_fd, 20 /* secs timeout */, 1);
374 if (res) {
375 fprintf(stderr, "INQUIRY do_scsi_pt() submission error\n");
376 res = pt_err(res);
377 goto err;
378 }
379 cat = get_scsi_pt_result_category(ptp);
380 if (SCSI_PT_RESULT_GOOD != cat) {
381 fprintf(stderr, "INQUIRY do_scsi_pt() category problem\n");
382 res = pt_cat_no_good(cat, ptp, sense_buffer);
383 goto err;
384 }
385
386 /* Good, so fetch Product ID from response, copy to 'b' */
387 if (b_mlen > 0) {
388 if (b_mlen > 16) {
389 memcpy(b, inqBuff + 16, 16);
390 b[16] = '\0';
391 } else {
392 memcpy(b, inqBuff + 16, b_mlen - 1);
393 b[b_mlen - 1] = '\0';
394 }
395 }
396 err:
397 if (ptp)
398 destruct_scsi_pt_obj(ptp);
399 close(sg_fd);
400 return res;
401 }
402
403 static void
work_thread(const char * dev_name,unsigned int lba,int id,int block,int excl,bool all_readers,int num,int wait_ms)404 work_thread(const char * dev_name, unsigned int lba, int id, int block,
405 int excl, bool all_readers, int num, int wait_ms)
406 {
407 unsigned int thr_odd_count = 0;
408 unsigned int thr_ebusy_count = 0;
409 int k, res;
410 int reader = ((id > 0) || (all_readers));
411
412 {
413 lock_guard<mutex> lg(console_mutex);
414
415 cerr << "Enter work_thread id=" << id << " excl=" << excl << " block="
416 << block << " reader=" << reader << endl;
417 }
418 for (k = 0; k < num; ++k) {
419 res = do_rd_inc_wr_twice(dev_name, reader, lba, block, excl,
420 wait_ms, thr_ebusy_count);
421 if (res < 0)
422 break;
423 if (res)
424 ++thr_odd_count;
425 }
426 {
427 lock_guard<mutex> lg(console_mutex);
428
429 if (k < num)
430 cerr << "thread id=" << id << " FAILed at iteration: " << k
431 << '\n';
432 else
433 cerr << "thread id=" << id << " normal exit" << '\n';
434 }
435
436 {
437 lock_guard<mutex> lg(odd_count_mutex);
438
439 odd_count += thr_odd_count;
440 ebusy_count += thr_ebusy_count;
441 }
442 }
443
444
445 int
main(int argc,char * argv[])446 main(int argc, char * argv[])
447 {
448 int k, res;
449 int block = 0;
450 int force = 0;
451 unsigned int lba = DEF_LBA;
452 int num_per_thread = DEF_NUM_PER_THREAD;
453 bool all_readers = false;
454 int num_threads = DEF_NUM_THREADS;
455 int wait_ms = DEF_WAIT_MS;
456 int exclude_o_excl = 0;
457 char * dev_name = NULL;
458 char b[64];
459
460 for (k = 1; k < argc; ++k) {
461 if (0 == memcmp("-b", argv[k], 2))
462 ++block;
463 else if (0 == memcmp("-f", argv[k], 2))
464 ++force;
465 else if (0 == memcmp("-h", argv[k], 2)) {
466 usage();
467 return 0;
468 } else if (0 == memcmp("-l", argv[k], 2)) {
469 ++k;
470 if ((k < argc) && isdigit(*argv[k]))
471 lba = (unsigned int)atoi(argv[k]);
472 else
473 break;
474 } else if (0 == memcmp("-n", argv[k], 2)) {
475 ++k;
476 if ((k < argc) && isdigit(*argv[k]))
477 num_per_thread = atoi(argv[k]);
478 else
479 break;
480 } else if (0 == memcmp("-t", argv[k], 2)) {
481 ++k;
482 if ((k < argc) && isdigit(*argv[k]))
483 num_threads = atoi(argv[k]);
484 else
485 break;
486 } else if (0 == memcmp("-R", argv[k], 2))
487 all_readers = true;
488 else if (0 == memcmp("-V", argv[k], 2)) {
489 printf("%s version: %s\n", util_name, version_str);
490 return 0;
491 } else if (0 == memcmp("-w", argv[k], 2)) {
492 ++k;
493 if ((k < argc) && (isdigit(*argv[k]) || ('-' == *argv[k]))) {
494 if ('-' == *argv[k])
495 wait_ms = - atoi(argv[k] + 1);
496 else
497 wait_ms = atoi(argv[k]);
498 } else
499 break;
500 } else if (0 == memcmp("-x", argv[k], 2))
501 ++exclude_o_excl;
502 else if (*argv[k] == '-') {
503 printf("Unrecognized switch: %s\n", argv[k]);
504 dev_name = NULL;
505 break;
506 }
507 else if (! dev_name)
508 dev_name = argv[k];
509 else {
510 printf("too many arguments\n");
511 dev_name = 0;
512 break;
513 }
514 }
515 if (0 == dev_name) {
516 usage();
517 return 1;
518 }
519 try {
520
521 if (! force) {
522 res = do_inquiry_prod_id(dev_name, block, wait_ms, ebusy_count,
523 b, sizeof(b));
524 if (res) {
525 fprintf(stderr, "INQUIRY failed on %s\n", dev_name);
526 return 1;
527 }
528 // For safety, since <lba> written to, only permit scsi_debug
529 // devices. Bypass this with '-f' option.
530 if (0 != memcmp("scsi_debug", b, 10)) {
531 fprintf(stderr, "Since this utility writes to LBA %d, only "
532 "devices with scsi_debug\nproduct ID accepted.\n",
533 lba);
534 return 2;
535 }
536 }
537
538 vector<thread *> vt;
539
540 for (k = 0; k < num_threads; ++k) {
541 int excl = ((0 == k) && (! exclude_o_excl)) ? 1 : 0;
542
543 thread * tp = new thread {work_thread, dev_name, lba, k, block,
544 excl, all_readers, num_per_thread,
545 wait_ms};
546 vt.push_back(tp);
547 }
548
549 for (k = 0; k < (int)vt.size(); ++k)
550 vt[k]->join();
551
552 for (k = 0; k < (int)vt.size(); ++k)
553 delete vt[k];
554
555 cout << "Expecting odd count of 0, got " << odd_count << endl;
556 cout << "Number of EBUSYs: " << ebusy_count << endl;
557
558 }
559 catch(system_error& e) {
560 cerr << "got a system_error exception: " << e.what() << '\n';
561 auto ec = e.code();
562 cerr << "category: " << ec.category().name() << '\n';
563 cerr << "value: " << ec.value() << '\n';
564 cerr << "message: " << ec.message() << '\n';
565 cerr << "\nNote: if g++ may need '-pthread' or similar in "
566 "compile/link line" << '\n';
567 }
568 catch(...) {
569 cerr << "got another exception: " << '\n';
570 }
571 return 0;
572 }
573