1 /*
2 * Copyright (c) 2013-2014 Douglas Gilbert.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <iostream>
30 #include <vector>
31 #include <system_error>
32 #include <thread>
33 #include <mutex>
34 #include <chrono>
35
36 #include <unistd.h>
37 #include <fcntl.h>
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <string.h>
41 #include <errno.h>
42 #include <ctype.h>
43 #include <sys/ioctl.h>
44 #include <sys/types.h>
45 #include <sys/stat.h>
46 #include "sg_lib.h"
47 #include "sg_pt.h"
48
49 static const char * version_str = "1.07 20140828";
50 static const char * util_name = "sg_tst_excl2";
51
52 /* This is a test program for checking O_EXCL on open() works. It uses
53 * multiple threads and can be run as multiple processes and attempts
54 * to "break" O_EXCL. The strategy is to open a device O_EXCL|O_NONBLOCK
55 * and do a double increment on a LB then close it. Prior to the first
56 * increment, the value is checked for even or odd. Assuming the count
57 * starts as an even (typically 0) then it should remain even. Odd instances
58 * are counted and reported at the end of the program, after all threads
59 * have completed.
60 *
61 * This is C++ code with some things from C++11 (e.g. threads) and was
62 * only just able to compile (when some things were reverted) with gcc/g++
63 * version 4.7.3 found in Ubuntu 13.04 . C++11 "feature complete" support
64 * was not available until g++ version 4.8.1 and that is only currently
65 * found in Fedora 19 .
66 *
67 * The build uses various object files from the <sg3_utils>/lib directory
68 * which is assumed to be a sibling of this examples directory. Those
69 * object files in the lib directory can be built with:
70 * cd <sg3_utils> ; ./configure ; cd lib; make
71 * Then to build sg_tst_excl2 concatenate the next 3 lines:
72 * g++ -Wall -std=c++11 -pthread -I ../include ../lib/sg_lib.o
73 * ../lib/sg_lib_data.o ../lib/sg_pt_linux.o -o sg_tst_excl2
74 * sg_tst_excl2.cpp
75 * Alternatively use 'make -f Makefile.cplus sg_tst_excl2'
76 *
77 * BEWARE: this utility modifies a logical block (default LBA 1000) on the
78 * given device.
79 *
80 * Test breaks sg driver in lk 3.10.4 but works with proposed fix so should
81 * work soon thereafter. Works on standard block driver (e.g. /dev/sdc) in
82 * lk 3.10.4 . Fails on bsg driver in lk 3.10.4 because it ignores the
83 * O_EXCL flag (and that is unlikely to change).
84 *
85 */
86
87 using namespace std;
88 using namespace std::chrono;
89
90 #define DEF_NUM_PER_THREAD 200
91 #define DEF_NUM_THREADS 4
92 #define DEF_WAIT_MS 0 /* 0: yield; -1: don't wait; -2: sleep(0) */
93
94 #define DEF_LBA 1000
95
96 #define EBUFF_SZ 256
97
98
99 static mutex odd_count_mutex;
100 static mutex console_mutex;
101 static unsigned int odd_count;
102 static unsigned int ebusy_count;
103
104
105 static void
usage(void)106 usage(void)
107 {
108 printf("Usage: %s [-b] [-f] [-h] [-l <lba>] [-n <n_per_thr>] "
109 "[-t <num_thrs>]\n"
110 " [-V] [-w <wait_ms>] [-x] "
111 "<disk_device>\n", util_name);
112 printf(" where\n");
113 printf(" -b block on open (def: O_NONBLOCK)\n");
114 printf(" -f force: any SCSI disk (def: only "
115 "scsi_debug)\n");
116 printf(" WARNING: <lba> written to\n");
117 printf(" -h print this usage message then exit\n");
118 printf(" -l <lba> logical block to increment (def: %u)\n",
119 DEF_LBA);
120 printf(" -n <n_per_thr> number of loops per thread "
121 "(def: %d)\n", DEF_NUM_PER_THREAD);
122 printf(" -t <num_thrs> number of threads (def: %d)\n",
123 DEF_NUM_THREADS);
124 printf(" -V print version number then exit\n");
125 printf(" -w <wait_ms> >0: sleep_for(<wait_ms>); =0: "
126 "yield(); -1: no\n"
127 " wait; -2: sleep(0) (def: %d)\n",
128 DEF_WAIT_MS);
129 printf(" -x don't use O_EXCL on first thread "
130 "(def: use\n"
131 " O_EXCL on all threads)\n\n");
132 printf("Test O_EXCL open flag with pass-through drivers. Each "
133 "open/close cycle with\nthe O_EXCL flag does a double increment "
134 "on lba (using its first 4 bytes).\n");
135 }
136
137 /* Assumed a lock (mutex) held when pt_err() is called */
138 static int
pt_err(int res)139 pt_err(int res)
140 {
141 if (res < 0)
142 fprintf(stderr, " pass through os error: %s\n", safe_strerror(-res));
143 else if (SCSI_PT_DO_BAD_PARAMS == res)
144 fprintf(stderr, " bad pass through setup\n");
145 else if (SCSI_PT_DO_TIMEOUT == res)
146 fprintf(stderr, " pass through timeout\n");
147 else
148 fprintf(stderr, " do_scsi_pt error=%d\n", res);
149 return -1;
150 }
151
152 /* Assumed a lock (mutex) held when pt_cat_no_good() is called */
153 static int
pt_cat_no_good(int cat,struct sg_pt_base * ptp,const unsigned char * sbp)154 pt_cat_no_good(int cat, struct sg_pt_base * ptp, const unsigned char * sbp)
155 {
156 int slen;
157 char b[256];
158 const int bl = (int)sizeof(b);
159
160 switch (cat) {
161 case SCSI_PT_RESULT_STATUS: /* other than GOOD and CHECK CONDITION */
162 sg_get_scsi_status_str(get_scsi_pt_status_response(ptp), bl, b);
163 fprintf(stderr, " scsi status: %s\n", b);
164 break;
165 case SCSI_PT_RESULT_SENSE:
166 slen = get_scsi_pt_sense_len(ptp);
167 sg_get_sense_str("", sbp, slen, 1, bl, b);
168 fprintf(stderr, "%s", b);
169 break;
170 case SCSI_PT_RESULT_TRANSPORT_ERR:
171 get_scsi_pt_transport_err_str(ptp, bl, b);
172 fprintf(stderr, " transport: %s", b);
173 break;
174 case SCSI_PT_RESULT_OS_ERR:
175 get_scsi_pt_os_err_str(ptp, bl, b);
176 fprintf(stderr, " os: %s", b);
177 break;
178 default:
179 fprintf(stderr, " unknown pt result category (%d)\n", cat);
180 break;
181 }
182 return -1;
183 }
184
185 #define READ16_REPLY_LEN 512
186 #define READ16_CMD_LEN 16
187 #define WRITE16_REPLY_LEN 512
188 #define WRITE16_CMD_LEN 16
189
190 /* Opens dev_name and spins if busy (i.e. gets EBUSY), sleeping for
191 * wait_ms milliseconds if wait_ms is positive. Reads lba and treats the
192 * first 4 bytes as an int (SCSI endian), increments it and writes it back.
193 * Repeats so that happens twice. Then closes dev_name. If an error occurs
194 * returns -1 else returns 0 if first int read is even otherwise returns 1. */
195 static int
do_rd_inc_wr_twice(const char * dev_name,unsigned int lba,int block,int excl,int wait_ms,unsigned int & ebusys)196 do_rd_inc_wr_twice(const char * dev_name, unsigned int lba, int block,
197 int excl, int wait_ms, unsigned int & ebusys)
198 {
199 int k, sg_fd, res, cat;
200 int odd = 0;
201 unsigned int u = 0;
202 struct sg_pt_base * ptp = NULL;
203 unsigned char r16CmdBlk [READ16_CMD_LEN] =
204 {0x88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0};
205 unsigned char w16CmdBlk [WRITE16_CMD_LEN] =
206 {0x8a, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0};
207 unsigned char sense_buffer[64];
208 unsigned char lb[READ16_REPLY_LEN];
209 char ebuff[EBUFF_SZ];
210 int open_flags = O_RDWR;
211
212 r16CmdBlk[6] = w16CmdBlk[6] = (lba >> 24) & 0xff;
213 r16CmdBlk[7] = w16CmdBlk[7] = (lba >> 16) & 0xff;
214 r16CmdBlk[8] = w16CmdBlk[8] = (lba >> 8) & 0xff;
215 r16CmdBlk[9] = w16CmdBlk[9] = lba & 0xff;
216 if (! block)
217 open_flags |= O_NONBLOCK;
218 if (excl)
219 open_flags |= O_EXCL;
220
221 while (((sg_fd = scsi_pt_open_flags(dev_name, open_flags, 0)) < 0) &&
222 (-EBUSY == sg_fd)) {
223 ++ebusys;
224 if (wait_ms > 0)
225 this_thread::sleep_for(milliseconds{wait_ms});
226 else if (0 == wait_ms)
227 this_thread::yield(); // thread yield
228 else if (-2 == wait_ms)
229 sleep(0); // process yield ??
230 }
231 if (sg_fd < 0) {
232 snprintf(ebuff, EBUFF_SZ,
233 "do_rd_inc_wr_twice: error opening file: %s", dev_name);
234 {
235 lock_guard<mutex> lg(console_mutex);
236
237 perror(ebuff);
238 }
239 return -1;
240 }
241
242 ptp = construct_scsi_pt_obj();
243 for (k = 0; k < 2; ++k) {
244 /* Prepare READ_16 command */
245 clear_scsi_pt_obj(ptp);
246 set_scsi_pt_cdb(ptp, r16CmdBlk, sizeof(r16CmdBlk));
247 set_scsi_pt_sense(ptp, sense_buffer, sizeof(sense_buffer));
248 set_scsi_pt_data_in(ptp, lb, READ16_REPLY_LEN);
249 res = do_scsi_pt(ptp, sg_fd, 20 /* secs timeout */, 1);
250 if (res) {
251 {
252 lock_guard<mutex> lg(console_mutex);
253
254 fprintf(stderr, "READ_16 do_scsi_pt() submission error\n");
255 res = pt_err(res);
256 }
257 goto err;
258 }
259 cat = get_scsi_pt_result_category(ptp);
260 if (SCSI_PT_RESULT_GOOD != cat) {
261 {
262 lock_guard<mutex> lg(console_mutex);
263
264 fprintf(stderr, "READ_16 do_scsi_pt() category problem\n");
265 res = pt_cat_no_good(cat, ptp, sense_buffer);
266 }
267 goto err;
268 }
269
270 u = (lb[0] << 24) + (lb[1] << 16) + (lb[2] << 8) + lb[3];
271 // Assuming u starts test as even (probably 0), expect it to stay even
272 if (0 == k)
273 odd = (1 == (u % 2));
274 ++u;
275 lb[0] = (u >> 24) & 0xff;
276 lb[1] = (u >> 16) & 0xff;
277 lb[2] = (u >> 8) & 0xff;
278 lb[3] = u & 0xff;
279
280 if (wait_ms > 0) /* allow daylight for bad things ... */
281 this_thread::sleep_for(milliseconds{wait_ms});
282 else if (0 == wait_ms)
283 this_thread::yield(); // thread yield
284 else if (-2 == wait_ms)
285 sleep(0); // process yield ??
286
287 /* Prepare WRITE_16 command */
288 clear_scsi_pt_obj(ptp);
289 set_scsi_pt_cdb(ptp, w16CmdBlk, sizeof(w16CmdBlk));
290 set_scsi_pt_sense(ptp, sense_buffer, sizeof(sense_buffer));
291 set_scsi_pt_data_out(ptp, lb, WRITE16_REPLY_LEN);
292 res = do_scsi_pt(ptp, sg_fd, 20 /* secs timeout */, 1);
293 if (res) {
294 {
295 lock_guard<mutex> lg(console_mutex);
296
297 fprintf(stderr, "WRITE_16 do_scsi_pt() submission error\n");
298 res = pt_err(res);
299 }
300 goto err;
301 }
302 cat = get_scsi_pt_result_category(ptp);
303 if (SCSI_PT_RESULT_GOOD != cat) {
304 {
305 lock_guard<mutex> lg(console_mutex);
306
307 fprintf(stderr, "WRITE_16 do_scsi_pt() category problem\n");
308 res = pt_cat_no_good(cat, ptp, sense_buffer);
309 }
310 goto err;
311 }
312 }
313 err:
314 if (ptp)
315 destruct_scsi_pt_obj(ptp);
316 scsi_pt_close_device(sg_fd);
317 return odd;
318 }
319
320
321
322 #define INQ_REPLY_LEN 96
323 #define INQ_CMD_LEN 6
324
325 /* Send INQUIRY and fetches response. If okay puts PRODUCT ID field
326 * in b (up to m_blen bytes). Does not use O_EXCL flag. Returns 0 on success,
327 * else -1 . */
328 static int
do_inquiry_prod_id(const char * dev_name,int block,int wait_ms,unsigned int & ebusys,char * b,int b_mlen)329 do_inquiry_prod_id(const char * dev_name, int block, int wait_ms,
330 unsigned int & ebusys, char * b, int b_mlen)
331 {
332 int sg_fd, res, cat;
333 struct sg_pt_base * ptp = NULL;
334 unsigned char inqCmdBlk [INQ_CMD_LEN] =
335 {0x12, 0, 0, 0, INQ_REPLY_LEN, 0};
336 unsigned char inqBuff[INQ_REPLY_LEN];
337 unsigned char sense_buffer[64];
338 char ebuff[EBUFF_SZ];
339 int open_flags = O_RDWR; /* since O_EXCL | O_RDONLY gives EPERM */
340
341 if (! block)
342 open_flags |= O_NONBLOCK;
343 while (((sg_fd = scsi_pt_open_flags(dev_name, open_flags, 0)) < 0) &&
344 (-EBUSY == sg_fd)) {
345 ++ebusys;
346 if (wait_ms > 0)
347 this_thread::sleep_for(milliseconds{wait_ms});
348 else if (0 == wait_ms)
349 this_thread::yield(); // thread yield
350 else if (-2 == wait_ms)
351 sleep(0); // process yield ??
352 }
353 if (sg_fd < 0) {
354 snprintf(ebuff, EBUFF_SZ,
355 "do_inquiry_prod_id: error opening file: %s", dev_name);
356 perror(ebuff);
357 return -1;
358 }
359 /* Prepare INQUIRY command */
360 ptp = construct_scsi_pt_obj();
361 clear_scsi_pt_obj(ptp);
362 set_scsi_pt_cdb(ptp, inqCmdBlk, sizeof(inqCmdBlk));
363 set_scsi_pt_sense(ptp, sense_buffer, sizeof(sense_buffer));
364 set_scsi_pt_data_in(ptp, inqBuff, INQ_REPLY_LEN);
365 res = do_scsi_pt(ptp, sg_fd, 20 /* secs timeout */, 1);
366 if (res) {
367 {
368 lock_guard<mutex> lg(console_mutex);
369
370 fprintf(stderr, "INQUIRY do_scsi_pt() submission error\n");
371 res = pt_err(res);
372 }
373 goto err;
374 }
375 cat = get_scsi_pt_result_category(ptp);
376 if (SCSI_PT_RESULT_GOOD != cat) {
377 {
378 lock_guard<mutex> lg(console_mutex);
379
380 fprintf(stderr, "INQUIRY do_scsi_pt() category problem\n");
381 res = pt_cat_no_good(cat, ptp, sense_buffer);
382 }
383 goto err;
384 }
385
386 /* Good, so fetch Product ID from response, copy to 'b' */
387 if (b_mlen > 0) {
388 if (b_mlen > 16) {
389 memcpy(b, inqBuff + 16, 16);
390 b[16] = '\0';
391 } else {
392 memcpy(b, inqBuff + 16, b_mlen - 1);
393 b[b_mlen - 1] = '\0';
394 }
395 }
396 err:
397 if (ptp)
398 destruct_scsi_pt_obj(ptp);
399 close(sg_fd);
400 return 0;
401 }
402
403 static void
work_thread(const char * dev_name,unsigned int lba,int id,int block,int excl,int num,int wait_ms)404 work_thread(const char * dev_name, unsigned int lba, int id, int block,
405 int excl, int num, int wait_ms)
406 {
407 unsigned int thr_odd_count = 0;
408 unsigned int thr_ebusy_count = 0;
409 int k, res;
410
411 {
412 lock_guard<mutex> lg(console_mutex);
413
414 cerr << "Enter work_thread id=" << id << " excl=" << excl << " block="
415 << block << endl;
416 }
417 for (k = 0; k < num; ++k) {
418 res = do_rd_inc_wr_twice(dev_name, lba, block, excl, wait_ms,
419 thr_ebusy_count);
420 if (res < 0)
421 break;
422 if (res)
423 ++thr_odd_count;
424 }
425 {
426 lock_guard<mutex> lg(console_mutex);
427
428 if (k < num)
429 cerr << "thread id=" << id << " FAILed at iteration: " << k <<
430 '\n';
431 else
432 cerr << "thread id=" << id << " normal exit" << '\n';
433 }
434
435 {
436 lock_guard<mutex> lg(odd_count_mutex);
437
438 odd_count += thr_odd_count;
439 ebusy_count += thr_ebusy_count;
440 }
441 }
442
443
444 int
main(int argc,char * argv[])445 main(int argc, char * argv[])
446 {
447 int k, res;
448 int block = 0;
449 int force = 0;
450 unsigned int lba = DEF_LBA;
451 int num_per_thread = DEF_NUM_PER_THREAD;
452 int num_threads = DEF_NUM_THREADS;
453 int wait_ms = DEF_WAIT_MS;
454 int exclude_o_excl = 0;
455 char * dev_name = NULL;
456 char b[64];
457
458 for (k = 1; k < argc; ++k) {
459 if (0 == memcmp("-b", argv[k], 2))
460 ++block;
461 else if (0 == memcmp("-f", argv[k], 2))
462 ++force;
463 else if (0 == memcmp("-h", argv[k], 2)) {
464 usage();
465 return 0;
466 } else if (0 == memcmp("-l", argv[k], 2)) {
467 ++k;
468 if ((k < argc) && isdigit(*argv[k]))
469 lba = (unsigned int)atoi(argv[k]);
470 else
471 break;
472 } else if (0 == memcmp("-n", argv[k], 2)) {
473 ++k;
474 if ((k < argc) && isdigit(*argv[k]))
475 num_per_thread = atoi(argv[k]);
476 else
477 break;
478 } else if (0 == memcmp("-t", argv[k], 2)) {
479 ++k;
480 if ((k < argc) && isdigit(*argv[k]))
481 num_threads = atoi(argv[k]);
482 else
483 break;
484 } else if (0 == memcmp("-V", argv[k], 2)) {
485 printf("%s version: %s\n", util_name, version_str);
486 return 0;
487 } else if (0 == memcmp("-w", argv[k], 2)) {
488 ++k;
489 if ((k < argc) && (isdigit(*argv[k]) || ('-' == *argv[k]))) {
490 if ('-' == *argv[k])
491 wait_ms = - atoi(argv[k] + 1);
492 else
493 wait_ms = atoi(argv[k]);
494 } else
495 break;
496 } else if (0 == memcmp("-x", argv[k], 2))
497 ++exclude_o_excl;
498 else if (*argv[k] == '-') {
499 printf("Unrecognized switch: %s\n", argv[k]);
500 dev_name = NULL;
501 break;
502 }
503 else if (! dev_name)
504 dev_name = argv[k];
505 else {
506 printf("too many arguments\n");
507 dev_name = 0;
508 break;
509 }
510 }
511 if (0 == dev_name) {
512 usage();
513 return 1;
514 }
515 try {
516
517 if (! force) {
518 res = do_inquiry_prod_id(dev_name, block, wait_ms, ebusy_count,
519 b, sizeof(b));
520 if (res) {
521 fprintf(stderr, "INQUIRY failed on %s\n", dev_name);
522 return 1;
523 }
524 // For safety, since <lba> written to, only permit scsi_debug
525 // devices. Bypass this with '-f' option.
526 if (0 != memcmp("scsi_debug", b, 10)) {
527 fprintf(stderr, "Since this utility writes to LBA %d, only "
528 "devices with scsi_debug\nproduct ID accepted.\n",
529 lba);
530 return 2;
531 }
532 }
533
534 vector<thread *> vt;
535
536 for (k = 0; k < num_threads; ++k) {
537 int excl = ((0 == k) && exclude_o_excl) ? 0 : 1;
538
539 thread * tp = new thread {work_thread, dev_name, lba, k, block,
540 excl, num_per_thread, wait_ms};
541 vt.push_back(tp);
542 }
543
544 for (k = 0; k < (int)vt.size(); ++k)
545 vt[k]->join();
546
547 for (k = 0; k < (int)vt.size(); ++k)
548 delete vt[k];
549
550 cout << "Expecting odd count of 0, got " << odd_count << endl;
551 cout << "Number of EBUSYs: " << ebusy_count << endl;
552
553 }
554 catch(system_error& e) {
555 cerr << "got a system_error exception: " << e.what() << '\n';
556 auto ec = e.code();
557 cerr << "category: " << ec.category().name() << '\n';
558 cerr << "value: " << ec.value() << '\n';
559 cerr << "message: " << ec.message() << '\n';
560 cerr << "\nNote: if g++ may need '-pthread' or similar in "
561 "compile/link line" << '\n';
562 }
563 catch(...) {
564 cerr << "got another exception: " << '\n';
565 }
566 return 0;
567 }
568