1 /*
2 * Copyright (c) 2013-2022 Douglas Gilbert.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * SPDX-License-Identifier: BSD-2-Clause
27 */
28
29 #include <iostream>
30 #include <vector>
31 #include <system_error>
32 #include <thread>
33 #include <mutex>
34 #include <chrono>
35
36 #include <unistd.h>
37 #include <fcntl.h>
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <string.h>
41 #include <errno.h>
42 #include <ctype.h>
43 #include <sys/ioctl.h>
44 #include <sys/types.h>
45 #include <sys/stat.h>
46 #include "sg_lib.h"
47 #include "sg_pt.h"
48 #include "sg_unaligned.h"
49
50 static const char * version_str = "1.11 20220425";
51 static const char * util_name = "sg_tst_excl2";
52
53 /* This is a test program for checking O_EXCL on open() works. It uses
54 * multiple threads and can be run as multiple processes and attempts
55 * to "break" O_EXCL. The strategy is to open a device O_EXCL|O_NONBLOCK
56 * and do a double increment on a LB then close it. Prior to the first
57 * increment, the value is checked for even or odd. Assuming the count
58 * starts as an even (typically 0) then it should remain even. Odd instances
59 * are counted and reported at the end of the program, after all threads
60 * have completed.
61 *
62 * This is C++ code with some things from C++11 (e.g. threads) and was
63 * only just able to compile (when some things were reverted) with gcc/g++
64 * version 4.7.3 found in Ubuntu 13.04 . C++11 "feature complete" support
65 * was not available until g++ version 4.8.1 and that is only currently
66 * found in Fedora 19 .
67 *
68 * The build uses various object files from the <sg3_utils>/lib directory
69 * which is assumed to be a sibling of this examples directory. Those
70 * object files in the lib directory can be built with:
71 * cd <sg3_utils> ; ./configure ; cd lib; make
72 * Then:
73 * cd ../testing
74 * make sg_tst_excl2
75 *
76 * BEWARE: this utility modifies a logical block (default LBA 1000) on the
77 * given device.
78 *
79 */
80
81 using namespace std;
82 using namespace std::chrono;
83
84 #define DEF_NUM_PER_THREAD 200
85 #define DEF_NUM_THREADS 4
86 #define DEF_WAIT_MS 0 /* 0: yield; -1: don't wait; -2: sleep(0) */
87
88 #define DEF_LBA 1000
89
90 #define EBUFF_SZ 256
91
92
93 static mutex odd_count_mutex;
94 static mutex console_mutex;
95 static unsigned int odd_count;
96 static unsigned int ebusy_count;
97
98
99 static void
usage(void)100 usage(void)
101 {
102 printf("Usage: %s [-b] [-f] [-h] [-l <lba>] [-n <n_per_thr>] "
103 "[-t <num_thrs>]\n"
104 " [-V] [-w <wait_ms>] [-x] "
105 "<disk_device>\n", util_name);
106 printf(" where\n");
107 printf(" -b block on open (def: O_NONBLOCK)\n");
108 printf(" -f force: any SCSI disk (def: only "
109 "scsi_debug)\n");
110 printf(" WARNING: <lba> written to\n");
111 printf(" -h print this usage message then exit\n");
112 printf(" -l <lba> logical block to increment (def: %u)\n",
113 DEF_LBA);
114 printf(" -n <n_per_thr> number of loops per thread "
115 "(def: %d)\n", DEF_NUM_PER_THREAD);
116 printf(" -t <num_thrs> number of threads (def: %d)\n",
117 DEF_NUM_THREADS);
118 printf(" -V print version number then exit\n");
119 printf(" -w <wait_ms> >0: sleep_for(<wait_ms>); =0: "
120 "yield(); -1: no\n"
121 " wait; -2: sleep(0) (def: %d)\n",
122 DEF_WAIT_MS);
123 printf(" -x don't use O_EXCL on first thread "
124 "(def: use\n"
125 " O_EXCL on all threads)\n\n");
126 printf("Test O_EXCL open flag with pass-through drivers. Each "
127 "open/close cycle with\nthe O_EXCL flag does a double increment "
128 "on lba (using its first 4 bytes).\n");
129 }
130
131 /* Assumed a lock (mutex) held when pt_err() is called */
132 static int
pt_err(int res)133 pt_err(int res)
134 {
135 if (res < 0)
136 fprintf(stderr, " pass through os error: %s\n", safe_strerror(-res));
137 else if (SCSI_PT_DO_BAD_PARAMS == res)
138 fprintf(stderr, " bad pass through setup\n");
139 else if (SCSI_PT_DO_TIMEOUT == res)
140 fprintf(stderr, " pass through timeout\n");
141 else
142 fprintf(stderr, " do_scsi_pt error=%d\n", res);
143 return -1;
144 }
145
146 /* Assumed a lock (mutex) held when pt_cat_no_good() is called */
147 static int
pt_cat_no_good(int cat,struct sg_pt_base * ptp,const unsigned char * sbp)148 pt_cat_no_good(int cat, struct sg_pt_base * ptp, const unsigned char * sbp)
149 {
150 int slen;
151 char b[256];
152 const int bl = (int)sizeof(b);
153
154 switch (cat) {
155 case SCSI_PT_RESULT_STATUS: /* other than GOOD and CHECK CONDITION */
156 sg_get_scsi_status_str(get_scsi_pt_status_response(ptp), bl, b);
157 fprintf(stderr, " scsi status: %s\n", b);
158 break;
159 case SCSI_PT_RESULT_SENSE:
160 slen = get_scsi_pt_sense_len(ptp);
161 sg_get_sense_str("", sbp, slen, 1, bl, b);
162 fprintf(stderr, "%s", b);
163 break;
164 case SCSI_PT_RESULT_TRANSPORT_ERR:
165 get_scsi_pt_transport_err_str(ptp, bl, b);
166 fprintf(stderr, " transport: %s", b);
167 break;
168 case SCSI_PT_RESULT_OS_ERR:
169 get_scsi_pt_os_err_str(ptp, bl, b);
170 fprintf(stderr, " os: %s", b);
171 break;
172 default:
173 fprintf(stderr, " unknown pt result category (%d)\n", cat);
174 break;
175 }
176 return -1;
177 }
178
179 #define READ16_REPLY_LEN 512
180 #define READ16_CMD_LEN 16
181 #define WRITE16_REPLY_LEN 512
182 #define WRITE16_CMD_LEN 16
183
184 /* Opens dev_name and spins if busy (i.e. gets EBUSY), sleeping for
185 * wait_ms milliseconds if wait_ms is positive. Reads lba and treats the
186 * first 4 bytes as an int (SCSI endian), increments it and writes it back.
187 * Repeats so that happens twice. Then closes dev_name. If an error occurs
188 * returns -1 else returns 0 if first int read is even otherwise returns 1. */
189 static int
do_rd_inc_wr_twice(const char * dev_name,unsigned int lba,int block,int excl,int wait_ms,unsigned int & ebusys)190 do_rd_inc_wr_twice(const char * dev_name, unsigned int lba, int block,
191 int excl, int wait_ms, unsigned int & ebusys)
192 {
193 int k, sg_fd, res, cat;
194 int odd = 0;
195 unsigned int u = 0;
196 struct sg_pt_base * ptp = NULL;
197 unsigned char r16CmdBlk [READ16_CMD_LEN] =
198 {0x88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0};
199 unsigned char w16CmdBlk [WRITE16_CMD_LEN] =
200 {0x8a, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0};
201 unsigned char sense_buffer[64] SG_C_CPP_ZERO_INIT;
202 unsigned char lb[READ16_REPLY_LEN];
203 char ebuff[EBUFF_SZ];
204 int open_flags = O_RDWR;
205
206 sg_put_unaligned_be64(lba, r16CmdBlk + 2);
207 sg_put_unaligned_be64(lba, w16CmdBlk + 2);
208 if (! block)
209 open_flags |= O_NONBLOCK;
210 if (excl)
211 open_flags |= O_EXCL;
212
213 while (((sg_fd = scsi_pt_open_flags(dev_name, open_flags, 0)) < 0) &&
214 (-EBUSY == sg_fd)) {
215 ++ebusys;
216 if (wait_ms > 0)
217 this_thread::sleep_for(milliseconds{wait_ms});
218 else if (0 == wait_ms)
219 this_thread::yield(); // thread yield
220 else if (-2 == wait_ms)
221 sleep(0); // process yield ??
222 }
223 if (sg_fd < 0) {
224 snprintf(ebuff, EBUFF_SZ,
225 "do_rd_inc_wr_twice: error opening file: %s", dev_name);
226 {
227 lock_guard<mutex> lg(console_mutex);
228
229 perror(ebuff);
230 }
231 return -1;
232 }
233
234 ptp = construct_scsi_pt_obj();
235 for (k = 0; k < 2; ++k) {
236 /* Prepare READ_16 command */
237 clear_scsi_pt_obj(ptp);
238 set_scsi_pt_cdb(ptp, r16CmdBlk, sizeof(r16CmdBlk));
239 set_scsi_pt_sense(ptp, sense_buffer, sizeof(sense_buffer));
240 set_scsi_pt_data_in(ptp, lb, READ16_REPLY_LEN);
241 res = do_scsi_pt(ptp, sg_fd, 20 /* secs timeout */, 1);
242 if (res) {
243 {
244 lock_guard<mutex> lg(console_mutex);
245
246 fprintf(stderr, "READ_16 do_scsi_pt() submission error\n");
247 res = pt_err(res);
248 }
249 goto err;
250 }
251 cat = get_scsi_pt_result_category(ptp);
252 if (SCSI_PT_RESULT_GOOD != cat) {
253 {
254 lock_guard<mutex> lg(console_mutex);
255
256 fprintf(stderr, "READ_16 do_scsi_pt() category problem\n");
257 res = pt_cat_no_good(cat, ptp, sense_buffer);
258 }
259 goto err;
260 }
261
262 u = sg_get_unaligned_be32(lb);
263 // Assuming u starts test as even (probably 0), expect it to stay even
264 if (0 == k)
265 odd = (1 == (u % 2));
266 ++u;
267 sg_put_unaligned_be32(u, lb);
268
269 if (wait_ms > 0) /* allow daylight for bad things ... */
270 this_thread::sleep_for(milliseconds{wait_ms});
271 else if (0 == wait_ms)
272 this_thread::yield(); // thread yield
273 else if (-2 == wait_ms)
274 sleep(0); // process yield ??
275
276 /* Prepare WRITE_16 command */
277 clear_scsi_pt_obj(ptp);
278 set_scsi_pt_cdb(ptp, w16CmdBlk, sizeof(w16CmdBlk));
279 set_scsi_pt_sense(ptp, sense_buffer, sizeof(sense_buffer));
280 set_scsi_pt_data_out(ptp, lb, WRITE16_REPLY_LEN);
281 res = do_scsi_pt(ptp, sg_fd, 20 /* secs timeout */, 1);
282 if (res) {
283 {
284 lock_guard<mutex> lg(console_mutex);
285
286 fprintf(stderr, "WRITE_16 do_scsi_pt() submission error\n");
287 res = pt_err(res);
288 }
289 goto err;
290 }
291 cat = get_scsi_pt_result_category(ptp);
292 if (SCSI_PT_RESULT_GOOD != cat) {
293 {
294 lock_guard<mutex> lg(console_mutex);
295
296 fprintf(stderr, "WRITE_16 do_scsi_pt() category problem\n");
297 res = pt_cat_no_good(cat, ptp, sense_buffer);
298 }
299 goto err;
300 }
301 }
302 err:
303 if (ptp)
304 destruct_scsi_pt_obj(ptp);
305 scsi_pt_close_device(sg_fd);
306 return odd;
307 }
308
309
310
311 #define INQ_REPLY_LEN 96
312 #define INQ_CMD_LEN 6
313
314 /* Send INQUIRY and fetches response. If okay puts PRODUCT ID field
315 * in b (up to m_blen bytes). Does not use O_EXCL flag. Returns 0 on success,
316 * else -1 . */
317 static int
do_inquiry_prod_id(const char * dev_name,int block,int wait_ms,unsigned int & ebusys,char * b,int b_mlen)318 do_inquiry_prod_id(const char * dev_name, int block, int wait_ms,
319 unsigned int & ebusys, char * b, int b_mlen)
320 {
321 int sg_fd, res, cat;
322 struct sg_pt_base * ptp = NULL;
323 unsigned char inqCmdBlk [INQ_CMD_LEN] =
324 {0x12, 0, 0, 0, INQ_REPLY_LEN, 0};
325 unsigned char inqBuff[INQ_REPLY_LEN];
326 unsigned char sense_buffer[64] SG_C_CPP_ZERO_INIT;
327 char ebuff[EBUFF_SZ];
328 int open_flags = O_RDWR; /* since O_EXCL | O_RDONLY gives EPERM */
329
330 if (! block)
331 open_flags |= O_NONBLOCK;
332 while (((sg_fd = scsi_pt_open_flags(dev_name, open_flags, 0)) < 0) &&
333 (-EBUSY == sg_fd)) {
334 ++ebusys;
335 if (wait_ms > 0)
336 this_thread::sleep_for(milliseconds{wait_ms});
337 else if (0 == wait_ms)
338 this_thread::yield(); // thread yield
339 else if (-2 == wait_ms)
340 sleep(0); // process yield ??
341 }
342 if (sg_fd < 0) {
343 snprintf(ebuff, EBUFF_SZ,
344 "do_inquiry_prod_id: error opening file: %s", dev_name);
345 perror(ebuff);
346 return -1;
347 }
348 /* Prepare INQUIRY command */
349 ptp = construct_scsi_pt_obj();
350 clear_scsi_pt_obj(ptp);
351 set_scsi_pt_cdb(ptp, inqCmdBlk, sizeof(inqCmdBlk));
352 set_scsi_pt_sense(ptp, sense_buffer, sizeof(sense_buffer));
353 set_scsi_pt_data_in(ptp, inqBuff, INQ_REPLY_LEN);
354 res = do_scsi_pt(ptp, sg_fd, 20 /* secs timeout */, 1);
355 if (res) {
356 {
357 lock_guard<mutex> lg(console_mutex);
358
359 fprintf(stderr, "INQUIRY do_scsi_pt() submission error\n");
360 res = pt_err(res);
361 }
362 goto err;
363 }
364 cat = get_scsi_pt_result_category(ptp);
365 if (SCSI_PT_RESULT_GOOD != cat) {
366 {
367 lock_guard<mutex> lg(console_mutex);
368
369 fprintf(stderr, "INQUIRY do_scsi_pt() category problem\n");
370 res = pt_cat_no_good(cat, ptp, sense_buffer);
371 }
372 goto err;
373 }
374
375 /* Good, so fetch Product ID from response, copy to 'b' */
376 if (b_mlen > 0) {
377 if (b_mlen > 16) {
378 memcpy(b, inqBuff + 16, 16);
379 b[16] = '\0';
380 } else {
381 memcpy(b, inqBuff + 16, b_mlen - 1);
382 b[b_mlen - 1] = '\0';
383 }
384 }
385 err:
386 if (ptp)
387 destruct_scsi_pt_obj(ptp);
388 close(sg_fd);
389 return 0;
390 }
391
392 static void
work_thread(const char * dev_name,unsigned int lba,int id,int block,int excl,int num,int wait_ms)393 work_thread(const char * dev_name, unsigned int lba, int id, int block,
394 int excl, int num, int wait_ms)
395 {
396 unsigned int thr_odd_count = 0;
397 unsigned int thr_ebusy_count = 0;
398 int k, res;
399
400 {
401 lock_guard<mutex> lg(console_mutex);
402
403 cerr << "Enter work_thread id=" << id << " excl=" << excl << " block="
404 << block << endl;
405 }
406 for (k = 0; k < num; ++k) {
407 res = do_rd_inc_wr_twice(dev_name, lba, block, excl, wait_ms,
408 thr_ebusy_count);
409 if (res < 0)
410 break;
411 if (res)
412 ++thr_odd_count;
413 }
414 {
415 lock_guard<mutex> lg(console_mutex);
416
417 if (k < num)
418 cerr << "thread id=" << id << " FAILed at iteration: " << k <<
419 '\n';
420 else
421 cerr << "thread id=" << id << " normal exit" << '\n';
422 }
423
424 {
425 lock_guard<mutex> lg(odd_count_mutex);
426
427 odd_count += thr_odd_count;
428 ebusy_count += thr_ebusy_count;
429 }
430 }
431
432
433 int
main(int argc,char * argv[])434 main(int argc, char * argv[])
435 {
436 int k, res;
437 int block = 0;
438 int force = 0;
439 unsigned int lba = DEF_LBA;
440 int num_per_thread = DEF_NUM_PER_THREAD;
441 int num_threads = DEF_NUM_THREADS;
442 int wait_ms = DEF_WAIT_MS;
443 int exclude_o_excl = 0;
444 char * dev_name = NULL;
445 char b[64];
446
447 for (k = 1; k < argc; ++k) {
448 if (0 == memcmp("-b", argv[k], 2))
449 ++block;
450 else if (0 == memcmp("-f", argv[k], 2))
451 ++force;
452 else if (0 == memcmp("-h", argv[k], 2)) {
453 usage();
454 return 0;
455 } else if (0 == memcmp("-l", argv[k], 2)) {
456 ++k;
457 if ((k < argc) && isdigit(*argv[k]))
458 lba = (unsigned int)atoi(argv[k]);
459 else
460 break;
461 } else if (0 == memcmp("-n", argv[k], 2)) {
462 ++k;
463 if ((k < argc) && isdigit(*argv[k]))
464 num_per_thread = atoi(argv[k]);
465 else
466 break;
467 } else if (0 == memcmp("-t", argv[k], 2)) {
468 ++k;
469 if ((k < argc) && isdigit(*argv[k]))
470 num_threads = atoi(argv[k]);
471 else
472 break;
473 } else if (0 == memcmp("-V", argv[k], 2)) {
474 printf("%s version: %s\n", util_name, version_str);
475 return 0;
476 } else if (0 == memcmp("-w", argv[k], 2)) {
477 ++k;
478 if ((k < argc) && (isdigit(*argv[k]) || ('-' == *argv[k]))) {
479 if ('-' == *argv[k])
480 wait_ms = - atoi(argv[k] + 1);
481 else
482 wait_ms = atoi(argv[k]);
483 } else
484 break;
485 } else if (0 == memcmp("-x", argv[k], 2))
486 ++exclude_o_excl;
487 else if (*argv[k] == '-') {
488 printf("Unrecognized switch: %s\n", argv[k]);
489 dev_name = NULL;
490 break;
491 }
492 else if (! dev_name)
493 dev_name = argv[k];
494 else {
495 printf("too many arguments\n");
496 dev_name = 0;
497 break;
498 }
499 }
500 if (0 == dev_name) {
501 usage();
502 return 1;
503 }
504 try {
505
506 if (! force) {
507 res = do_inquiry_prod_id(dev_name, block, wait_ms, ebusy_count,
508 b, sizeof(b));
509 if (res) {
510 fprintf(stderr, "INQUIRY failed on %s\n", dev_name);
511 return 1;
512 }
513 // For safety, since <lba> written to, only permit scsi_debug
514 // devices. Bypass this with '-f' option.
515 if (0 != memcmp("scsi_debug", b, 10)) {
516 fprintf(stderr, "Since this utility writes to LBA %d, only "
517 "devices with scsi_debug\nproduct ID accepted.\n",
518 lba);
519 return 2;
520 }
521 }
522
523 vector<thread *> vt;
524
525 for (k = 0; k < num_threads; ++k) {
526 int excl = ((0 == k) && exclude_o_excl) ? 0 : 1;
527
528 thread * tp = new thread {work_thread, dev_name, lba, k, block,
529 excl, num_per_thread, wait_ms};
530 vt.push_back(tp);
531 }
532
533 for (k = 0; k < (int)vt.size(); ++k)
534 vt[k]->join();
535
536 for (k = 0; k < (int)vt.size(); ++k)
537 delete vt[k];
538
539 cout << "Expecting odd count of 0, got " << odd_count << endl;
540 cout << "Number of EBUSYs: " << ebusy_count << endl;
541
542 }
543 catch(system_error& e) {
544 cerr << "got a system_error exception: " << e.what() << '\n';
545 auto ec = e.code();
546 cerr << "category: " << ec.category().name() << '\n';
547 cerr << "value: " << ec.value() << '\n';
548 cerr << "message: " << ec.message() << '\n';
549 cerr << "\nNote: if g++ may need '-pthread' or similar in "
550 "compile/link line" << '\n';
551 }
552 catch(...) {
553 cerr << "got another exception: " << '\n';
554 }
555 return 0;
556 }
557