1 /*
2 * A utility program for copying files. Specialised for "files" that
3 * represent devices that understand the SCSI command set.
4 *
5 * Copyright (C) 2018-2022 D. Gilbert
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2, or (at your option)
9 * any later version.
10 *
11 * SPDX-License-Identifier: GPL-2.0-or-later
12 *
13 * This program is a specialisation of the Unix "dd" command in which
14 * one or both of the given files is a scsi generic device.
15 * A logical block size ('bs') is assumed to be 512 if not given. This
16 * program complains if 'ibs' or 'obs' are given with some other value
17 * than 'bs'. If 'if' is not given or 'if=-' then stdin is assumed. If
18 * 'of' is not given or 'of=-' then stdout assumed.
19 *
20 * A non-standard argument "bpt" (blocks per transfer) is added to control
21 * the maximum number of blocks in each transfer. The default value is 128.
22 * For example if "bs=512" and "bpt=32" then a maximum of 32 blocks (16 KiB
23 * in this case) are transferred to or from the sg device in a single SCSI
24 * command.
25 *
26 * This version is designed for the linux kernel 4 and 5 series.
27 *
28 * sg_mrq_dd uses C++ threads and MRQ (multiple requests (in one invocation))
29 * facilities in the sg version 4 driver to do "dd" type copies and verifies.
30 *
31 */
32
33 static const char * version_str = "1.44 20221020";
34
35 #define _XOPEN_SOURCE 600
36 #ifndef _GNU_SOURCE
37 #define _GNU_SOURCE 1
38 #endif
39
40 #include <unistd.h>
41 #include <fcntl.h>
42 #include <stdio.h>
43 #include <stdlib.h>
44 #include <stdarg.h>
45 #include <stdbool.h>
46 #include <string.h>
47 #include <ctype.h>
48 #include <errno.h>
49 #include <time.h> /* for nanosleep() */
50 #include <poll.h>
51 #include <limits.h>
52 // #include <pthread.h>
53 #include <signal.h>
54 #define __STDC_FORMAT_MACROS 1
55 #include <inttypes.h>
56 #include <sys/ioctl.h>
57 #include <sys/stat.h>
58 #include <sys/sysmacros.h>
59 #ifndef major
60 #include <sys/types.h>
61 #endif
62 #include <sys/time.h>
63 #include <linux/major.h> /* for MEM_MAJOR, SCSI_GENERIC_MAJOR, etc */
64 #include <linux/fs.h> /* for BLKSSZGET and friends */
65 #include <sys/mman.h> /* for mmap() system call */
66
67 #include <vector>
68 #include <array>
69 #include <atomic> // C++ header replacing <stdatomic.h>
70 #include <random>
71 #include <thread> // needed for std::this_thread::yield()
72 #include <mutex>
73 #include <condition_variable> // for infant_cv: copy/verify first segment
74 // single threaded
75 #include <chrono>
76
77 #ifdef HAVE_CONFIG_H
78 #include "config.h"
79 #endif
80
81 #ifdef HAVE_GETRANDOM
82 #include <sys/random.h> /* for getrandom() system call */
83 #endif
84
85 #ifndef HAVE_LINUX_SG_V4_HDR
86 /* Kernel uapi header contain __user decorations on user space pointers
87 * to indicate they are unsafe in the kernel space. However glibc takes
88 * all those __user decorations out from headers in /usr/include/linux .
89 * So to stop compile errors when directly importing include/uapi/scsi/sg.h
90 * undef __user before doing that include. */
91 #define __user
92
93 /* Want to block the original sg.h header from also being included. That
94 * causes lots of multiple definition errors. This will only work if this
95 * header is included _before_ the original sg.h header. */
96 #define _SCSI_GENERIC_H /* original kernel header guard */
97 #define _SCSI_SG_H /* glibc header guard */
98
99 #include "uapi_sg.h" /* local copy of include/uapi/scsi/sg.h */
100
101 #else
102 #define __user
103 #endif /* end of: ifndef HAVE_LINUX_SG_V4_HDR */
104
105 // C++ local header
106 #include "sg_scat_gath.h"
107
108 // C headers associated with sg3_utils library
109 #include "sg_lib.h"
110 #include "sg_cmds_basic.h"
111 #include "sg_io_linux.h"
112 #include "sg_unaligned.h"
113 #include "sg_pr2serr.h"
114
115
116 using namespace std;
117
118 // #ifdef __GNUC__
119 // #ifndef __clang__
120 // #pragma GCC diagnostic ignored "-Wclobbered"
121 // #endif
122 // #endif
123
124
125 #ifndef SGV4_FLAG_POLLED
126 #define SGV4_FLAG_POLLED 0x800
127 #endif
128
129 #define MAX_SGL_NUM_VAL (INT32_MAX - 1) /* should reduce for testing */
130 // #define MAX_SGL_NUM_VAL 7 /* should reduce for testing */
131 #if MAX_SGL_NUM_VAL > INT32_MAX
132 #error "MAX_SGL_NUM_VAL cannot exceed 2^31 - 1"
133 #endif
134
135 #define DEF_BLOCK_SIZE 512
136 #define DEF_BLOCKS_PER_TRANSFER 128
137 #define DEF_BLOCKS_PER_2048TRANSFER 32
138 #define DEF_SDT_ICT_MS 300
139 #define DEF_SDT_CRT_SEC 3
140 #define DEF_SCSI_CDB_SZ 10
141 #define MAX_SCSI_CDB_SZ 16 /* could be 32 */
142 #define PACK_ID_TID_MULTIPLIER (0x1000000) /* 16,777,216 */
143 #define MAX_SLICES 16 /* number of IFILE,OFILE pairs */
144 #define MAX_BPT_VALUE (1 << 24) /* used for maximum bs as well */
145 #define MAX_COUNT_SKIP_SEEK (1LL << 48) /* coverity wants upper bound */
146
147 #define SENSE_BUFF_LEN 64 /* Arbitrary, could be larger */
148 #define READ_CAP_REPLY_LEN 8
149 #define RCAP16_REPLY_LEN 32
150
151 #define DEF_TIMEOUT 60000 /* 60,000 millisecs == 60 seconds */
152
153 #define SGP_READ10 0x28
154 #define SGP_PRE_FETCH10 0x34
155 #define SGP_PRE_FETCH16 0x90
156 #define SGP_VERIFY10 0x2f
157 #define SGP_WRITE10 0x2a
158 #define DEF_NUM_THREADS 4
159 #define MAX_NUM_THREADS 1024 /* was SG_MAX_QUEUE with v3 driver */
160 #define DEF_MRQ_NUM 16
161
162 #define FT_UNKNOWN 0 /* yet to be checked */
163 #define FT_OTHER 1 /* filetype other than one of the following */
164 #define FT_SG 2 /* filetype is sg char device */
165 #define FT_DEV_NULL 4 /* either /dev/null, /dev/zero, or "." */
166 #define FT_ST 8 /* filetype is st char device (tape) */
167 #define FT_BLOCK 16 /* filetype is a block device */
168 #define FT_FIFO 32 /* fifo (named or unnamed pipe (stdout)) */
169 #define FT_CHAR 64 /* fifo (named or unnamed pipe (stdout)) */
170 #define FT_RANDOM_0_FF 128 /* iflag=00, iflag=ff and iflag=random
171 override if=IFILE */
172 #define FT_ERROR 256 /* couldn't "stat" file */
173
174 #define DEV_NULL_MINOR_NUM 3
175 #define DEV_ZERO_MINOR_NUM 5
176
177 #define EBUFF_SZ 768
178
179 #define PROC_SCSI_SG_VERSION "/proc/scsi/sg/version"
180 #define SYS_SCSI_SG_VERSION "/sys/module/sg/version"
181
182
183 struct flags_t {
184 bool append;
185 bool coe;
186 bool dio;
187 bool direct;
188 bool dpo;
189 bool dsync;
190 bool excl;
191 bool ff;
192 bool fua;
193 bool masync; /* more async sg v4 driver fd flag */
194 bool mout_if; /* META_OUT_IF flag at mrq level */
195 bool nocreat;
196 bool no_dur;
197 bool no_thresh;
198 bool no_waitq; /* dummy, no longer supported, just warn */
199 bool order_wr;
200 bool polled; /* was previously 'hipri' */
201 bool qhead;
202 bool qtail;
203 bool random;
204 bool serial;
205 bool same_fds;
206 bool wq_excl;
207 bool zero;
208 int cdl; /* command duration limits, 0 --> no cdl */
209 int mmap;
210 };
211
212 typedef pair<int64_t, int> get_next_res_t; /* LBA, num */
213 typedef array<uint8_t, MAX_SCSI_CDB_SZ> cdb_arr_t;
214
215 struct cp_ver_pair_t {
cp_ver_pair_tcp_ver_pair_t216 cp_ver_pair_t() {}
217
218 get_next_res_t get_next(int desired_num_blks);
219
220 enum class my_state {empty,
221 init,
222 underway,
223 ignore,
224 finished} state = {my_state::empty};
225
226 int my_index = 0;
227 int in_fd = -1;
228 int in_type = FT_UNKNOWN;
229 int out_fd = -1;
230 int out_type = FT_UNKNOWN;
231
232 int64_t dd_count = 0;
233 atomic<int64_t> next_count_pos {};
234 atomic<int64_t> in_rem_count {};
235 atomic<int64_t> out_rem_count {};
236 atomic<int> in_partial {};
237 atomic<int> out_partial {};
238 atomic<int> sum_of_resids {};
239 };
240
241 typedef array<cp_ver_pair_t, MAX_SLICES> cp_ver_arr_t;
242
243 /* There is one instance of this structure and it is at file scope so it is
244 * initialized to zero. The design of this copy multi-threaded copy algorithm
245 * attempts to have no locks on the fast path. Contention in gcoll.get_next()
246 * is resolved by the loser repeating its operation. Statistics and error
247 * information is held in each thread until it shuts down and contention
248 * can occur at that point. */
249 struct global_collection /* one instance visible to all threads */
250 {
251 cp_ver_arr_t cp_ver_arr;
252
253 /* get_next() is the pivotal function for multi-threaded safety. It can
254 * be safely called from all threads with the desired number of blocks
255 * (typically mrq*bpt) and this function returns a pair. The first pair
256 * value is the starting count value/index [0..dd_count) and the second
257 * pair value is the number of blocks to copy. If desired_num_blks is
258 * negative this flags an error has occurred. If the second value in the
259 * returned pair is 0 then the calling thread should shutdown; a
260 * negative value indicates an error has occurred (e.g. in another
261 * thread) and the calling thread should shutdown. */
262
263 int in0fd;
264 int64_t dd_count;
265 int in_type; /* expect all IFILEs to have same type */
266 int cdbsz_in;
267 int help;
268 struct flags_t in_flags;
269 atomic<int> in_partial; /* | */
270 off_t in_st_size; /* Only for FT_OTHER (regular) file */
271 int mrq_num; /* if user gives 0, set this to 1 */
272 int out0fd;
273 int out_type;
274 int cdbsz_out;
275 struct flags_t out_flags;
276 atomic<int> out_partial; /* | */
277 off_t out_st_size; /* Only for FT_OTHER (regular) file */
278 condition_variable infant_cv; /* after thread:0 does first segment */
279 mutex infant_mut;
280 int bs;
281 int bpt;
282 int cmd_timeout; /* in milliseconds */
283 int elem_sz;
284 int outregfd;
285 int outreg_type;
286 off_t outreg_st_size;
287 atomic<int> dio_incomplete_count;
288 atomic<int> sum_of_resids;
289 atomic<int> reason_res;
290 atomic<int> most_recent_pack_id;
291 uint32_t sdt_ict; /* stall detection; initial check time (milliseconds) */
292 uint32_t sdt_crt; /* check repetition time (seconds), after first stall */
293 int dry_run;
294 int verbose;
295 bool mrq_eq_0; /* true when user gives mrq=0 */
296 bool processed;
297 bool cdbsz_given;
298 bool cdl_given;
299 bool count_given;
300 bool ese;
301 bool flexible;
302 bool mrq_polled;
303 bool ofile_given;
304 bool unit_nanosec; /* default duration unit is millisecond */
305 bool verify; /* don't copy, verify like Unix: cmp */
306 bool prefetch; /* for verify: do PF(b),RD(a),V(b)_a_data */
307 vector<string> inf_v;
308 vector<string> outf_v;
309 const char * infp;
310 const char * outfp;
311 class scat_gath_list i_sgl;
312 class scat_gath_list o_sgl;
313 };
314
315 typedef struct request_element
316 { /* one instance per worker thread */
317 struct global_collection *clp;
318 bool has_share;
319 bool both_sg;
320 bool same_sg;
321 bool only_in_sg;
322 bool only_out_sg;
323 bool stop_after_write;
324 bool stop_now;
325 int id;
326 int bs;
327 int infd;
328 int outfd;
329 int outregfd;
330 uint8_t * buffp;
331 uint8_t * alloc_bp;
332 struct sg_io_v4 io_hdr4[2];
333 uint8_t cmd[MAX_SCSI_CDB_SZ];
334 uint8_t sb[SENSE_BUFF_LEN];
335 int dio_incomplete_count;
336 int mmap_active;
337 int rd_p_id;
338 int rep_count;
339 int rq_id;
340 int mmap_len;
341 int mrq_id;
342 int mrq_index;
343 int mrq_pack_id_off;
344 uint32_t a_mrq_din_blks;
345 uint32_t a_mrq_dout_blks;
346 int64_t in_follow_on;
347 int64_t out_follow_on;
348 int64_t in_local_count;
349 int64_t out_local_count;
350 int64_t in_rem_count;
351 int64_t out_rem_count;
352 int in_local_partial;
353 int out_local_partial;
354 int in_resid_bytes;
355 long seed;
356 #ifdef HAVE_SRAND48_R /* gcc extension. N.B. non-reentrant version slower */
357 struct drand48_data drand;/* opaque, used by srand48_r and mrand48_r */
358 #endif
359 } Rq_elem;
360
361 /* Additional parameters for sg_start_io() and sg_finish_io() */
362 struct sg_io_extra {
363 bool prefetch;
364 bool dout_is_split;
365 int hpv4_ind;
366 int blk_offset;
367 int blks;
368 };
369
370 #define MONO_MRQ_ID_INIT 0x10000
371
372
373
374 /* Use this class to wrap C++11 <random> features to produce uniform random
375 * unsigned ints in the range [lo, hi] (inclusive) given a_seed */
376 class Rand_uint {
377 public:
Rand_uint(unsigned int lo,unsigned int hi,unsigned int a_seed)378 Rand_uint(unsigned int lo, unsigned int hi, unsigned int a_seed)
379 : uid(lo, hi), dre(a_seed) { }
380 /* uid ctor takes inclusive range when integral type */
381
get()382 unsigned int get() { return uid(dre); }
383
384 private:
385 uniform_int_distribution<unsigned int> uid;
386 default_random_engine dre;
387 };
388
389 static atomic<int> num_ebusy(0);
390 static atomic<int> num_start_eagain(0);
391 static atomic<int> num_fin_eagain(0);
392 static atomic<int> num_miscompare(0);
393 static atomic<int> num_fallthru_sigusr2(0);
394 static atomic<bool> vb_first_time(true);
395
396 static sigset_t signal_set;
397 static sigset_t orig_signal_set;
398
399 static const char * sg_allow_dio = "/sys/module/sg/parameters/allow_dio";
400
401 static int do_both_sg_segment(Rq_elem * rep, scat_gath_iter & i_sg_it,
402 scat_gath_iter & o_sg_it, int seg_blks,
403 vector<cdb_arr_t> & a_cdb,
404 vector<struct sg_io_v4> & a_v4);
405 static int do_both_sg_segment_mrq0(Rq_elem * rep, scat_gath_iter & i_sg_it,
406 scat_gath_iter & o_sg_it, int seg_blks);
407 static int do_normal_sg_segment(Rq_elem * rep, scat_gath_iter & i_sg_it,
408 scat_gath_iter & o_sg_it, int seg_blks,
409 vector<cdb_arr_t> & a_cdb,
410 vector<struct sg_io_v4> & a_v4);
411 static int do_normal_normal_segment(Rq_elem * rep, scat_gath_iter & i_sg_it,
412 scat_gath_iter & o_sg_it, int seg_blks);
413
414 #define STRERR_BUFF_LEN 128
415
416 static mutex strerr_mut;
417
418 static bool have_sg_version = false;
419 static int sg_version = 0;
420 static bool sg_version_ge_40045 = false;
421 static atomic<bool> shutting_down{false};
422 static bool do_sync = false;
423 static int do_time = 1;
424 static struct global_collection gcoll;
425 static struct timeval start_tm;
426 static int num_threads = DEF_NUM_THREADS;
427 static bool after1 = false;
428 static int listen_t_tid;
429
430 static const char * my_name = "sg_mrq_dd: ";
431
432 // static const char * mrq_blk_s = "mrq: ordinary blocking";
433 static const char * mrq_svb_s = "mrq: shared variable blocking (svb)";
434 static const char * mrq_ob_s = "mrq: ordered blocking";
435 static const char * mrq_vb_s = "mrq: variable blocking";
436
437
438 #ifdef __GNUC__
439 static int pr2serr_lk(const char * fmt, ...)
440 __attribute__ ((format (printf, 1, 2)));
441 #else
442 static int pr2serr_lk(const char * fmt, ...);
443 #endif
444
445
446 static int
pr2serr_lk(const char * fmt,...)447 pr2serr_lk(const char * fmt, ...)
448 {
449 int n;
450 va_list args;
451 lock_guard<mutex> lk(strerr_mut);
452
453 va_start(args, fmt);
454 n = vfprintf(stderr, fmt, args);
455 va_end(args);
456 return n;
457 }
458
459 static void
usage(int pg_num)460 usage(int pg_num)
461 {
462 if (pg_num > 4)
463 goto page5;
464 if (pg_num > 3)
465 goto page4;
466 else if (pg_num > 2)
467 goto page3;
468 else if (pg_num > 1)
469 goto page2;
470
471 pr2serr("Usage: sg_mrq_dd [bs=BS] [conv=CONV] [count=COUNT] [ibs=BS] "
472 "[if=IFILE*]\n"
473 " [iflag=FLAGS] [obs=BS] [of=OFILE*] "
474 "[oflag=FLAGS]\n"
475 " [seek=SEEK] [skip=SKIP] [--help] [--verify] "
476 "[--version]\n\n");
477 pr2serr(" [bpt=BPT] [cdbsz=6|10|12|16] [cdl=CDL] "
478 "[dio=0|1]\n"
479 " [elemsz_kb=EKB] [ese=0|1] [fua=0|1|2|3] "
480 "[polled=NRQS]\n"
481 " [mrq=NRQS] [ofreg=OFREG] [sdt=SDT] "
482 "[sync=0|1]\n"
483 " [thr=THR] [time=0|1|2[,TO]] [verbose=VERB] "
484 "[--dry-run]\n"
485 " [--pre-fetch] [--verbose] [--version]\n\n"
486 " where: operands have the form name=value and are pecular to "
487 "'dd'\n"
488 " style commands, and options start with one or "
489 "two hyphens;\n"
490 " the main operands and options (shown in first group "
491 "above) are:\n"
492 " bs must be device logical block size (default "
493 "512)\n"
494 " conv comma separated list from: [nocreat,noerror,"
495 "notrunc,\n"
496 " null,sync]\n"
497 " count number of blocks to copy (def: device size)\n"
498 " if file(s) or device(s) to read from (def: "
499 "stdin)\n"
500 " iflag comma separated list from: [00,coe,dio,"
501 "direct,dpo,\n"
502 " dsync,excl,ff,fua,masync,mmap,mout_if,nodur,"
503 "null,\n"
504 " order,qhead,qtail,random,same_fds,serial,"
505 "wq_excl]\n"
506 " of file(s) or device(s) to write to (def: "
507 "/dev/null)\n"
508 " 'of=.' also outputs to /dev/null\n"
509 " oflag comma separated list from: [append,nocreat,\n"
510 " <<list from iflag>>]\n"
511 " seek block position to start writing to OFILE\n"
512 " skip block position to start reading from IFILE\n"
513 " --help|-h output this usage message then exit\n"
514 " --verify|-x do a verify (compare) operation [def: do a "
515 "copy]\n"
516 " --version|-V output version string then exit\n\n"
517 "Copy IFILE to OFILE, similar to dd command. A comma separated "
518 "list of files\n may be given for IFILE*, ditto for OFILE*. "
519 "This utility is specialized for\nSCSI devices and uses the "
520 "'multiple requests' (mrq) in a single invocation\nfacility in "
521 "version 4 of the sg driver unless mrq=0. Usually one or both\n"
522 "IFILE and OFILE will be sg devices. With the --verify option "
523 "it does a\nverify/compare operation instead of a copy. This "
524 "utility is Linux specific.\nUse '-hh', '-hhh', '-hhhh' or "
525 "'-hhhhh' for more information.\n"
526 );
527 return;
528 page2:
529 pr2serr("Syntax: sg_mrq_dd [operands] [options]\n\n"
530 " the lesser used operands and option are:\n\n"
531 " bpt is blocks_per_transfer (default is 128)\n"
532 " cdbsz size of SCSI READ, WRITE or VERIFY cdb_s "
533 "(default is 10)\n"
534 " cdl command duration limits value 0 to 7 (def: "
535 "0 (no cdl))\n"
536 " dio is direct IO, 1->attempt, 0->indirect IO (def)\n"
537 " elemsz_kb=EKB scatter gather list element size in "
538 "kibibytes;\n"
539 " must be power of two, >= page_size "
540 "(typically 4)\n"
541 " ese=0|1 exit on secondary error when 1, else continue\n"
542 " fua force unit access: 0->don't(def), 1->OFILE, "
543 "2->IFILE,\n"
544 " 3->OFILE+IFILE\n"
545 " ibs IFILE logical block size, cannot differ from "
546 "obs or bs\n"
547 " hipri same as polled=NRQS; name 'hipri' is deprecated\n"
548 " mrq NRQS is number of cmds placed in each sg "
549 "ioctl\n"
550 " (def: 16). Does not set mrq hipri flag.\n"
551 " if mrq=0 does one-by-one, blocking "
552 "ioctl(SG_IO)s\n"
553 " obs OFILE logical block size, cannot differ from "
554 "ibs or bs\n"
555 " ofreg OFREG is regular file or pipe to send what is "
556 "read from\n"
557 " polled similar to mrq=NRQS operand but also sets "
558 "polled flag\n"
559 " IFILE in the first half of each shared element\n"
560 " sdt stall detection times: CRT[,ICT]. CRT: check "
561 "repetition\n"
562 " time (after first) in seconds; ICT: initial "
563 "check time\n"
564 " in milliseconds. Default: 3,300 . Use CRT=0 "
565 "to disable\n"
566 " sync 0->no sync(def), 1->SYNCHRONIZE CACHE on OFILE "
567 "after copy\n"
568 " thr is number of threads, must be > 0, default 4, "
569 "max 1024\n"
570 " time 0->no timing; 1/2->millisec/nanosec precision "
571 "(def: 1);\n"
572 " TO is command timeout in seconds (def: 60)\n"
573 " verbose increase verbosity (def: VERB=0)\n"
574 " --dry-run|-d prepare but bypass copy/read\n"
575 " --prefetch|-p with verify: do pre-fetch first\n"
576 " --verbose|-v increase verbosity of utility\n\n"
577 "Use '-hhh', '-hhhh' or '-hhhhh' for more information about "
578 "flags.\n"
579 );
580 return;
581 page3:
582 pr2serr("Syntax: sg_mrq_dd [operands] [options]\n\n"
583 " where: 'iflag=<arg>' and 'oflag=<arg>' arguments are listed "
584 "below:\n\n"
585 " 00 use all zeros instead of if=IFILE (only in "
586 "iflag)\n"
587 " 00,ff generates blocks that contain own (32 bit be) "
588 "blk addr\n"
589 " append append output to OFILE (assumes OFILE is "
590 "regular file)\n"
591 " coe continue of error (reading, fills with zeros)\n"
592 " dio sets the SG_FLAG_DIRECT_IO in sg requests\n"
593 " direct sets the O_DIRECT flag on open()\n"
594 " dpo sets the DPO (disable page out) in SCSI READs "
595 "and WRITEs\n"
596 " dsync sets the O_SYNC flag on open()\n"
597 " excl sets the O_EXCL flag on open()\n"
598 " ff use all 0xff bytes instead of if=IFILE (only in "
599 "iflag)\n"
600 " fua sets the FUA (force unit access) in SCSI READs "
601 "and WRITEs\n"
602 " hipri same as 'polled'; name 'hipri' is deprecated\n"
603 " masync set 'more async' flag on this sg device\n"
604 " mmap setup mmap IO on IFILE or OFILE\n"
605 " mmap,mmap when used twice, doesn't call munmap()\n"
606 " mout_if set META_OUT_IF flag on control object\n"
607 " nocreat will fail rather than create OFILE\n"
608 " nodur turns off command duration calculations\n"
609 " no_thresh skip checking per fd max data xfer size\n"
610 " order require write ordering on sg->sg copy; only "
611 "for oflag\n"
612 " polled set POLLED flag and use blk_poll() for "
613 "completions\n"
614 " qhead queue new request at head of block queue\n"
615 " qtail queue new request at tail of block queue (def: "
616 "q at head)\n"
617 " random use random data instead of if=IFILE (only in "
618 "iflag)\n"
619 " same_fds each thread of a IOFILE pair uses same fds\n"
620 " serial serialize sg command execution (def: overlap)\n"
621 " wq_excl set SG_CTL_FLAGM_EXCL_WAITQ on this sg fd\n"
622 "\n"
623 "Copies IFILE to OFILE (and to OFILE2 if given). If IFILE and "
624 "OFILE are sg\ndevices 'shared' mode is selected. "
625 "When sharing, the data stays in a\nsingle "
626 "in-kernel buffer which is copied (or mmap-ed) to the user "
627 "space\nif the 'ofreg=OFREG' is given. Use '-hhhh' or '-hhhhh' "
628 "for more information.\n"
629 );
630 return;
631 page4:
632 pr2serr("pack_id:\n"
633 "These are ascending integers, starting at 1, associated with "
634 "each issued\nSCSI command. When both IFILE and OFILE are sg "
635 "devices, then the READ in\neach read-write pair is issued an "
636 "even pack_id and its WRITE pair is\ngiven the pack_id one "
637 "higher (i.e. an odd number). This enables a\n'dmesg -w' "
638 "user to see that progress is being "
639 "made.\n\n");
640 pr2serr("Debugging:\n"
641 "Apart from using one or more '--verbose' options which gets a "
642 "bit noisy\n'dmesg -w' can give a good overview "
643 "of what is happening.\nThat does a sg driver object tree "
644 "traversal that does minimal locking\nto make sure that each "
645 "traversal is 'safe'. So it is important to note\nthe whole "
646 "tree is not locked. This means for fast devices the overall\n"
647 "tree state may change while the traversal is occurring. For "
648 "example,\nit has been observed that both the read- and write- "
649 "sides of a request\nshare show they are in 'active' state "
650 "which should not be possible.\nIt occurs because the read-side "
651 "probably jumped out of active state and\nthe write-side "
652 "request entered it while some other nodes were being "
653 "printed.\n\n");
654 pr2serr("Busy state:\n"
655 "Busy state (abbreviated to 'bsy' in the dmesg "
656 "output)\nis entered during request setup and completion. It "
657 "is intended to be\na temporary state. It should not block "
658 "but does sometimes (e.g. in\nblock_get_request()). Even so "
659 "that blockage should be short and if not\nthere is a "
660 "problem.\n\n");
661 pr2serr("--verify :\n"
662 "For comparing IFILE with OFILE. Does repeated sequences of: "
663 "READ(ifile)\nand uses data returned to send to VERIFY(ofile, "
664 "BYTCHK=1). So the OFILE\ndevice/disk is doing the actual "
665 "comparison. Stops on first miscompare\nunless oflag=coe is "
666 "given\n\n");
667 pr2serr("--prefetch :\n"
668 "Used with --verify option. Prepends a PRE-FETCH(ofile, IMMED) "
669 "to verify\nsequence. This should speed the trailing VERIFY by "
670 "making sure that\nthe data it needs for the comparison is "
671 "already in its cache.\n");
672 return;
673 page5:
674 pr2serr(" IFILE and/or OFILE lists\n\n"
675 "For dd, its if= operand takes a single file (or device), ditto "
676 "for the of=\noperand. This utility extends that to "
677 "allowing a comma separated list\nof files. Ideally if multiple "
678 "IFILEs are given, the same number of OFILEs\nshould be given. "
679 "Simple expansions occur to make the list lengths equal\n"
680 "(e.g. if 5 IFILEs are given but no OFILEs, then OFILEs is "
681 "expanded to 5\n'/dev/null' files). IFILE,OFILE pairs with "
682 "the same list position are\ncalled a 'slice'. Each slice is "
683 "processed (i.e. copy or verify) in one or\nmore threads. The "
684 "number of threads must be >= the number of slices. Best\nif "
685 "the number of threads is an integer multiple of the number of "
686 "slices.\nThe file type of multiple IFILEs must be the same, "
687 "ditto for OFILEs.\nSupport for slices is for testing rather "
688 "than a general mechanism.\n");
689 }
690
691 static void
lk_print_command_len(const char * prefix,uint8_t * cmdp,int len,bool lock)692 lk_print_command_len(const char *prefix, uint8_t * cmdp, int len, bool lock)
693 {
694 if (lock) {
695 lock_guard<mutex> lk(strerr_mut);
696
697 if (prefix && *prefix)
698 fputs(prefix, stderr);
699 sg_print_command_len(cmdp, len);
700 } else {
701 if (prefix && *prefix)
702 fputs(prefix, stderr);
703 sg_print_command_len(cmdp, len);
704 }
705 }
706
707 static void
lk_chk_n_print4(const char * leadin,const struct sg_io_v4 * h4p,bool raw_sinfo)708 lk_chk_n_print4(const char * leadin, const struct sg_io_v4 * h4p,
709 bool raw_sinfo)
710 {
711 lock_guard<mutex> lk(strerr_mut);
712
713 if (h4p->usr_ptr) {
714 const cdb_arr_t * cdbp = (const cdb_arr_t *)h4p->usr_ptr;
715
716 pr2serr("Failed cdb: ");
717 sg_print_command(cdbp->data());
718 } else
719 pr2serr("cdb: <null>\n");
720 sg_linux_sense_print(leadin, h4p->device_status, h4p->transport_status,
721 h4p->driver_status, (const uint8_t *)h4p->response,
722 h4p->response_len, raw_sinfo);
723 }
724
725 static void
hex2stderr_lk(const uint8_t * b_str,int len,int no_ascii)726 hex2stderr_lk(const uint8_t * b_str, int len, int no_ascii)
727 {
728 lock_guard<mutex> lk(strerr_mut);
729
730 hex2stderr(b_str, len, no_ascii);
731 }
732
733 static int
system_wrapper(const char * cmd)734 system_wrapper(const char * cmd)
735 {
736 int res;
737
738 res = system(cmd);
739 if (WIFSIGNALED(res) &&
740 (WTERMSIG(res) == SIGINT || WTERMSIG(res) == SIGQUIT))
741 raise(WTERMSIG(res));
742 return WEXITSTATUS(res);
743 }
744
745 /* Flags decoded into abbreviations for those that are set, separated by
746 * '|' . */
747 static char *
sg_flags_str(int flags,int b_len,char * b)748 sg_flags_str(int flags, int b_len, char * b)
749 {
750 int n = 0;
751
752 if ((b_len < 1) || (! b))
753 return b;
754 b[0] = '\0';
755 if (SG_FLAG_DIRECT_IO & flags) { /* 0x1 */
756 n += sg_scnpr(b + n, b_len - n, "DIO|");
757 if (n >= b_len)
758 goto fini;
759 }
760 if (SG_FLAG_MMAP_IO & flags) { /* 0x4 */
761 n += sg_scnpr(b + n, b_len - n, "MMAP|");
762 if (n >= b_len)
763 goto fini;
764 }
765 if (SGV4_FLAG_YIELD_TAG & flags) { /* 0x8 */
766 n += sg_scnpr(b + n, b_len - n, "YTAG|");
767 if (n >= b_len)
768 goto fini;
769 }
770 if (SG_FLAG_Q_AT_TAIL & flags) { /* 0x10 */
771 n += sg_scnpr(b + n, b_len - n, "QTAI|");
772 if (n >= b_len)
773 goto fini;
774 }
775 if (SG_FLAG_Q_AT_HEAD & flags) { /* 0x20 */
776 n += sg_scnpr(b + n, b_len - n, "QHEA|");
777 if (n >= b_len)
778 goto fini;
779 }
780 if (SGV4_FLAG_DOUT_OFFSET & flags) { /* 0x40 */
781 n += sg_scnpr(b + n, b_len - n, "DOFF|");
782 if (n >= b_len)
783 goto fini;
784 }
785 if (SGV4_FLAG_EVENTFD & flags) { /* 0x80 */
786 n += sg_scnpr(b + n, b_len - n, "EVFD|");
787 if (n >= b_len)
788 goto fini;
789 }
790 if (SGV4_FLAG_COMPLETE_B4 & flags) { /* 0x100 */
791 n += sg_scnpr(b + n, b_len - n, "CPL_B4|");
792 if (n >= b_len)
793 goto fini;
794 }
795 if (SGV4_FLAG_SIGNAL & flags) { /* 0x200 */
796 n += sg_scnpr(b + n, b_len - n, "SIGNAL|");
797 if (n >= b_len)
798 goto fini;
799 }
800 if (SGV4_FLAG_IMMED & flags) { /* 0x400 */
801 n += sg_scnpr(b + n, b_len - n, "IMM|");
802 if (n >= b_len)
803 goto fini;
804 }
805 if (SGV4_FLAG_POLLED & flags) { /* 0x800 */
806 n += sg_scnpr(b + n, b_len - n, "POLLED|");
807 if (n >= b_len)
808 goto fini;
809 }
810 if (SGV4_FLAG_STOP_IF & flags) { /* 0x1000 */
811 n += sg_scnpr(b + n, b_len - n, "STOPIF|");
812 if (n >= b_len)
813 goto fini;
814 }
815 if (SGV4_FLAG_DEV_SCOPE & flags) { /* 0x2000 */
816 n += sg_scnpr(b + n, b_len - n, "DEV_SC|");
817 if (n >= b_len)
818 goto fini;
819 }
820 if (SGV4_FLAG_SHARE & flags) { /* 0x4000 */
821 n += sg_scnpr(b + n, b_len - n, "SHARE|");
822 if (n >= b_len)
823 goto fini;
824 }
825 if (SGV4_FLAG_DO_ON_OTHER & flags) { /* 0x8000 */
826 n += sg_scnpr(b + n, b_len - n, "DO_OTH|");
827 if (n >= b_len)
828 goto fini;
829 }
830 if (SGV4_FLAG_NO_DXFER & flags) { /* 0x10000 */
831 n += sg_scnpr(b + n, b_len - n, "NOXFER|");
832 if (n >= b_len)
833 goto fini;
834 }
835 if (SGV4_FLAG_KEEP_SHARE & flags) { /* 0x20000 */
836 n += sg_scnpr(b + n, b_len - n, "KEEP_SH|");
837 if (n >= b_len)
838 goto fini;
839 }
840 if (SGV4_FLAG_MULTIPLE_REQS & flags) { /* 0x40000 */
841 n += sg_scnpr(b + n, b_len - n, "MRQS|");
842 if (n >= b_len)
843 goto fini;
844 }
845 if (SGV4_FLAG_ORDERED_WR & flags) { /* 0x80000 */
846 n += sg_scnpr(b + n, b_len - n, "OWR|");
847 if (n >= b_len)
848 goto fini;
849 }
850 if (SGV4_FLAG_REC_ORDER & flags) { /* 0x100000 */
851 n += sg_scnpr(b + n, b_len - n, "REC_O|");
852 if (n >= b_len)
853 goto fini;
854 }
855 if (SGV4_FLAG_META_OUT_IF & flags) { /* 0x200000 */
856 n += sg_scnpr(b + n, b_len - n, "MOUT_IF|");
857 if (n >= b_len)
858 goto fini;
859 }
860 if (0 == n)
861 n += sg_scnpr(b + n, b_len - n, "<none>");
862 fini:
863 if (n < b_len) { /* trim trailing '\' */
864 if ('|' == b[n - 1])
865 b[n - 1] = '\0';
866 } else if ('|' == b[b_len - 1])
867 b[b_len - 1] = '\0';
868 return b;
869 }
870
871 /* Info field decoded into abbreviations for those bits that are set,
872 * separated by '|' . */
873 static char *
sg_info_str(int info,int b_len,char * b)874 sg_info_str(int info, int b_len, char * b)
875 {
876 int n = 0;
877
878 if ((b_len < 1) || (! b))
879 return b;
880 b[0] = '\0';
881 if (SG_INFO_CHECK & info) { /* 0x1 */
882 n += sg_scnpr(b + n, b_len - n, "CHK|");
883 if (n >= b_len)
884 goto fini;
885 }
886 if (SG_INFO_DIRECT_IO & info) { /* 0x2 */
887 n += sg_scnpr(b + n, b_len - n, "DIO|");
888 if (n >= b_len)
889 goto fini;
890 }
891 if (SG_INFO_MIXED_IO & info) { /* 0x4 */
892 n += sg_scnpr(b + n, b_len - n, "MIO|");
893 if (n >= b_len)
894 goto fini;
895 }
896 if (SG_INFO_DEVICE_DETACHING & info) { /* 0x8 */
897 n += sg_scnpr(b + n, b_len - n, "DETA|");
898 if (n >= b_len)
899 goto fini;
900 }
901 if (SG_INFO_ABORTED & info) { /* 0x10 */
902 n += sg_scnpr(b + n, b_len - n, "ABRT|");
903 if (n >= b_len)
904 goto fini;
905 }
906 if (SG_INFO_MRQ_FINI & info) { /* 0x20 */
907 n += sg_scnpr(b + n, b_len - n, "MRQF|");
908 if (n >= b_len)
909 goto fini;
910 }
911 fini:
912 if (n < b_len) { /* trim trailing '\' */
913 if ('|' == b[n - 1])
914 b[n - 1] = '\0';
915 } else if ('|' == b[b_len - 1])
916 b[b_len - 1] = '\0';
917 return b;
918 }
919
920 static void
v4hdr_out_lk(const char * leadin,const sg_io_v4 * h4p,int id,bool chk_info)921 v4hdr_out_lk(const char * leadin, const sg_io_v4 * h4p, int id, bool chk_info)
922 {
923 lock_guard<mutex> lk(strerr_mut);
924 char b[80];
925
926 if (leadin)
927 pr2serr("%s [id=%d]:\n", leadin, id);
928 if (('Q' != h4p->guard) || (0 != h4p->protocol) ||
929 (0 != h4p->subprotocol))
930 pr2serr(" <<<sg_io_v4 _NOT_ properly set>>>\n");
931 pr2serr(" pointers: cdb=%s sense=%s din=%p dout=%p\n",
932 (h4p->request ? "y" : "NULL"), (h4p->response ? "y" : "NULL"),
933 (void *)h4p->din_xferp, (void *)h4p->dout_xferp);
934 pr2serr(" lengths: cdb=%u sense=%u din=%u dout=%u\n",
935 h4p->request_len, h4p->max_response_len, h4p->din_xfer_len,
936 h4p->dout_xfer_len);
937 pr2serr(" flags=0x%x request_extra{pack_id}=%d\n",
938 h4p->flags, h4p->request_extra);
939 pr2serr(" flags set: %s\n", sg_flags_str(h4p->flags, sizeof(b), b));
940 pr2serr(" %s OUT:\n", leadin);
941 pr2serr(" response_len=%d driver/transport/device_status="
942 "0x%x/0x%x/0x%x\n", h4p->response_len, h4p->driver_status,
943 h4p->transport_status, h4p->device_status);
944 pr2serr(" info=0x%x din_resid=%u dout_resid=%u spare_out=%u "
945 "dur=%u\n",
946 h4p->info, h4p->din_resid, h4p->dout_resid, h4p->spare_out,
947 h4p->duration);
948 if (chk_info && (SG_INFO_CHECK & h4p->info))
949 pr2serr(" >>>> info: %s\n", sg_info_str(h4p->info, sizeof(b), b));
950 }
951
952 static void
fetch_sg_version(void)953 fetch_sg_version(void)
954 {
955 FILE * fp;
956 char b[96];
957
958 have_sg_version = false;
959 sg_version = 0;
960 fp = fopen(PROC_SCSI_SG_VERSION, "r");
961 if (fp && fgets(b, sizeof(b) - 1, fp)) {
962 if (1 == sscanf(b, "%d", &sg_version))
963 have_sg_version = !!sg_version;
964 } else {
965 int j, k, l;
966
967 if (fp)
968 fclose(fp);
969 fp = fopen(SYS_SCSI_SG_VERSION, "r");
970 if (fp && fgets(b, sizeof(b) - 1, fp)) {
971 if (3 == sscanf(b, "%d.%d.%d", &j, &k, &l)) {
972 sg_version = (j * 10000) + (k * 100) + l;
973 have_sg_version = !!sg_version;
974 }
975 }
976 if (NULL == fp)
977 pr2serr("The sg driver may not be loaded\n");
978 }
979 if (fp)
980 fclose(fp);
981 }
982
983 static void
calc_duration_throughput(int contin)984 calc_duration_throughput(int contin)
985 {
986 struct timeval end_tm, res_tm;
987 double a, b;
988
989 gettimeofday(&end_tm, NULL);
990 res_tm.tv_sec = end_tm.tv_sec - start_tm.tv_sec;
991 res_tm.tv_usec = end_tm.tv_usec - start_tm.tv_usec;
992 if (res_tm.tv_usec < 0) {
993 --res_tm.tv_sec;
994 res_tm.tv_usec += 1000000;
995 }
996 a = res_tm.tv_sec;
997 a += (0.000001 * res_tm.tv_usec);
998
999 b = 0.0;
1000 for (auto && cvp : gcoll.cp_ver_arr) {
1001 if (cvp.state == cp_ver_pair_t::my_state::empty)
1002 break;
1003 b += (double)(cvp.dd_count - cvp.out_rem_count.load());
1004 }
1005 b *= (double)gcoll.bs;
1006 pr2serr("time to %s data %s %d.%06d secs",
1007 (gcoll.verify ? "verify" : "copy"), (contin ? "so far" : "was"),
1008 (int)res_tm.tv_sec, (int)res_tm.tv_usec);
1009 if ((a > 0.00001) && (b > 511))
1010 pr2serr(", %.2f MB/sec\n", b / (a * 1000000.0));
1011 else
1012 pr2serr("\n");
1013 }
1014
1015 static void
print_stats(const char * str)1016 print_stats(const char * str)
1017 {
1018 bool show_slice = ((gcoll.cp_ver_arr.size() > 1) &&
1019 (gcoll.cp_ver_arr[1].state !=
1020 cp_ver_pair_t::my_state::empty));
1021 int k = 0;
1022 int64_t infull, outfull;
1023
1024 for (auto && cvp : gcoll.cp_ver_arr) {
1025 ++k;
1026 if (cvp.state == cp_ver_pair_t::my_state::empty)
1027 break;
1028 if (cvp.state == cp_ver_pair_t::my_state::ignore) {
1029 pr2serr(">>> IGNORING slice: %d\n", k);
1030 continue;
1031 }
1032 if (show_slice)
1033 pr2serr(">>> slice: %d\n", k);
1034 if (0 != cvp.out_rem_count.load())
1035 pr2serr(" remaining block count=%" PRId64 "\n",
1036 cvp.out_rem_count.load());
1037 infull = cvp.dd_count - cvp.in_rem_count.load();
1038 pr2serr("%s%" PRId64 "+%d records in\n", str,
1039 infull, cvp.in_partial.load());
1040
1041 if (cvp.out_type == FT_DEV_NULL)
1042 pr2serr("%s0+0 records out\n", str);
1043 else {
1044 outfull = cvp.dd_count - cvp.out_rem_count.load();
1045 pr2serr("%s%" PRId64 "+%d records %s\n", str,
1046 outfull, cvp.out_partial.load(),
1047 (gcoll.verify ? "verified" : "out"));
1048 }
1049 }
1050 }
1051
1052 static void
interrupt_handler(int sig)1053 interrupt_handler(int sig)
1054 {
1055 struct sigaction sigact;
1056
1057 sigact.sa_handler = SIG_DFL;
1058 sigemptyset(&sigact.sa_mask);
1059 sigact.sa_flags = 0;
1060 sigaction(sig, &sigact, NULL);
1061 pr2serr("Interrupted by signal,");
1062 if (do_time > 0)
1063 calc_duration_throughput(0);
1064 print_stats("");
1065 kill(getpid(), sig);
1066 }
1067
1068 static void
siginfo_handler(int sig)1069 siginfo_handler(int sig)
1070 {
1071 if (sig) { ; } /* unused, dummy to suppress warning */
1072 pr2serr("Progress report, continuing ...\n");
1073 if (do_time > 0)
1074 calc_duration_throughput(1);
1075 print_stats(" ");
1076 }
1077
1078 /* Usually this signal (SIGUSR2) will be caught by the timed wait in the
1079 * sig_listen_thread thread but some might slip through while the timed
1080 * wait is being re-armed or after that thread is finished. This handler
1081 * acts as a backstop. */
1082 static void
siginfo2_handler(int sig)1083 siginfo2_handler(int sig)
1084 {
1085 if (sig) { ; } /* unused, dummy to suppress warning */
1086 ++num_fallthru_sigusr2;
1087 }
1088
1089 static void
install_handler(int sig_num,void (* sig_handler)(int sig))1090 install_handler(int sig_num, void (*sig_handler) (int sig))
1091 {
1092 struct sigaction sigact;
1093 sigaction (sig_num, NULL, &sigact);
1094 if (sigact.sa_handler != SIG_IGN)
1095 {
1096 sigact.sa_handler = sig_handler;
1097 sigemptyset (&sigact.sa_mask);
1098 sigact.sa_flags = 0;
1099 sigaction (sig_num, &sigact, NULL);
1100 }
1101 }
1102
1103 /* Make safe_strerror() thread safe */
1104 static char *
tsafe_strerror(int code,char * ebp)1105 tsafe_strerror(int code, char * ebp)
1106 {
1107 lock_guard<mutex> lk(strerr_mut);
1108 char * cp;
1109
1110 cp = safe_strerror(code);
1111 strncpy(ebp, cp, STRERR_BUFF_LEN);
1112 ebp[STRERR_BUFF_LEN - 1] = '\0';
1113 return ebp;
1114 }
1115
1116
1117 static int
dd_filetype(const char * filename,off_t & st_size)1118 dd_filetype(const char * filename, off_t & st_size)
1119 {
1120 struct stat st;
1121 size_t len = strlen(filename);
1122
1123 if ((1 == len) && ('.' == filename[0]))
1124 return FT_DEV_NULL;
1125 if (stat(filename, &st) < 0)
1126 return FT_ERROR;
1127 if (S_ISCHR(st.st_mode)) {
1128 if ((MEM_MAJOR == major(st.st_rdev)) &&
1129 ((DEV_NULL_MINOR_NUM == minor(st.st_rdev)) ||
1130 (DEV_ZERO_MINOR_NUM == minor(st.st_rdev))))
1131 return FT_DEV_NULL; /* treat /dev/null + /dev/zero the same */
1132 if (SCSI_GENERIC_MAJOR == major(st.st_rdev))
1133 return FT_SG;
1134 if (SCSI_TAPE_MAJOR == major(st.st_rdev))
1135 return FT_ST;
1136 return FT_CHAR;
1137 } else if (S_ISBLK(st.st_mode))
1138 return FT_BLOCK;
1139 else if (S_ISFIFO(st.st_mode))
1140 return FT_FIFO;
1141 st_size = st.st_size;
1142 return FT_OTHER;
1143 }
1144
1145 /* Returns reserved_buffer_size/mmap_size if success, else 0 for failure */
1146 static int
sg_prepare_resbuf(int fd,struct global_collection * clp,bool is_in,uint8_t ** mmpp)1147 sg_prepare_resbuf(int fd, struct global_collection *clp, bool is_in,
1148 uint8_t **mmpp)
1149 {
1150 static bool done = false;
1151 bool no_dur = is_in ? clp->in_flags.no_dur : clp->out_flags.no_dur;
1152 bool masync = is_in ? clp->in_flags.masync : clp->out_flags.masync;
1153 bool wq_excl = is_in ? clp->in_flags.wq_excl : clp->out_flags.wq_excl;
1154 bool skip_thresh = is_in ? clp->in_flags.no_thresh :
1155 clp->out_flags.no_thresh;
1156 int elem_sz = clp->elem_sz;
1157 int res, t, num, err;
1158 uint8_t *mmp;
1159 struct sg_extended_info sei {};
1160 struct sg_extended_info * seip = &sei;
1161
1162 res = ioctl(fd, SG_GET_VERSION_NUM, &t);
1163 if ((res < 0) || (t < 40000)) {
1164 if (ioctl(fd, SG_GET_RESERVED_SIZE, &num) < 0) {
1165 perror("SG_GET_RESERVED_SIZE ioctl failed");
1166 return 0;
1167 }
1168 if (! done) {
1169 done = true;
1170 pr2serr_lk("%ssg driver prior to 4.0.00, reduced functionality\n",
1171 my_name);
1172 }
1173 goto bypass;
1174 }
1175 if (elem_sz >= 4096) {
1176 seip->sei_rd_mask |= SG_SEIM_SGAT_ELEM_SZ;
1177 res = ioctl(fd, SG_SET_GET_EXTENDED, seip);
1178 if (res < 0)
1179 pr2serr_lk("sg_mrq_dd: %s: SG_SET_GET_EXTENDED(SGAT_ELEM_SZ) rd "
1180 "error: %s\n", __func__, strerror(errno));
1181 if (elem_sz != (int)seip->sgat_elem_sz) {
1182 seip->sei_wr_mask |= SG_SEIM_SGAT_ELEM_SZ;
1183 seip->sgat_elem_sz = elem_sz;
1184 res = ioctl(fd, SG_SET_GET_EXTENDED, seip);
1185 if (res < 0)
1186 pr2serr_lk("sg_mrq_dd: %s: SG_SET_GET_EXTENDED(SGAT_ELEM_SZ) "
1187 "wr error: %s\n", __func__, strerror(errno));
1188 }
1189 }
1190 if (no_dur || masync || skip_thresh) {
1191 seip->sei_wr_mask |= SG_SEIM_CTL_FLAGS;
1192 if (no_dur) {
1193 seip->ctl_flags_wr_mask |= SG_CTL_FLAGM_NO_DURATION;
1194 seip->ctl_flags |= SG_CTL_FLAGM_NO_DURATION;
1195 }
1196 if (masync) {
1197 seip->ctl_flags_wr_mask |= SG_CTL_FLAGM_MORE_ASYNC;
1198 seip->ctl_flags |= SG_CTL_FLAGM_MORE_ASYNC;
1199 }
1200 if (wq_excl) {
1201 seip->ctl_flags_wr_mask |= SG_CTL_FLAGM_EXCL_WAITQ;
1202 seip->ctl_flags |= SG_CTL_FLAGM_EXCL_WAITQ;
1203 }
1204 if (skip_thresh) {
1205 seip->tot_fd_thresh = 0;
1206 sei.sei_wr_mask |= SG_SEIM_TOT_FD_THRESH;
1207 }
1208 res = ioctl(fd, SG_SET_GET_EXTENDED, seip);
1209 if (res < 0)
1210 pr2serr_lk("sg_mrq_dd: %s: SG_SET_GET_EXTENDED(NO_DURATION) "
1211 "error: %s\n", __func__, strerror(errno));
1212 }
1213 bypass:
1214 num = clp->bs * clp->bpt;
1215 res = ioctl(fd, SG_SET_RESERVED_SIZE, &num);
1216 if (res < 0) {
1217 perror("sg_mrq_dd: SG_SET_RESERVED_SIZE error");
1218 return 0;
1219 } else {
1220 int nn;
1221
1222 res = ioctl(fd, SG_GET_RESERVED_SIZE, &nn);
1223 if (res < 0) {
1224 perror("sg_mrq_dd: SG_GET_RESERVED_SIZE error");
1225 return 0;
1226 }
1227 if (nn < num) {
1228 pr2serr_lk("%s: SG_GET_RESERVED_SIZE shows size truncated, "
1229 "wanted %d got %d\n", __func__, num, nn);
1230 return 0;
1231 }
1232 if (mmpp) {
1233 mmp = (uint8_t *)mmap(NULL, num, PROT_READ | PROT_WRITE,
1234 MAP_SHARED, fd, 0);
1235 if (MAP_FAILED == mmp) {
1236 err = errno;
1237 pr2serr_lk("sg_mrq_dd: %s: sz=%d, fd=%d, mmap() failed: %s\n",
1238 __func__, num, fd, strerror(err));
1239 return 0;
1240 }
1241 *mmpp = mmp;
1242 }
1243 }
1244 t = 1;
1245 res = ioctl(fd, SG_SET_FORCE_PACK_ID, &t);
1246 if (res < 0)
1247 perror("sg_mrq_dd: SG_SET_FORCE_PACK_ID error");
1248 if (clp->unit_nanosec) {
1249 seip->sei_wr_mask |= SG_SEIM_CTL_FLAGS;
1250 seip->ctl_flags_wr_mask |= SG_CTL_FLAGM_TIME_IN_NS;
1251 seip->ctl_flags |= SG_CTL_FLAGM_TIME_IN_NS;
1252 if (ioctl(fd, SG_SET_GET_EXTENDED, seip) < 0) {
1253 res = -1;
1254 pr2serr_lk("ioctl(EXTENDED(TIME_IN_NS)) failed, errno=%d %s\n",
1255 errno, strerror(errno));
1256 }
1257 }
1258 if (clp->verbose) {
1259 t = 1;
1260 /* more info in the kernel log */
1261 res = ioctl(fd, SG_SET_DEBUG, &t);
1262 if (res < 0)
1263 perror("sg_mrq_dd: SG_SET_DEBUG error");
1264 }
1265 return (res < 0) ? 0 : num;
1266 }
1267
1268 static int
sg_in_open(struct global_collection * clp,const string & inf,uint8_t ** mmpp,int * mmap_lenp)1269 sg_in_open(struct global_collection *clp, const string & inf, uint8_t **mmpp,
1270 int * mmap_lenp)
1271 {
1272 int fd, err, n;
1273 int flags = O_RDWR;
1274 char ebuff[EBUFF_SZ];
1275 const char * fnp = inf.c_str();
1276
1277 if (clp->in_flags.direct)
1278 flags |= O_DIRECT;
1279 if (clp->in_flags.excl)
1280 flags |= O_EXCL;
1281 if (clp->in_flags.dsync)
1282 flags |= O_SYNC;
1283
1284 if ((fd = open(fnp, flags)) < 0) {
1285 err = errno;
1286 snprintf(ebuff, EBUFF_SZ, "%s: could not open %s for sg reading",
1287 __func__, fnp);
1288 perror(ebuff);
1289 return -sg_convert_errno(err);
1290 }
1291 n = sg_prepare_resbuf(fd, clp, true, mmpp);
1292 if (n <= 0) {
1293 close(fd);
1294 return -SG_LIB_FILE_ERROR;
1295 }
1296 if (mmap_lenp)
1297 *mmap_lenp = n;
1298 return fd;
1299 }
1300
1301 static int
sg_out_open(struct global_collection * clp,const string & outf,uint8_t ** mmpp,int * mmap_lenp)1302 sg_out_open(struct global_collection *clp, const string & outf,
1303 uint8_t **mmpp, int * mmap_lenp)
1304 {
1305 int fd, err, n;
1306 int flags = O_RDWR;
1307 char ebuff[EBUFF_SZ];
1308 const char * fnp = outf.c_str();
1309
1310 if (clp->out_flags.direct)
1311 flags |= O_DIRECT;
1312 if (clp->out_flags.excl)
1313 flags |= O_EXCL;
1314 if (clp->out_flags.dsync)
1315 flags |= O_SYNC;
1316
1317 if ((fd = open(fnp, flags)) < 0) {
1318 err = errno;
1319 snprintf(ebuff, EBUFF_SZ, "%s: could not open %s for sg %s",
1320 __func__, fnp, (clp->verify ? "verifying" : "writing"));
1321 perror(ebuff);
1322 return -sg_convert_errno(err);
1323 }
1324 n = sg_prepare_resbuf(fd, clp, false, mmpp);
1325 if (n <= 0) {
1326 close(fd);
1327 return -SG_LIB_FILE_ERROR;
1328 }
1329 if (mmap_lenp)
1330 *mmap_lenp = n;
1331 return fd;
1332 }
1333
1334 static int
reg_file_open(struct global_collection * clp,const string & fn_s,bool for_wr)1335 reg_file_open(struct global_collection *clp, const string & fn_s,
1336 bool for_wr)
1337 {
1338 int fd, flags;
1339 char ebuff[EBUFF_SZ];
1340
1341 if (for_wr) {
1342 flags = O_WRONLY;
1343 if (! clp->out_flags.nocreat)
1344 flags |= O_CREAT;
1345 if (clp->out_flags.append)
1346 flags |= O_APPEND;
1347 } else
1348 flags = O_RDONLY;
1349 if (clp->in_flags.direct)
1350 flags |= O_DIRECT;
1351 if (clp->in_flags.excl)
1352 flags |= O_EXCL;
1353 if (clp->in_flags.dsync)
1354 flags |= O_SYNC;
1355
1356 if (for_wr)
1357 fd = open(fn_s.c_str(), flags, 0666);
1358 else
1359 fd = open(fn_s.c_str(), flags);
1360 if (fd < 0) {
1361 int err = errno;
1362 snprintf(ebuff, EBUFF_SZ, "%scould not open %s for %sing ",
1363 my_name, fn_s.c_str(), (for_wr ? "writ" : "read"));
1364 perror(ebuff);
1365 return -err;
1366 }
1367 return fd;
1368 }
1369
1370 get_next_res_t
get_next(int desired_num_blks)1371 cp_ver_pair_t::get_next(int desired_num_blks)
1372 {
1373 int64_t expected, desired;
1374
1375 if (desired_num_blks <= 0) {
1376 if (desired_num_blks < 0) {
1377 if (next_count_pos.load() >= 0) /* flag error detection */
1378 next_count_pos.store(desired_num_blks);
1379 }
1380 return make_pair(next_count_pos.load(), 0);
1381 }
1382
1383 expected = next_count_pos.load();
1384 do { /* allowed to race with other threads */
1385 if (expected < 0)
1386 return make_pair(0, (int)expected);
1387 else if (expected >= dd_count)
1388 return make_pair(expected, 0); /* clean finish */
1389 desired = expected + desired_num_blks;
1390 if (desired > dd_count)
1391 desired = dd_count;
1392 } while (! next_count_pos.compare_exchange_strong(expected, desired));
1393 return make_pair(expected, desired - expected);
1394 }
1395
1396 /* Return of 0 -> success, see sg_ll_read_capacity*() otherwise */
1397 static int
scsi_read_capacity(int sg_fd,int64_t * num_sect,int * sect_sz)1398 scsi_read_capacity(int sg_fd, int64_t * num_sect, int * sect_sz)
1399 {
1400 int res;
1401 uint8_t rcBuff[RCAP16_REPLY_LEN] = {};
1402
1403 res = sg_ll_readcap_10(sg_fd, 0, 0, rcBuff, READ_CAP_REPLY_LEN, false, 0);
1404 if (0 != res)
1405 goto bad;
1406
1407 if ((0xff == rcBuff[0]) && (0xff == rcBuff[1]) && (0xff == rcBuff[2]) &&
1408 (0xff == rcBuff[3])) {
1409
1410 res = sg_ll_readcap_16(sg_fd, 0, 0, rcBuff, RCAP16_REPLY_LEN, false,
1411 0);
1412 if (0 != res)
1413 goto bad;
1414 *num_sect = sg_get_unaligned_be64(rcBuff + 0) + 1;
1415 *sect_sz = sg_get_unaligned_be32(rcBuff + 8);
1416 } else {
1417 /* take care not to sign extend values > 0x7fffffff */
1418 *num_sect = (int64_t)sg_get_unaligned_be32(rcBuff + 0) + 1;
1419 *sect_sz = sg_get_unaligned_be32(rcBuff + 4);
1420 }
1421 return 0;
1422 bad:
1423 *num_sect = 0;
1424 *sect_sz = 0;
1425 return res;
1426 }
1427
1428 /* Return of 0 -> success, -1 -> failure. BLKGETSIZE64, BLKGETSIZE and */
1429 /* BLKSSZGET macros problematic (from <linux/fs.h> or <sys/mount.h>). */
1430 static int
read_blkdev_capacity(int sg_fd,int64_t * num_sect,int * sect_sz)1431 read_blkdev_capacity(int sg_fd, int64_t * num_sect, int * sect_sz)
1432 {
1433 #ifdef BLKSSZGET
1434 if ((ioctl(sg_fd, BLKSSZGET, sect_sz) < 0) && (*sect_sz > 0)) {
1435 perror("BLKSSZGET ioctl error");
1436 return -1;
1437 } else {
1438 #ifdef BLKGETSIZE64
1439 uint64_t ull;
1440
1441 if (ioctl(sg_fd, BLKGETSIZE64, &ull) < 0) {
1442
1443 perror("BLKGETSIZE64 ioctl error");
1444 return -1;
1445 }
1446 *num_sect = ((int64_t)ull / (int64_t)*sect_sz);
1447 #else
1448 unsigned long ul;
1449
1450 if (ioctl(sg_fd, BLKGETSIZE, &ul) < 0) {
1451 perror("BLKGETSIZE ioctl error");
1452 return -1;
1453 }
1454 *num_sect = (int64_t)ul;
1455 #endif
1456 }
1457 return 0;
1458 #else
1459 *num_sect = 0;
1460 *sect_sz = 0;
1461 return -1;
1462 #endif
1463 }
1464
1465 static void
flag_all_stop(struct global_collection * clp)1466 flag_all_stop(struct global_collection * clp)
1467 {
1468 for (auto && elem : clp->cp_ver_arr) {
1469 if (elem.state == cp_ver_pair_t::my_state::empty)
1470 break;
1471 elem.next_count_pos.store(-1);
1472 }
1473 }
1474
1475 /* Has an infinite loop doing a timed wait for any signals in signal_set.
1476 * After each timeout (300 ms) checks if the most_recent_pack_id atomic
1477 * integer has changed. If not after another two timeouts announces a stall
1478 * has been detected. If shutting down atomic is true breaks out of loop and
1479 * shuts down this thread. Other than that, this thread is normally cancelled
1480 * by the main thread, after other threads have exited. */
1481 static void
sig_listen_thread(struct global_collection * clp)1482 sig_listen_thread(struct global_collection * clp)
1483 {
1484 bool stall_reported = false;
1485 int prev_pack_id = 0;
1486 int sig_number, pack_id;
1487 uint32_t ict_ms = (clp->sdt_ict ? clp->sdt_ict : DEF_SDT_ICT_MS);
1488 struct timespec ts;
1489 struct timespec * tsp = &ts;
1490
1491 tsp->tv_sec = ict_ms / 1000;
1492 tsp->tv_nsec = (ict_ms % 1000) * 1000 * 1000; /* DEF_SDT_ICT_MS */
1493 listen_t_tid = gettid(); // to facilitate sending SIGUSR2 to exit
1494 while (1) {
1495 sig_number = sigtimedwait(&signal_set, NULL, tsp);
1496 if (sig_number < 0) {
1497 int err = errno;
1498
1499 /* EAGAIN implies a timeout */
1500 if ((EAGAIN == err) && (clp->sdt_crt > 0)) {
1501 pack_id = clp->most_recent_pack_id.load();
1502 if ((pack_id > 0) && (pack_id == prev_pack_id)) {
1503 if (! stall_reported) {
1504 stall_reported = true;
1505 tsp->tv_sec = clp->sdt_crt;
1506 tsp->tv_nsec = 0;
1507 pr2serr_lk("%s: first stall at pack_id=%d detected\n",
1508 __func__, pack_id);
1509 } else
1510 pr2serr_lk("%s: subsequent stall at pack_id=%d\n",
1511 __func__, pack_id);
1512 // following command assumes linux bash or similar shell
1513 system_wrapper("cat /proc/scsi/sg/debug >> /dev/stderr\n");
1514 // system_wrapper("/usr/bin/dmesg\n");
1515 } else
1516 prev_pack_id = pack_id;
1517 } else if (EAGAIN != err)
1518 pr2serr_lk("%s: sigtimedwait() errno=%d\n", __func__, err);
1519 }
1520 if (SIGINT == sig_number) {
1521 pr2serr_lk("%sinterrupted by SIGINT\n", my_name);
1522 flag_all_stop(clp);
1523 shutting_down.store(true);
1524 sigprocmask(SIG_SETMASK, &orig_signal_set, NULL);
1525 raise(SIGINT);
1526 break;
1527 }
1528 if (SIGUSR2 == sig_number) {
1529 if (clp->verbose > 2)
1530 pr2serr_lk("%s: SIGUSR2 received\n", __func__);
1531 break;
1532 } if (shutting_down)
1533 break;
1534 } /* end of while loop */
1535 if (clp->verbose > 3)
1536 pr2serr_lk("%s: exiting\n", __func__);
1537 }
1538
1539 static bool
sg_share_prepare(int write_side_fd,int read_side_fd,int id,bool vb_b)1540 sg_share_prepare(int write_side_fd, int read_side_fd, int id, bool vb_b)
1541 {
1542 struct sg_extended_info sei {};
1543 struct sg_extended_info * seip = &sei;
1544
1545 seip->sei_wr_mask |= SG_SEIM_SHARE_FD;
1546 seip->sei_rd_mask |= SG_SEIM_SHARE_FD;
1547 seip->share_fd = read_side_fd;
1548 if (ioctl(write_side_fd, SG_SET_GET_EXTENDED, seip) < 0) {
1549 pr2serr_lk("tid=%d: ioctl(EXTENDED(shared_fd=%d), failed "
1550 "errno=%d %s\n", id, read_side_fd, errno,
1551 strerror(errno));
1552 return false;
1553 }
1554 if (vb_b)
1555 pr2serr_lk("%s: tid=%d: ioctl(EXTENDED(shared_fd)) ok, "
1556 "read_side_fd=%d, write_side_fd=%d\n", __func__, id,
1557 read_side_fd, write_side_fd);
1558 return true;
1559 }
1560
1561 static void
sg_take_snap(int sg_fd,int id,bool vb_b)1562 sg_take_snap(int sg_fd, int id, bool vb_b)
1563 {
1564 struct sg_extended_info sei {};
1565 struct sg_extended_info * seip = &sei;
1566
1567 seip->sei_wr_mask |= SG_SEIM_CTL_FLAGS;
1568 seip->sei_rd_mask |= SG_SEIM_CTL_FLAGS;
1569 seip->ctl_flags_wr_mask |= SG_CTL_FLAGM_SNAP_DEV;
1570 seip->ctl_flags &= ~SG_CTL_FLAGM_SNAP_DEV; /* 0 --> append */
1571 if (ioctl(sg_fd, SG_SET_GET_EXTENDED, seip) < 0) {
1572 pr2serr_lk("tid=%d: ioctl(EXTENDED(SNAP_DEV), failed errno=%d %s\n",
1573 id, errno, strerror(errno));
1574 return;
1575 }
1576 if (vb_b)
1577 pr2serr_lk("tid=%d: ioctl(SNAP_DEV) ok\n", id);
1578 }
1579
1580 // <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
1581 /* Each thread's "main" function */
1582 static void
read_write_thread(struct global_collection * clp,int thr_idx,int slice_idx,bool singleton)1583 read_write_thread(struct global_collection * clp, int thr_idx, int slice_idx,
1584 bool singleton)
1585 {
1586 Rq_elem rel {};
1587 Rq_elem * rep = &rel;
1588 int n, sz, fd, vb, err, seg_blks;
1589 int res = 0;
1590 int num_sg = 0;
1591 bool own_infd = false;
1592 bool in_is_sg, in_mmap, out_is_sg, out_mmap;
1593 bool own_outfd = false;
1594 bool only_one_sg = false;
1595 struct cp_ver_pair_t & cvp = clp->cp_ver_arr[slice_idx];
1596 class scat_gath_iter i_sg_it(clp->i_sgl);
1597 class scat_gath_iter o_sg_it(clp->o_sgl);
1598 const string & inf = clp->inf_v[slice_idx];
1599 const string & outf = clp->outf_v[slice_idx];
1600 vector<cdb_arr_t> a_cdb;
1601 vector<struct sg_io_v4> a_v4;
1602
1603 vb = clp->verbose;
1604 sz = clp->mrq_num * clp->bpt * clp->bs;
1605 in_is_sg = (FT_SG == clp->in_type);
1606 in_mmap = (in_is_sg && (clp->in_flags.mmap > 0));
1607 out_is_sg = (FT_SG == clp->out_type);
1608 out_mmap = (out_is_sg && (clp->out_flags.mmap > 0));
1609 rep->clp = clp;
1610 rep->id = thr_idx;
1611 rep->bs = clp->bs;
1612
1613 if (in_is_sg && out_is_sg)
1614 rep->both_sg = true;
1615 else if (in_is_sg || out_is_sg) {
1616 only_one_sg = true;
1617 if (in_is_sg)
1618 rep->only_in_sg = true;
1619 else
1620 rep->only_out_sg = true;
1621 }
1622
1623 if (vb > 2) {
1624 pr2serr_lk("%d <-- Starting worker thread, slice=%d\n", thr_idx,
1625 slice_idx);
1626 if (vb > 3)
1627 pr2serr_lk(" %s ---> %s\n", inf.c_str(), outf.c_str());
1628 }
1629 if (! (rep->both_sg || in_mmap)) {
1630 rep->buffp = sg_memalign(sz, 0 /* page align */, &rep->alloc_bp,
1631 false);
1632 if (NULL == rep->buffp) {
1633 pr2serr_lk("Failed to allocate %d bytes, exiting\n", sz);
1634 return;
1635 }
1636 }
1637 rep->infd = clp->in0fd;
1638 rep->outfd = clp->out0fd;
1639 rep->outregfd = clp->outregfd;
1640 rep->rep_count = 0;
1641 rep->in_follow_on = -1;
1642 rep->out_follow_on = -1;
1643 if (cvp.state == cp_ver_pair_t::my_state::init)
1644 cvp.state = cp_ver_pair_t::my_state::underway;
1645 if (FT_OTHER == cvp.in_type) {
1646 fd = reg_file_open(clp, inf, false);
1647 if (fd < 0) {
1648 pr2serr_lk("[%d]: unable to open IFILE of slice=%d\n", thr_idx,
1649 slice_idx);
1650 return;
1651 }
1652 rep->infd = fd;
1653 }
1654 if (FT_OTHER == cvp.out_type) {
1655 fd = reg_file_open(clp, outf, true);
1656 if (fd < 0) {
1657 pr2serr_lk("[%d]: unable to open OFILE of slice=%d\n", thr_idx,
1658 slice_idx);
1659 return;
1660 }
1661 rep->outfd = fd;
1662 }
1663
1664 if (rep->infd == rep->outfd) {
1665 if (in_is_sg)
1666 rep->same_sg = true;
1667 }
1668 if (clp->in_flags.random) {
1669 #ifdef HAVE_GETRANDOM
1670 ssize_t ssz = getrandom(&rep->seed, sizeof(rep->seed), GRND_NONBLOCK);
1671
1672 if (ssz < (ssize_t)sizeof(rep->seed)) {
1673 pr2serr_lk("[%d] %s: getrandom() failed, ret=%d\n", thr_idx,
1674 __func__, (int)ssz);
1675 rep->seed = (long)time(NULL);
1676 }
1677 #else
1678 rep->seed = (long)time(NULL); /* use seconds since epoch as proxy */
1679 #endif
1680 if (vb > 1)
1681 pr2serr_lk("[%d] %s: seed=%ld\n", thr_idx, __func__, rep->seed);
1682 #ifdef HAVE_SRAND48_R
1683 srand48_r(rep->seed, &rep->drand);
1684 #else
1685 srand48(rep->seed);
1686 #endif
1687 }
1688
1689 if (in_is_sg && inf.size()) {
1690 if ((clp->in_flags.same_fds || (0 == thr_idx)) &&
1691 (cvp.in_fd >= 0))
1692 fd = cvp.in_fd;
1693 else {
1694 fd = sg_in_open(clp, inf, (in_mmap ? &rep->buffp : NULL),
1695 (in_mmap ? &rep->mmap_len : NULL));
1696 if (fd < 0)
1697 goto fini;
1698 own_infd = true;
1699 if (cvp.in_fd < 0)
1700 cvp.in_fd = fd;
1701 }
1702 rep->infd = fd;
1703 rep->mmap_active = in_mmap ? clp->in_flags.mmap : 0;
1704 if (in_mmap && (vb > 4))
1705 pr2serr_lk("[%d] %s: mmap buffp=%p\n", thr_idx, __func__,
1706 rep->buffp);
1707 ++num_sg;
1708 if (vb > 2)
1709 pr2serr_lk("[%d]: opened local sg IFILE\n", thr_idx);
1710 }
1711 if (out_is_sg && outf.size()) {
1712 if ((clp->out_flags.same_fds || (0 == thr_idx)) &&
1713 (cvp.out_fd >= 0))
1714 fd = cvp.out_fd;
1715 else {
1716 fd = sg_out_open(clp, outf, (out_mmap ? &rep->buffp : NULL),
1717 (out_mmap ? &rep->mmap_len : NULL));
1718 if (fd < 0)
1719 goto fini;
1720 own_outfd = true;
1721 if (cvp.out_fd < 0)
1722 cvp.out_fd = fd;
1723 }
1724 rep->outfd = fd;
1725 if (! rep->mmap_active)
1726 rep->mmap_active = out_mmap ? clp->out_flags.mmap : 0;
1727 if (out_mmap && (vb > 4))
1728 pr2serr_lk("[%d]: mmap buffp=%p\n", thr_idx, rep->buffp);
1729 ++num_sg;
1730 if (vb > 2)
1731 pr2serr_lk("[%d]: opened local sg OFILE\n", thr_idx);
1732 }
1733 if (vb > 2) {
1734 if (in_is_sg && (! own_infd))
1735 pr2serr_lk("[%d]: using global sg IFILE, fd=%d\n", thr_idx,
1736 rep->infd);
1737 if (out_is_sg && (! own_outfd))
1738 pr2serr_lk("[%d]: using global sg OFILE, fd=%d\n", thr_idx,
1739 rep->outfd);
1740 }
1741 if (rep->both_sg)
1742 rep->has_share = sg_share_prepare(rep->outfd, rep->infd, thr_idx,
1743 vb > 9);
1744 if (vb > 9)
1745 pr2serr_lk("[%d]: has_share=%s\n", thr_idx,
1746 (rep->has_share ? "true" : "false"));
1747 // share_and_ofreg = (rep->has_share && (rep->outregfd >= 0));
1748
1749 /* vvvvvvvvvvvvvv Main segment copy loop vvvvvvvvvvvvvvvvvvvvvvv */
1750 while (! shutting_down) {
1751 get_next_res_t gnr = cvp.get_next(clp->mrq_num * clp->bpt);
1752
1753 seg_blks = gnr.second;
1754 if (seg_blks <= 0) {
1755 if (seg_blks < 0)
1756 res = -seg_blks;
1757 else
1758 cvp.state = cp_ver_pair_t::my_state::finished;
1759 break;
1760 }
1761 if (! i_sg_it.set_by_blk_idx(gnr.first)) {
1762 lock_guard<mutex> lk(strerr_mut);
1763
1764 pr2serr_lk("[%d]: input set_by_blk_idx() failed\n", thr_idx);
1765 i_sg_it.dbg_print("input after set_by_blk_idx", false, vb > 5);
1766 res = 2;
1767 break;
1768 }
1769 if (! o_sg_it.set_by_blk_idx(gnr.first)) {
1770 pr2serr_lk("[%d]: output set_by_blk_idx() failed\n", thr_idx);
1771 res = 3;
1772 break;
1773 }
1774 if (rep->both_sg) {
1775 uint32_t nn = (2 * clp->mrq_num) + 4;
1776
1777 if (a_cdb.capacity() < nn)
1778 a_cdb.reserve(nn);
1779 if (a_v4.capacity() < nn)
1780 a_v4.reserve(nn);
1781 if (clp->mrq_eq_0)
1782 res = do_both_sg_segment_mrq0(rep, i_sg_it, o_sg_it,
1783 seg_blks);
1784 else
1785 res = do_both_sg_segment(rep, i_sg_it, o_sg_it, seg_blks,
1786 a_cdb, a_v4);
1787 if (res < 0)
1788 break;
1789 } else if (only_one_sg) {
1790 uint32_t nn = clp->mrq_num + 4;
1791
1792 if (a_cdb.capacity() < nn)
1793 a_cdb.reserve(nn);
1794 if (a_v4.capacity() < nn)
1795 a_v4.reserve(nn);
1796 res = do_normal_sg_segment(rep, i_sg_it, o_sg_it, seg_blks, a_cdb,
1797 a_v4);
1798 if (res < 0)
1799 break;
1800 } else {
1801 res = do_normal_normal_segment(rep, i_sg_it, o_sg_it, seg_blks);
1802 if (res < 0)
1803 break;
1804 }
1805 if (singleton) {
1806 {
1807 lock_guard<mutex> lk(clp->infant_mut);
1808
1809 clp->processed = true;
1810 } /* this unlocks lk */
1811 clp->infant_cv.notify_one();
1812 singleton = false;
1813 }
1814 if (rep->stop_after_write || rep->stop_now) {
1815 shutting_down = true;
1816 break;
1817 }
1818 } /* ^^^^^^^^^^ end of main while loop which copies segments ^^^^^^ */
1819
1820 if (shutting_down) {
1821 if (vb > 3)
1822 pr2serr_lk("%s: t=%d: shutting down\n", __func__, rep->id);
1823 goto fini;
1824 }
1825 if (singleton) {
1826 {
1827 lock_guard<mutex> lk(clp->infant_mut);
1828
1829 clp->processed = true;
1830 } /* this unlocks lk */
1831 clp->infant_cv.notify_one();
1832 }
1833 if (res < 0) {
1834 if (seg_blks >= 0)
1835 cvp.get_next(-1); /* flag error to main */
1836 pr2serr_lk("%s: t=%d: aborting, res=%d\n", __func__, rep->id, res);
1837 }
1838
1839 fini:
1840
1841 if ((1 == rep->mmap_active) && (rep->mmap_len > 0)) {
1842 if (munmap(rep->buffp, rep->mmap_len) < 0) {
1843 err = errno;
1844 char bb[64];
1845
1846 pr2serr_lk("thread=%d: munmap() failed: %s\n", rep->id,
1847 tsafe_strerror(err, bb));
1848 }
1849 if (vb > 4)
1850 pr2serr_lk("thread=%d: munmap(%p, %d)\n", rep->id, rep->buffp,
1851 rep->mmap_len);
1852 rep->mmap_active = 0;
1853 }
1854
1855 if (own_infd && (rep->infd >= 0)) {
1856 if (vb && in_is_sg) {
1857 if (ioctl(rep->infd, SG_GET_NUM_WAITING, &n) >= 0) {
1858 if (n > 0)
1859 pr2serr_lk("%s: tid=%d: num_waiting=%d prior close(in)\n",
1860 __func__, rep->id, n);
1861 } else {
1862 err = errno;
1863 pr2serr_lk("%s: [%d] ioctl(SG_GET_NUM_WAITING) errno=%d: "
1864 "%s\n", __func__, rep->id, err, strerror(err));
1865 }
1866 }
1867 close(rep->infd);
1868 }
1869 if (own_outfd && (rep->outfd >= 0)) {
1870 if (vb && out_is_sg) {
1871 if (ioctl(rep->outfd, SG_GET_NUM_WAITING, &n) >= 0) {
1872 if (n > 0)
1873 pr2serr_lk("%s: tid=%d: num_waiting=%d prior "
1874 "close(out)\n", __func__, rep->id, n);
1875 } else {
1876 err = errno;
1877 pr2serr_lk("%s: [%d] ioctl(SG_GET_NUM_WAITING) errno=%d: "
1878 "%s\n", __func__, rep->id, err, strerror(err));
1879 }
1880 }
1881 close(rep->outfd);
1882 }
1883 /* pass stats back to read-side */
1884 if (vb > 3)
1885 pr2serr_lk("%s: [%d] leaving: in/out local count=%" PRId64 "/%"
1886 PRId64 "\n", __func__, rep->id, rep->in_local_count,
1887 rep->out_local_count);
1888 cvp.in_rem_count -= rep->in_local_count;
1889 cvp.out_rem_count -= rep->out_local_count;
1890 cvp.in_partial += rep->in_local_partial;
1891 cvp.out_partial += rep->out_local_partial;
1892 cvp.sum_of_resids += rep->in_resid_bytes;
1893 if (rep->alloc_bp)
1894 free(rep->alloc_bp);
1895 }
1896
1897 /* N.B. Returns 'blocks' is successful, lesser positive number if there was
1898 * a short read, or an error code which is negative. */
1899 static int
normal_in_rd(Rq_elem * rep,int64_t lba,int blocks,int d_boff)1900 normal_in_rd(Rq_elem * rep, int64_t lba, int blocks, int d_boff)
1901 {
1902 struct global_collection * clp = rep->clp;
1903 int res, err;
1904 int id = rep->id;
1905 uint8_t * bp;
1906 char strerr_buff[STRERR_BUFF_LEN];
1907
1908 if (clp->verbose > 4)
1909 pr2serr_lk("[%d] %s: lba=%" PRIu64 ", blocks=%d, d_boff=%d\n", id,
1910 __func__, lba, blocks, d_boff);
1911 if (FT_RANDOM_0_FF == clp->in_type) {
1912 int k, j;
1913 const int jbump = sizeof(uint32_t);
1914 long rn;
1915 uint8_t * bp;
1916
1917 if (clp->in_flags.zero && clp->in_flags.ff && (rep->bs >= 4)) {
1918 uint32_t pos = (uint32_t)lba;
1919 uint32_t off;
1920
1921 for (k = 0, off = 0; k < blocks; ++k, off += rep->bs, ++pos) {
1922 for (j = 0; j < (rep->bs - 3); j += 4)
1923 sg_put_unaligned_be32(pos, rep->buffp + off + j);
1924 }
1925 } else if (clp->in_flags.zero)
1926 memset(rep->buffp + d_boff, 0, blocks * rep->bs);
1927 else if (clp->in_flags.ff)
1928 memset(rep->buffp + d_boff, 0xff, blocks * rep->bs);
1929 else {
1930 bp = rep->buffp + d_boff;
1931 for (k = 0; k < blocks; ++k, bp += rep->bs) {
1932 for (j = 0; j < rep->bs; j += jbump) {
1933 /* mrand48 takes uniformly from [-2^31, 2^31) */
1934 #ifdef HAVE_SRAND48_R
1935 mrand48_r(&rep->drand, &rn);
1936 #else
1937 rn = mrand48();
1938 #endif
1939 *((uint32_t *)(bp + j)) = (uint32_t)rn;
1940 }
1941 }
1942 }
1943 return blocks;
1944 }
1945
1946 if (clp->in_type != FT_FIFO) {
1947 int64_t pos = lba * rep->bs;
1948
1949 if (rep->in_follow_on != pos) {
1950 if (lseek64(rep->infd, pos, SEEK_SET) < 0) {
1951 err = errno;
1952 pr2serr_lk("[%d] %s: >> lseek64(%" PRId64 "): %s\n", id,
1953 __func__, pos, safe_strerror(err));
1954 return -err;
1955 }
1956 rep->in_follow_on = pos;
1957 }
1958 }
1959 bp = rep->buffp + d_boff;
1960 while (((res = read(rep->infd, bp, blocks * rep->bs)) < 0) &&
1961 ((EINTR == errno) || (EAGAIN == errno)))
1962 std::this_thread::yield();/* another thread may be able to progress */
1963 if (res < 0) {
1964 err = errno;
1965 if (clp->in_flags.coe) {
1966 memset(bp, 0, blocks * rep->bs);
1967 pr2serr_lk("[%d] %s : >> substituted zeros for in blk=%" PRId64
1968 " for %d bytes, %s\n", id, __func__, lba,
1969 blocks * rep->bs,
1970 tsafe_strerror(err, strerr_buff));
1971 res = blocks * rep->bs;
1972 } else {
1973 pr2serr_lk("[%d] %s: error in normal read, %s\n", id, __func__,
1974 tsafe_strerror(err, strerr_buff));
1975 return -err;
1976 }
1977 }
1978 rep->in_follow_on += res;
1979 if (res < blocks * rep->bs) {
1980 blocks = res / rep->bs;
1981 if ((res % rep->bs) > 0) {
1982 rep->in_local_partial++;
1983 rep->in_resid_bytes = res % rep->bs;
1984 }
1985 }
1986 return blocks;
1987 }
1988
1989 /* N.B. Returns 'blocks' is successful, lesser positive number if there was
1990 * a short write, or an error code which is negative. */
1991 static int
normal_out_wr(Rq_elem * rep,int64_t lba,int blocks,int d_boff)1992 normal_out_wr(Rq_elem * rep, int64_t lba, int blocks, int d_boff)
1993 {
1994 int res, err;
1995 int id = rep->id;
1996 struct global_collection * clp = rep->clp;
1997 uint8_t * bp = rep->buffp + d_boff;
1998 char strerr_buff[STRERR_BUFF_LEN];
1999
2000 if (clp->verbose > 4)
2001 pr2serr_lk("[%d] %s: lba=%" PRIu64 ", blocks=%d, d_boff=%d\n", id,
2002 __func__, lba, blocks, d_boff);
2003
2004 if (clp->in_type != FT_FIFO) {
2005 int64_t pos = lba * rep->bs;
2006
2007 if (rep->out_follow_on != pos) {
2008 if (lseek64(rep->outfd, pos, SEEK_SET) < 0) {
2009 err = errno;
2010 pr2serr_lk("[%d] %s: >> lseek64(%" PRId64 "): %s\n", id,
2011 __func__, pos, safe_strerror(err));
2012 return -err;
2013 }
2014 rep->out_follow_on = pos;
2015 }
2016 }
2017 while (((res = write(rep->outfd, bp, blocks * rep->bs))
2018 < 0) && ((EINTR == errno) || (EAGAIN == errno)))
2019 std::this_thread::yield();/* another thread may be able to progress */
2020 if (res < 0) {
2021 err = errno;
2022 if (clp->out_flags.coe) {
2023 pr2serr_lk("[%d] %s: >> ignored error for out lba=%" PRId64
2024 " for %d bytes, %s\n", id, __func__, lba,
2025 blocks * rep->bs, tsafe_strerror(err, strerr_buff));
2026 res = blocks * rep->bs;
2027 }
2028 else {
2029 pr2serr_lk("[%d] %s: error normal write, %s\n", id, __func__,
2030 tsafe_strerror(err, strerr_buff));
2031 return -err;
2032 }
2033 }
2034 rep->out_follow_on += res;
2035 if (res < blocks * rep->bs) {
2036 blocks = res / rep->bs;
2037 if ((res % rep->bs) > 0) {
2038 blocks++;
2039 rep->out_local_partial++;
2040 }
2041 }
2042 return blocks;
2043 }
2044
2045 static int
extra_out_wr(Rq_elem * rep,int num_bytes,int d_boff)2046 extra_out_wr(Rq_elem * rep, int num_bytes, int d_boff)
2047 {
2048 int res, err;
2049 int id = rep->id;
2050 struct global_collection * clp = rep->clp;
2051 uint8_t * bp = rep->buffp + d_boff;
2052 char strerr_buff[STRERR_BUFF_LEN];
2053
2054 if (clp->verbose > 4)
2055 pr2serr_lk("[%d] %s: num_bytes=%d, d_boff=%d\n", id, __func__,
2056 num_bytes, d_boff);
2057
2058 while (((res = write(clp->out0fd, bp, num_bytes))
2059 < 0) && ((EINTR == errno) || (EAGAIN == errno)))
2060 std::this_thread::yield();/* another thread may be able to progress */
2061 if (res < 0) {
2062 err = errno;
2063 pr2serr_lk("[%d] %s: error normal write, %s\n", id, __func__,
2064 tsafe_strerror(err, strerr_buff));
2065 return -err;
2066 }
2067 if (res > 0)
2068 rep->out_local_partial++;
2069 return res;
2070 }
2071
2072 static int
sg_build_scsi_cdb(uint8_t * cdbp,int cdb_sz,unsigned int blocks,int64_t start_block,bool ver_true,bool write_true,bool fua,bool dpo,int cdl)2073 sg_build_scsi_cdb(uint8_t * cdbp, int cdb_sz, unsigned int blocks,
2074 int64_t start_block, bool ver_true, bool write_true,
2075 bool fua, bool dpo, int cdl)
2076 {
2077 bool normal_rw = true;
2078 int rd_opcode[] = {0x8, 0x28, 0xa8, 0x88};
2079 int ve_opcode[] = {0xff /* no VER(6) */, 0x2f, 0xaf, 0x8f};
2080 int wr_opcode[] = {0xa, 0x2a, 0xaa, 0x8a};
2081 int sz_ind;
2082
2083 memset(cdbp, 0, cdb_sz);
2084 if (ver_true) { /* only support VERIFY(10) */
2085 if (cdb_sz < 10) {
2086 pr2serr_lk("%s only support VERIFY(10)\n", my_name);
2087 return 1;
2088 }
2089 cdb_sz = 10;
2090 fua = false;
2091 cdbp[1] |= 0x2; /* BYTCHK=1 --> sending dout for comparison */
2092 cdbp[0] = ve_opcode[1];
2093 normal_rw = false;
2094 }
2095 if (dpo)
2096 cdbp[1] |= 0x10;
2097 if (fua)
2098 cdbp[1] |= 0x8;
2099 switch (cdb_sz) {
2100 case 6:
2101 sz_ind = 0;
2102 cdbp[0] = (uint8_t)(write_true ? wr_opcode[sz_ind] :
2103 rd_opcode[sz_ind]);
2104 sg_put_unaligned_be24(0x1fffff & start_block, cdbp + 1);
2105 cdbp[4] = (256 == blocks) ? 0 : (uint8_t)blocks;
2106 if (blocks > 256) {
2107 pr2serr_lk("%sfor 6 byte commands, maximum number of blocks is "
2108 "256\n", my_name);
2109 return 1;
2110 }
2111 if ((start_block + blocks - 1) & (~0x1fffff)) {
2112 pr2serr_lk("%sfor 6 byte commands, can't address blocks beyond "
2113 "%d\n", my_name, 0x1fffff);
2114 return 1;
2115 }
2116 if (dpo || fua) {
2117 pr2serr_lk("%sfor 6 byte commands, neither dpo nor fua bits "
2118 "supported\n", my_name);
2119 return 1;
2120 }
2121 break;
2122 case 10:
2123 if (! ver_true) {
2124 sz_ind = 1;
2125 cdbp[0] = (uint8_t)(write_true ? wr_opcode[sz_ind] :
2126 rd_opcode[sz_ind]);
2127 }
2128 sg_put_unaligned_be32((uint32_t)start_block, cdbp + 2);
2129 sg_put_unaligned_be16((uint16_t)blocks, cdbp + 7);
2130 if (blocks & (~0xffff)) {
2131 pr2serr_lk("%sfor 10 byte commands, maximum number of blocks is "
2132 "%d\n", my_name, 0xffff);
2133 return 1;
2134 }
2135 break;
2136 case 12:
2137 sz_ind = 2;
2138 cdbp[0] = (uint8_t)(write_true ? wr_opcode[sz_ind] :
2139 rd_opcode[sz_ind]);
2140 sg_put_unaligned_be32((uint32_t)start_block, cdbp + 2);
2141 sg_put_unaligned_be32((uint32_t)blocks, cdbp + 6);
2142 break;
2143 case 16:
2144 sz_ind = 3;
2145 cdbp[0] = (uint8_t)(write_true ? wr_opcode[sz_ind] :
2146 rd_opcode[sz_ind]);
2147 sg_put_unaligned_be64((uint64_t)start_block, cdbp + 2);
2148 sg_put_unaligned_be32((uint32_t)blocks, cdbp + 10);
2149 if (normal_rw && (cdl > 0)) {
2150 if (cdl & 0x4)
2151 cdbp[1] |= 0x1;
2152 if (cdl & 0x3)
2153 cdbp[14] |= ((cdl & 0x3) << 6);
2154 }
2155 break;
2156 default:
2157 pr2serr_lk("%sexpected cdb size of 6, 10, 12, or 16 but got %d\n",
2158 my_name, cdb_sz);
2159 return 1;
2160 }
2161 return 0;
2162 }
2163
2164 static int
process_mrq_response(Rq_elem * rep,const struct sg_io_v4 * ctl_v4p,const struct sg_io_v4 * a_v4p,int num_mrq,uint32_t & good_inblks,uint32_t & good_outblks,bool & last_err_on_in)2165 process_mrq_response(Rq_elem * rep, const struct sg_io_v4 * ctl_v4p,
2166 const struct sg_io_v4 * a_v4p, int num_mrq,
2167 uint32_t & good_inblks, uint32_t & good_outblks,
2168 bool & last_err_on_in)
2169 {
2170 struct global_collection * clp = rep->clp;
2171 bool ok, all_good;
2172 bool sb_in_co = !!(ctl_v4p->response);
2173 int id = rep->id;
2174 int resid = ctl_v4p->din_resid;
2175 int sres = ctl_v4p->spare_out;
2176 int n_subm = num_mrq - ctl_v4p->dout_resid;
2177 int n_cmpl = ctl_v4p->info;
2178 int n_good = 0;
2179 int hole_count = 0;
2180 int cat = 0;
2181 int vb = clp->verbose;
2182 int k, j, f1, slen;
2183 char b[160];
2184
2185 good_inblks = 0;
2186 good_outblks = 0;
2187 if (vb > 2)
2188 pr2serr_lk("[thread_id=%d] %s: num_mrq=%d, n_subm=%d, n_cmpl=%d\n",
2189 id, __func__, num_mrq, n_subm, n_cmpl);
2190 if (n_subm < 0) {
2191 pr2serr_lk("[%d] co.dout_resid(%d) > num_mrq(%d)\n", id,
2192 ctl_v4p->dout_resid, num_mrq);
2193 return -1;
2194 }
2195 if (n_cmpl != (num_mrq - resid))
2196 pr2serr_lk("[%d] co.info(%d) != (num_mrq(%d) - co.din_resid(%d))\n"
2197 "will use co.info\n", id, n_cmpl, num_mrq, resid);
2198 if (n_cmpl > n_subm) {
2199 pr2serr_lk("[%d] n_cmpl(%d) > n_subm(%d), use n_subm for both\n",
2200 id, n_cmpl, n_subm);
2201 n_cmpl = n_subm;
2202 }
2203 if (sres) {
2204 pr2serr_lk("[%d] secondary error: %s [%d], info=0x%x\n", id,
2205 strerror(sres), sres, ctl_v4p->info);
2206 if (E2BIG == sres) {
2207 sg_take_snap(rep->infd, id, true);
2208 sg_take_snap(rep->outfd, id, true);
2209 }
2210 }
2211 /* Check if those submitted have finished or not. N.B. If there has been
2212 * an error then there may be "holes" (i.e. info=0x0) in the array due
2213 * to completions being out-of-order. */
2214 for (k = 0, j = 0; ((k < num_mrq) && (j < n_subm));
2215 ++k, j += f1, ++a_v4p) {
2216 slen = a_v4p->response_len;
2217 if (! (SG_INFO_MRQ_FINI & a_v4p->info))
2218 ++hole_count;
2219 ok = true;
2220 f1 = !!(a_v4p->info); /* want to skip n_subm count if info is 0x0 */
2221 if (SG_INFO_CHECK & a_v4p->info) {
2222 if ((0 == k) && (SGV4_FLAG_META_OUT_IF & ctl_v4p->flags) &&
2223 (UINT32_MAX == a_v4p->info)) {
2224 hole_count = 0;
2225 n_good = num_mrq;
2226 good_inblks = rep->a_mrq_din_blks;
2227 good_outblks = rep->a_mrq_dout_blks;
2228 break;
2229 }
2230 ok = false;
2231 pr2serr_lk("[%d] a_v4[%d]: SG_INFO_CHECK set [%s]\n", id, k,
2232 sg_info_str(a_v4p->info, sizeof(b), b));
2233 }
2234 if (sg_scsi_status_is_bad(a_v4p->device_status) ||
2235 a_v4p->transport_status || a_v4p->driver_status) {
2236 ok = false;
2237 last_err_on_in = ! (a_v4p->flags & SGV4_FLAG_DO_ON_OTHER);
2238 if (SAM_STAT_CHECK_CONDITION != a_v4p->device_status) {
2239 pr2serr_lk("[%d] a_v4[%d]:\n", id, k);
2240 if (vb)
2241 lk_chk_n_print4(" >>", a_v4p, vb > 4);
2242 }
2243 }
2244 if (slen > 0) {
2245 struct sg_scsi_sense_hdr ssh;
2246 const uint8_t *sbp = (const uint8_t *)
2247 (sb_in_co ? ctl_v4p->response : a_v4p->response);
2248
2249 if (sg_scsi_normalize_sense(sbp, slen, &ssh) &&
2250 (ssh.response_code >= 0x70)) {
2251 if (ssh.response_code & 0x1) {
2252 ok = true;
2253 last_err_on_in = false;
2254 } else
2255 cat = sg_err_category_sense(sbp, slen);
2256 if (SPC_SK_MISCOMPARE == ssh.sense_key)
2257 ++num_miscompare;
2258
2259 pr2serr_lk("[%d] a_v4[%d]:\n", id, k);
2260 if (vb)
2261 lk_chk_n_print4(" >>", a_v4p, vb > 4);
2262 }
2263 } else if (! ok)
2264 cat = SG_LIB_CAT_OTHER;
2265 if (ok && f1) {
2266 ++n_good;
2267 if (a_v4p->dout_xfer_len >= (uint32_t)rep->bs)
2268 good_outblks += (a_v4p->dout_xfer_len - a_v4p->dout_resid) /
2269 rep->bs;
2270 if (a_v4p->din_xfer_len >= (uint32_t)rep->bs)
2271 good_inblks += (a_v4p->din_xfer_len - a_v4p->din_resid) /
2272 rep->bs;
2273 }
2274 if (! ok) {
2275 if ((a_v4p->dout_xfer_len > 0) || (! clp->in_flags.coe))
2276 rep->stop_after_write = true;
2277 }
2278 } /* end of request array scan loop */
2279 if ((n_subm == num_mrq) || (vb < 3))
2280 goto fini;
2281 pr2serr_lk("[%d] checking response array _beyond_ number of "
2282 "submissions [%d] to num_mrq:\n", id, k);
2283 for (all_good = true; k < num_mrq; ++k, ++a_v4p) {
2284 if (SG_INFO_MRQ_FINI & a_v4p->info) {
2285 pr2serr_lk("[%d] a_v4[%d]: unexpected SG_INFO_MRQ_FINI set [%s]\n",
2286 id, k, sg_info_str(a_v4p->info, sizeof(b), b));
2287 all_good = false;
2288 }
2289 if (a_v4p->device_status || a_v4p->transport_status ||
2290 a_v4p->driver_status) {
2291 pr2serr_lk("[%d] a_v4[%d]:\n", id, k);
2292 lk_chk_n_print4(" ", a_v4p, vb > 4);
2293 all_good = false;
2294 }
2295 }
2296 if (all_good)
2297 pr2serr_lk(" ... all good\n");
2298 fini:
2299 if (cat > 0)
2300 clp->reason_res.store(cat);
2301 return n_good;
2302 }
2303
2304 /* Returns number of blocks successfully processed or a negative error
2305 * number. */
2306 static int
sg_half_segment_mrq0(Rq_elem * rep,scat_gath_iter & sg_it,bool is_wr,int seg_blks,uint8_t * dp)2307 sg_half_segment_mrq0(Rq_elem * rep, scat_gath_iter & sg_it, bool is_wr,
2308 int seg_blks, uint8_t *dp)
2309 {
2310 int k, res, fd, pack_id_base, id, rflags;
2311 int num, kk, lin_blks, cdbsz, err;
2312 uint32_t q_blks = 0;
2313 struct global_collection * clp = rep->clp;
2314 cdb_arr_t t_cdb {};
2315 struct sg_io_v4 t_v4 {};
2316 struct sg_io_v4 * t_v4p = &t_v4;
2317 struct flags_t * flagsp = is_wr ? &clp->out_flags : &clp->in_flags;
2318 int vb = clp->verbose;
2319
2320 id = rep->id;
2321 pack_id_base = id * PACK_ID_TID_MULTIPLIER;
2322 rflags = 0;
2323 fd = is_wr ? rep->outfd : rep->infd;
2324 if (flagsp->mmap && (rep->outregfd >= 0))
2325 rflags |= SGV4_FLAG_MMAP_IO;
2326 if (flagsp->dio)
2327 rflags |= SGV4_FLAG_DIRECT_IO;
2328 if (flagsp->qhead)
2329 rflags |= SGV4_FLAG_Q_AT_HEAD;
2330 if (flagsp->qtail)
2331 rflags |= SGV4_FLAG_Q_AT_TAIL;
2332 if (flagsp->polled)
2333 rflags |= SGV4_FLAG_POLLED;
2334
2335 for (k = 0, num = 0; seg_blks > 0; ++k, seg_blks -= num) {
2336 kk = min<int>(seg_blks, clp->bpt);
2337 lin_blks = sg_it.linear_for_n_blks(kk);
2338 num = lin_blks;
2339 if (num <= 0) {
2340 res = 0;
2341 pr2serr_lk("[%d] %s: unexpected num=%d\n", id, __func__, num);
2342 break;
2343 }
2344
2345 /* First build the command/request for the read-side */
2346 cdbsz = is_wr ? clp->cdbsz_out : clp->cdbsz_in;
2347 res = sg_build_scsi_cdb(t_cdb.data(), cdbsz, num, sg_it.current_lba(),
2348 false, is_wr, flagsp->fua, flagsp->dpo,
2349 flagsp->cdl);
2350 if (res) {
2351 pr2serr_lk("[%d] %s: sg_build_scsi_cdb() failed\n", id, __func__);
2352 break;
2353 } else if (vb > 3)
2354 lk_print_command_len("cdb: ", t_cdb.data(), cdbsz, true);
2355
2356 t_v4p->guard = 'Q';
2357 t_v4p->request = (uint64_t)t_cdb.data();
2358 t_v4p->usr_ptr = t_v4p->request;
2359 t_v4p->response = (uint64_t)rep->sb;
2360 t_v4p->max_response_len = sizeof(rep->sb);
2361 t_v4p->flags = rflags;
2362 t_v4p->request_len = cdbsz;
2363 if (is_wr) {
2364 t_v4p->dout_xfer_len = num * rep->bs;
2365 t_v4p->dout_xferp = (uint64_t)(dp + (q_blks * rep->bs));
2366 t_v4p->din_xfer_len = 0;
2367 } else {
2368 t_v4p->din_xfer_len = num * rep->bs;
2369 t_v4p->din_xferp = (uint64_t)(dp + (q_blks * rep->bs));
2370 t_v4p->dout_xfer_len = 0;
2371 }
2372 t_v4p->timeout = clp->cmd_timeout;
2373 t_v4p->request_extra = pack_id_base + ++rep->mrq_pack_id_off;
2374 clp->most_recent_pack_id.store(t_v4p->request_extra);
2375 mrq0_again:
2376 res = ioctl(fd, SG_IO, t_v4p);
2377 err = errno;
2378 if (vb > 5)
2379 v4hdr_out_lk("sg_half_segment_mrq0: >> after ioctl(SG_IO)",
2380 t_v4p, id, false);
2381 if (res < 0) {
2382 if (E2BIG == err)
2383 sg_take_snap(fd, id, true);
2384 else if (EBUSY == err) {
2385 ++num_ebusy;
2386 std::this_thread::yield();/* so other threads can progress */
2387 goto mrq0_again;
2388 }
2389 pr2serr_lk("[%d] %s: ioctl(SG_IO)-->%d, errno=%d: %s\n", id,
2390 __func__, res, err, strerror(err));
2391 return -err;
2392 }
2393 if (t_v4p->device_status || t_v4p->transport_status ||
2394 t_v4p->driver_status) {
2395 rep->stop_now = true;
2396 pr2serr_lk("[%d] t_v4[%d]:\n", id, k);
2397 lk_chk_n_print4(" ", t_v4p, vb > 4);
2398 return q_blks;
2399 }
2400 q_blks += num;
2401 sg_it.add_blks(num);
2402 }
2403 return q_blks;
2404 }
2405
2406 /* Returns number of blocks successfully processed or a negative error
2407 * number. */
2408 static int
sg_half_segment(Rq_elem * rep,scat_gath_iter & sg_it,bool is_wr,int seg_blks,uint8_t * dp,vector<cdb_arr_t> & a_cdb,vector<struct sg_io_v4> & a_v4)2409 sg_half_segment(Rq_elem * rep, scat_gath_iter & sg_it, bool is_wr,
2410 int seg_blks, uint8_t *dp, vector<cdb_arr_t> & a_cdb,
2411 vector<struct sg_io_v4> & a_v4)
2412 {
2413 int num_mrq, k, res, fd, mrq_pack_id_base, id, b_len, rflags;
2414 int num, kk, lin_blks, cdbsz, num_good, err;
2415 int o_seg_blks = seg_blks;
2416 uint32_t in_fin_blks, out_fin_blks;
2417 uint32_t mrq_q_blks = 0;
2418 uint32_t in_mrq_q_blks = 0;
2419 uint32_t out_mrq_q_blks = 0;
2420 const int max_cdb_sz = MAX_SCSI_CDB_SZ;
2421 struct sg_io_v4 * a_v4p;
2422 struct sg_io_v4 ctl_v4 {}; /* MRQ control object */
2423 struct global_collection * clp = rep->clp;
2424 const char * iosub_str = "SG_IOSUBMIT(variable blocking)";
2425 char b[80];
2426 cdb_arr_t t_cdb {};
2427 struct sg_io_v4 t_v4 {};
2428 struct sg_io_v4 * t_v4p = &t_v4;
2429 struct flags_t * flagsp = is_wr ? &clp->out_flags : &clp->in_flags;
2430 bool serial = flagsp->serial;
2431 bool err_on_in = false;
2432 int vb = clp->verbose;
2433
2434 id = rep->id;
2435 b_len = sizeof(b);
2436 if (serial)
2437 iosub_str = "SG_IO(ordered blocking)";
2438
2439 a_cdb.clear();
2440 a_v4.clear();
2441 rep->a_mrq_din_blks = 0;
2442 rep->a_mrq_dout_blks = 0;
2443 mrq_pack_id_base = id * PACK_ID_TID_MULTIPLIER;
2444
2445 rflags = 0;
2446 if (flagsp->mmap && (rep->outregfd >= 0))
2447 rflags |= SGV4_FLAG_MMAP_IO;
2448 if (flagsp->dio)
2449 rflags |= SGV4_FLAG_DIRECT_IO;
2450 if (flagsp->qhead)
2451 rflags |= SGV4_FLAG_Q_AT_HEAD;
2452 if (flagsp->qtail)
2453 rflags |= SGV4_FLAG_Q_AT_TAIL;
2454 if (flagsp->polled)
2455 rflags |= SGV4_FLAG_POLLED;
2456
2457 for (k = 0, num = 0; seg_blks > 0; ++k, seg_blks -= num) {
2458 kk = min<int>(seg_blks, clp->bpt);
2459 lin_blks = sg_it.linear_for_n_blks(kk);
2460 num = lin_blks;
2461 if (num <= 0) {
2462 res = 0;
2463 pr2serr_lk("[%d] %s: unexpected num=%d\n", id, __func__, num);
2464 break;
2465 }
2466
2467 /* First build the command/request for the read-side */
2468 cdbsz = is_wr ? clp->cdbsz_out : clp->cdbsz_in;
2469 res = sg_build_scsi_cdb(t_cdb.data(), cdbsz, num, sg_it.current_lba(),
2470 false, is_wr, flagsp->fua, flagsp->dpo,
2471 flagsp->cdl);
2472 if (res) {
2473 pr2serr_lk("[%d] %s: sg_build_scsi_cdb() failed\n", id, __func__);
2474 break;
2475 } else if (vb > 3)
2476 lk_print_command_len("cdb: ", t_cdb.data(), cdbsz, true);
2477 a_cdb.push_back(t_cdb);
2478
2479 t_v4p->guard = 'Q';
2480 t_v4p->flags = rflags;
2481 t_v4p->request_len = cdbsz;
2482 t_v4p->response = (uint64_t)rep->sb;
2483 t_v4p->max_response_len = sizeof(rep->sb);
2484 t_v4p->flags = rflags;
2485 t_v4p->usr_ptr = (uint64_t)&a_cdb[a_cdb.size() - 1];
2486 if (is_wr) {
2487 rep->a_mrq_dout_blks += num;
2488 t_v4p->dout_xfer_len = num * rep->bs;
2489 t_v4p->dout_xferp = (uint64_t)(dp + (mrq_q_blks * rep->bs));
2490 t_v4p->din_xfer_len = 0;
2491 } else {
2492 rep->a_mrq_din_blks += num;
2493 t_v4p->din_xfer_len = num * rep->bs;
2494 t_v4p->din_xferp = (uint64_t)(dp + (mrq_q_blks * rep->bs));
2495 t_v4p->dout_xfer_len = 0;
2496 }
2497 t_v4p->timeout = clp->cmd_timeout;
2498 mrq_q_blks += num;
2499 t_v4p->request_extra = mrq_pack_id_base + ++rep->mrq_pack_id_off;
2500 clp->most_recent_pack_id.store(t_v4p->request_extra);
2501 a_v4.push_back(t_v4);
2502
2503 sg_it.add_blks(num);
2504 }
2505
2506 if (rep->only_in_sg)
2507 fd = rep->infd;
2508 else if (rep->only_out_sg)
2509 fd = rep->outfd;
2510 else {
2511 pr2serr_lk("[%d] %s: why am I here? No sg devices\n", id, __func__);
2512 return -EINVAL;
2513 }
2514 num_mrq = a_v4.size();
2515 a_v4p = a_v4.data();
2516 res = 0;
2517 ctl_v4.guard = 'Q';
2518 ctl_v4.request_len = a_cdb.size() * max_cdb_sz;
2519 ctl_v4.request = (uint64_t)a_cdb.data();
2520 ctl_v4.max_response_len = sizeof(rep->sb);
2521 ctl_v4.response = (uint64_t)rep->sb;
2522 ctl_v4.flags = SGV4_FLAG_MULTIPLE_REQS;
2523 if (! flagsp->coe)
2524 ctl_v4.flags |= SGV4_FLAG_STOP_IF;
2525 if (clp->mrq_polled)
2526 ctl_v4.flags |= SGV4_FLAG_POLLED;
2527 if (clp->in_flags.mout_if || clp->out_flags.mout_if) {
2528 ctl_v4.flags |= SGV4_FLAG_META_OUT_IF;
2529 if (num_mrq > 0)
2530 a_v4[0].info = UINT32_MAX;
2531 }
2532 ctl_v4.dout_xferp = (uint64_t)a_v4.data(); /* request array */
2533 ctl_v4.dout_xfer_len = a_v4.size() * sizeof(struct sg_io_v4);
2534 ctl_v4.din_xferp = (uint64_t)a_v4.data(); /* response array */
2535 ctl_v4.din_xfer_len = a_v4.size() * sizeof(struct sg_io_v4);
2536 if (false /* allow_mrq_abort */) {
2537 ctl_v4.request_extra = mrq_pack_id_base + ++rep->mrq_pack_id_off;
2538 clp->most_recent_pack_id.store(ctl_v4.request_extra);
2539 }
2540
2541 if (vb && vb_first_time.load()) {
2542 pr2serr_lk("First controlling object output by ioctl(%s), flags: "
2543 "%s\n", iosub_str, sg_flags_str(ctl_v4.flags, b_len, b));
2544 vb_first_time.store(false);
2545 } else if (vb > 4) {
2546 pr2serr_lk("[%d] %s: >> Control object _before_ ioctl(%s):\n", id,
2547 __func__, iosub_str);
2548 }
2549 if (vb > 4) {
2550 if (vb > 5)
2551 hex2stderr_lk((const uint8_t *)&ctl_v4, sizeof(ctl_v4), 1);
2552 v4hdr_out_lk(">> Control object before", &ctl_v4, id, false);
2553 }
2554
2555 try_again:
2556 if (!after1 && (vb > 1)) {
2557 after1 = true;
2558 pr2serr_lk("%s: %s\n", __func__, serial ? mrq_ob_s : mrq_vb_s);
2559 }
2560 if (serial)
2561 res = ioctl(fd, SG_IO, &ctl_v4);
2562 else
2563 res = ioctl(fd, SG_IOSUBMIT, &ctl_v4); /* overlapping commands */
2564 if (res < 0) {
2565 err = errno;
2566 if (E2BIG == err)
2567 sg_take_snap(fd, id, true);
2568 else if (EBUSY == err) {
2569 ++num_ebusy;
2570 std::this_thread::yield();/* allow another thread to progress */
2571 goto try_again;
2572 }
2573 pr2serr_lk("[%d] %s: ioctl(%s, %s)-->%d, errno=%d: %s\n", id,
2574 __func__, iosub_str, sg_flags_str(ctl_v4.flags, b_len, b),
2575 res, err, strerror(err));
2576 return -err;
2577 }
2578 if (vb > 4) {
2579 pr2serr_lk("%s: >> Control object after ioctl(%s) seg_blks=%d:\n",
2580 __func__, iosub_str, o_seg_blks);
2581 if (vb > 5)
2582 hex2stderr_lk((const uint8_t *)&ctl_v4, sizeof(ctl_v4), 1);
2583 v4hdr_out_lk(">> Control object after", &ctl_v4, id, false);
2584 if (vb > 5) {
2585 for (k = 0; k < num_mrq; ++k) {
2586 if ((vb > 6) || a_v4p[k].info) {
2587 snprintf(b, b_len, "a_v4[%d/%d]", k, num_mrq);
2588 v4hdr_out_lk(b, (a_v4p + k), id, true);
2589 }
2590 }
2591 }
2592 }
2593 num_good = process_mrq_response(rep, &ctl_v4, a_v4p, num_mrq, in_fin_blks,
2594 out_fin_blks, err_on_in);
2595 if (is_wr)
2596 out_mrq_q_blks = mrq_q_blks;
2597 else
2598 in_mrq_q_blks = mrq_q_blks;
2599 if (vb > 2)
2600 pr2serr_lk("%s: >>> seg_blks=%d, num_good=%d, in_q/fin blks=%u/%u; "
2601 "out_q/fin blks=%u/%u\n", __func__, o_seg_blks, num_good,
2602 in_mrq_q_blks, in_fin_blks, out_mrq_q_blks, out_fin_blks);
2603
2604 if (clp->ese) {
2605 int sres = ctl_v4.spare_out;
2606
2607 if (sres != 0) {
2608 clp->reason_res.store(sg_convert_errno(sres));
2609 pr2serr_lk("Exit due to secondary error [%d]\n", sres);
2610 return -sres;
2611 }
2612 }
2613 if (num_good < 0)
2614 return -ENODATA;
2615 else {
2616 if (num_good < num_mrq) {
2617 int resid_blks = in_mrq_q_blks - in_fin_blks;
2618
2619 if (resid_blks > 0) {
2620 rep->in_rem_count += resid_blks;
2621 rep->stop_after_write = ! (err_on_in && clp->in_flags.coe);
2622 }
2623
2624 resid_blks = out_mrq_q_blks - out_fin_blks;
2625 if (resid_blks > 0) {
2626 rep->out_rem_count += resid_blks;
2627 rep->stop_after_write = ! (! err_on_in && clp->out_flags.coe);
2628 }
2629 }
2630 }
2631 return is_wr ? out_fin_blks : in_fin_blks;
2632 }
2633
2634 /* Returns number of blocks successfully processed or a negative error
2635 * number. */
2636 static int
do_normal_normal_segment(Rq_elem * rep,scat_gath_iter & i_sg_it,scat_gath_iter & o_sg_it,int seg_blks)2637 do_normal_normal_segment(Rq_elem * rep, scat_gath_iter & i_sg_it,
2638 scat_gath_iter & o_sg_it, int seg_blks)
2639 {
2640 int k, kk, res, id, num, d_off;
2641 int o_seg_blks = seg_blks;
2642 uint32_t in_fin_blks = 0;
2643 uint32_t out_fin_blks = 0;
2644 struct global_collection * clp = rep->clp;
2645
2646 id = rep->id;
2647 d_off = 0;
2648 for (k = 0; seg_blks > 0; ++k, seg_blks -= num, d_off += num) {
2649 kk = min<int>(seg_blks, clp->bpt);
2650 num = i_sg_it.linear_for_n_blks(kk);
2651 res = normal_in_rd(rep, i_sg_it.current_lba(), num,
2652 d_off * rep->bs);
2653 if (res < 0) {
2654 pr2serr_lk("[%d] %s: normal in failed d_off=%d, err=%d\n",
2655 id, __func__, d_off, -res);
2656 break;
2657 }
2658 i_sg_it.add_blks(res);
2659 if (res < num) {
2660 d_off += res;
2661 rep->stop_after_write = true;
2662 break;
2663 }
2664 }
2665 seg_blks = d_off;
2666 in_fin_blks = seg_blks;
2667
2668 if (FT_DEV_NULL == clp->out_type)
2669 goto fini;
2670 d_off = 0;
2671 for (k = 0; seg_blks > 0; ++k, seg_blks -= num, d_off += num) {
2672 kk = min<int>(seg_blks, clp->bpt);
2673 num = o_sg_it.linear_for_n_blks(kk);
2674 res = normal_out_wr(rep, o_sg_it.current_lba(), num,
2675 d_off * rep->bs);
2676 if (res < num) {
2677 if (res < 0) {
2678 pr2serr_lk("[%d] %s: normal out failed d_off=%d, err=%d\n",
2679 id, __func__, d_off, -res);
2680 break;
2681 }
2682 }
2683 o_sg_it.add_blks(res);
2684 if (res < num) {
2685 d_off += res;
2686 rep->stop_after_write = true;
2687 break;
2688 }
2689 }
2690 if (rep->in_resid_bytes > 0) {
2691 res = extra_out_wr(rep, rep->in_resid_bytes, d_off * rep->bs);
2692 if (res < 0)
2693 pr2serr_lk("[%d] %s: extr out failed d_off=%d, err=%d\n", id,
2694 __func__, d_off, -res);
2695 rep->in_resid_bytes = 0;
2696 }
2697 seg_blks = d_off;
2698 out_fin_blks = seg_blks;
2699
2700 fini:
2701 rep->in_local_count += in_fin_blks;
2702 rep->out_local_count += out_fin_blks;
2703
2704 if ((in_fin_blks + out_fin_blks) < (uint32_t)o_seg_blks) {
2705 int resid_blks = o_seg_blks - in_fin_blks;
2706
2707 if (resid_blks > 0)
2708 rep->in_rem_count += resid_blks;
2709 resid_blks = o_seg_blks - out_fin_blks;
2710 if (resid_blks > 0)
2711 rep->out_rem_count += resid_blks;
2712 }
2713 return res < 0 ? res : (min<int>(in_fin_blks, out_fin_blks));
2714 }
2715
2716 /* Returns number of blocks successfully processed or a negative error
2717 * number. */
2718 static int
do_normal_sg_segment(Rq_elem * rep,scat_gath_iter & i_sg_it,scat_gath_iter & o_sg_it,int seg_blks,vector<cdb_arr_t> & a_cdb,vector<struct sg_io_v4> & a_v4)2719 do_normal_sg_segment(Rq_elem * rep, scat_gath_iter & i_sg_it,
2720 scat_gath_iter & o_sg_it, int seg_blks,
2721 vector<cdb_arr_t> & a_cdb,
2722 vector<struct sg_io_v4> & a_v4)
2723 {
2724 bool in_is_normal = ! rep->only_in_sg;
2725 int k, kk, res, id, num, d_off;
2726 int o_seg_blks = seg_blks;
2727 uint32_t in_fin_blks = 0;
2728 uint32_t out_fin_blks = 0;
2729 struct global_collection * clp = rep->clp;
2730
2731 id = rep->id;
2732 a_cdb.clear();
2733 a_v4.clear();
2734
2735 if (in_is_normal) { /* in: normal --> out : sg */
2736 d_off = 0;
2737 for (k = 0; seg_blks > 0; ++k, seg_blks -= num, d_off += num) {
2738 kk = min<int>(seg_blks, clp->bpt);
2739 num = i_sg_it.linear_for_n_blks(kk);
2740 res = normal_in_rd(rep, i_sg_it.current_lba(), num,
2741 d_off * rep->bs);
2742 if (res < 0) {
2743 pr2serr_lk("[%d] %s: normal in failed d_off=%d, err=%d\n",
2744 id, __func__, d_off, -res);
2745 break;
2746 }
2747 i_sg_it.add_blks(res);
2748 if (res < num) {
2749 d_off += res;
2750 rep->stop_after_write = true;
2751 break;
2752 }
2753 }
2754 seg_blks = d_off;
2755 in_fin_blks = seg_blks;
2756
2757 if (rep->in_resid_bytes > 0) {
2758 ++seg_blks;
2759 rep->in_resid_bytes = 0;
2760 }
2761 if (clp->mrq_eq_0)
2762 res = sg_half_segment_mrq0(rep, o_sg_it, true /* is_wr */,
2763 seg_blks, rep->buffp);
2764 else
2765 res = sg_half_segment(rep, o_sg_it, true /* is_wr */, seg_blks,
2766 rep->buffp, a_cdb, a_v4);
2767 if (res < seg_blks) {
2768 if (res < 0) {
2769 pr2serr_lk("[%d] %s: sg out failed d_off=%d, err=%d\n",
2770 id, __func__, d_off, -res);
2771 goto fini;
2772 }
2773 rep->stop_after_write = true;
2774 }
2775 seg_blks = res;
2776 out_fin_blks = seg_blks;
2777
2778 } else { /* in: sg --> out: normal */
2779 if (clp->mrq_eq_0)
2780 res = sg_half_segment_mrq0(rep, i_sg_it, false, seg_blks,
2781 rep->buffp);
2782 else
2783 res = sg_half_segment(rep, i_sg_it, false, seg_blks, rep->buffp,
2784 a_cdb, a_v4);
2785 if (res < seg_blks) {
2786 if (res < 0) {
2787 pr2serr_lk("[%d] %s: sg in failed, err=%d\n", id, __func__,
2788 -res);
2789 goto fini;
2790 }
2791 rep->stop_after_write = true;
2792 }
2793 seg_blks = res;
2794 in_fin_blks = seg_blks;
2795
2796 if (FT_DEV_NULL == clp->out_type) {
2797 out_fin_blks = seg_blks;/* so finish logic doesn't suspect ... */
2798 goto bypass;
2799 }
2800 d_off = 0;
2801 for (k = 0; seg_blks > 0; ++k, seg_blks -= num, d_off += num) {
2802 kk = min<int>(seg_blks, clp->bpt);
2803 num = o_sg_it.linear_for_n_blks(kk);
2804 res = normal_out_wr(rep, o_sg_it.current_lba(), num,
2805 d_off * rep->bs);
2806 if (res < num) {
2807 if (res < 0) {
2808 pr2serr_lk("[%d] %s: normal out failed d_off=%d, err=%d\n",
2809 id, __func__, d_off, -res);
2810 break;
2811 }
2812 }
2813 o_sg_it.add_blks(res);
2814 if (res < num) {
2815 d_off += res;
2816 rep->stop_after_write = true;
2817 break;
2818 }
2819 }
2820 seg_blks = d_off;
2821 out_fin_blks = seg_blks;
2822 }
2823 bypass:
2824 rep->in_local_count += in_fin_blks;
2825 rep->out_local_count += out_fin_blks;
2826
2827 if ((in_fin_blks + out_fin_blks) < (uint32_t)o_seg_blks) {
2828 int resid_blks = o_seg_blks - in_fin_blks;
2829
2830 if (resid_blks > 0)
2831 rep->in_rem_count += resid_blks;
2832 resid_blks = o_seg_blks - out_fin_blks;
2833 if (resid_blks > 0)
2834 rep->out_rem_count += resid_blks;
2835 }
2836 fini:
2837 return res < 0 ? res : (min<int>(in_fin_blks, out_fin_blks));
2838 }
2839
2840 /* This function sets up a multiple request (mrq) transaction and sends it
2841 * to the pass-through. Returns number of blocks processed (==seg_blks for
2842 * all good) or a negative error number. */
2843 static int
do_both_sg_segment_mrq0(Rq_elem * rep,scat_gath_iter & i_sg_it,scat_gath_iter & o_sg_it,int seg_blks)2844 do_both_sg_segment_mrq0(Rq_elem * rep, scat_gath_iter & i_sg_it,
2845 scat_gath_iter & o_sg_it, int seg_blks)
2846 {
2847 int k, kk, res, pack_id_base, id, iflags, oflags;
2848 int num, i_lin_blks, o_lin_blks, cdbsz, err;
2849 uint32_t in_fin_blks = 0;
2850 uint32_t out_fin_blks = 0;
2851 struct global_collection * clp = rep->clp;
2852 int vb = clp->verbose;
2853 cdb_arr_t t_cdb {};
2854 struct sg_io_v4 t_v4 {};
2855 struct sg_io_v4 * t_v4p = &t_v4;
2856 struct flags_t * iflagsp = &clp->in_flags;
2857 struct flags_t * oflagsp = &clp->out_flags;
2858 const char * const a_ioctl_s = "do_both_sg_segment_mrq0: after "
2859 "ioctl(SG_IO)";
2860
2861 id = rep->id;
2862 pack_id_base = id * PACK_ID_TID_MULTIPLIER;
2863
2864 iflags = SGV4_FLAG_SHARE;
2865 if (iflagsp->mmap && (rep->outregfd >= 0))
2866 iflags |= SGV4_FLAG_MMAP_IO;
2867 else
2868 iflags |= SGV4_FLAG_NO_DXFER;
2869 if (iflagsp->dio)
2870 iflags |= SGV4_FLAG_DIRECT_IO;
2871 if (iflagsp->qhead)
2872 iflags |= SGV4_FLAG_Q_AT_HEAD;
2873 if (iflagsp->qtail)
2874 iflags |= SGV4_FLAG_Q_AT_TAIL;
2875 if (iflagsp->polled)
2876 iflags |= SGV4_FLAG_POLLED;
2877
2878 oflags = SGV4_FLAG_SHARE | SGV4_FLAG_NO_DXFER;
2879 if (oflagsp->dio)
2880 oflags |= SGV4_FLAG_DIRECT_IO;
2881 if (oflagsp->qhead)
2882 oflags |= SGV4_FLAG_Q_AT_HEAD;
2883 if (oflagsp->qtail)
2884 oflags |= SGV4_FLAG_Q_AT_TAIL;
2885 if (oflagsp->polled)
2886 oflags |= SGV4_FLAG_POLLED;
2887
2888 for (k = 0; seg_blks > 0; ++k, seg_blks -= num) {
2889 kk = min<int>(seg_blks, clp->bpt);
2890 i_lin_blks = i_sg_it.linear_for_n_blks(kk);
2891 o_lin_blks = o_sg_it.linear_for_n_blks(kk);
2892 num = min<int>(i_lin_blks, o_lin_blks);
2893 if (num <= 0) {
2894 res = 0;
2895 pr2serr_lk("[%d] %s: min(i_lin_blks=%d o_lin_blks=%d) < 1\n", id,
2896 __func__, i_lin_blks, o_lin_blks);
2897 break;
2898 }
2899
2900 /* First build the command/request for the read-side*/
2901 cdbsz = clp->cdbsz_in;
2902 res = sg_build_scsi_cdb(t_cdb.data(), cdbsz, num,
2903 i_sg_it.current_lba(), false, false,
2904 iflagsp->fua, iflagsp->dpo, iflagsp->cdl);
2905 if (res) {
2906 pr2serr_lk("%s: t=%d: input sg_build_scsi_cdb() failed\n",
2907 __func__, id);
2908 break;
2909 } else if (vb > 3)
2910 lk_print_command_len("input cdb: ", t_cdb.data(), cdbsz, true);
2911
2912 t_v4p->guard = 'Q';
2913 t_v4p->request = (uint64_t)t_cdb.data();
2914 t_v4p->usr_ptr = t_v4p->request;
2915 t_v4p->response = (uint64_t)rep->sb;
2916 t_v4p->max_response_len = sizeof(rep->sb);
2917 t_v4p->flags = iflags;
2918 t_v4p->request_len = cdbsz;
2919 t_v4p->din_xfer_len = num * rep->bs;
2920 t_v4p->dout_xfer_len = 0;
2921 t_v4p->timeout = clp->cmd_timeout;
2922 t_v4p->request_extra = pack_id_base + ++rep->mrq_pack_id_off;
2923 clp->most_recent_pack_id.store(t_v4p->request_extra);
2924 mrq0_again:
2925 res = ioctl(rep->infd, SG_IO, t_v4p);
2926 err = errno;
2927 if (vb > 5)
2928 v4hdr_out_lk(a_ioctl_s, t_v4p, id, false);
2929 if (res < 0) {
2930 if (E2BIG == err)
2931 sg_take_snap(rep->infd, id, true);
2932 else if (EBUSY == err) {
2933 ++num_ebusy;
2934 std::this_thread::yield();/* so other threads can progress */
2935 goto mrq0_again;
2936 }
2937 pr2serr_lk("[%d] %s: ioctl(SG_IO, read-side)-->%d, errno=%d: "
2938 "%s\n", id, __func__, res, err, strerror(err));
2939 return -err;
2940 }
2941 if (t_v4p->device_status || t_v4p->transport_status ||
2942 t_v4p->driver_status) {
2943 rep->stop_now = true;
2944 pr2serr_lk("[%d] t_v4[%d]:\n", id, k);
2945 lk_chk_n_print4(" ", t_v4p, vb > 4);
2946 return min<int>(in_fin_blks, out_fin_blks);
2947 }
2948 rep->in_local_count += num;
2949 in_fin_blks += num;
2950
2951 /* Now build the command/request for write-side (WRITE or VERIFY) */
2952 cdbsz = clp->cdbsz_out;
2953 res = sg_build_scsi_cdb(t_cdb.data(), cdbsz, num,
2954 o_sg_it.current_lba(), clp->verify, true,
2955 oflagsp->fua, oflagsp->dpo, oflagsp->cdl);
2956 if (res) {
2957 pr2serr_lk("%s: t=%d: output sg_build_scsi_cdb() failed\n",
2958 __func__, id);
2959 break;
2960 } else if (vb > 3)
2961 lk_print_command_len("output cdb: ", t_cdb.data(), cdbsz, true);
2962
2963 t_v4p->guard = 'Q';
2964 t_v4p->request = (uint64_t)t_cdb.data();
2965 t_v4p->usr_ptr = t_v4p->request;
2966 t_v4p->response = (uint64_t)rep->sb;
2967 t_v4p->max_response_len = sizeof(rep->sb);
2968 t_v4p->flags = oflags;
2969 t_v4p->request_len = cdbsz;
2970 t_v4p->din_xfer_len = 0;
2971 t_v4p->dout_xfer_len = num * rep->bs;
2972 t_v4p->timeout = clp->cmd_timeout;
2973 t_v4p->request_extra = pack_id_base + ++rep->mrq_pack_id_off;
2974 clp->most_recent_pack_id.store(t_v4p->request_extra);
2975 mrq0_again2:
2976 res = ioctl(rep->outfd, SG_IO, t_v4p);
2977 err = errno;
2978 if (vb > 5)
2979 v4hdr_out_lk(a_ioctl_s, t_v4p, id, false);
2980 if (res < 0) {
2981 if (E2BIG == err)
2982 sg_take_snap(rep->outfd, id, true);
2983 else if (EBUSY == err) {
2984 ++num_ebusy;
2985 std::this_thread::yield();/* so other threads can progress */
2986 goto mrq0_again2;
2987 }
2988 pr2serr_lk("[%d] %s: ioctl(SG_IO, write-side)-->%d, errno=%d: "
2989 "%s\n", id, __func__, res, err, strerror(err));
2990 return -err;
2991 }
2992 if (t_v4p->device_status || t_v4p->transport_status ||
2993 t_v4p->driver_status) {
2994 rep->stop_now = true;
2995 pr2serr_lk("[%d] t_v4[%d]:\n", id, k);
2996 lk_chk_n_print4(" ", t_v4p, vb > 4);
2997 return min<int>(in_fin_blks, out_fin_blks);
2998 }
2999 rep->out_local_count += num;
3000 out_fin_blks += num;
3001
3002 i_sg_it.add_blks(num);
3003 o_sg_it.add_blks(num);
3004 }
3005 return min<int>(in_fin_blks, out_fin_blks);
3006 }
3007
3008 /* This function sets up a multiple request (mrq) transaction and sends it
3009 * to the pass-through. Returns number of blocks processed (==seg_blks for
3010 * all good) or a negative error number. */
3011 static int
do_both_sg_segment(Rq_elem * rep,scat_gath_iter & i_sg_it,scat_gath_iter & o_sg_it,int seg_blks,vector<cdb_arr_t> & a_cdb,vector<struct sg_io_v4> & a_v4)3012 do_both_sg_segment(Rq_elem * rep, scat_gath_iter & i_sg_it,
3013 scat_gath_iter & o_sg_it, int seg_blks,
3014 vector<cdb_arr_t> & a_cdb,
3015 vector<struct sg_io_v4> & a_v4)
3016 {
3017 bool err_on_in = false;
3018 int num_mrq, k, res, fd, mrq_pack_id_base, id, b_len, iflags, oflags;
3019 int num, kk, i_lin_blks, o_lin_blks, cdbsz, num_good, err;
3020 int o_seg_blks = seg_blks;
3021 uint32_t in_fin_blks = 0;
3022 uint32_t out_fin_blks = 0;;
3023 uint32_t in_mrq_q_blks = 0;
3024 uint32_t out_mrq_q_blks = 0;
3025 const int max_cdb_sz = MAX_SCSI_CDB_SZ;
3026 struct sg_io_v4 * a_v4p;
3027 struct sg_io_v4 ctl_v4 {}; /* MRQ control object */
3028 struct global_collection * clp = rep->clp;
3029 const char * iosub_str = "SG_IOSUBMIT(svb)";
3030 char b[80];
3031 cdb_arr_t t_cdb {};
3032 struct sg_io_v4 t_v4 {};
3033 struct sg_io_v4 * t_v4p = &t_v4;
3034 struct flags_t * iflagsp = &clp->in_flags;
3035 struct flags_t * oflagsp = &clp->out_flags;
3036 int vb = clp->verbose;
3037
3038 id = rep->id;
3039 b_len = sizeof(b);
3040
3041 a_cdb.clear();
3042 a_v4.clear();
3043 rep->a_mrq_din_blks = 0;
3044 rep->a_mrq_dout_blks = 0;
3045 mrq_pack_id_base = id * PACK_ID_TID_MULTIPLIER;
3046
3047 iflags = SGV4_FLAG_SHARE;
3048 if (iflagsp->mmap && (rep->outregfd >= 0))
3049 iflags |= SGV4_FLAG_MMAP_IO;
3050 else
3051 iflags |= SGV4_FLAG_NO_DXFER;
3052 if (iflagsp->dio)
3053 iflags |= SGV4_FLAG_DIRECT_IO;
3054 if (iflagsp->qhead)
3055 iflags |= SGV4_FLAG_Q_AT_HEAD;
3056 if (iflagsp->qtail)
3057 iflags |= SGV4_FLAG_Q_AT_TAIL;
3058 if (iflagsp->polled)
3059 iflags |= SGV4_FLAG_POLLED;
3060
3061 oflags = SGV4_FLAG_SHARE | SGV4_FLAG_NO_DXFER;
3062 if (oflagsp->dio)
3063 oflags |= SGV4_FLAG_DIRECT_IO;
3064 if (oflagsp->qhead)
3065 oflags |= SGV4_FLAG_Q_AT_HEAD;
3066 if (oflagsp->qtail)
3067 oflags |= SGV4_FLAG_Q_AT_TAIL;
3068 if (oflagsp->polled)
3069 oflags |= SGV4_FLAG_POLLED;
3070 oflags |= SGV4_FLAG_DO_ON_OTHER;
3071
3072 for (k = 0; seg_blks > 0; ++k, seg_blks -= num) {
3073 kk = min<int>(seg_blks, clp->bpt);
3074 i_lin_blks = i_sg_it.linear_for_n_blks(kk);
3075 o_lin_blks = o_sg_it.linear_for_n_blks(kk);
3076 num = min<int>(i_lin_blks, o_lin_blks);
3077 if (num <= 0) {
3078 res = 0;
3079 pr2serr_lk("[%d] %s: min(i_lin_blks=%d o_lin_blks=%d) < 1\n", id,
3080 __func__, i_lin_blks, o_lin_blks);
3081 break;
3082 }
3083
3084 /* First build the command/request for the read-side*/
3085 cdbsz = clp->cdbsz_in;
3086 res = sg_build_scsi_cdb(t_cdb.data(), cdbsz, num,
3087 i_sg_it.current_lba(), false, false,
3088 iflagsp->fua, iflagsp->dpo, iflagsp->cdl);
3089 if (res) {
3090 pr2serr_lk("%s: t=%d: input sg_build_scsi_cdb() failed\n",
3091 __func__, id);
3092 break;
3093 } else if (vb > 3)
3094 lk_print_command_len("input cdb: ", t_cdb.data(), cdbsz, true);
3095 a_cdb.push_back(t_cdb);
3096
3097 t_v4p->guard = 'Q';
3098 t_v4p->flags = iflags;
3099 t_v4p->request_len = cdbsz;
3100 t_v4p->response = (uint64_t)rep->sb;
3101 t_v4p->max_response_len = sizeof(rep->sb);
3102 t_v4p->usr_ptr = (uint64_t)&a_cdb[a_cdb.size() - 1];
3103 t_v4p->din_xfer_len = num * rep->bs;
3104 rep->a_mrq_din_blks += num;
3105 t_v4p->dout_xfer_len = 0;
3106 t_v4p->timeout = clp->cmd_timeout;
3107 in_mrq_q_blks += num;
3108 t_v4p->request_extra = mrq_pack_id_base + ++rep->mrq_pack_id_off;
3109 clp->most_recent_pack_id.store(t_v4p->request_extra);
3110 a_v4.push_back(t_v4);
3111
3112 /* Now build the command/request for write-side (WRITE or VERIFY) */
3113 cdbsz = clp->cdbsz_out;
3114 res = sg_build_scsi_cdb(t_cdb.data(), cdbsz, num,
3115 o_sg_it.current_lba(), clp->verify, true,
3116 oflagsp->fua, oflagsp->dpo, oflagsp->cdl);
3117 if (res) {
3118 pr2serr_lk("%s: t=%d: output sg_build_scsi_cdb() failed\n",
3119 __func__, id);
3120 break;
3121 } else if (vb > 3)
3122 lk_print_command_len("output cdb: ", t_cdb.data(), cdbsz, true);
3123 a_cdb.push_back(t_cdb);
3124 t_v4p->guard = 'Q';
3125 t_v4p->flags = oflags;
3126 t_v4p->request_len = cdbsz;
3127 t_v4p->response = (uint64_t)rep->sb;
3128 t_v4p->max_response_len = sizeof(rep->sb);
3129 t_v4p->usr_ptr = (uint64_t)&a_cdb[a_cdb.size() - 1];
3130 t_v4p->din_xfer_len = 0;
3131 t_v4p->dout_xfer_len = num * rep->bs;
3132 rep->a_mrq_dout_blks += num;
3133 t_v4p->timeout = clp->cmd_timeout;
3134 out_mrq_q_blks += num;
3135 t_v4p->request_extra = mrq_pack_id_base + ++rep->mrq_pack_id_off;
3136 clp->most_recent_pack_id.store(t_v4p->request_extra);
3137 a_v4.push_back(t_v4);
3138
3139 i_sg_it.add_blks(num);
3140 o_sg_it.add_blks(num);
3141 }
3142
3143 if (vb > 6) {
3144 pr2serr_lk("%s: t=%d: a_v4 array contents:\n", __func__, id);
3145 hex2stderr_lk((const uint8_t *)a_v4.data(),
3146 a_v4.size() * sizeof(struct sg_io_v4), 1);
3147 }
3148 if (rep->both_sg || rep->same_sg)
3149 fd = rep->infd; /* assume share to rep->outfd */
3150 else {
3151 pr2serr_lk("[%d] %s: why am I here? Want 2 sg devices\n", id,
3152 __func__);
3153 res = -1;
3154 goto fini;
3155 }
3156 num_mrq = a_v4.size();
3157 a_v4p = a_v4.data();
3158 res = 0;
3159 ctl_v4.guard = 'Q';
3160 ctl_v4.request_len = a_cdb.size() * max_cdb_sz;
3161 ctl_v4.request = (uint64_t)a_cdb.data();
3162 ctl_v4.max_response_len = sizeof(rep->sb);
3163 ctl_v4.response = (uint64_t)rep->sb;
3164 ctl_v4.flags = SGV4_FLAG_MULTIPLE_REQS | SGV4_FLAG_SHARE;
3165 if (! (iflagsp->coe || oflagsp->coe))
3166 ctl_v4.flags |= SGV4_FLAG_STOP_IF;
3167 if ((! clp->verify) && clp->out_flags.order_wr)
3168 ctl_v4.flags |= SGV4_FLAG_ORDERED_WR;
3169 if (clp->mrq_polled)
3170 ctl_v4.flags |= SGV4_FLAG_POLLED;
3171 if (clp->in_flags.mout_if || clp->out_flags.mout_if) {
3172 ctl_v4.flags |= SGV4_FLAG_META_OUT_IF;
3173 if (num_mrq > 0)
3174 a_v4[0].info = UINT32_MAX;
3175 }
3176 ctl_v4.dout_xferp = (uint64_t)a_v4.data(); /* request array */
3177 ctl_v4.dout_xfer_len = a_v4.size() * sizeof(struct sg_io_v4);
3178 ctl_v4.din_xferp = (uint64_t)a_v4.data(); /* response array */
3179 ctl_v4.din_xfer_len = a_v4.size() * sizeof(struct sg_io_v4);
3180 if (false /* allow_mrq_abort */) {
3181 ctl_v4.request_extra = mrq_pack_id_base + ++rep->mrq_pack_id_off;
3182 clp->most_recent_pack_id.store(ctl_v4.request_extra);
3183 }
3184
3185 if (vb && vb_first_time.load()) {
3186 pr2serr_lk("First controlling object output by ioctl(%s), flags: "
3187 "%s\n", iosub_str, sg_flags_str(ctl_v4.flags, b_len, b));
3188 vb_first_time.store(false);
3189 } else if (vb > 4)
3190 pr2serr_lk("%s: >> Control object _before_ ioctl(%s):\n", __func__,
3191 iosub_str);
3192 if (vb > 4) {
3193 if (vb > 5)
3194 hex2stderr_lk((const uint8_t *)&ctl_v4, sizeof(ctl_v4), 1);
3195 v4hdr_out_lk(">> Control object before", &ctl_v4, id, false);
3196 }
3197
3198 try_again:
3199 if (!after1 && (vb > 1)) {
3200 after1 = true;
3201 pr2serr_lk("%s: %s\n", __func__, mrq_svb_s);
3202 }
3203 res = ioctl(fd, SG_IOSUBMIT, &ctl_v4);
3204 if (res < 0) {
3205 err = errno;
3206 if (E2BIG == err)
3207 sg_take_snap(fd, id, true);
3208 else if (EBUSY == err) {
3209 ++num_ebusy;
3210 std::this_thread::yield();/* allow another thread to progress */
3211 goto try_again;
3212 }
3213 pr2serr_lk("%s: ioctl(%s, %s)-->%d, errno=%d: %s\n", __func__,
3214 iosub_str, sg_flags_str(ctl_v4.flags, b_len, b), res, err,
3215 strerror(err));
3216 res = -err;
3217 goto fini;
3218 }
3219 if (vb > 4) {
3220 pr2serr_lk("%s: >> Control object after ioctl(%s) seg_blks=%d:\n",
3221 __func__, iosub_str, o_seg_blks);
3222 if (vb > 5)
3223 hex2stderr_lk((const uint8_t *)&ctl_v4, sizeof(ctl_v4), 1);
3224 v4hdr_out_lk(">> Control object after", &ctl_v4, id, false);
3225 if (vb > 5) {
3226 for (k = 0; k < num_mrq; ++k) {
3227 if ((vb > 6) || a_v4p[k].info) {
3228 snprintf(b, b_len, "a_v4[%d/%d]", k, num_mrq);
3229 v4hdr_out_lk(b, (a_v4p + k), id, true);
3230 }
3231 }
3232 }
3233 }
3234 num_good = process_mrq_response(rep, &ctl_v4, a_v4p, num_mrq, in_fin_blks,
3235 out_fin_blks, err_on_in);
3236 if (vb > 2)
3237 pr2serr_lk("%s: >>> seg_blks=%d, num_good=%d, in_q/fin blks=%u/%u; "
3238 "out_q/fin blks=%u/%u\n", __func__, o_seg_blks, num_good,
3239 in_mrq_q_blks, in_fin_blks, out_mrq_q_blks, out_fin_blks);
3240
3241 if (clp->ese) {
3242 int sres = ctl_v4.spare_out;
3243
3244 if (sres != 0) {
3245 clp->reason_res.store(sg_convert_errno(sres));
3246 pr2serr_lk("Exit due to secondary error [%d]\n", sres);
3247 return -sres;
3248 }
3249 }
3250 if (num_good < 0)
3251 res = -ENODATA;
3252 else {
3253 rep->in_local_count += in_fin_blks;
3254 rep->out_local_count += out_fin_blks;
3255
3256 if (num_good < num_mrq) { /* reduced number completed */
3257 int resid_blks = in_mrq_q_blks - in_fin_blks;
3258
3259 if (resid_blks > 0) {
3260 rep->in_rem_count += resid_blks;
3261 rep->stop_after_write = ! (err_on_in && clp->in_flags.coe);
3262 }
3263
3264 resid_blks = out_mrq_q_blks - out_fin_blks;
3265 if (resid_blks > 0) {
3266 rep->out_rem_count += resid_blks;
3267 rep->stop_after_write = ! ((! err_on_in) &&
3268 clp->out_flags.coe);
3269 }
3270 }
3271 }
3272 fini:
3273 return res < 0 ? res : (min<int>(in_fin_blks, out_fin_blks));
3274 }
3275
3276 #if 0
3277 /* Returns number found and (partially) processed. 'num' is the number of
3278 * completions to wait for when > 0. When 'num' is zero check all inflight
3279 * request on 'fd' and return quickly if none completed (i.e. don't wait)
3280 * If error return negative errno and if no request inflight or waiting
3281 * then return -9999 . */
3282 static int
3283 sg_blk_poll(int fd, int num)
3284 {
3285 int res;
3286 struct sg_extended_info sei {};
3287 struct sg_extended_info * seip = &sei;
3288
3289 seip->sei_rd_mask |= SG_SEIM_BLK_POLL;
3290 seip->sei_wr_mask |= SG_SEIM_BLK_POLL;
3291 seip->num = (num < 0) ? 0 : num;
3292 res = ioctl(fd, SG_SET_GET_EXTENDED, seip);
3293 if (res < 0) {
3294 pr2serr_lk("%s: SG_SET_GET_EXTENDED(BLK_POLL) error: %s\n",
3295 __func__, strerror(errno));
3296 return res;
3297 }
3298 return (seip->num == -1) ? -9999 : seip->num;
3299 }
3300 #endif
3301
3302 /* Returns the number of times 'ch' is found in string 's' given the
3303 * string's length. */
3304 static int
num_chs_in_str(const char * s,int slen,int ch)3305 num_chs_in_str(const char * s, int slen, int ch)
3306 {
3307 int res = 0;
3308
3309 while (--slen >= 0) {
3310 if (ch == s[slen])
3311 ++res;
3312 }
3313 return res;
3314 }
3315
3316 /* Returns the number of times either 'ch1' or 'ch2' is found in
3317 * string 's' given the string's length. */
3318 int
num_either_ch_in_str(const char * s,int slen,int ch1,int ch2)3319 num_either_ch_in_str(const char * s, int slen, int ch1, int ch2)
3320 {
3321 int k;
3322 int res = 0;
3323
3324 while (--slen >= 0) {
3325 k = s[slen];
3326 if ((ch1 == k) || (ch2 == k))
3327 ++res;
3328 }
3329 return res;
3330 }
3331
3332 /* Allocates and then populates a scatter gether list (array) and returns
3333 * it via *sgl_pp. Return of 0 is okay, else error number (in which case
3334 * NULL is written to *sgl_pp) . */
3335 static int
skip_seek(struct global_collection * clp,const char * key,const char * buf,bool is_skip,bool ignore_verbose)3336 skip_seek(struct global_collection *clp, const char * key, const char * buf,
3337 bool is_skip, bool ignore_verbose)
3338 {
3339 bool def_hex = false;
3340 int len;
3341 int vb = clp->verbose; /* needs to appear before skip/seek= on cl */
3342 int64_t ll;
3343 const char * cp;
3344 class scat_gath_list & either_list = is_skip ? clp->i_sgl : clp->o_sgl;
3345
3346 if (ignore_verbose)
3347 vb = 0;
3348 len = (int)strlen(buf);
3349 if ((('-' == buf[0]) && (1 == len)) || ((len > 1) && ('@' == buf[0])) ||
3350 ((len > 2) && ('H' == toupper(buf[0])) && ('@' == buf[1]))) {
3351 if ('H' == toupper(buf[0])) {
3352 cp = buf + 2;
3353 def_hex = true;
3354 } else if ('-' == buf[0])
3355 cp = buf;
3356 else
3357 cp = buf + 1;
3358 if (! either_list.load_from_file(cp, def_hex, clp->flexible, true)) {
3359 pr2serr("bad argument to '%s=' [err=%d]\n", key,
3360 either_list.m_errno);
3361 return SG_LIB_SYNTAX_ERROR;
3362 }
3363 } else if (num_either_ch_in_str(buf, len, ',', ' ') > 0) {
3364 if (! either_list.load_from_cli(buf, vb > 0)) {
3365 pr2serr("bad command line argument to '%s='\n", key);
3366 return SG_LIB_SYNTAX_ERROR;
3367 }
3368 } else { /* single number on command line (e.g. skip=1234) */
3369 ll = sg_get_llnum(buf);
3370 if ((ll < 0) || (ll > MAX_COUNT_SKIP_SEEK)) {
3371 pr2serr("bad argument to '%s='\n", key);
3372 return SG_LIB_SYNTAX_ERROR;
3373 }
3374 either_list.append_1or(0, ll);
3375 if (vb > 1)
3376 pr2serr("%s: singleton, half a degenerate sgl element\n", key);
3377 }
3378
3379 either_list.sum_scan(key, vb > 3 /* bool show_sgl */, vb > 1);
3380 return 0;
3381 }
3382
3383 static bool
process_flags(const char * arg,struct flags_t * fp)3384 process_flags(const char * arg, struct flags_t * fp)
3385 {
3386 char buff[256];
3387 char * cp;
3388 char * np;
3389
3390 strncpy(buff, arg, sizeof(buff));
3391 buff[sizeof(buff) - 1] = '\0';
3392 if ('\0' == buff[0]) {
3393 pr2serr("no flag found\n");
3394 return false;
3395 }
3396 cp = buff;
3397 do {
3398 np = strchr(cp, ',');
3399 if (np)
3400 *np++ = '\0';
3401 if (0 == strcmp(cp, "00"))
3402 fp->zero = true;
3403 else if (0 == strcmp(cp, "append"))
3404 fp->append = true;
3405 else if (0 == strcmp(cp, "coe"))
3406 fp->coe = true;
3407 else if (0 == strcmp(cp, "dio"))
3408 fp->dio = true;
3409 else if (0 == strcmp(cp, "direct"))
3410 fp->direct = true;
3411 else if (0 == strcmp(cp, "dpo"))
3412 fp->dpo = true;
3413 else if (0 == strcmp(cp, "dsync"))
3414 fp->dsync = true;
3415 else if (0 == strcmp(cp, "excl"))
3416 fp->excl = true;
3417 else if (0 == strcmp(cp, "ff"))
3418 fp->ff = true;
3419 else if (0 == strcmp(cp, "fua"))
3420 fp->fua = true;
3421 else if (0 == strcmp(cp, "hipri"))
3422 fp->polled = true;
3423 else if (0 == strcmp(cp, "masync"))
3424 fp->masync = true;
3425 else if (0 == strcmp(cp, "mmap"))
3426 ++fp->mmap; /* mmap > 1 stops munmap() being called */
3427 else if (0 == strcmp(cp, "nocreat"))
3428 fp->nocreat = true;
3429 else if (0 == strcmp(cp, "nodur"))
3430 fp->no_dur = true;
3431 else if (0 == strcmp(cp, "no_dur"))
3432 fp->no_dur = true;
3433 else if (0 == strcmp(cp, "no-dur"))
3434 fp->no_dur = true;
3435 else if (0 == strcmp(cp, "nothresh"))
3436 fp->no_thresh = true;
3437 else if (0 == strcmp(cp, "no_thresh"))
3438 fp->no_thresh = true;
3439 else if (0 == strcmp(cp, "no-thresh"))
3440 fp->no_thresh = true;
3441 else if (0 == strcmp(cp, "noxfer"))
3442 ; /* accept but ignore */
3443 else if (0 == strcmp(cp, "null"))
3444 ;
3445 else if (0 == strcmp(cp, "ordered"))
3446 fp->order_wr = true;
3447 else if (0 == strcmp(cp, "order"))
3448 fp->order_wr = true;
3449 else if (0 == strcmp(cp, "polled"))
3450 fp->polled = true;
3451 else if (0 == strcmp(cp, "qhead"))
3452 fp->qhead = true;
3453 else if (0 == strcmp(cp, "qtail"))
3454 fp->qtail = true;
3455 else if (0 == strcmp(cp, "random"))
3456 fp->random = true;
3457 else if ((0 == strcmp(cp, "mout_if")) || (0 == strcmp(cp, "mout-if")))
3458 fp->mout_if = true;
3459 else if ((0 == strcmp(cp, "same_fds")) ||
3460 (0 == strcmp(cp, "same-fds")))
3461 fp->same_fds = true;
3462 else if (0 == strcmp(cp, "serial"))
3463 fp->serial = true;
3464 else if (0 == strcmp(cp, "swait"))
3465 ; /* accept but ignore */
3466 else if (0 == strcmp(cp, "wq_excl"))
3467 fp->wq_excl = true;
3468 else {
3469 pr2serr("unrecognised flag: %s\n", cp);
3470 return false;
3471 }
3472 cp = np;
3473 } while (cp);
3474 return true;
3475 }
3476
3477 /* Process arguments given to 'conv=" option. Returns 0 on success,
3478 * 1 on error. */
3479 static int
process_conv(const char * arg,struct flags_t * ifp,struct flags_t * ofp)3480 process_conv(const char * arg, struct flags_t * ifp, struct flags_t * ofp)
3481 {
3482 char buff[256];
3483 char * cp;
3484 char * np;
3485
3486 strncpy(buff, arg, sizeof(buff));
3487 buff[sizeof(buff) - 1] = '\0';
3488 if ('\0' == buff[0]) {
3489 pr2serr("no conversions found\n");
3490 return 1;
3491 }
3492 cp = buff;
3493 do {
3494 np = strchr(cp, ',');
3495 if (np)
3496 *np++ = '\0';
3497 if (0 == strcmp(cp, "nocreat"))
3498 ofp->nocreat = true;
3499 else if (0 == strcmp(cp, "noerror"))
3500 ifp->coe = true; /* will still fail on write error */
3501 else if (0 == strcmp(cp, "notrunc"))
3502 ; /* this is the default action of sg_dd so ignore */
3503 else if (0 == strcmp(cp, "null"))
3504 ;
3505 else if (0 == strcmp(cp, "sync"))
3506 ; /* dd(susv4): pad errored block(s) with zeros but sg_dd does
3507 * that by default. Typical dd use: 'conv=noerror,sync' */
3508 else {
3509 pr2serr("unrecognised flag: %s\n", cp);
3510 return 1;
3511 }
3512 cp = np;
3513 } while (cp);
3514 return 0;
3515 }
3516
3517 #define STR_SZ 1024
3518 #define INOUTF_SZ 512
3519
3520 static int
parse_cmdline_sanity(int argc,char * argv[],struct global_collection * clp,char * outregf)3521 parse_cmdline_sanity(int argc, char * argv[], struct global_collection * clp,
3522 char * outregf)
3523 {
3524 bool contra = false;
3525 bool verbose_given = false;
3526 bool version_given = false;
3527 bool verify_given = false;
3528 bool bpt_given = false;
3529 int ibs = 0;
3530 int obs = 0;
3531 int ret = 0;
3532 int k, keylen, n, res;
3533 char str[STR_SZ];
3534 char * key;
3535 char * buf;
3536 char * skip_buf = NULL;
3537 char * seek_buf = NULL;
3538 const char * cp;
3539 const char * ccp;
3540
3541 for (k = 1; k < argc; k++) {
3542 if (argv[k]) {
3543 strncpy(str, argv[k], STR_SZ);
3544 str[STR_SZ - 1] = '\0';
3545 } else
3546 continue;
3547
3548 for (key = str, buf = key; *buf && *buf != '=';)
3549 buf++;
3550 if (*buf)
3551 *buf++ = '\0';
3552 keylen = strlen(key);
3553 if (0 == strcmp(key, "bpt")) {
3554 clp->bpt = sg_get_num(buf);
3555 if ((clp->bpt < 0) || (clp->bpt > MAX_BPT_VALUE)) {
3556 pr2serr("%sbad argument to 'bpt='\n", my_name);
3557 goto syn_err;
3558 }
3559 bpt_given = true;
3560 } else if (0 == strcmp(key, "bs")) {
3561 clp->bs = sg_get_num(buf);
3562 if ((clp->bs < 0) || (clp->bs > MAX_BPT_VALUE)) {
3563 pr2serr("%sbad argument to 'bs='\n", my_name);
3564 goto syn_err;
3565 }
3566 } else if (0 == strcmp(key, "cdbsz")) {
3567 ccp = strchr(buf, ',');
3568 n = sg_get_num(buf);
3569 if ((n < 0) || (n > 32)) {
3570 pr2serr("%s: bad argument to 'cdbsz=', expect 6, 10, 12 or "
3571 "16\n", my_name);
3572 goto syn_err;
3573 }
3574 clp->cdbsz_in = n;
3575 if (ccp) {
3576 n = sg_get_num(ccp + 1);
3577 if ((n < 0) || (n > 32)) {
3578 pr2serr("%s: bad second argument to 'cdbsz=', expect 6, "
3579 "10, 12 or 16\n", my_name);
3580 goto syn_err;
3581 }
3582 }
3583 clp->cdbsz_out = n;
3584 clp->cdbsz_given = true;
3585 } else if (0 == strcmp(key, "cdl")) {
3586 ccp = strchr(buf, ',');
3587 n = sg_get_num(buf);
3588 if ((n < 0) || (n > 7)) {
3589 pr2serr("%s: bad argument to 'cdl=', expect 0 to 7\n",
3590 my_name);
3591 goto syn_err;
3592 }
3593 clp->in_flags.cdl = n;
3594 if (ccp) {
3595 n = sg_get_num(ccp + 1);
3596 if ((n < 0) || (n > 7)) {
3597 pr2serr("%s: bad second argument to 'cdl=', expect 0 "
3598 "to 7\n", my_name);
3599 goto syn_err;
3600 }
3601 }
3602 clp->out_flags.cdl = n;
3603 clp->cdl_given = true;
3604 } else if (0 == strcmp(key, "coe")) {
3605 /* not documented, for compat with sgh_dd */
3606 clp->in_flags.coe = !! sg_get_num(buf);
3607 clp->out_flags.coe = clp->in_flags.coe;
3608 } else if (0 == strcmp(key, "conv")) {
3609 if (process_conv(buf, &clp->in_flags, &clp->out_flags)) {
3610 pr2serr("%s: bad argument to 'conv='\n", my_name);
3611 goto syn_err;
3612 }
3613 } else if (0 == strcmp(key, "count")) {
3614 if (clp->count_given) {
3615 pr2serr("second 'count=' argument detected, only one "
3616 "please\n");
3617 contra = true;
3618 goto syn_err;
3619 }
3620 if (0 != strcmp("-1", buf)) {
3621 clp->dd_count = sg_get_llnum(buf);
3622 if ((clp->dd_count < 0) ||
3623 (clp->dd_count > MAX_COUNT_SKIP_SEEK)) {
3624 pr2serr("%sbad argument to 'count='\n", my_name);
3625 goto syn_err;
3626 }
3627 } /* treat 'count=-1' as calculate count (same as not given) */
3628 clp->count_given = true;
3629 } else if (0 == strcmp(key, "dio")) {
3630 clp->in_flags.dio = !! sg_get_num(buf);
3631 clp->out_flags.dio = clp->in_flags.dio;
3632 } else if (0 == strcmp(key, "elemsz_kb")) {
3633 n = sg_get_num(buf);
3634 if ((n < 1) || (n > (MAX_BPT_VALUE / 1024))) {
3635 pr2serr("elemsz_kb=EKB wants an integer > 0\n");
3636 goto syn_err;
3637 }
3638 if (n & (n - 1)) {
3639 pr2serr("elemsz_kb=EKB wants EKB to be power of 2\n");
3640 goto syn_err;
3641 }
3642 clp->elem_sz = n * 1024;
3643 } else if (0 == strcmp(key, "ese")) {
3644 n = sg_get_num(buf);
3645 if (n < 0) {
3646 pr2serr("ese= wants 0 (default) or 1\n");
3647 goto syn_err;
3648 }
3649 clp->ese = !!n;
3650 } else if (0 == strcmp(key, "fua")) {
3651 n = sg_get_num(buf);
3652 if (n & 1)
3653 clp->out_flags.fua = true;
3654 if (n & 2)
3655 clp->in_flags.fua = true;
3656 } else if (0 == strcmp(key, "ibs")) {
3657 ibs = sg_get_num(buf);
3658 if ((ibs < 0) || (ibs > MAX_BPT_VALUE)) {
3659 pr2serr("%sbad argument to 'ibs='\n", my_name);
3660 goto syn_err;
3661 }
3662 } else if (0 == strcmp(key, "if")) {
3663 if (clp->inf_v.size() > 0) {
3664 pr2serr("Second 'if=' argument??\n");
3665 goto syn_err;
3666 } else {
3667 cp = buf;
3668 while ((ccp = strchr(cp, ','))) {
3669 clp->inf_v.push_back(string(cp , ccp - cp));
3670 cp = ccp + 1;
3671 }
3672 clp->inf_v.push_back(string(cp , strlen(cp)));
3673 }
3674 } else if (0 == strcmp(key, "iflag")) {
3675 if (! process_flags(buf, &clp->in_flags)) {
3676 pr2serr("%sbad argument to 'iflag='\n", my_name);
3677 goto syn_err;
3678 }
3679 } else if ((0 == strcmp(key, "hipri")) ||
3680 (0 == strcmp(key, "mrq")) ||
3681 (0 == strcmp(key, "polled"))) {
3682 if (isdigit(buf[0]))
3683 cp = buf;
3684 else {
3685 pr2serr("%sonly mrq=NRQS or polled=NRQS which is a number "
3686 "allowed here\n", my_name);
3687 goto syn_err;
3688 }
3689 clp->mrq_num = sg_get_num(cp);
3690 if (clp->mrq_num < 0) {
3691 pr2serr("%sbad argument to 'mrq='\n", my_name);
3692 goto syn_err;
3693 }
3694 if (0 == clp->mrq_num) {
3695 clp->mrq_eq_0 = true;
3696 clp->mrq_num = 1;
3697 pr2serr("note: send single, non-mrq commands\n");
3698 }
3699 if ('m' != key[0])
3700 clp->mrq_polled = true;
3701 } else if ((0 == strcmp(key, "no_waitq")) ||
3702 (0 == strcmp(key, "no-waitq"))) {
3703 n = sg_get_num(buf);
3704 if (-1 == n) {
3705 pr2serr("%sbad argument to 'no_waitq=', expect 0 or 1\n",
3706 my_name);
3707 goto syn_err;
3708 }
3709 clp->in_flags.no_waitq = true;
3710 clp->out_flags.no_waitq = true;
3711 } else if (0 == strcmp(key, "obs")) {
3712 obs = sg_get_num(buf);
3713 if ((obs < 0) || (obs > MAX_BPT_VALUE)) {
3714 pr2serr("%sbad argument to 'obs='\n", my_name);
3715 goto syn_err;
3716 }
3717 } else if (strcmp(key, "ofreg") == 0) {
3718 if ('\0' != outregf[0]) {
3719 pr2serr("Second OFREG argument??\n");
3720 contra = true;
3721 goto syn_err;
3722 } else {
3723 memcpy(outregf, buf, INOUTF_SZ);
3724 outregf[INOUTF_SZ - 1] = '\0'; /* noisy compiler */
3725 }
3726 } else if (strcmp(key, "of") == 0) {
3727 if (clp->outf_v.size() > 0) {
3728 pr2serr("Second 'of=' argument??\n");
3729 goto syn_err;
3730 } else {
3731 cp = buf;
3732 while ((ccp = strchr(cp, ','))) {
3733 clp->outf_v.push_back(string(cp , ccp - cp));
3734 cp = ccp + 1;
3735 }
3736 clp->outf_v.push_back(string(cp , strlen(cp)));
3737 }
3738 } else if (0 == strcmp(key, "oflag")) {
3739 if (! process_flags(buf, &clp->out_flags)) {
3740 pr2serr("%sbad argument to 'oflag='\n", my_name);
3741 goto syn_err;
3742 }
3743 } else if (0 == strcmp(key, "sdt")) {
3744 ccp = strchr(buf, ',');
3745 n = sg_get_num(buf);
3746 if (n < 0) {
3747 pr2serr("%sbad argument to 'sdt=CRT[,ICT]'\n", my_name);
3748 goto syn_err;
3749 }
3750 clp->sdt_crt = n;
3751 if (ccp) {
3752 n = sg_get_num(ccp + 1);
3753 if (n < 0) {
3754 pr2serr("%sbad 2nd argument to 'sdt=CRT,ICT'\n",
3755 my_name);
3756 goto syn_err;
3757 }
3758 clp->sdt_ict = n;
3759 }
3760 } else if (0 == strcmp(key, "seek")) {
3761 n = strlen(buf);
3762 if (n < 1) {
3763 pr2serr("%sneed argument to 'seek='\n", my_name);
3764 goto syn_err;
3765 }
3766 seek_buf = (char *)calloc(n + 16, 1);
3767 if (NULL == seek_buf)
3768 goto syn_err;
3769 memcpy(seek_buf, buf, n + 1);
3770 } else if (0 == strcmp(key, "skip")) {
3771 n = strlen(buf);
3772 if (n < 1) {
3773 pr2serr("%sneed argument to 'skip='\n", my_name);
3774 goto syn_err;
3775 }
3776 skip_buf = (char *)calloc(n + 16, 1);
3777 if (NULL == skip_buf)
3778 goto syn_err;
3779 memcpy(skip_buf, buf, n + 1);
3780 } else if (0 == strcmp(key, "sync"))
3781 do_sync = !! sg_get_num(buf);
3782 else if (0 == strcmp(key, "thr")) {
3783 num_threads = sg_get_num(buf);
3784 if ((num_threads < 0) || (num_threads > MAX_BPT_VALUE)) {
3785 pr2serr("%sneed argument to 'skip='\n", my_name);
3786 goto syn_err;
3787 }
3788 } else if (0 == strcmp(key, "time")) {
3789 ccp = strchr(buf, ',');
3790 do_time = sg_get_num(buf);
3791 if (do_time < 0) {
3792 pr2serr("%sbad argument to 'time=0|1|2'\n", my_name);
3793 goto syn_err;
3794 }
3795 if (ccp) {
3796 n = sg_get_num(ccp + 1);
3797 if ((n < 0) || (n > (MAX_BPT_VALUE / 1000))) {
3798 pr2serr("%sbad argument to 'time=0|1|2,TO'\n", my_name);
3799 goto syn_err;
3800 }
3801 clp->cmd_timeout = n ? (n * 1000) : DEF_TIMEOUT;
3802 }
3803 } else if (0 == strncmp(key, "verb", 4))
3804 clp->verbose = sg_get_num(buf);
3805 else if ((keylen > 1) && ('-' == key[0]) && ('-' != key[1])) {
3806 res = 0;
3807 n = num_chs_in_str(key + 1, keylen - 1, 'd');
3808 clp->dry_run += n;
3809 res += n;
3810 n = num_chs_in_str(key + 1, keylen - 1, 'h');
3811 clp->help += n;
3812 res += n;
3813 n = num_chs_in_str(key + 1, keylen - 1, 'p');
3814 if (n > 0)
3815 clp->prefetch = true;
3816 res += n;
3817 n = num_chs_in_str(key + 1, keylen - 1, 'v');
3818 if (n > 0)
3819 verbose_given = true;
3820 clp->verbose += n; /* -v ---> --verbose */
3821 res += n;
3822 n = num_chs_in_str(key + 1, keylen - 1, 'V');
3823 if (n > 0)
3824 version_given = true;
3825 res += n;
3826 n = num_chs_in_str(key + 1, keylen - 1, 'x');
3827 if (n > 0)
3828 verify_given = true;
3829 res += n;
3830
3831 if (res < (keylen - 1)) {
3832 pr2serr("Unrecognised short option in '%s', try '--help'\n",
3833 key);
3834 goto syn_err;
3835 }
3836 } else if ((0 == strncmp(key, "--dry-run", 9)) ||
3837 (0 == strncmp(key, "--dry_run", 9)))
3838 ++clp->dry_run;
3839 else if ((0 == strncmp(key, "--help", 6)) ||
3840 (0 == strcmp(key, "-?")))
3841 ++clp->help;
3842 else if ((0 == strncmp(key, "--prefetch", 10)) ||
3843 (0 == strncmp(key, "--pre-fetch", 11)))
3844 clp->prefetch = true;
3845 else if (0 == strncmp(key, "--verb", 6)) {
3846 verbose_given = true;
3847 ++clp->verbose; /* --verbose */
3848 } else if (0 == strncmp(key, "--veri", 6))
3849 verify_given = true;
3850 else if (0 == strncmp(key, "--vers", 6))
3851 version_given = true;
3852 else {
3853 pr2serr("Unrecognized option '%s'\n", key);
3854 pr2serr("For more information use '--help'\n");
3855 goto syn_err;
3856 }
3857 } /* end of parsing for loop */
3858
3859 if (skip_buf) {
3860 res = skip_seek(clp, "skip", skip_buf, true /* skip */, false);
3861 free(skip_buf);
3862 skip_buf = NULL;
3863 if (res) {
3864 pr2serr("%sbad argument to 'seek='\n", my_name);
3865 goto syn_err;
3866 }
3867 }
3868 if (seek_buf) {
3869 res = skip_seek(clp, "seek", seek_buf, false /* skip */, false);
3870 free(seek_buf);
3871 seek_buf = NULL;
3872 if (res) {
3873 pr2serr("%sbad argument to 'seek='\n", my_name);
3874 goto syn_err;
3875 }
3876 }
3877 /* heap usage should be all freed up now */
3878
3879 #ifdef DEBUG
3880 pr2serr("In DEBUG mode, ");
3881 if (verbose_given && version_given) {
3882 pr2serr("but override: '-vV' given, zero verbose and continue\n");
3883 verbose_given = false;
3884 version_given = false;
3885 clp->verbose = 0;
3886 } else if (! verbose_given) {
3887 pr2serr("set '-vv'\n");
3888 clp->verbose = 2;
3889 } else
3890 pr2serr("keep verbose=%d\n", clp->verbose);
3891 #else
3892 if (verbose_given && version_given)
3893 pr2serr("Not in DEBUG mode, so '-vV' has no special action\n");
3894 #endif
3895 if (version_given) {
3896 pr2serr("%s%s\n", my_name, version_str);
3897 ret = SG_LIB_OK_FALSE;
3898 goto oth_err;
3899 }
3900 if (clp->help > 0) {
3901 usage(clp->help);
3902 ret = SG_LIB_OK_FALSE;
3903 goto oth_err;
3904 }
3905 if (clp->bs <= 0) {
3906 clp->bs = DEF_BLOCK_SIZE;
3907 pr2serr("Assume default 'bs' ((logical) block size) of %d bytes\n",
3908 clp->bs);
3909 }
3910 if (verify_given) {
3911 pr2serr("Doing verify/cmp rather than copy\n");
3912 clp->verify = true;
3913 }
3914 if ((ibs && (ibs != clp->bs)) || (obs && (obs != clp->bs))) {
3915 pr2serr("If 'ibs' or 'obs' given must be same as 'bs'\n");
3916 usage(0);
3917 goto syn_err;
3918 }
3919 if (clp->out_flags.append) {
3920 if ((clp->o_sgl.lowest_lba > 0) ||
3921 (clp->o_sgl.linearity != SGL_LINEAR)) {
3922 pr2serr("Can't use both append and seek switches\n");
3923 goto syn_err;
3924 }
3925 if (verify_given) {
3926 pr2serr("Can't use both append and verify switches\n");
3927 goto syn_err;
3928 }
3929 }
3930 if (clp->bpt < 1) {
3931 pr2serr("bpt must be greater than 0\n");
3932 goto syn_err;
3933 }
3934 if (clp->in_flags.mmap && clp->out_flags.mmap) {
3935 pr2serr("mmap flag on both IFILE and OFILE doesn't work\n");
3936 goto syn_err;
3937 }
3938 /* defaulting transfer size to 128*2048 for CD/DVDs is too large
3939 * for the block layer in lk 2.6 and results in an EIO on the
3940 * SG_IO ioctl. So reduce it in that case. */
3941 if ((clp->bs >= 2048) && (! bpt_given))
3942 clp->bpt = DEF_BLOCKS_PER_2048TRANSFER;
3943 if (clp->in_flags.order_wr && (! clp->out_flags.order_wr))
3944 pr2serr("Warning iflag=order is ignored, use with oflag=\n");
3945 if ((num_threads < 1) || (num_threads > MAX_NUM_THREADS)) {
3946 pr2serr("too few or too many threads requested\n");
3947 usage(1);
3948 goto syn_err;
3949 }
3950 clp->unit_nanosec = (do_time > 1) || !!getenv("SG3_UTILS_LINUX_NANO");
3951 return 0;
3952
3953 syn_err:
3954 if (seek_buf)
3955 free(seek_buf);
3956 if (skip_buf)
3957 free(skip_buf);
3958 return contra ? SG_LIB_CONTRADICT : SG_LIB_SYNTAX_ERROR;
3959 oth_err:
3960 if (seek_buf)
3961 free(seek_buf);
3962 if (skip_buf)
3963 free(skip_buf);
3964 return ret;
3965 }
3966
3967 static int
calc_count(struct global_collection * clp,const char * inf,int64_t & in_num_sect,const char * outf,int64_t & out_num_sect)3968 calc_count(struct global_collection * clp, const char * inf,
3969 int64_t & in_num_sect, const char * outf, int64_t & out_num_sect)
3970 {
3971 int in_sect_sz, out_sect_sz, res;
3972
3973 if (clp->dd_count < 0) {
3974 in_num_sect = -1;
3975 out_num_sect = -1;
3976 }
3977 if (FT_SG == clp->in_type) {
3978 res = scsi_read_capacity(clp->in0fd, &in_num_sect, &in_sect_sz);
3979 if (2 == res) {
3980 pr2serr("Unit attention, media changed(in), continuing\n");
3981 res = scsi_read_capacity(clp->in0fd, &in_num_sect,
3982 &in_sect_sz);
3983 }
3984 if (0 != res) {
3985 if (res == SG_LIB_CAT_INVALID_OP)
3986 pr2serr("read capacity not supported on %s\n", inf);
3987 else if (res == SG_LIB_CAT_NOT_READY)
3988 pr2serr("read capacity failed, %s not ready\n", inf);
3989 else
3990 pr2serr("Unable to read capacity on %s\n", inf);
3991 return SG_LIB_FILE_ERROR;
3992 } else if (clp->bs != in_sect_sz) {
3993 pr2serr(">> warning: logical block size on %s confusion: "
3994 "bs=%d, device claims=%d\n", clp->infp, clp->bs,
3995 in_sect_sz);
3996 return SG_LIB_FILE_ERROR;
3997 }
3998 }
3999 if (FT_SG == clp->out_type) {
4000 res = scsi_read_capacity(clp->out0fd, &out_num_sect, &out_sect_sz);
4001 if (2 == res) {
4002 pr2serr("Unit attention, media changed(out), continuing\n");
4003 res = scsi_read_capacity(clp->out0fd, &out_num_sect,
4004 &out_sect_sz);
4005 }
4006 if (0 != res) {
4007 if (res == SG_LIB_CAT_INVALID_OP)
4008 pr2serr("read capacity not supported on %s\n", outf);
4009 else if (res == SG_LIB_CAT_NOT_READY)
4010 pr2serr("read capacity failed, %s not ready\n", outf);
4011 else
4012 pr2serr("Unable to read capacity on %s\n", outf);
4013 out_num_sect = -1;
4014 return SG_LIB_FILE_ERROR;
4015 } else if (clp->bs != out_sect_sz) {
4016 pr2serr(">> warning: logical block size on %s confusion: "
4017 "bs=%d, device claims=%d\n", clp->outfp, clp->bs,
4018 out_sect_sz);
4019 return SG_LIB_FILE_ERROR;
4020 }
4021 }
4022
4023 if (clp->dd_count < 0) {
4024 if (FT_SG == clp->in_type)
4025 ;
4026 else if (FT_BLOCK == clp->in_type) {
4027 if (0 != read_blkdev_capacity(clp->in0fd, &in_num_sect,
4028 &in_sect_sz)) {
4029 pr2serr("Unable to read block capacity on %s\n", inf);
4030 in_num_sect = -1;
4031 }
4032 if (clp->bs != in_sect_sz) {
4033 pr2serr("logical block size on %s confusion; bs=%d, from "
4034 "device=%d\n", inf, clp->bs, in_sect_sz);
4035 in_num_sect = -1;
4036 }
4037 }
4038
4039 if (FT_SG == clp->out_type)
4040 ;
4041 else if (FT_BLOCK == clp->out_type) {
4042 if (0 != read_blkdev_capacity(clp->out0fd, &out_num_sect,
4043 &out_sect_sz)) {
4044 pr2serr("Unable to read block capacity on %s\n", outf);
4045 out_num_sect = -1;
4046 }
4047 if (clp->bs != out_sect_sz) {
4048 pr2serr("logical block size on %s confusion: bs=%d, from "
4049 "device=%d\n", outf, clp->bs, out_sect_sz);
4050 out_num_sect = -1;
4051 }
4052 }
4053 }
4054 return 0;
4055 }
4056
4057 static int
do_count_work(struct global_collection * clp,const char * inf,int64_t & in_num_sect,const char * outf,int64_t & out_num_sect)4058 do_count_work(struct global_collection * clp, const char * inf,
4059 int64_t & in_num_sect, const char * outf,
4060 int64_t & out_num_sect)
4061 {
4062 int res;
4063 class scat_gath_list * isglp = &clp->i_sgl;
4064 class scat_gath_list * osglp = &clp->o_sgl;
4065
4066 res = calc_count(clp, inf, in_num_sect, outf, out_num_sect);
4067 if (res)
4068 return res;
4069
4070 if ((-1 == in_num_sect) && (FT_OTHER == clp->in_type)) {
4071 in_num_sect = clp->in_st_size / clp->bs;
4072 if (clp->in_st_size % clp->bs) {
4073 ++in_num_sect;
4074 pr2serr("Warning: the file size of %s is not a multiple of BS "
4075 "[%d]\n", inf, clp->bs);
4076 }
4077 }
4078 if ((in_num_sect > 0) && (isglp->high_lba_p1 > in_num_sect)) {
4079 pr2serr("%shighest LBA [0x%" PRIx64 "] exceeds input length: %"
4080 PRIx64 " blocks\n", my_name, isglp->high_lba_p1 - 1,
4081 in_num_sect);
4082 return SG_LIB_CAT_OTHER;
4083 }
4084 if ((out_num_sect > 0) && (osglp->high_lba_p1 > out_num_sect)) {
4085 pr2serr("%shighest LBA [0x%" PRIx64 "] exceeds output length: %"
4086 PRIx64 " blocks\n", my_name, osglp->high_lba_p1 - 1,
4087 out_num_sect);
4088 return SG_LIB_CAT_OTHER;
4089 }
4090
4091 if (isglp->sum_hard || osglp->sum_hard) {
4092 int64_t ccount;
4093
4094 if (isglp->sum_hard && osglp->sum_hard) {
4095 if (isglp->sum != osglp->sum) {
4096 pr2serr("%stwo hard sgl_s, sum of blocks differ: in=%" PRId64
4097 ", out=%" PRId64 "\n", my_name , isglp->sum,
4098 osglp->sum);
4099 return SG_LIB_CAT_OTHER;
4100 }
4101 ccount = isglp->sum;
4102 } else if (isglp->sum_hard) {
4103 if (osglp->sum > isglp->sum) {
4104 pr2serr("%soutput sgl already too many blocks [%" PRId64
4105 "]\n", my_name, osglp->sum);
4106 return SG_LIB_CAT_OTHER;
4107 }
4108 if (osglp->linearity != SGL_NON_MONOTONIC)
4109 osglp->append_1or(isglp->sum - osglp->sum);
4110 else {
4111 pr2serr("%soutput sgl non-montonic: can't extend\n",
4112 my_name);
4113 return SG_LIB_CAT_OTHER;
4114 }
4115 ccount = isglp->sum;
4116 } else { /* only osglp hard */
4117 if (isglp->sum > osglp->sum) {
4118 pr2serr("%sinput sgl already too many blocks [%" PRId64
4119 "]\n", my_name, isglp->sum);
4120 return SG_LIB_CAT_OTHER;
4121 }
4122 if (isglp->linearity != SGL_NON_MONOTONIC)
4123 isglp->append_1or(osglp->sum - isglp->sum);
4124 else {
4125 pr2serr("%sinput sgl non-monotonic: can't extend\n",
4126 my_name);
4127 return SG_LIB_CAT_OTHER;
4128 }
4129 ccount = osglp->sum;
4130 }
4131 if (SG_COUNT_INDEFINITE == clp->dd_count)
4132 clp->dd_count = ccount;
4133 else if (ccount != clp->dd_count) {
4134 pr2serr("%scount=COUNT disagrees with scatter gather list "
4135 "length [%" PRId64 "]\n", my_name, ccount);
4136 return SG_LIB_CAT_OTHER;
4137 }
4138 } else if (clp->dd_count != 0) { /* and both input and output are soft */
4139 int64_t iposs = INT64_MAX;
4140 int64_t oposs = INT64_MAX;
4141
4142 if (clp->dd_count > 0) {
4143 if (isglp->sum > clp->dd_count) {
4144 pr2serr("%sskip sgl sum [%" PRId64 "] exceeds COUNT\n",
4145 my_name, isglp->sum);
4146 return SG_LIB_CAT_OTHER;
4147 }
4148 if (osglp->sum > clp->dd_count) {
4149 pr2serr("%sseek sgl sum [%" PRId64 "] exceeds COUNT\n",
4150 my_name, osglp->sum);
4151 return SG_LIB_CAT_OTHER;
4152 }
4153 goto fini;
4154 }
4155
4156 /* clp->dd_count == SG_COUNT_INDEFINITE */
4157 if (in_num_sect > 0)
4158 iposs = in_num_sect + isglp->sum - isglp->high_lba_p1;
4159 if (out_num_sect > 0)
4160 oposs = out_num_sect + osglp->sum - osglp->high_lba_p1;
4161 clp->dd_count = iposs < oposs ? iposs : oposs;
4162 if (INT64_MAX == clp->dd_count) {
4163 pr2serr("%scan't deduce count=COUNT, please supply one\n",
4164 my_name);
4165 return SG_LIB_CAT_OTHER;
4166 }
4167 if (isglp->sum > clp->dd_count) {
4168 pr2serr("%sdeduced COUNT [%" PRId64 "] exceeds skip sgl sum\n",
4169 my_name, clp->dd_count);
4170 return SG_LIB_CAT_OTHER;
4171 }
4172 if (osglp->sum > clp->dd_count) {
4173 pr2serr("%sdeduced COUNT [%" PRId64 "] exceeds seek sgl sum\n",
4174 my_name, clp->dd_count);
4175 return SG_LIB_CAT_OTHER;
4176 }
4177 }
4178 if (clp->dd_count == 0)
4179 return 0;
4180 fini:
4181 if (clp->dd_count > isglp->sum)
4182 isglp->append_1or(clp->dd_count - isglp->sum);
4183 if (clp->dd_count > osglp->sum)
4184 osglp->append_1or(clp->dd_count - osglp->sum);
4185 return 0;
4186 }
4187
4188
4189 int
main(int argc,char * argv[])4190 main(int argc, char * argv[])
4191 {
4192 bool fail_after_cli = false;
4193 bool ifile_given = true;
4194 // char inf[INOUTF_SZ];
4195 // char outf[INOUTF_SZ];
4196 char outregf[INOUTF_SZ];
4197 int res, k, err;
4198 size_t num_ifiles, num_ofiles, num_slices, inf0_sz;
4199 int64_t in_num_sect = -1;
4200 int64_t out_num_sect = -1;
4201 const char * ccp = NULL;
4202 const char * cc2p;
4203 struct global_collection * clp = &gcoll;
4204 thread sig_listen_thr;
4205 vector<thread> work_thr_v;
4206 vector<thread> listen_thr_v;
4207 char ebuff[EBUFF_SZ];
4208 #if 0 /* SG_LIB_ANDROID */
4209 struct sigaction actions;
4210
4211 memset(&actions, 0, sizeof(actions));
4212 sigemptyset(&actions.sa_mask);
4213 actions.sa_flags = 0;
4214 actions.sa_handler = thread_exit_handler;
4215 sigaction(SIGUSR1, &actions, NULL);
4216 sigaction(SIGUSR2, &actions, NULL);
4217 #endif
4218 /* memset(clp, 0, sizeof(*clp)); */
4219 clp->dd_count = SG_COUNT_INDEFINITE;
4220 clp->bpt = DEF_BLOCKS_PER_TRANSFER;
4221 clp->cmd_timeout = DEF_TIMEOUT;
4222 clp->sdt_ict = DEF_SDT_ICT_MS;
4223 clp->sdt_crt = DEF_SDT_CRT_SEC;
4224 clp->in_type = FT_FIFO;
4225 /* change dd's default: if of=OFILE not given, assume /dev/null */
4226 clp->out_type = FT_DEV_NULL;
4227 clp->cdbsz_in = DEF_SCSI_CDB_SZ;
4228 clp->cdbsz_out = DEF_SCSI_CDB_SZ;
4229 clp->mrq_num = DEF_MRQ_NUM;
4230 // inf[0] = '\0';
4231 // outf[0] = '\0';
4232 outregf[0] = '\0';
4233 fetch_sg_version();
4234 if (sg_version >= 40045)
4235 sg_version_ge_40045 = true;
4236 else {
4237 pr2serr(">>> %srequires an sg driver version of 4.0.45 or later\n\n",
4238 my_name);
4239 fail_after_cli = true;
4240 }
4241
4242 res = parse_cmdline_sanity(argc, argv, clp, outregf);
4243 if (SG_LIB_OK_FALSE == res)
4244 return 0;
4245 if (res)
4246 return res;
4247 if (fail_after_cli) {
4248 pr2serr("%scommand line parsing was okay but sg driver is too old\n",
4249 my_name);
4250 return SG_LIB_SYNTAX_ERROR;
4251 }
4252
4253 install_handler(SIGINT, interrupt_handler);
4254 install_handler(SIGQUIT, interrupt_handler);
4255 install_handler(SIGPIPE, interrupt_handler);
4256 install_handler(SIGUSR1, siginfo_handler);
4257 install_handler(SIGUSR2, siginfo2_handler);
4258
4259 num_ifiles = clp->inf_v.size();
4260 num_ofiles = clp->outf_v.size();
4261 if (num_ifiles > MAX_SLICES) {
4262 pr2serr("%sonly support %d slices but given %zd IFILEs\n", my_name,
4263 MAX_SLICES, num_ifiles);
4264 return SG_LIB_SYNTAX_ERROR;
4265 }
4266 if (num_ofiles > MAX_SLICES) {
4267 pr2serr("%sonly support %d slices but given %zd OFILEs\n", my_name,
4268 MAX_SLICES, num_ifiles);
4269 return SG_LIB_SYNTAX_ERROR;
4270 }
4271 if (0 == num_ofiles) {
4272 if (0 == num_ifiles) {
4273 pr2serr("%sexpect either if= or of= to be given\n", my_name);
4274 return SG_LIB_SYNTAX_ERROR;
4275 }
4276 for (k = 0; k < (int)num_ifiles; ++k)
4277 clp->outf_v.push_back("."); /* same as /dev/null */
4278 }
4279 if (0 == num_ifiles) {
4280 ifile_given = false;
4281 for (k = 0; k < (int)num_ofiles; ++k)
4282 clp->inf_v.push_back("");
4283 }
4284 if ((num_ifiles > 1) && (num_ofiles > 1) && (num_ifiles != num_ofiles)) {
4285 pr2serr("%snumber of IFILEs [%zd] and number of OFILEs [%zd] > 1 "
4286 "and unequal\n", my_name, num_ifiles, num_ofiles);
4287 return SG_LIB_SYNTAX_ERROR;
4288 }
4289 if ((num_ifiles > 1) && (1 == num_ofiles)) {
4290 /* if many IFILEs and one OFILE, replicate OFILE till same size */
4291 for (k = 1; k < (int)num_ifiles; ++k)
4292 clp->outf_v.push_back(clp->outf_v[0]);
4293 num_ofiles = clp->outf_v.size();
4294 } else if ((num_ofiles > 1) && (1 == num_ifiles)) {
4295 /* if many OFILEs and one IFILE, replicate IFILE till same size */
4296 for (k = 1; k < (int)num_ofiles; ++k)
4297 clp->inf_v.push_back(clp->inf_v[0]);
4298 num_ifiles = clp->inf_v.size();
4299 }
4300 num_slices = (num_ifiles > num_ofiles) ? num_ifiles : num_ofiles;
4301 if ((int)num_slices > num_threads) {
4302 pr2serr("%sNumber of slices [%zd] exceeds number of threads [%d].\n",
4303 my_name, num_slices, num_threads);
4304 pr2serr("Number of threads needs to be increased.\n");
4305 return SG_LIB_SYNTAX_ERROR;
4306 }
4307 k = 0;
4308 for (auto && cvp : clp->cp_ver_arr) {
4309 if (k >= (int)num_slices)
4310 break;
4311 cvp.my_index = k++;
4312 cvp.state = cp_ver_pair_t::my_state::init;
4313 }
4314 clp->in0fd = STDIN_FILENO;
4315 clp->out0fd = STDOUT_FILENO;
4316 if (clp->in_flags.ff && clp->in_flags.zero) {
4317 ccp = "<addr_as_data>";
4318 cc2p = "addr_as_data";
4319 } else if (clp->in_flags.ff) {
4320 ccp = "<0xff bytes>";
4321 cc2p = "ff";
4322 } else if (clp->in_flags.random) {
4323 ccp = "<random>";
4324 cc2p = "random";
4325 } else if (clp->in_flags.zero) {
4326 ccp = "<zero bytes>";
4327 cc2p = "00";
4328 }
4329 inf0_sz = clp->inf_v.size() ? clp->inf_v[0].size() : 0;
4330 if (ccp) {
4331 if (ifile_given) {
4332 pr2serr("%siflag=%s and if=%s contradict\n", my_name, cc2p,
4333 clp->inf_v[0].c_str());
4334 return SG_LIB_CONTRADICT;
4335 }
4336 for (auto && cvp : clp->cp_ver_arr) {
4337 if (cvp.state == cp_ver_pair_t::my_state::empty)
4338 break;
4339 cvp.in_type = FT_RANDOM_0_FF;
4340 }
4341 clp->in_type = FT_RANDOM_0_FF;
4342 clp->infp = ccp;
4343 clp->in0fd = -1;
4344 } else if (inf0_sz && ('-' != clp->inf_v[0].c_str()[0])) {
4345 const string & inf_s = clp->inf_v[0];
4346 const char * infp = inf_s.c_str();
4347
4348 clp->in_type = dd_filetype(infp, clp->in_st_size);
4349 if (FT_ERROR == clp->in_type) {
4350 pr2serr("%sunable to access %s\n", my_name, infp);
4351 return SG_LIB_FILE_ERROR;
4352 } else if (FT_ST == clp->in_type) {
4353 pr2serr("%sunable to use scsi tape device %s\n", my_name, infp);
4354 return SG_LIB_FILE_ERROR;
4355 } else if (FT_CHAR == clp->in_type) {
4356 pr2serr("%sunable to use unknown char device %s\n", my_name, infp);
4357 return SG_LIB_FILE_ERROR;
4358 } else if (FT_SG == clp->in_type) {
4359 clp->in0fd = sg_in_open(clp, inf_s, NULL, NULL);
4360 if (clp->in0fd < 0)
4361 return -clp->in0fd;
4362 } else {
4363 clp->in0fd = reg_file_open(clp, infp, false /* read */);
4364 if (clp->in0fd < 0)
4365 return sg_convert_errno(-clp->in0fd);
4366 }
4367 clp->infp = infp;
4368 }
4369 if (clp->cdl_given && (! clp->cdbsz_given)) {
4370 bool changed = false;
4371
4372 if ((clp->cdbsz_in < 16) && (clp->in_flags.cdl > 0)) {
4373 clp->cdbsz_in = 16;
4374 changed = true;
4375 }
4376 if ((clp->cdbsz_out < 16) && (! clp->verify) &&
4377 (clp->out_flags.cdl > 0)) {
4378 clp->cdbsz_out = 16;
4379 changed = true;
4380 }
4381 if (changed)
4382 pr2serr(">> increasing cdbsz to 16 due to cdl > 0\n");
4383 }
4384 if ((clp->verbose > 0) &&
4385 (clp->in_flags.no_waitq || clp->out_flags.no_waitq))
4386 pr2serr("no_waitq=<n> operand is now ignored\n");
4387 if (clp->outf_v.size()) {
4388 const string & outf_s = clp->outf_v[0].c_str();
4389 const char * outfp = outf_s.c_str();
4390
4391 clp->ofile_given = true;
4392 if ('-' == outfp[0])
4393 clp->out_type = FT_FIFO;
4394 else
4395 clp->out_type = dd_filetype(outfp, clp->out_st_size);
4396
4397 if ((FT_SG != clp->out_type) && clp->verify) {
4398 pr2serr("%s --verify only supported by sg OFILEs\n", my_name);
4399 return SG_LIB_FILE_ERROR;
4400 }
4401 if (FT_FIFO == clp->out_type)
4402 ;
4403 else if (FT_ST == clp->out_type) {
4404 pr2serr("%sunable to use scsi tape device %s\n", my_name, outfp);
4405 return SG_LIB_FILE_ERROR;
4406 } else if (FT_CHAR == clp->out_type) {
4407 pr2serr("%sunable to use unknown char device %s\n", my_name,
4408 outfp);
4409 return SG_LIB_FILE_ERROR;
4410 } else if (FT_SG == clp->out_type) {
4411 clp->out0fd = sg_out_open(clp, outf_s, NULL, NULL);
4412 if (clp->out0fd < 0)
4413 return -clp->out0fd;
4414 } else if (FT_DEV_NULL == clp->out_type)
4415 clp->out0fd = -1; /* don't bother opening */
4416 else {
4417 clp->out0fd = reg_file_open(clp, outfp, true /* write */);
4418 if (clp->out0fd < 0)
4419 return sg_convert_errno(-clp->out0fd);
4420 }
4421 clp->outfp = outfp;
4422 }
4423 if (clp->verify && (clp->out_type == FT_DEV_NULL)) {
4424 pr2serr("Can't do verify when OFILE not given\n");
4425 return SG_LIB_SYNTAX_ERROR;
4426 }
4427
4428 if ((FT_SG == clp->in_type) && (FT_SG == clp->out_type)) {
4429 if (clp->in_flags.serial || clp->out_flags.serial)
4430 pr2serr("serial flag ignored when both IFILE and OFILE are sg "
4431 "devices\n");
4432 if (clp->in_flags.order_wr && (num_threads > 1))
4433 pr2serr("Warning: write ordering only guaranteed for single "
4434 "thread\n");
4435 } else if (clp->in_flags.order_wr)
4436 pr2serr("Warning: oflag=order only active on sg->sg copies\n");
4437
4438 if (outregf[0]) {
4439 int ftyp = dd_filetype(outregf, clp->outreg_st_size);
4440
4441 clp->outreg_type = ftyp;
4442 if (! ((FT_OTHER == ftyp) || (FT_ERROR == ftyp) ||
4443 (FT_DEV_NULL == ftyp))) {
4444 pr2serr("File: %s can only be regular file or pipe (or "
4445 "/dev/null)\n", outregf);
4446 return SG_LIB_SYNTAX_ERROR;
4447 }
4448 if ((clp->outregfd = open(outregf, O_WRONLY | O_CREAT, 0666)) < 0) {
4449 err = errno;
4450 snprintf(ebuff, EBUFF_SZ, "could not open %s for writing",
4451 outregf);
4452 perror(ebuff);
4453 return sg_convert_errno(err);
4454 }
4455 if (clp->verbose > 1)
4456 pr2serr("ofreg=%s opened okay, fd=%d\n", outregf, clp->outregfd);
4457 if (FT_ERROR == ftyp)
4458 clp->outreg_type = FT_OTHER; /* regular file created */
4459 } else
4460 clp->outregfd = -1;
4461
4462 if ((STDIN_FILENO == clp->in0fd) && (STDOUT_FILENO == clp->out0fd)) {
4463 pr2serr("Won't default both IFILE to stdin _and_ OFILE to "
4464 "/dev/null\n");
4465 pr2serr("For more information use '--help'\n");
4466 return SG_LIB_SYNTAX_ERROR;
4467 }
4468 if ((clp->in_type == FT_FIFO) && (! clp->i_sgl.is_pipe_suitable())) {
4469 pr2serr("The skip= argument is not suitable for a pipe\n");
4470 return SG_LIB_SYNTAX_ERROR;
4471 }
4472 if ((clp->out_type == FT_FIFO) && (! clp->o_sgl.is_pipe_suitable())) {
4473 pr2serr("The seek= argument is not suitable for a pipe\n");
4474 return SG_LIB_SYNTAX_ERROR;
4475 }
4476 res = do_count_work(clp, clp->inf_v[0].c_str(), in_num_sect,
4477 clp->outf_v[0].c_str(), out_num_sect);
4478 if (res)
4479 return res;
4480
4481 if (clp->verbose > 2)
4482 pr2serr("Start of loop, count=%" PRId64 ", in_num_sect=%" PRId64
4483 ", out_num_sect=%" PRId64 "\n", clp->dd_count, in_num_sect,
4484 out_num_sect);
4485 if (clp->dd_count < 0) {
4486 pr2serr("Couldn't calculate count, please give one\n");
4487 return SG_LIB_CAT_OTHER;
4488 }
4489 if (! clp->cdbsz_given) {
4490 if ((FT_SG == clp->in_type) && (MAX_SCSI_CDB_SZ != clp->cdbsz_in) &&
4491 ((clp->i_sgl.high_lba_p1 > UINT_MAX) || (clp->bpt > USHRT_MAX))) {
4492 pr2serr("Note: SCSI command size increased to 16 bytes (for "
4493 "'if')\n");
4494 clp->cdbsz_in = MAX_SCSI_CDB_SZ;
4495 }
4496 if ((FT_SG == clp->out_type) && (MAX_SCSI_CDB_SZ != clp->cdbsz_out) &&
4497 ((clp->o_sgl.high_lba_p1 > UINT_MAX) || (clp->bpt > USHRT_MAX))) {
4498 pr2serr("Note: SCSI command size increased to 16 bytes (for "
4499 "'of')\n");
4500 clp->cdbsz_out = MAX_SCSI_CDB_SZ;
4501 }
4502 }
4503
4504 for (auto && cvp : clp->cp_ver_arr) {
4505 cvp.in_type = clp->in_type;
4506 cvp.out_type = clp->out_type;
4507 cvp.dd_count = clp->dd_count;
4508 cvp.in_rem_count = clp->dd_count;
4509 cvp.out_rem_count = clp->dd_count;
4510 }
4511
4512 if (clp->dry_run > 0) {
4513 pr2serr("Due to --dry-run option, bypass copy/read\n");
4514 goto fini;
4515 }
4516 if (! clp->ofile_given)
4517 pr2serr("of=OFILE not given so only read from IFILE, to output to "
4518 "stdout use 'of=-'\n");
4519
4520 sigemptyset(&signal_set);
4521 sigaddset(&signal_set, SIGINT);
4522 sigaddset(&signal_set, SIGUSR2);
4523
4524 res = sigprocmask(SIG_BLOCK, &signal_set, &orig_signal_set);
4525 if (res < 0) {
4526 pr2serr("sigprocmask failed: %s\n", safe_strerror(errno));
4527 goto fini;
4528 }
4529
4530 listen_thr_v.emplace_back(sig_listen_thread, clp);
4531
4532 if (do_time) {
4533 start_tm.tv_sec = 0;
4534 start_tm.tv_usec = 0;
4535 gettimeofday(&start_tm, NULL);
4536 }
4537
4538 /* vvvvvvvvvvv Start worker threads vvvvvvvvvvvvvvvvvvvvvvvv */
4539 if (num_threads > 0) {
4540 auto & cvp = clp->cp_ver_arr[0];
4541
4542 cvp.in_fd = clp->in0fd;
4543 cvp.out_fd = clp->out0fd;
4544
4545 /* launch "infant" thread to catch early mortality, if any */
4546 work_thr_v.emplace_back(read_write_thread, clp, 0, 0, true);
4547 {
4548 unique_lock<mutex> lk(clp->infant_mut);
4549 clp->infant_cv.wait(lk, []{ return gcoll.processed; });
4550 }
4551 if (clp->cp_ver_arr[0].next_count_pos.load() < 0) {
4552 /* infant thread error-ed out, join with it */
4553 for (auto & t : work_thr_v) {
4554 if (t.joinable())
4555 t.join();
4556 }
4557 goto jump;
4558 }
4559
4560 /* now start the rest of the threads */
4561 for (k = 1; k < num_threads; ++k)
4562 work_thr_v.emplace_back(read_write_thread, clp, k,
4563 k % (int)num_slices, false);
4564
4565 /* now wait for worker threads to finish */
4566 for (auto & t : work_thr_v) {
4567 if (t.joinable())
4568 t.join();
4569 }
4570 } /* worker threads hereafter have all exited */
4571 jump:
4572 if (do_time && (start_tm.tv_sec || start_tm.tv_usec))
4573 calc_duration_throughput(0);
4574
4575 if (do_sync) {
4576 if (FT_SG == clp->out_type) {
4577 pr2serr_lk(">> Synchronizing cache on %s\n",
4578 (clp->outf_v.size() ? clp->outf_v[0].c_str() : "" ));
4579 res = sg_ll_sync_cache_10(clp->out0fd, 0, 0, 0, 0, 0, false, 0);
4580 if (SG_LIB_CAT_UNIT_ATTENTION == res) {
4581 pr2serr_lk("Unit attention(out), continuing\n");
4582 res = sg_ll_sync_cache_10(clp->out0fd, 0, 0, 0, 0, 0, false,
4583 0);
4584 }
4585 if (0 != res)
4586 pr2serr_lk("Unable to synchronize cache\n");
4587 }
4588 }
4589
4590 shutting_down = true;
4591 for (auto & t : listen_thr_v) {
4592 if (t.joinable()) {
4593 t.detach();
4594 if (listen_t_tid > 0)
4595 kill(listen_t_tid, SIGUSR2);
4596 // t.~thread(); /* kill listening thread; doesn't work */
4597 }
4598 std::this_thread::yield(); // not enough it seems
4599 { /* allow time for SIGUSR2 signal to get through */
4600 struct timespec tspec = {0, 1000000}; /* 1 msec */
4601 struct timespec rem;
4602
4603 while ((nanosleep(&tspec, &rem) < 0) && (EINTR == errno))
4604 tspec = rem;
4605 }
4606 }
4607
4608 fini:
4609 if ((STDIN_FILENO != clp->in0fd) && (clp->in0fd >= 0))
4610 close(clp->in0fd);
4611 if ((STDOUT_FILENO != clp->out0fd) && (FT_DEV_NULL != clp->out_type) &&
4612 (clp->out0fd >= 0))
4613 close(clp->out0fd);
4614 if ((clp->outregfd >= 0) && (STDOUT_FILENO != clp->outregfd) &&
4615 (FT_DEV_NULL != clp->outreg_type))
4616 close(clp->outregfd);
4617 print_stats("");
4618 if (clp->dio_incomplete_count.load()) {
4619 int fd;
4620 char c;
4621
4622 pr2serr(">> Direct IO requested but incomplete %d times\n",
4623 clp->dio_incomplete_count.load());
4624 if ((fd = open(sg_allow_dio, O_RDONLY)) >= 0) {
4625 if (1 == read(fd, &c, 1)) {
4626 if ('0' == c)
4627 pr2serr(">>> %s set to '0' but should be set to '1' for "
4628 "direct IO\n", sg_allow_dio);
4629 }
4630 close(fd);
4631 }
4632 }
4633
4634 k = 0;
4635 for (auto && cvp : gcoll.cp_ver_arr) {
4636 if (cvp.state == cp_ver_pair_t::my_state::empty)
4637 break;
4638 ++k;
4639 if (cvp.sum_of_resids.load())
4640 pr2serr(">> slice: %d, Non-zero sum of residual counts=%d\n",
4641 k, cvp.sum_of_resids.load());
4642 }
4643 if (clp->verbose && (num_start_eagain > 0))
4644 pr2serr("Number of start EAGAINs: %d\n", num_start_eagain.load());
4645 if (clp->verbose && (num_fin_eagain > 0))
4646 pr2serr("Number of finish EAGAINs: %d\n", num_fin_eagain.load());
4647 if (clp->verbose && (num_ebusy > 0))
4648 pr2serr("Number of EBUSYs: %d\n", num_ebusy.load());
4649 if (clp->verbose && (num_miscompare > 0))
4650 pr2serr("Number of miscompare%s: %d\n",
4651 (num_miscompare > 1) ? "s" : "", num_miscompare.load());
4652 if (clp->verify && (SG_LIB_CAT_MISCOMPARE == res))
4653 pr2serr("Verify/compare failed due to miscompare\n");
4654 if (0 == res)
4655 res = clp->reason_res.load();
4656 sigprocmask(SIG_SETMASK, &orig_signal_set, NULL);
4657 if (clp->verbose) {
4658 int num_sigusr2 = num_fallthru_sigusr2.load();
4659 if (num_sigusr2 > 0)
4660 pr2serr("Number of fall-through SIGUSR2 signals caught: %d\n",
4661 num_sigusr2);
4662 }
4663 return (res >= 0) ? res : SG_LIB_CAT_OTHER;
4664 }
4665