1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*******************************************************************************
3 * This file contains main functions related to iSCSI DataSequenceInOrder=No
4 * and DataPDUInOrder=No.
6 * (c) Copyright 2007-2013 Datera, Inc.
8 * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
10 ******************************************************************************/
12 #include <linux/slab.h>
13 #include <linux/random.h>
15 #include <target/iscsi/iscsi_target_core.h>
16 #include "iscsi_target_util.h"
17 #include "iscsi_target_tpg.h"
18 #include "iscsi_target_seq_pdu_list.h"
21 static void iscsit_dump_seq_list(struct iscsi_cmd *cmd)
24 struct iscsi_seq *seq;
26 pr_debug("Dumping Sequence List for ITT: 0x%08x:\n",
29 for (i = 0; i < cmd->seq_count; i++) {
30 seq = &cmd->seq_list[i];
31 pr_debug("i: %d, pdu_start: %d, pdu_count: %d,"
32 " offset: %d, xfer_len: %d, seq_send_order: %d,"
33 " seq_no: %d\n", i, seq->pdu_start, seq->pdu_count,
34 seq->offset, seq->xfer_len, seq->seq_send_order,
39 static void iscsit_dump_pdu_list(struct iscsi_cmd *cmd)
42 struct iscsi_pdu *pdu;
44 pr_debug("Dumping PDU List for ITT: 0x%08x:\n",
47 for (i = 0; i < cmd->pdu_count; i++) {
48 pdu = &cmd->pdu_list[i];
49 pr_debug("i: %d, offset: %d, length: %d,"
50 " pdu_send_order: %d, seq_no: %d\n", i, pdu->offset,
51 pdu->length, pdu->pdu_send_order, pdu->seq_no);
55 static void iscsit_dump_seq_list(struct iscsi_cmd *cmd) {}
56 static void iscsit_dump_pdu_list(struct iscsi_cmd *cmd) {}
59 static void iscsit_ordered_seq_lists(
60 struct iscsi_cmd *cmd,
65 for (i = 0; i < cmd->seq_count; i++) {
66 if (cmd->seq_list[i].type != SEQTYPE_NORMAL)
68 cmd->seq_list[i].seq_send_order = seq_count++;
72 static void iscsit_ordered_pdu_lists(
73 struct iscsi_cmd *cmd,
76 u32 i, pdu_send_order = 0, seq_no = 0;
78 for (i = 0; i < cmd->pdu_count; i++) {
80 if (cmd->pdu_list[i].seq_no == seq_no) {
81 cmd->pdu_list[i].pdu_send_order = pdu_send_order++;
91 * Generate count random values into array.
92 * Use 0x80000000 to mark generates valued in array[].
94 static void iscsit_create_random_array(u32 *array, u32 count)
103 for (i = 0; i < count; i++) {
105 get_random_bytes(&j, sizeof(u32));
106 j = (1 + (int) (9999 + 1) - j) % count;
107 for (k = 0; k < i + 1; k++) {
109 if ((array[k] & 0x80000000) && (array[k] == j))
115 for (i = 0; i < count; i++)
116 array[i] &= ~0x80000000;
119 static int iscsit_randomize_pdu_lists(
120 struct iscsi_cmd *cmd,
124 u32 *array, pdu_count, seq_count = 0, seq_no = 0, seq_offset = 0;
126 for (pdu_count = 0; pdu_count < cmd->pdu_count; pdu_count++) {
128 if (cmd->pdu_list[pdu_count].seq_no == seq_no) {
132 array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);
134 pr_err("Unable to allocate memory"
135 " for random array.\n");
138 iscsit_create_random_array(array, seq_count);
140 for (i = 0; i < seq_count; i++)
141 cmd->pdu_list[seq_offset+i].pdu_send_order = array[i];
145 seq_offset += seq_count;
152 array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);
154 pr_err("Unable to allocate memory for"
158 iscsit_create_random_array(array, seq_count);
160 for (i = 0; i < seq_count; i++)
161 cmd->pdu_list[seq_offset+i].pdu_send_order = array[i];
169 static int iscsit_randomize_seq_lists(
170 struct iscsi_cmd *cmd,
174 u32 *array, seq_count = cmd->seq_count;
176 if ((type == PDULIST_IMMEDIATE) || (type == PDULIST_UNSOLICITED))
178 else if (type == PDULIST_IMMEDIATE_AND_UNSOLICITED)
184 array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);
186 pr_err("Unable to allocate memory for random array.\n");
189 iscsit_create_random_array(array, seq_count);
191 for (i = 0; i < cmd->seq_count; i++) {
192 if (cmd->seq_list[i].type != SEQTYPE_NORMAL)
194 cmd->seq_list[i].seq_send_order = array[j++];
201 static void iscsit_determine_counts_for_list(
202 struct iscsi_cmd *cmd,
203 struct iscsi_build_list *bl,
207 int check_immediate = 0;
208 u32 burstlength = 0, offset = 0;
209 u32 unsolicited_data_length = 0;
211 struct iscsi_conn *conn = cmd->conn;
213 if (cmd->se_cmd.data_direction == DMA_TO_DEVICE)
214 mdsl = cmd->conn->conn_ops->MaxXmitDataSegmentLength;
216 mdsl = cmd->conn->conn_ops->MaxRecvDataSegmentLength;
218 if ((bl->type == PDULIST_IMMEDIATE) ||
219 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
222 if ((bl->type == PDULIST_UNSOLICITED) ||
223 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
224 unsolicited_data_length = min(cmd->se_cmd.data_length,
225 conn->sess->sess_ops->FirstBurstLength);
227 while (offset < cmd->se_cmd.data_length) {
230 if (check_immediate) {
232 offset += bl->immediate_data_length;
234 if (unsolicited_data_length)
235 unsolicited_data_length -=
236 bl->immediate_data_length;
239 if (unsolicited_data_length > 0) {
240 if ((offset + mdsl) >= cmd->se_cmd.data_length) {
241 unsolicited_data_length -=
242 (cmd->se_cmd.data_length - offset);
243 offset += (cmd->se_cmd.data_length - offset);
247 >= conn->sess->sess_ops->FirstBurstLength) {
248 unsolicited_data_length -=
249 (conn->sess->sess_ops->FirstBurstLength -
251 offset += (conn->sess->sess_ops->FirstBurstLength -
259 unsolicited_data_length -= mdsl;
262 if ((offset + mdsl) >= cmd->se_cmd.data_length) {
263 offset += (cmd->se_cmd.data_length - offset);
266 if ((burstlength + mdsl) >=
267 conn->sess->sess_ops->MaxBurstLength) {
268 offset += (conn->sess->sess_ops->MaxBurstLength -
282 * Builds PDU and/or Sequence list, called while DataSequenceInOrder=No
283 * or DataPDUInOrder=No.
285 static int iscsit_do_build_pdu_and_seq_lists(
286 struct iscsi_cmd *cmd,
287 struct iscsi_build_list *bl)
289 int check_immediate = 0, datapduinorder, datasequenceinorder;
290 u32 burstlength = 0, offset = 0, i = 0, mdsl;
291 u32 pdu_count = 0, seq_no = 0, unsolicited_data_length = 0;
292 struct iscsi_conn *conn = cmd->conn;
293 struct iscsi_pdu *pdu = cmd->pdu_list;
294 struct iscsi_seq *seq = cmd->seq_list;
296 if (cmd->se_cmd.data_direction == DMA_TO_DEVICE)
297 mdsl = cmd->conn->conn_ops->MaxXmitDataSegmentLength;
299 mdsl = cmd->conn->conn_ops->MaxRecvDataSegmentLength;
301 datapduinorder = conn->sess->sess_ops->DataPDUInOrder;
302 datasequenceinorder = conn->sess->sess_ops->DataSequenceInOrder;
304 if ((bl->type == PDULIST_IMMEDIATE) ||
305 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
308 if ((bl->type == PDULIST_UNSOLICITED) ||
309 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
310 unsolicited_data_length = min(cmd->se_cmd.data_length,
311 conn->sess->sess_ops->FirstBurstLength);
313 while (offset < cmd->se_cmd.data_length) {
315 if (!datapduinorder) {
316 pdu[i].offset = offset;
317 pdu[i].seq_no = seq_no;
319 if (!datasequenceinorder && (pdu_count == 1)) {
320 seq[seq_no].pdu_start = i;
321 seq[seq_no].seq_no = seq_no;
322 seq[seq_no].offset = offset;
323 seq[seq_no].orig_offset = offset;
326 if (check_immediate) {
328 if (!datapduinorder) {
329 pdu[i].type = PDUTYPE_IMMEDIATE;
330 pdu[i++].length = bl->immediate_data_length;
332 if (!datasequenceinorder) {
333 seq[seq_no].type = SEQTYPE_IMMEDIATE;
334 seq[seq_no].pdu_count = 1;
335 seq[seq_no].xfer_len =
336 bl->immediate_data_length;
338 offset += bl->immediate_data_length;
341 if (unsolicited_data_length)
342 unsolicited_data_length -=
343 bl->immediate_data_length;
346 if (unsolicited_data_length > 0) {
347 if ((offset + mdsl) >= cmd->se_cmd.data_length) {
348 if (!datapduinorder) {
349 pdu[i].type = PDUTYPE_UNSOLICITED;
351 (cmd->se_cmd.data_length - offset);
353 if (!datasequenceinorder) {
354 seq[seq_no].type = SEQTYPE_UNSOLICITED;
355 seq[seq_no].pdu_count = pdu_count;
356 seq[seq_no].xfer_len = (burstlength +
357 (cmd->se_cmd.data_length - offset));
359 unsolicited_data_length -=
360 (cmd->se_cmd.data_length - offset);
361 offset += (cmd->se_cmd.data_length - offset);
364 if ((offset + mdsl) >=
365 conn->sess->sess_ops->FirstBurstLength) {
366 if (!datapduinorder) {
367 pdu[i].type = PDUTYPE_UNSOLICITED;
369 (conn->sess->sess_ops->FirstBurstLength -
372 if (!datasequenceinorder) {
373 seq[seq_no].type = SEQTYPE_UNSOLICITED;
374 seq[seq_no].pdu_count = pdu_count;
375 seq[seq_no].xfer_len = (burstlength +
376 (conn->sess->sess_ops->FirstBurstLength -
379 unsolicited_data_length -=
380 (conn->sess->sess_ops->FirstBurstLength -
382 offset += (conn->sess->sess_ops->FirstBurstLength -
390 if (!datapduinorder) {
391 pdu[i].type = PDUTYPE_UNSOLICITED;
392 pdu[i++].length = mdsl;
396 unsolicited_data_length -= mdsl;
399 if ((offset + mdsl) >= cmd->se_cmd.data_length) {
400 if (!datapduinorder) {
401 pdu[i].type = PDUTYPE_NORMAL;
402 pdu[i].length = (cmd->se_cmd.data_length - offset);
404 if (!datasequenceinorder) {
405 seq[seq_no].type = SEQTYPE_NORMAL;
406 seq[seq_no].pdu_count = pdu_count;
407 seq[seq_no].xfer_len = (burstlength +
408 (cmd->se_cmd.data_length - offset));
410 offset += (cmd->se_cmd.data_length - offset);
413 if ((burstlength + mdsl) >=
414 conn->sess->sess_ops->MaxBurstLength) {
415 if (!datapduinorder) {
416 pdu[i].type = PDUTYPE_NORMAL;
418 (conn->sess->sess_ops->MaxBurstLength -
421 if (!datasequenceinorder) {
422 seq[seq_no].type = SEQTYPE_NORMAL;
423 seq[seq_no].pdu_count = pdu_count;
424 seq[seq_no].xfer_len = (burstlength +
425 (conn->sess->sess_ops->MaxBurstLength -
428 offset += (conn->sess->sess_ops->MaxBurstLength -
436 if (!datapduinorder) {
437 pdu[i].type = PDUTYPE_NORMAL;
438 pdu[i++].length = mdsl;
444 if (!datasequenceinorder) {
445 if (bl->data_direction & ISCSI_PDU_WRITE) {
446 if (bl->randomize & RANDOM_R2T_OFFSETS) {
447 if (iscsit_randomize_seq_lists(cmd, bl->type)
451 iscsit_ordered_seq_lists(cmd, bl->type);
452 } else if (bl->data_direction & ISCSI_PDU_READ) {
453 if (bl->randomize & RANDOM_DATAIN_SEQ_OFFSETS) {
454 if (iscsit_randomize_seq_lists(cmd, bl->type)
458 iscsit_ordered_seq_lists(cmd, bl->type);
461 iscsit_dump_seq_list(cmd);
463 if (!datapduinorder) {
464 if (bl->data_direction & ISCSI_PDU_WRITE) {
465 if (bl->randomize & RANDOM_DATAOUT_PDU_OFFSETS) {
466 if (iscsit_randomize_pdu_lists(cmd, bl->type)
470 iscsit_ordered_pdu_lists(cmd, bl->type);
471 } else if (bl->data_direction & ISCSI_PDU_READ) {
472 if (bl->randomize & RANDOM_DATAIN_PDU_OFFSETS) {
473 if (iscsit_randomize_pdu_lists(cmd, bl->type)
477 iscsit_ordered_pdu_lists(cmd, bl->type);
480 iscsit_dump_pdu_list(cmd);
486 int iscsit_build_pdu_and_seq_lists(
487 struct iscsi_cmd *cmd,
488 u32 immediate_data_length)
490 struct iscsi_build_list bl;
491 u32 pdu_count = 0, seq_count = 1;
492 struct iscsi_conn *conn = cmd->conn;
493 struct iscsi_pdu *pdu = NULL;
494 struct iscsi_seq *seq = NULL;
496 struct iscsi_session *sess = conn->sess;
497 struct iscsi_node_attrib *na;
500 * Do nothing if no OOO shenanigans
502 if (sess->sess_ops->DataSequenceInOrder &&
503 sess->sess_ops->DataPDUInOrder)
506 if (cmd->data_direction == DMA_NONE)
509 na = iscsit_tpg_get_node_attrib(sess);
510 memset(&bl, 0, sizeof(struct iscsi_build_list));
512 if (cmd->data_direction == DMA_FROM_DEVICE) {
513 bl.data_direction = ISCSI_PDU_READ;
514 bl.type = PDULIST_NORMAL;
515 if (na->random_datain_pdu_offsets)
516 bl.randomize |= RANDOM_DATAIN_PDU_OFFSETS;
517 if (na->random_datain_seq_offsets)
518 bl.randomize |= RANDOM_DATAIN_SEQ_OFFSETS;
520 bl.data_direction = ISCSI_PDU_WRITE;
521 bl.immediate_data_length = immediate_data_length;
522 if (na->random_r2t_offsets)
523 bl.randomize |= RANDOM_R2T_OFFSETS;
525 if (!cmd->immediate_data && !cmd->unsolicited_data)
526 bl.type = PDULIST_NORMAL;
527 else if (cmd->immediate_data && !cmd->unsolicited_data)
528 bl.type = PDULIST_IMMEDIATE;
529 else if (!cmd->immediate_data && cmd->unsolicited_data)
530 bl.type = PDULIST_UNSOLICITED;
531 else if (cmd->immediate_data && cmd->unsolicited_data)
532 bl.type = PDULIST_IMMEDIATE_AND_UNSOLICITED;
535 iscsit_determine_counts_for_list(cmd, &bl, &seq_count, &pdu_count);
537 if (!conn->sess->sess_ops->DataSequenceInOrder) {
538 seq = kcalloc(seq_count, sizeof(struct iscsi_seq), GFP_ATOMIC);
540 pr_err("Unable to allocate struct iscsi_seq list\n");
544 cmd->seq_count = seq_count;
547 if (!conn->sess->sess_ops->DataPDUInOrder) {
548 pdu = kcalloc(pdu_count, sizeof(struct iscsi_pdu), GFP_ATOMIC);
550 pr_err("Unable to allocate struct iscsi_pdu list.\n");
555 cmd->pdu_count = pdu_count;
558 return iscsit_do_build_pdu_and_seq_lists(cmd, &bl);
561 struct iscsi_pdu *iscsit_get_pdu_holder(
562 struct iscsi_cmd *cmd,
567 struct iscsi_pdu *pdu = NULL;
569 if (!cmd->pdu_list) {
570 pr_err("struct iscsi_cmd->pdu_list is NULL!\n");
574 pdu = &cmd->pdu_list[0];
576 for (i = 0; i < cmd->pdu_count; i++)
577 if ((pdu[i].offset == offset) && (pdu[i].length == length))
580 pr_err("Unable to locate PDU holder for ITT: 0x%08x, Offset:"
581 " %u, Length: %u\n", cmd->init_task_tag, offset, length);
585 struct iscsi_pdu *iscsit_get_pdu_holder_for_seq(
586 struct iscsi_cmd *cmd,
587 struct iscsi_seq *seq)
590 struct iscsi_conn *conn = cmd->conn;
591 struct iscsi_pdu *pdu = NULL;
593 if (!cmd->pdu_list) {
594 pr_err("struct iscsi_cmd->pdu_list is NULL!\n");
598 if (conn->sess->sess_ops->DataSequenceInOrder) {
600 pdu = &cmd->pdu_list[cmd->pdu_start];
602 for (i = 0; pdu[i].seq_no != cmd->seq_no; i++) {
603 pr_debug("pdu[i].seq_no: %d, pdu[i].pdu"
604 "_send_order: %d, pdu[i].offset: %d,"
605 " pdu[i].length: %d\n", pdu[i].seq_no,
606 pdu[i].pdu_send_order, pdu[i].offset,
609 if (pdu[i].pdu_send_order == cmd->pdu_send_order) {
610 cmd->pdu_send_order++;
615 cmd->pdu_start += cmd->pdu_send_order;
616 cmd->pdu_send_order = 0;
619 if (cmd->pdu_start < cmd->pdu_count)
622 pr_err("Command ITT: 0x%08x unable to locate"
623 " struct iscsi_pdu for cmd->pdu_send_order: %u.\n",
624 cmd->init_task_tag, cmd->pdu_send_order);
628 pr_err("struct iscsi_seq is NULL!\n");
632 pr_debug("seq->pdu_start: %d, seq->pdu_count: %d,"
633 " seq->seq_no: %d\n", seq->pdu_start, seq->pdu_count,
636 pdu = &cmd->pdu_list[seq->pdu_start];
638 if (seq->pdu_send_order == seq->pdu_count) {
639 pr_err("Command ITT: 0x%08x seq->pdu_send"
640 "_order: %u equals seq->pdu_count: %u\n",
641 cmd->init_task_tag, seq->pdu_send_order,
646 for (i = 0; i < seq->pdu_count; i++) {
647 if (pdu[i].pdu_send_order == seq->pdu_send_order) {
648 seq->pdu_send_order++;
653 pr_err("Command ITT: 0x%08x unable to locate iscsi"
654 "_pdu_t for seq->pdu_send_order: %u.\n",
655 cmd->init_task_tag, seq->pdu_send_order);
662 struct iscsi_seq *iscsit_get_seq_holder(
663 struct iscsi_cmd *cmd,
669 if (!cmd->seq_list) {
670 pr_err("struct iscsi_cmd->seq_list is NULL!\n");
674 for (i = 0; i < cmd->seq_count; i++) {
675 pr_debug("seq_list[i].orig_offset: %d, seq_list[i]."
676 "xfer_len: %d, seq_list[i].seq_no %u\n",
677 cmd->seq_list[i].orig_offset, cmd->seq_list[i].xfer_len,
678 cmd->seq_list[i].seq_no);
680 if ((cmd->seq_list[i].orig_offset +
681 cmd->seq_list[i].xfer_len) >=
683 return &cmd->seq_list[i];
686 pr_err("Unable to locate Sequence holder for ITT: 0x%08x,"
687 " Offset: %u, Length: %u\n", cmd->init_task_tag, offset,