1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /******************************************************************************* 3 * This file contains the iSCSI Target DataIN value generation functions. 4 * 5 * (c) Copyright 2007-2013 Datera, Inc. 6 * 7 * Author: Nicholas A. Bellinger <nab@linux-iscsi.org> 8 * 9 ******************************************************************************/ 10 11 #include <linux/slab.h> 12 #include <scsi/iscsi_proto.h> 13 #include <target/iscsi/iscsi_target_core.h> 14 #include "iscsi_target_seq_pdu_list.h" 15 #include "iscsi_target_erl1.h" 16 #include "iscsi_target_util.h" 17 #include "iscsi_target.h" 18 #include "iscsi_target_datain_values.h" 19 20 struct iscsi_datain_req *iscsit_allocate_datain_req(void) 21 { 22 struct iscsi_datain_req *dr; 23 24 dr = kmem_cache_zalloc(lio_dr_cache, GFP_ATOMIC); 25 if (!dr) { 26 pr_err("Unable to allocate memory for" 27 " struct iscsi_datain_req\n"); 28 return NULL; 29 } 30 INIT_LIST_HEAD(&dr->cmd_datain_node); 31 32 return dr; 33 } 34 35 void iscsit_attach_datain_req(struct iscsi_cmd *cmd, struct iscsi_datain_req *dr) 36 { 37 spin_lock(&cmd->datain_lock); 38 list_add_tail(&dr->cmd_datain_node, &cmd->datain_list); 39 spin_unlock(&cmd->datain_lock); 40 } 41 42 void iscsit_free_datain_req(struct iscsi_cmd *cmd, struct iscsi_datain_req *dr) 43 { 44 spin_lock(&cmd->datain_lock); 45 list_del(&dr->cmd_datain_node); 46 spin_unlock(&cmd->datain_lock); 47 48 kmem_cache_free(lio_dr_cache, dr); 49 } 50 51 void iscsit_free_all_datain_reqs(struct iscsi_cmd *cmd) 52 { 53 struct iscsi_datain_req *dr, *dr_tmp; 54 55 spin_lock(&cmd->datain_lock); 56 list_for_each_entry_safe(dr, dr_tmp, &cmd->datain_list, cmd_datain_node) { 57 list_del(&dr->cmd_datain_node); 58 kmem_cache_free(lio_dr_cache, dr); 59 } 60 spin_unlock(&cmd->datain_lock); 61 } 62 63 struct iscsi_datain_req *iscsit_get_datain_req(struct iscsi_cmd *cmd) 64 { 65 if (list_empty(&cmd->datain_list)) { 66 pr_err("cmd->datain_list is empty for ITT:" 67 " 0x%08x\n", cmd->init_task_tag); 68 return NULL; 69 } 70 71 return list_first_entry(&cmd->datain_list, struct iscsi_datain_req, 72 cmd_datain_node); 73 } 74 75 /* 76 * For Normal and Recovery DataSequenceInOrder=Yes and DataPDUInOrder=Yes. 77 */ 78 static struct iscsi_datain_req *iscsit_set_datain_values_yes_and_yes( 79 struct iscsi_cmd *cmd, 80 struct iscsi_datain *datain) 81 { 82 u32 next_burst_len, read_data_done, read_data_left; 83 struct iscsi_conn *conn = cmd->conn; 84 struct iscsi_datain_req *dr; 85 86 dr = iscsit_get_datain_req(cmd); 87 if (!dr) 88 return NULL; 89 90 if (dr->recovery && dr->generate_recovery_values) { 91 if (iscsit_create_recovery_datain_values_datasequenceinorder_yes( 92 cmd, dr) < 0) 93 return NULL; 94 95 dr->generate_recovery_values = 0; 96 } 97 98 next_burst_len = (!dr->recovery) ? 99 cmd->next_burst_len : dr->next_burst_len; 100 read_data_done = (!dr->recovery) ? 101 cmd->read_data_done : dr->read_data_done; 102 103 read_data_left = (cmd->se_cmd.data_length - read_data_done); 104 if (!read_data_left) { 105 pr_err("ITT: 0x%08x read_data_left is zero!\n", 106 cmd->init_task_tag); 107 return NULL; 108 } 109 110 if ((read_data_left <= conn->conn_ops->MaxRecvDataSegmentLength) && 111 (read_data_left <= (conn->sess->sess_ops->MaxBurstLength - 112 next_burst_len))) { 113 datain->length = read_data_left; 114 115 datain->flags |= (ISCSI_FLAG_CMD_FINAL | ISCSI_FLAG_DATA_STATUS); 116 if (conn->sess->sess_ops->ErrorRecoveryLevel > 0) 117 datain->flags |= ISCSI_FLAG_DATA_ACK; 118 } else { 119 if ((next_burst_len + 120 conn->conn_ops->MaxRecvDataSegmentLength) < 121 conn->sess->sess_ops->MaxBurstLength) { 122 datain->length = 123 conn->conn_ops->MaxRecvDataSegmentLength; 124 next_burst_len += datain->length; 125 } else { 126 datain->length = (conn->sess->sess_ops->MaxBurstLength - 127 next_burst_len); 128 next_burst_len = 0; 129 130 datain->flags |= ISCSI_FLAG_CMD_FINAL; 131 if (conn->sess->sess_ops->ErrorRecoveryLevel > 0) 132 datain->flags |= ISCSI_FLAG_DATA_ACK; 133 } 134 } 135 136 datain->data_sn = (!dr->recovery) ? cmd->data_sn++ : dr->data_sn++; 137 datain->offset = read_data_done; 138 139 if (!dr->recovery) { 140 cmd->next_burst_len = next_burst_len; 141 cmd->read_data_done += datain->length; 142 } else { 143 dr->next_burst_len = next_burst_len; 144 dr->read_data_done += datain->length; 145 } 146 147 if (!dr->recovery) { 148 if (datain->flags & ISCSI_FLAG_DATA_STATUS) 149 dr->dr_complete = DATAIN_COMPLETE_NORMAL; 150 151 return dr; 152 } 153 154 if (!dr->runlength) { 155 if (datain->flags & ISCSI_FLAG_DATA_STATUS) { 156 dr->dr_complete = 157 (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ? 158 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY : 159 DATAIN_COMPLETE_CONNECTION_RECOVERY; 160 } 161 } else { 162 if ((dr->begrun + dr->runlength) == dr->data_sn) { 163 dr->dr_complete = 164 (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ? 165 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY : 166 DATAIN_COMPLETE_CONNECTION_RECOVERY; 167 } 168 } 169 170 return dr; 171 } 172 173 /* 174 * For Normal and Recovery DataSequenceInOrder=No and DataPDUInOrder=Yes. 175 */ 176 static struct iscsi_datain_req *iscsit_set_datain_values_no_and_yes( 177 struct iscsi_cmd *cmd, 178 struct iscsi_datain *datain) 179 { 180 u32 offset, read_data_done, read_data_left, seq_send_order; 181 struct iscsi_conn *conn = cmd->conn; 182 struct iscsi_datain_req *dr; 183 struct iscsi_seq *seq; 184 185 dr = iscsit_get_datain_req(cmd); 186 if (!dr) 187 return NULL; 188 189 if (dr->recovery && dr->generate_recovery_values) { 190 if (iscsit_create_recovery_datain_values_datasequenceinorder_no( 191 cmd, dr) < 0) 192 return NULL; 193 194 dr->generate_recovery_values = 0; 195 } 196 197 read_data_done = (!dr->recovery) ? 198 cmd->read_data_done : dr->read_data_done; 199 seq_send_order = (!dr->recovery) ? 200 cmd->seq_send_order : dr->seq_send_order; 201 202 read_data_left = (cmd->se_cmd.data_length - read_data_done); 203 if (!read_data_left) { 204 pr_err("ITT: 0x%08x read_data_left is zero!\n", 205 cmd->init_task_tag); 206 return NULL; 207 } 208 209 seq = iscsit_get_seq_holder_for_datain(cmd, seq_send_order); 210 if (!seq) 211 return NULL; 212 213 seq->sent = 1; 214 215 if (!dr->recovery && !seq->next_burst_len) 216 seq->first_datasn = cmd->data_sn; 217 218 offset = (seq->offset + seq->next_burst_len); 219 220 if ((offset + conn->conn_ops->MaxRecvDataSegmentLength) >= 221 cmd->se_cmd.data_length) { 222 datain->length = (cmd->se_cmd.data_length - offset); 223 datain->offset = offset; 224 225 datain->flags |= ISCSI_FLAG_CMD_FINAL; 226 if (conn->sess->sess_ops->ErrorRecoveryLevel > 0) 227 datain->flags |= ISCSI_FLAG_DATA_ACK; 228 229 seq->next_burst_len = 0; 230 seq_send_order++; 231 } else { 232 if ((seq->next_burst_len + 233 conn->conn_ops->MaxRecvDataSegmentLength) < 234 conn->sess->sess_ops->MaxBurstLength) { 235 datain->length = 236 conn->conn_ops->MaxRecvDataSegmentLength; 237 datain->offset = (seq->offset + seq->next_burst_len); 238 239 seq->next_burst_len += datain->length; 240 } else { 241 datain->length = (conn->sess->sess_ops->MaxBurstLength - 242 seq->next_burst_len); 243 datain->offset = (seq->offset + seq->next_burst_len); 244 245 datain->flags |= ISCSI_FLAG_CMD_FINAL; 246 if (conn->sess->sess_ops->ErrorRecoveryLevel > 0) 247 datain->flags |= ISCSI_FLAG_DATA_ACK; 248 249 seq->next_burst_len = 0; 250 seq_send_order++; 251 } 252 } 253 254 if ((read_data_done + datain->length) == cmd->se_cmd.data_length) 255 datain->flags |= ISCSI_FLAG_DATA_STATUS; 256 257 datain->data_sn = (!dr->recovery) ? cmd->data_sn++ : dr->data_sn++; 258 if (!dr->recovery) { 259 cmd->seq_send_order = seq_send_order; 260 cmd->read_data_done += datain->length; 261 } else { 262 dr->seq_send_order = seq_send_order; 263 dr->read_data_done += datain->length; 264 } 265 266 if (!dr->recovery) { 267 if (datain->flags & ISCSI_FLAG_CMD_FINAL) 268 seq->last_datasn = datain->data_sn; 269 if (datain->flags & ISCSI_FLAG_DATA_STATUS) 270 dr->dr_complete = DATAIN_COMPLETE_NORMAL; 271 272 return dr; 273 } 274 275 if (!dr->runlength) { 276 if (datain->flags & ISCSI_FLAG_DATA_STATUS) { 277 dr->dr_complete = 278 (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ? 279 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY : 280 DATAIN_COMPLETE_CONNECTION_RECOVERY; 281 } 282 } else { 283 if ((dr->begrun + dr->runlength) == dr->data_sn) { 284 dr->dr_complete = 285 (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ? 286 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY : 287 DATAIN_COMPLETE_CONNECTION_RECOVERY; 288 } 289 } 290 291 return dr; 292 } 293 294 /* 295 * For Normal and Recovery DataSequenceInOrder=Yes and DataPDUInOrder=No. 296 */ 297 static struct iscsi_datain_req *iscsit_set_datain_values_yes_and_no( 298 struct iscsi_cmd *cmd, 299 struct iscsi_datain *datain) 300 { 301 u32 next_burst_len, read_data_done, read_data_left; 302 struct iscsi_conn *conn = cmd->conn; 303 struct iscsi_datain_req *dr; 304 struct iscsi_pdu *pdu; 305 306 dr = iscsit_get_datain_req(cmd); 307 if (!dr) 308 return NULL; 309 310 if (dr->recovery && dr->generate_recovery_values) { 311 if (iscsit_create_recovery_datain_values_datasequenceinorder_yes( 312 cmd, dr) < 0) 313 return NULL; 314 315 dr->generate_recovery_values = 0; 316 } 317 318 next_burst_len = (!dr->recovery) ? 319 cmd->next_burst_len : dr->next_burst_len; 320 read_data_done = (!dr->recovery) ? 321 cmd->read_data_done : dr->read_data_done; 322 323 read_data_left = (cmd->se_cmd.data_length - read_data_done); 324 if (!read_data_left) { 325 pr_err("ITT: 0x%08x read_data_left is zero!\n", 326 cmd->init_task_tag); 327 return dr; 328 } 329 330 pdu = iscsit_get_pdu_holder_for_seq(cmd, NULL); 331 if (!pdu) 332 return dr; 333 334 if ((read_data_done + pdu->length) == cmd->se_cmd.data_length) { 335 pdu->flags |= (ISCSI_FLAG_CMD_FINAL | ISCSI_FLAG_DATA_STATUS); 336 if (conn->sess->sess_ops->ErrorRecoveryLevel > 0) 337 pdu->flags |= ISCSI_FLAG_DATA_ACK; 338 339 next_burst_len = 0; 340 } else { 341 if ((next_burst_len + conn->conn_ops->MaxRecvDataSegmentLength) < 342 conn->sess->sess_ops->MaxBurstLength) 343 next_burst_len += pdu->length; 344 else { 345 pdu->flags |= ISCSI_FLAG_CMD_FINAL; 346 if (conn->sess->sess_ops->ErrorRecoveryLevel > 0) 347 pdu->flags |= ISCSI_FLAG_DATA_ACK; 348 349 next_burst_len = 0; 350 } 351 } 352 353 pdu->data_sn = (!dr->recovery) ? cmd->data_sn++ : dr->data_sn++; 354 if (!dr->recovery) { 355 cmd->next_burst_len = next_burst_len; 356 cmd->read_data_done += pdu->length; 357 } else { 358 dr->next_burst_len = next_burst_len; 359 dr->read_data_done += pdu->length; 360 } 361 362 datain->flags = pdu->flags; 363 datain->length = pdu->length; 364 datain->offset = pdu->offset; 365 datain->data_sn = pdu->data_sn; 366 367 if (!dr->recovery) { 368 if (datain->flags & ISCSI_FLAG_DATA_STATUS) 369 dr->dr_complete = DATAIN_COMPLETE_NORMAL; 370 371 return dr; 372 } 373 374 if (!dr->runlength) { 375 if (datain->flags & ISCSI_FLAG_DATA_STATUS) { 376 dr->dr_complete = 377 (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ? 378 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY : 379 DATAIN_COMPLETE_CONNECTION_RECOVERY; 380 } 381 } else { 382 if ((dr->begrun + dr->runlength) == dr->data_sn) { 383 dr->dr_complete = 384 (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ? 385 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY : 386 DATAIN_COMPLETE_CONNECTION_RECOVERY; 387 } 388 } 389 390 return dr; 391 } 392 393 /* 394 * For Normal and Recovery DataSequenceInOrder=No and DataPDUInOrder=No. 395 */ 396 static struct iscsi_datain_req *iscsit_set_datain_values_no_and_no( 397 struct iscsi_cmd *cmd, 398 struct iscsi_datain *datain) 399 { 400 u32 read_data_done, read_data_left, seq_send_order; 401 struct iscsi_conn *conn = cmd->conn; 402 struct iscsi_datain_req *dr; 403 struct iscsi_pdu *pdu; 404 struct iscsi_seq *seq = NULL; 405 406 dr = iscsit_get_datain_req(cmd); 407 if (!dr) 408 return NULL; 409 410 if (dr->recovery && dr->generate_recovery_values) { 411 if (iscsit_create_recovery_datain_values_datasequenceinorder_no( 412 cmd, dr) < 0) 413 return NULL; 414 415 dr->generate_recovery_values = 0; 416 } 417 418 read_data_done = (!dr->recovery) ? 419 cmd->read_data_done : dr->read_data_done; 420 seq_send_order = (!dr->recovery) ? 421 cmd->seq_send_order : dr->seq_send_order; 422 423 read_data_left = (cmd->se_cmd.data_length - read_data_done); 424 if (!read_data_left) { 425 pr_err("ITT: 0x%08x read_data_left is zero!\n", 426 cmd->init_task_tag); 427 return NULL; 428 } 429 430 seq = iscsit_get_seq_holder_for_datain(cmd, seq_send_order); 431 if (!seq) 432 return NULL; 433 434 seq->sent = 1; 435 436 if (!dr->recovery && !seq->next_burst_len) 437 seq->first_datasn = cmd->data_sn; 438 439 pdu = iscsit_get_pdu_holder_for_seq(cmd, seq); 440 if (!pdu) 441 return NULL; 442 443 if (seq->pdu_send_order == seq->pdu_count) { 444 pdu->flags |= ISCSI_FLAG_CMD_FINAL; 445 if (conn->sess->sess_ops->ErrorRecoveryLevel > 0) 446 pdu->flags |= ISCSI_FLAG_DATA_ACK; 447 448 seq->next_burst_len = 0; 449 seq_send_order++; 450 } else 451 seq->next_burst_len += pdu->length; 452 453 if ((read_data_done + pdu->length) == cmd->se_cmd.data_length) 454 pdu->flags |= ISCSI_FLAG_DATA_STATUS; 455 456 pdu->data_sn = (!dr->recovery) ? cmd->data_sn++ : dr->data_sn++; 457 if (!dr->recovery) { 458 cmd->seq_send_order = seq_send_order; 459 cmd->read_data_done += pdu->length; 460 } else { 461 dr->seq_send_order = seq_send_order; 462 dr->read_data_done += pdu->length; 463 } 464 465 datain->flags = pdu->flags; 466 datain->length = pdu->length; 467 datain->offset = pdu->offset; 468 datain->data_sn = pdu->data_sn; 469 470 if (!dr->recovery) { 471 if (datain->flags & ISCSI_FLAG_CMD_FINAL) 472 seq->last_datasn = datain->data_sn; 473 if (datain->flags & ISCSI_FLAG_DATA_STATUS) 474 dr->dr_complete = DATAIN_COMPLETE_NORMAL; 475 476 return dr; 477 } 478 479 if (!dr->runlength) { 480 if (datain->flags & ISCSI_FLAG_DATA_STATUS) { 481 dr->dr_complete = 482 (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ? 483 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY : 484 DATAIN_COMPLETE_CONNECTION_RECOVERY; 485 } 486 } else { 487 if ((dr->begrun + dr->runlength) == dr->data_sn) { 488 dr->dr_complete = 489 (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ? 490 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY : 491 DATAIN_COMPLETE_CONNECTION_RECOVERY; 492 } 493 } 494 495 return dr; 496 } 497 498 struct iscsi_datain_req *iscsit_get_datain_values( 499 struct iscsi_cmd *cmd, 500 struct iscsi_datain *datain) 501 { 502 struct iscsi_conn *conn = cmd->conn; 503 504 if (conn->sess->sess_ops->DataSequenceInOrder && 505 conn->sess->sess_ops->DataPDUInOrder) 506 return iscsit_set_datain_values_yes_and_yes(cmd, datain); 507 else if (!conn->sess->sess_ops->DataSequenceInOrder && 508 conn->sess->sess_ops->DataPDUInOrder) 509 return iscsit_set_datain_values_no_and_yes(cmd, datain); 510 else if (conn->sess->sess_ops->DataSequenceInOrder && 511 !conn->sess->sess_ops->DataPDUInOrder) 512 return iscsit_set_datain_values_yes_and_no(cmd, datain); 513 else if (!conn->sess->sess_ops->DataSequenceInOrder && 514 !conn->sess->sess_ops->DataPDUInOrder) 515 return iscsit_set_datain_values_no_and_no(cmd, datain); 516 517 return NULL; 518 } 519 EXPORT_SYMBOL(iscsit_get_datain_values); 520