From 10868c491d1d16fe2879fdba30a95d855fa936a6 Mon Sep 17 00:00:00 2001
From: Xie Yongji <xieyongji@bytedance.com>
Date: Tue, 2 Jun 2020 20:15:50 +0800
Subject: [PATCH] libiscsi: Avoid discontinuities in cmdsn ordering in some
 cases

We should plug the cmdsn gap in order to continue
to use the session when the pdus is cancelled before
sending out.

Signed-off-by: Xie Yongji <xieyongji@bytedance.com>
---
 lib/iscsi-command.c | 26 ++++++++++++++++++++++----
 lib/nop.c           |  3 ++-
 lib/pdu.c           | 15 +++++++++++++++
 3 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/lib/iscsi-command.c b/lib/iscsi-command.c
index 87e2336..12948c4 100644
--- a/lib/iscsi-command.c
+++ b/lib/iscsi-command.c
@@ -271,7 +271,7 @@ iscsi_scsi_command_async(struct iscsi_context *iscsi, int lun,
 	iscsi_pdu_set_expxferlen(pdu, task->expxferlen);
 
 	/* cmdsn */
-	iscsi_pdu_set_cmdsn(pdu, iscsi->cmdsn++);
+	iscsi_pdu_set_cmdsn(pdu, iscsi->cmdsn);
 
 	/* cdb */
 	iscsi_pdu_set_cdb(pdu, task);
@@ -285,6 +285,7 @@ iscsi_scsi_command_async(struct iscsi_context *iscsi, int lun,
 		iscsi->drv->free_pdu(iscsi, pdu);
 		return -1;
 	}
+	iscsi->cmdsn++;
 
 	/* The F flag is not set. This means we haven't sent all the unsolicited
 	 * data yet. Sent as much as we are allowed as a train of DATA-OUT PDUs.
@@ -2667,6 +2668,9 @@ iscsi_scsi_cancel_task(struct iscsi_context *iscsi,
 		       struct scsi_task *task)
 {
 	struct iscsi_pdu *pdu;
+	struct iscsi_pdu *next_pdu;
+	uint32_t cmdsn_gap = 0;
+	int ret = -1;
 
 	for (pdu = iscsi->waitpdu; pdu; pdu = pdu->next) {
 		if (pdu->itt == task->itt) {
@@ -2679,18 +2683,32 @@ iscsi_scsi_cancel_task(struct iscsi_context *iscsi,
 			return 0;
 		}
 	}
-	for (pdu = iscsi->outqueue; pdu; pdu = pdu->next) {
+	for (pdu = iscsi->outqueue; pdu; pdu = next_pdu) {
+		next_pdu = pdu->next;
+
+		if (cmdsn_gap > 0) {
+			iscsi_pdu_set_cmdsn(pdu, pdu->cmdsn - cmdsn_gap);
+		}
+
 		if (pdu->itt == task->itt) {
 			ISCSI_LIST_REMOVE(&iscsi->outqueue, pdu);
 			if (pdu->callback) {
 				pdu->callback(iscsi, SCSI_STATUS_CANCELLED, NULL,
 				      pdu->private_data);
 			}
+			if (!(pdu->outdata.data[0] & ISCSI_PDU_IMMEDIATE) &&
+			    (pdu->outdata.data[0] & 0x3f) != ISCSI_PDU_DATA_OUT) {
+				iscsi->cmdsn--;
+				cmdsn_gap++;
+			}
 			iscsi->drv->free_pdu(iscsi, pdu);
-			return 0;
+			ret = 0;
+			if (!cmdsn_gap) {
+				break;
+			}
 		}
 	}
-	return -1;
+	return ret;
 }
 
 void
diff --git a/lib/nop.c b/lib/nop.c
index 2137530..2c1391a 100644
--- a/lib/nop.c
+++ b/lib/nop.c
@@ -64,7 +64,7 @@ iscsi_nop_out_async(struct iscsi_context *iscsi, iscsi_command_cb cb,
 	iscsi_pdu_set_lun(pdu, 0);
 
 	/* cmdsn */
-	iscsi_pdu_set_cmdsn(pdu, iscsi->cmdsn++);
+	iscsi_pdu_set_cmdsn(pdu, iscsi->cmdsn);
 
 	pdu->callback     = cb;
 	pdu->private_data = private_data;
@@ -83,6 +83,7 @@ iscsi_nop_out_async(struct iscsi_context *iscsi, iscsi_command_cb cb,
 		return -1;
 	}
 
+	iscsi->cmdsn++;
 	iscsi->nops_in_flight++;
 	ISCSI_LOG(iscsi, (iscsi->nops_in_flight > 1) ? 1 : 6,
 	          "NOP Out Send (nops_in_flight: %d, pdu->cmdsn %08x, pdu->itt %08x, pdu->ttt %08x, iscsi->maxcmdsn %08x, iscsi->expcmdsn %08x)",
diff --git a/lib/pdu.c b/lib/pdu.c
index 8c8fc99..73d2f83 100644
--- a/lib/pdu.c
+++ b/lib/pdu.c
@@ -719,10 +719,14 @@ iscsi_timeout_scan(struct iscsi_context *iscsi)
 	struct iscsi_pdu *pdu;
 	struct iscsi_pdu *next_pdu;
 	time_t t = time(NULL);
+	uint32_t cmdsn_gap = 0;
 
 	for (pdu = iscsi->outqueue; pdu; pdu = next_pdu) {
 		next_pdu = pdu->next;
 
+                if (cmdsn_gap > 0) {
+			iscsi_pdu_set_cmdsn(pdu, pdu->cmdsn - cmdsn_gap);
+                }
 		if (pdu->scsi_timeout == 0) {
 			/* no timeout for this pdu */
 			continue;
@@ -731,6 +735,11 @@ iscsi_timeout_scan(struct iscsi_context *iscsi)
 			/* not expired yet */
 			continue;
 		}
+		if (!(pdu->outdata.data[0] & ISCSI_PDU_IMMEDIATE) &&
+		    (pdu->outdata.data[0] & 0x3f) != ISCSI_PDU_DATA_OUT) {
+			iscsi->cmdsn--;
+			cmdsn_gap++;
+		}
 		ISCSI_LIST_REMOVE(&iscsi->outqueue, pdu);
 		iscsi_set_error(iscsi, "command timed out");
 		iscsi_dump_pdu_header(iscsi, pdu->outdata.data);
@@ -783,6 +792,12 @@ iscsi_cancel_pdus(struct iscsi_context *iscsi)
 			              NULL, pdu->private_data);
 		}
 		iscsi->drv->free_pdu(iscsi, pdu);
+		if (!(pdu->outdata.data[0] & ISCSI_PDU_IMMEDIATE) &&
+		    (pdu->outdata.data[0] & 0x3f) != ISCSI_PDU_DATA_OUT) {
+			iscsi->cmdsn--;
+		}
+
+
 	}
 	while ((pdu = iscsi->waitpdu)) {
 		ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu);