Merge pull request #123 from plieven/asyncv3

add support for async messages

include/iscsi-private.h

@@ -80,6 +80,7 @@ struct iscsi_context {
 
 	int fd;
 	int is_connected;
+	int is_corked;
 
 	int tcp_user_timeout;
 	int tcp_keepcnt;
@@ -125,6 +126,7 @@ struct iscsi_context {
 	int no_auto_reconnect;
 	int reconnect_deferred;
 	int reconnect_max_retries;
+	int pending_reconnect;
 
 	int log_level;
 	iscsi_log_fn log_fn;
@@ -188,8 +190,9 @@ enum iscsi_opcode {
 	ISCSI_PDU_DATA_IN                        = 0x25,
 	ISCSI_PDU_LOGOUT_RESPONSE                = 0x26,
 	ISCSI_PDU_R2T                            = 0x31,
+	ISCSI_PDU_ASYNC_MSG                      = 0x32,
 	ISCSI_PDU_REJECT                         = 0x3f,
-	ISCSI_PDU_NO_PDU	                 = 0xff
+	ISCSI_PDU_NO_PDU                         = 0xff
 };
 
 struct iscsi_scsi_cbdata {
@@ -209,6 +212,13 @@ struct iscsi_pdu {
  * This includes any DATA-OUT PDU as well as all NOPs.
  */
 #define ISCSI_PDU_DROP_ON_RECONNECT	0x00000004
+/* stop sending after this PDU has been sent */
+#define ISCSI_PDU_CORK_WHEN_SENT	0x00000008
+/* put this immediate delivery PDU in front of outqueue.
+ * This is currently only used for immediate logout requests
+ * as answer to an async logout event. */
+#define ISCSI_PDU_URGENT_DELIVERY	0x00000010
+
 
 	uint32_t flags;
 
@@ -339,14 +349,18 @@ iscsi_log_message(struct iscsi_context *iscsi, int level, const char *format, ..
 void
 iscsi_add_to_outqueue(struct iscsi_context *iscsi, struct iscsi_pdu *pdu);
 
-int
+int iscsi_serial32_compare(uint32_t s1, uint32_t s2);
-iscsi_serial32_compare(uint32_t s1, uint32_t s2);
+void iscsi_adjust_statsn(struct iscsi_context *iscsi, struct iscsi_in_pdu *in);
+void iscsi_adjust_maxexpcmdsn(struct iscsi_context *iscsi, struct iscsi_in_pdu *in);
 
-uint32_t
+uint32_t iscsi_itt_post_increment(struct iscsi_context *iscsi);
-iscsi_itt_post_increment(struct iscsi_context *iscsi);
 
 void iscsi_timeout_scan(struct iscsi_context *iscsi);
 
+int
+iscsi_logout_async_internal(struct iscsi_context *iscsi, iscsi_command_cb cb,
+		  void *private_data, uint32_t flags);
+
 #ifdef __cplusplus
 }
 #endif

lib/iscsi-command.c

@@ -208,7 +208,7 @@ iscsi_scsi_command_async(struct iscsi_context *iscsi, int lun,
 		return -1;
 	}
 
-	if (iscsi->is_loggedin == 0) {
+	if (iscsi->is_loggedin == 0 && !iscsi->pending_reconnect) {
 		iscsi_set_error(iscsi, "Trying to send command while "
 				"not logged in.");
 		return -1;
@@ -347,23 +347,12 @@ int
 iscsi_process_scsi_reply(struct iscsi_context *iscsi, struct iscsi_pdu *pdu,
 			 struct iscsi_in_pdu *in)
 {
-	uint32_t statsn, maxcmdsn, expcmdsn, flags, status;
+	uint32_t flags, status;
 	struct iscsi_scsi_cbdata *scsi_cbdata = &pdu->scsi_cbdata;
 	struct scsi_task *task = scsi_cbdata->task;
 
-	statsn = scsi_get_uint32(&in->hdr[24]);
+	iscsi_adjust_statsn(iscsi, in);
-	if (statsn > iscsi->statsn) {
+	iscsi_adjust_maxexpcmdsn(iscsi, in);
-		iscsi->statsn = statsn;
-	}
-
-	maxcmdsn = scsi_get_uint32(&in->hdr[32]);
-	if (iscsi_serial32_compare(maxcmdsn, iscsi->maxcmdsn) > 0) {
-		iscsi->maxcmdsn = maxcmdsn;
-	}
-	expcmdsn = scsi_get_uint32(&in->hdr[28]);
-	if (iscsi_serial32_compare(expcmdsn, iscsi->expcmdsn) > 0) {
-		iscsi->expcmdsn = expcmdsn;
-	}
 
 	flags = in->hdr[1];
 	if ((flags&ISCSI_PDU_DATA_FINAL) == 0) {
@@ -489,24 +478,13 @@ int
 iscsi_process_scsi_data_in(struct iscsi_context *iscsi, struct iscsi_pdu *pdu,
 			   struct iscsi_in_pdu *in, int *is_finished)
 {
-	uint32_t statsn, maxcmdsn, expcmdsn, flags, status;
+	uint32_t flags, status;
 	struct iscsi_scsi_cbdata *scsi_cbdata = &pdu->scsi_cbdata;
 	struct scsi_task *task = scsi_cbdata->task;
 	int dsl;
 
-	statsn = scsi_get_uint32(&in->hdr[24]);
+	iscsi_adjust_statsn(iscsi, in);
-	if (statsn > iscsi->statsn) {
+	iscsi_adjust_maxexpcmdsn(iscsi, in);
-		iscsi->statsn = statsn;
-	}
-
-	maxcmdsn = scsi_get_uint32(&in->hdr[32]);
-	if (iscsi_serial32_compare(maxcmdsn, iscsi->maxcmdsn) > 0) {
-		iscsi->maxcmdsn = maxcmdsn;
-	}
-	expcmdsn = scsi_get_uint32(&in->hdr[28]);
-	if (iscsi_serial32_compare(expcmdsn, iscsi->expcmdsn) > 0) {
-		iscsi->expcmdsn = expcmdsn;
-	}
 
 	flags = in->hdr[1];
 	if ((flags&ISCSI_PDU_DATA_ACK_REQUESTED) != 0) {
@@ -579,20 +557,13 @@ int
 iscsi_process_r2t(struct iscsi_context *iscsi, struct iscsi_pdu *pdu,
 			 struct iscsi_in_pdu *in)
 {
-	uint32_t ttt, offset, len, maxcmdsn, expcmdsn;
+	uint32_t ttt, offset, len;
 
 	ttt    = scsi_get_uint32(&in->hdr[20]);
 	offset = scsi_get_uint32(&in->hdr[40]);
 	len    = scsi_get_uint32(&in->hdr[44]);
 
-	maxcmdsn = scsi_get_uint32(&in->hdr[32]);
+	iscsi_adjust_maxexpcmdsn(iscsi, in);
-	if (iscsi_serial32_compare(maxcmdsn, iscsi->maxcmdsn) > 0) {
-		iscsi->maxcmdsn = maxcmdsn;
-	}
-	expcmdsn = scsi_get_uint32(&in->hdr[28]);
-	if (iscsi_serial32_compare(expcmdsn, iscsi->expcmdsn) > 0) {
-		iscsi->expcmdsn = expcmdsn;
-	}
 
 	pdu->datasn = 0;
 	iscsi_send_data_out(iscsi, pdu, ttt, offset, len);

lib/login.c

@@ -965,22 +965,14 @@ int
 iscsi_process_login_reply(struct iscsi_context *iscsi, struct iscsi_pdu *pdu,
 			  struct iscsi_in_pdu *in)
 {
-  uint32_t status, maxcmdsn, expcmdsn;
+	uint32_t status;
 	char *ptr = (char *)in->data;
 	int size = in->data_pos;
 
 	status = scsi_get_uint16(&in->hdr[36]);
 
-	iscsi->statsn = scsi_get_uint16(&in->hdr[24]);
+	iscsi_adjust_statsn(iscsi, in);
-
+	iscsi_adjust_maxexpcmdsn(iscsi, in);
-	maxcmdsn = scsi_get_uint32(&in->hdr[32]);
-	if (iscsi_serial32_compare(maxcmdsn, iscsi->maxcmdsn) > 0) {
-		iscsi->maxcmdsn = maxcmdsn;
-	}
-	expcmdsn = scsi_get_uint32(&in->hdr[28]);
-	if (iscsi_serial32_compare(expcmdsn, iscsi->expcmdsn) > 0) {
-		iscsi->expcmdsn = expcmdsn;
-	}
 
 	/* XXX here we should parse the data returned in case the target
 	 * renegotiated some some parameters.
@@ -1117,10 +1109,9 @@ iscsi_process_login_reply(struct iscsi_context *iscsi, struct iscsi_pdu *pdu,
 	return 0;
 }
 
-
 int
-iscsi_logout_async(struct iscsi_context *iscsi, iscsi_command_cb cb,
+iscsi_logout_async_internal(struct iscsi_context *iscsi, iscsi_command_cb cb,
-		   void *private_data)
+		   void *private_data, uint32_t flags)
 {
 	struct iscsi_pdu *pdu;
 
@@ -1154,6 +1145,7 @@ iscsi_logout_async(struct iscsi_context *iscsi, iscsi_command_cb cb,
 
 	pdu->callback     = cb;
 	pdu->private_data = private_data;
+	pdu->flags |= ISCSI_PDU_CORK_WHEN_SENT | flags;
 
 	if (iscsi_queue_pdu(iscsi, pdu) != 0) {
 		iscsi_set_error(iscsi, "Out-of-memory: failed to queue iscsi "
@@ -1165,20 +1157,18 @@ iscsi_logout_async(struct iscsi_context *iscsi, iscsi_command_cb cb,
 	return 0;
 }
 
+int
+iscsi_logout_async(struct iscsi_context *iscsi, iscsi_command_cb cb,
+		   void *private_data)
+{
+	return iscsi_logout_async_internal(iscsi, cb, private_data, 0);
+}
+
 int
 iscsi_process_logout_reply(struct iscsi_context *iscsi, struct iscsi_pdu *pdu,
 struct iscsi_in_pdu *in)
 {
-  uint32_t maxcmdsn, expcmdsn;
+	iscsi_adjust_maxexpcmdsn(iscsi, in);
-
-	maxcmdsn = scsi_get_uint32(&in->hdr[32]);
-	if (iscsi_serial32_compare(maxcmdsn, iscsi->maxcmdsn) > 0) {
-		iscsi->maxcmdsn = maxcmdsn;
-	}
-	expcmdsn = scsi_get_uint32(&in->hdr[28]);
-	if (iscsi_serial32_compare(expcmdsn, iscsi->expcmdsn) > 0) {
-		iscsi->expcmdsn = expcmdsn;
-	}
 
 	iscsi->is_loggedin = 0;
 	pdu->callback(iscsi, SCSI_STATUS_GOOD, NULL, pdu->private_data);

lib/pdu.c

@@ -69,6 +69,31 @@ iscsi_itt_post_increment(struct iscsi_context *iscsi) {
 	return old_itt;
 }
 
+void iscsi_adjust_statsn(struct iscsi_context *iscsi, struct iscsi_in_pdu *in) {
+	uint32_t statsn = scsi_get_uint32(&in->hdr[24]);
+	uint32_t itt = scsi_get_uint32(&in->hdr[16]);
+	
+	if (itt == 0xffffffff) {
+		/* target will not increase statsn if itt == 0xffffffff */
+		statsn--;
+	}
+	
+	if (iscsi_serial32_compare(statsn, iscsi->statsn) > 0) {
+		iscsi->statsn = statsn;
+	}
+}
+
+void iscsi_adjust_maxexpcmdsn(struct iscsi_context *iscsi, struct iscsi_in_pdu *in) {
+	uint32_t maxcmdsn = scsi_get_uint32(&in->hdr[32]);
+	if (iscsi_serial32_compare(maxcmdsn, iscsi->maxcmdsn) > 0) {
+		iscsi->maxcmdsn = maxcmdsn;
+	}
+	uint32_t expcmdsn = scsi_get_uint32(&in->hdr[28]);
+	if (iscsi_serial32_compare(expcmdsn, iscsi->expcmdsn) > 0) {
+		iscsi->expcmdsn = expcmdsn;
+	}
+}
+
 void iscsi_dump_pdu_header(struct iscsi_context *iscsi, unsigned char *data) {
 	char dump[ISCSI_RAW_HEADER_SIZE*3+1]={0};
 	int i;
@@ -147,6 +172,10 @@ iscsi_free_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu)
 	}
 	pdu->indata.data = NULL;
 
+	if (iscsi->outqueue_current == pdu) {
+		iscsi->outqueue_current = NULL;
+	}
+
 	iscsi_sfree(iscsi, pdu);
 }
 
@@ -297,14 +326,10 @@ int iscsi_process_target_nop_in(struct iscsi_context *iscsi,
 				struct iscsi_in_pdu *in)
 {
 	uint32_t ttt;
-	uint32_t statsn;
 
 	ttt = scsi_get_uint32(&in->hdr[20]);
 
-	statsn = scsi_get_uint32(&in->hdr[24]);
+	iscsi_adjust_statsn(iscsi, in);
-	if (statsn > iscsi->statsn) {
-		iscsi->statsn = statsn;
-	}
 
 	/* if the server does not want a response */
 	if (ttt == 0xffffffff) {
@@ -317,12 +342,25 @@ int iscsi_process_target_nop_in(struct iscsi_context *iscsi,
 }
 
 
+static void iscsi_reconnect_after_logout(struct iscsi_context *iscsi, int status,
+                        void *command_data _U_, void *opaque _U_)
+{
+	if (status) {
+		ISCSI_LOG(iscsi, 1, "logout failed: %s", iscsi_get_error(iscsi));
+	} else {
+		ISCSI_LOG(iscsi, 2, "logout was successful");
+	}
+	iscsi->pending_reconnect = 1;
+}
+
+
 int iscsi_process_reject(struct iscsi_context *iscsi,
 				struct iscsi_in_pdu *in)
 {
 	int size = in->data_pos;
 	uint32_t itt;
 	struct iscsi_pdu *pdu;
+	uint8_t reason = in->hdr[2];
 
 	if (size < ISCSI_RAW_HEADER_SIZE) {
 		iscsi_set_error(iscsi, "size of REJECT payload is too small."
@@ -331,6 +369,14 @@ int iscsi_process_reject(struct iscsi_context *iscsi,
 		return -1;
 	}
 
+	if (reason == ISCSI_REJECT_WAITING_FOR_LOGOUT) {
+		ISCSI_LOG(iscsi, 1, "target rejects request with reason: %s",  iscsi_reject_reason_str(reason));
+		iscsi_logout_async_internal(iscsi, iscsi_reconnect_after_logout, NULL, ISCSI_PDU_DROP_ON_RECONNECT|ISCSI_PDU_URGENT_DELIVERY);
+		return 0;
+	}
+
+	iscsi_set_error(iscsi, "Request was rejected with reason: 0x%02x (%s)", reason, iscsi_reject_reason_str(reason));
+
 	itt = scsi_get_uint32(&in->data[16]);
 
 	if (iscsi->log_level > 1) {
@@ -350,8 +396,10 @@ int iscsi_process_reject(struct iscsi_context *iscsi,
 		return -1;
 	}
 
-	pdu->callback(iscsi, SCSI_STATUS_ERROR, NULL,
+	if (pdu->callback) {
-			      pdu->private_data);
+		pdu->callback(iscsi, SCSI_STATUS_ERROR, NULL,
+					pdu->private_data);
+	}
 
 	ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu);
 	iscsi_free_pdu(iscsi, pdu);
@@ -376,13 +424,36 @@ iscsi_process_pdu(struct iscsi_context *iscsi, struct iscsi_in_pdu *in)
 		return -1;
 	}
 
-	if (opcode == ISCSI_PDU_REJECT) {
+	if (opcode == ISCSI_PDU_ASYNC_MSG) {
-		iscsi_set_error(iscsi, "Request was rejected with reason: 0x%02x (%s)", in->hdr[2], iscsi_reject_reason_str(in->hdr[2]));
+		uint8_t event = in->hdr[36];
-
+		uint16_t param1 = scsi_get_uint16(&in->hdr[38]); 
-		if (iscsi_process_reject(iscsi, in) != 0) {
+		uint16_t param2 = scsi_get_uint16(&in->hdr[40]); 
+		uint16_t param3 = scsi_get_uint16(&in->hdr[42]); 
+		switch (event) {
+		case 0x1:
+			ISCSI_LOG(iscsi, 2, "target requests logout within %u seconds", param3);
+			iscsi_logout_async_internal(iscsi, iscsi_reconnect_after_logout, NULL, ISCSI_PDU_DROP_ON_RECONNECT|ISCSI_PDU_URGENT_DELIVERY);
+			return 0;
+		case 0x2:
+			ISCSI_LOG(iscsi, 2, "target will drop this connection. Time2Wait is %u seconds", param2);
+			iscsi->last_reconnect = time(NULL) + param2;
+			return 0;
+		case 0x3:
+			ISCSI_LOG(iscsi, 2, "target will drop all connections of this session. Time2Wait is %u seconds", param2);
+			iscsi->last_reconnect = time(NULL) + param2;
+			return 0;
+		case 0x4:
+			ISCSI_LOG(iscsi, 2, "target requests parameter renogitiation.");
+			iscsi_logout_async_internal(iscsi, iscsi_reconnect_after_logout, NULL, ISCSI_PDU_DROP_ON_RECONNECT);
+			return 0;
+		default:
+			ISCSI_LOG(iscsi, 1, "unhandled async event %u: param1 %u param2 %u param3 %u", event, param1, param2, param3);
 			return -1;
 		}
-		return 0;
+	}
+
+	if (opcode == ISCSI_PDU_REJECT) {
+		return iscsi_process_reject(iscsi, in);
 	}
 
 	if (opcode == ISCSI_PDU_NOP_IN && itt == 0xffffffff) {

lib/socket.c

@@ -94,7 +94,8 @@ iscsi_add_to_outqueue(struct iscsi_context *iscsi, struct iscsi_pdu *pdu)
 	 * ensure that pakets with the same CmdSN are kept in FIFO order.
 	 */
 	do {
-		if (iscsi_serial32_compare(pdu->cmdsn, current->cmdsn) < 0) {
+		if (iscsi_serial32_compare(pdu->cmdsn, current->cmdsn) < 0 ||
+			pdu->flags & ISCSI_PDU_URGENT_DELIVERY) {
 			/* insert PDU before the current */
 			if (last != NULL) {
 				last->next=pdu;
@@ -611,8 +612,15 @@ iscsi_write_to_socket(struct iscsi_context *iscsi)
 
 	while (iscsi->outqueue != NULL || iscsi->outqueue_current != NULL) {
 		if (iscsi->outqueue_current == NULL) {
-			if (iscsi_serial32_compare(iscsi->outqueue->cmdsn, iscsi->maxcmdsn) > 0) {
+			if (iscsi->is_corked) {
-				/* stop sending. maxcmdsn is reached */
+				/* connection is corked we are not allowed to send
+				 * additional PDUs */
+				return 0;
+			}
+			
+			if (iscsi_serial32_compare(iscsi->outqueue->cmdsn, iscsi->maxcmdsn) > 0
+				&& !(iscsi->outqueue->outdata.data[0] & ISCSI_PDU_IMMEDIATE)) {
+				/* stop sending for non-immediate PDUs. maxcmdsn is reached */
 				return 0;
 			}
 			/* pop first element of the outqueue */
@@ -698,10 +706,12 @@ iscsi_write_to_socket(struct iscsi_context *iscsi)
 		if (pdu->payload_written != total) {
 			return 0;
 		}
-
 		if (pdu->flags & ISCSI_PDU_DELETE_WHEN_SENT) {
 			iscsi_free_pdu(iscsi, pdu);
 		}
+		if (pdu->flags & ISCSI_PDU_CORK_WHEN_SENT) {
+			iscsi->is_corked = 1;
+		}
 		iscsi->outqueue_current = NULL;
 	}
 	return 0;
@@ -725,6 +735,10 @@ iscsi_service(struct iscsi_context *iscsi, int revents)
 		return 0;
 	}
 
+	if (iscsi->pending_reconnect) {
+		iscsi_reconnect(iscsi);
+	}
+
 	if (revents & POLLERR) {
 		int err = 0;
 		socklen_t err_size = sizeof(err);

lib/task_mgmt.c

@@ -92,18 +92,11 @@ int
 iscsi_process_task_mgmt_reply(struct iscsi_context *iscsi, struct iscsi_pdu *pdu,
 			    struct iscsi_in_pdu *in)
 {
-	uint32_t response, maxcmdsn, expcmdsn;
+	uint32_t response;
 
 	response = in->hdr[2];
 
-	maxcmdsn = scsi_get_uint32(&in->hdr[32]);
+	iscsi_adjust_maxexpcmdsn(iscsi, in);
-	if (iscsi_serial32_compare(maxcmdsn, iscsi->maxcmdsn) > 0) {
-		iscsi->maxcmdsn = maxcmdsn;
-	}
-	expcmdsn = scsi_get_uint32(&in->hdr[28]);
-	if (iscsi_serial32_compare(expcmdsn, iscsi->expcmdsn) > 0) {
-		iscsi->expcmdsn = expcmdsn;
-	}
 
 	pdu->callback(iscsi, SCSI_STATUS_GOOD, &response, pdu->private_data);