From 237729545a274939e564cb43218e754d30348920 Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Thu, 6 Dec 2012 10:03:01 +0100 Subject: [PATCH 1/4] NOP count in flight of client generated NOP-Outs This patch adds the abilitiy to libiscsi to count the number of consecutive outstanding NOPs. With this ability its fairly easy to implement a keepalive check with NOPs in your application. Periodically (e.g. every 5 secs) create a NOP-Out with: iscsi_nop_out_async(iscsi, NULL, NULL, 0, NULL); At that time check the number of consecutive missing NOP-Ins with iscsi_get_nops_in_flight(iscsi) > N. Where N is the number of missing NOP-Ins you will allow. Please note that it is legitime for the Target to ignore a NOP if the load is very high as those packet are mark as IMMEDIATE. Signed-off-by: Peter Lieven --- include/iscsi-private.h | 3 ++- include/iscsi.h | 8 ++++++++ lib/libiscsi.def | 1 + lib/libiscsi.syms | 1 + lib/nop.c | 14 ++++++++++++++ 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/include/iscsi-private.h b/include/iscsi-private.h index 3307250..07b8ee5 100644 --- a/include/iscsi-private.h +++ b/include/iscsi-private.h @@ -95,6 +95,7 @@ struct iscsi_context { int is_loggedin; int is_reconnecting; int bind_interfaces_cnt; + int nops_in_flight; int chap_a; int chap_i; @@ -122,7 +123,7 @@ struct iscsi_context { int no_auto_reconnect; int reconnect_deferred; int reconnect_max_retries; - + int log_level; iscsi_log_fn log_fn; diff --git a/include/iscsi.h b/include/iscsi.h index 162756d..971393f 100644 --- a/include/iscsi.h +++ b/include/iscsi.h @@ -35,6 +35,7 @@ struct sockaddr; /* FEATURES */ #define LIBISCSI_FEATURE_IOVECTOR (1) +#define LIBISCSI_FEATURE_NOP_COUNTER (1) #define MAX_STRING_SIZE (255) @@ -474,10 +475,17 @@ struct iscsi_discovery_address { * structure containing the data returned from * the server. * ISCSI_STATUS_CANCELLED: Discovery was aborted. Command_data is NULL. + * + * The callback may be NULL if you only want to let libiscsi count the in-flight + * NOPs. */ EXTERN int iscsi_nop_out_async(struct iscsi_context *iscsi, iscsi_command_cb cb, unsigned char *data, int len, void *private_data); + +/* read out the number of consecutive nop outs that did not receive an answer */ +EXTERN int iscsi_get_nops_in_flight(struct iscsi_context *iscsi); + struct scsi_task; enum iscsi_task_mgmt_funcs { diff --git a/lib/libiscsi.def b/lib/libiscsi.def index b7220da..5e8bedf 100644 --- a/lib/libiscsi.def +++ b/lib/libiscsi.def @@ -14,6 +14,7 @@ iscsi_get_fd iscsi_get_lba_status_sync iscsi_get_lba_status_task iscsi_get_target_address +iscsi_get_nops_in_flight iscsi_inquiry_sync iscsi_inquiry_task iscsi_is_logged_in diff --git a/lib/libiscsi.syms b/lib/libiscsi.syms index 5e410c4..8c1f0a8 100644 --- a/lib/libiscsi.syms +++ b/lib/libiscsi.syms @@ -12,6 +12,7 @@ iscsi_get_fd iscsi_get_lba_status_sync iscsi_get_lba_status_task iscsi_get_target_address +iscsi_get_nops_in_flight iscsi_inquiry_sync iscsi_inquiry_task iscsi_is_logged_in diff --git a/lib/nop.c b/lib/nop.c index 102df7c..597fe7c 100644 --- a/lib/nop.c +++ b/lib/nop.c @@ -81,6 +81,8 @@ iscsi_nop_out_async(struct iscsi_context *iscsi, iscsi_command_cb cb, return -1; } + iscsi->nops_in_flight++; + return 0; } @@ -130,6 +132,12 @@ iscsi_process_nop_out_reply(struct iscsi_context *iscsi, struct iscsi_pdu *pdu, { struct iscsi_data data; + iscsi->nops_in_flight = 0; + + if (pdu->callback == NULL) { + return 0; + } + data.data = NULL; data.size = 0; @@ -137,7 +145,13 @@ iscsi_process_nop_out_reply(struct iscsi_context *iscsi, struct iscsi_pdu *pdu, data.data = in->data; data.size = in->data_pos; } + pdu->callback(iscsi, SCSI_STATUS_GOOD, &data, pdu->private_data); return 0; } + +int iscsi_get_nops_in_flight(struct iscsi_context *iscsi) +{ + return iscsi->nops_in_flight; +} From 5f18c7270636aea08eb60c64be9f349cd67b8d21 Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Thu, 6 Dec 2012 10:27:40 +0100 Subject: [PATCH 2/4] RECONNCT fix broken reconnect for iov out vectors We where modifying out_offset and out_len in iscsi_write_to_socket(). If the packet that was being sent before reconnect was a write command the was a significant change that out_offset and out_len where already touched. When requeing the packet after reconnect we where sending garbage! Signed-off-by: Peter Lieven --- include/iscsi-private.h | 3 ++- lib/connect.c | 2 ++ lib/socket.c | 9 ++++----- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/include/iscsi-private.h b/include/iscsi-private.h index 07b8ee5..68b4463 100644 --- a/include/iscsi-private.h +++ b/include/iscsi-private.h @@ -218,7 +218,8 @@ struct iscsi_pdu { struct iscsi_data outdata; /* Header for PDU to send */ uint32_t out_offset; /* Offset into data-out iovector */ - uint32_t out_len; /* Amount of data to sent */ + uint32_t out_len; /* Amount of data to sent starting at out_offset */ + uint32_t out_written; /* Number of bytes written to socket */ struct iscsi_data indata; diff --git a/lib/connect.c b/lib/connect.c index b36f5c9..0275b17 100644 --- a/lib/connect.c +++ b/lib/connect.c @@ -308,6 +308,7 @@ try_again: iscsi->statsn++; pdu->written = 0; + pdu->out_written = 0; iscsi_add_to_outqueue(iscsi, pdu); } @@ -341,6 +342,7 @@ try_again: iscsi->statsn++; pdu->written = 0; + pdu->out_written = 0; iscsi_add_to_outqueue(iscsi, pdu); } diff --git a/lib/socket.c b/lib/socket.c index 048802a..5a8ff60 100644 --- a/lib/socket.c +++ b/lib/socket.c @@ -517,11 +517,11 @@ iscsi_write_to_socket(struct iscsi_context *iscsi) } /* Write any iovectors that might have been passed to us */ - while (pdu->out_len > 0) { + while (pdu->out_written < pdu->out_len) { unsigned char *buf; - count = pdu->out_len; - buf = iscsi_get_user_out_buffer(iscsi, pdu, pdu->out_offset, &count); + count = pdu->out_len - pdu->out_written; + buf = iscsi_get_user_out_buffer(iscsi, pdu, pdu->out_offset + pdu->out_written, &count); if (buf == NULL) { iscsi_set_error(iscsi, "Can't find iovector data for DATA-OUT"); return -1; @@ -539,8 +539,7 @@ iscsi_write_to_socket(struct iscsi_context *iscsi) "socket :%d", errno); return -1; } - pdu->out_offset += count; - pdu->out_len -= count; + pdu->out_written += count; } if (pdu->written == total) { From 154760a00e657fdd2cc70669602dad71cdb41724 Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Thu, 6 Dec 2012 10:34:54 +0100 Subject: [PATCH 3/4] RECONNECT remove redundant requeue code We first merge waitpdu and outqueue queues and then requeue the elements. iscsi_add_to_outqueue() will sort them accordingly so this is no problem. Signed-off-by: Peter Lieven --- lib/connect.c | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/lib/connect.c b/lib/connect.c index 0275b17..da3b5d8 100644 --- a/lib/connect.c +++ b/lib/connect.c @@ -288,6 +288,12 @@ try_again: goto try_again; } + while (old_iscsi->outqueue) { + struct iscsi_pdu *pdu = old_iscsi->outqueue; + SLIST_REMOVE(&old_iscsi->outqueue, pdu); + SLIST_ADD_END(&old_iscsi->waitpdu, pdu); + } + while (old_iscsi->waitpdu) { struct iscsi_pdu *pdu = old_iscsi->waitpdu; @@ -296,30 +302,6 @@ try_again: continue; } - pdu->itt = iscsi_itt_post_increment(iscsi); - iscsi_pdu_set_itt(pdu, pdu->itt); - - /* do not increase cmdsn for PDUs marked for immediate delivery - * this will result in a protocol error */ - pdu->cmdsn = (pdu->outdata.data[0] & ISCSI_PDU_IMMEDIATE)?iscsi->cmdsn:iscsi->cmdsn++; - iscsi_pdu_set_cmdsn(pdu, pdu->cmdsn); - - iscsi_pdu_set_expstatsn(pdu, iscsi->statsn); - iscsi->statsn++; - - pdu->written = 0; - pdu->out_written = 0; - iscsi_add_to_outqueue(iscsi, pdu); - } - - while (old_iscsi->outqueue) { - struct iscsi_pdu *pdu = old_iscsi->outqueue; - - SLIST_REMOVE(&old_iscsi->outqueue, pdu); - if (pdu->itt == 0xffffffff) { - continue; - } - if (pdu->flags & ISCSI_PDU_DROP_ON_RECONNECT) { /* We dont want to requeue things like DATA-OUT since these guys * will be reissued automatically anyway once the corresponding From 36be387547aa5a5ea29091d9eb57a20b00cad031 Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Thu, 6 Dec 2012 10:38:28 +0100 Subject: [PATCH 4/4] SOCKET remove useless code At this point in iscsi_write_to_socket() pdu->written can only be equal to total. Signed-off-by: Peter Lieven --- lib/socket.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/socket.c b/lib/socket.c index 5a8ff60..c7f9c4f 100644 --- a/lib/socket.c +++ b/lib/socket.c @@ -542,12 +542,10 @@ iscsi_write_to_socket(struct iscsi_context *iscsi) pdu->out_written += count; } - if (pdu->written == total) { - if (pdu->flags & ISCSI_PDU_DELETE_WHEN_SENT) { - iscsi_free_pdu(iscsi, pdu); - } - iscsi->outqueue_current = NULL; + if (pdu->flags & ISCSI_PDU_DELETE_WHEN_SENT) { + iscsi_free_pdu(iscsi, pdu); } + iscsi->outqueue_current = NULL; } return 0; }