From ad9cd56b2d36075e8ba8b6c54c8993a6eee77157 Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Fri, 19 Oct 2012 23:40:12 +0200 Subject: [PATCH 01/17] Add setters for TCP keepalive values This patch adds 3 functions to set the 3 keepalive values TCP_KEEPIDLE, TCP_KEEPCNT and TCP_KEEPINTVL. The values have to be set after iscsi context creation and are then configured on the socket on each new connection. --- include/iscsi-private.h | 3 +++ include/iscsi.h | 22 ++++++++++++++++++++++ lib/connect.c | 3 +++ lib/init.c | 4 ++++ lib/libiscsi.def | 3 +++ lib/libiscsi.syms | 3 +++ lib/socket.c | 20 +++++++++++++++++++- 7 files changed, 57 insertions(+), 1 deletion(-) diff --git a/include/iscsi-private.h b/include/iscsi-private.h index 010eeb8..d0a73f6 100644 --- a/include/iscsi-private.h +++ b/include/iscsi-private.h @@ -86,6 +86,9 @@ struct iscsi_context { int is_connected; int tcp_user_timeout; + int tcp_keepcnt; + int tcp_keepintvl; + int tcp_keepidle; int current_phase; int next_phase; diff --git a/include/iscsi.h b/include/iscsi.h index 70881f6..30aa5ea 100644 --- a/include/iscsi.h +++ b/include/iscsi.h @@ -974,6 +974,28 @@ iscsi_set_debug(struct iscsi_context *iscsi, int level); EXTERN void iscsi_set_tcp_user_timeout(struct iscsi_context *iscsi, int timeout_ms); +/* + * This function is to set the TCP_KEEPIDLE option. It has to be called after iscsi + * context creation. + */ +EXTERN void +iscsi_set_tcp_keepidle(struct iscsi_context *iscsi, int value); + +/* + * This function is to set the TCP_KEEPCNT option. It has to be called after iscsi + * context creation. + */ +EXTERN void +iscsi_set_tcp_keepcnt(struct iscsi_context *iscsi, int value); + +/* + * This function is to set the TCP_KEEPINTVL option. It has to be called after iscsi + * context creation. + */ +EXTERN void +iscsi_set_tcp_keepintvl(struct iscsi_context *iscsi, int value); + + #ifdef __cplusplus } #endif diff --git a/lib/connect.c b/lib/connect.c index 492f65a..2f5b003 100644 --- a/lib/connect.c +++ b/lib/connect.c @@ -234,6 +234,9 @@ try_again: iscsi->debug = old_iscsi->debug; iscsi->tcp_user_timeout = old_iscsi->tcp_user_timeout; + iscsi->tcp_keepidle = old_iscsi->tcp_keepidle; + iscsi->tcp_keepcnt = old_iscsi->tcp_keepcnt; + iscsi->tcp_keepintvl = old_iscsi->tcp_keepintvl; if (iscsi_full_connect_sync(iscsi, iscsi->portal, iscsi->lun) != 0) { iscsi_destroy_context(iscsi); diff --git a/lib/init.c b/lib/init.c index 9c6bcac..61852f2 100644 --- a/lib/init.c +++ b/lib/init.c @@ -71,6 +71,10 @@ iscsi_create_context(const char *initiator_name) iscsi->use_immediate_data = ISCSI_IMMEDIATE_DATA_YES; iscsi->want_header_digest = ISCSI_HEADER_DIGEST_NONE_CRC32C; + iscsi->tcp_keepcnt=3; + iscsi->tcp_keepintvl=30; + iscsi->tcp_keepidle=30; + return iscsi; } diff --git a/lib/libiscsi.def b/lib/libiscsi.def index 56ca61a..00cef83 100644 --- a/lib/libiscsi.def +++ b/lib/libiscsi.def @@ -73,6 +73,9 @@ iscsi_set_session_type iscsi_set_targetname iscsi_set_tcp_keepalive iscsi_set_tcp_user_timeout +iscsi_set_tcp_keepidle +iscsi_set_tcp_keepcnt +iscsi_set_tcp_keepintvl iscsi_startstopunit_sync iscsi_startstopunit_task iscsi_synchronizecache10_sync diff --git a/lib/libiscsi.syms b/lib/libiscsi.syms index dba697a..47fb0dc 100644 --- a/lib/libiscsi.syms +++ b/lib/libiscsi.syms @@ -71,6 +71,9 @@ iscsi_set_session_type iscsi_set_targetname iscsi_set_tcp_keepalive iscsi_set_tcp_user_timeout +iscsi_set_tcp_keepidle +iscsi_set_tcp_keepcnt +iscsi_set_tcp_keepintvl iscsi_startstopunit_sync iscsi_startstopunit_task iscsi_synchronizecache10_sync diff --git a/lib/socket.c b/lib/socket.c index 9912976..930a1eb 100644 --- a/lib/socket.c +++ b/lib/socket.c @@ -160,7 +160,7 @@ iscsi_connect_async(struct iscsi_context *iscsi, const char *portal, set_nonblocking(iscsi->fd); - iscsi_set_tcp_keepalive(iscsi, 30, 3, 30); + iscsi_set_tcp_keepalive(iscsi, iscsi->tcp_keepidle, iscsi->tcp_keepcnt, iscsi->tcp_keepintvl); if (iscsi->tcp_user_timeout > 0) { set_tcp_user_timeout(iscsi); @@ -563,6 +563,24 @@ void iscsi_set_tcp_user_timeout(struct iscsi_context *iscsi, int timeout_ms) DPRINTF(iscsi,2,"TCP_USER_TIMEOUT will be set to %dms on next socket creation",timeout_ms); } +void iscsi_set_tcp_keepidle(struct iscsi_context *iscsi, int value) +{ + iscsi->tcp_keepidle=value; + DPRINTF(iscsi,2,"TCP_KEEPIDLE will be set to %d on next socket creation",value); +} + +void iscsi_set_tcp_keepcnt(struct iscsi_context *iscsi, int value) +{ + iscsi->tcp_keepcnt=value; + DPRINTF(iscsi,2,"TCP_KEEPCNT will be set to %d on next socket creation",value); +} + +void iscsi_set_tcp_keepintvl(struct iscsi_context *iscsi, int value) +{ + iscsi->tcp_keepintvl=value; + DPRINTF(iscsi,2,"TCP_KEEPINTVL will be set to %d on next socket creation",value); +} + int iscsi_set_tcp_keepalive(struct iscsi_context *iscsi, int idle, int count, int interval) { int level, value; From 236aaa011fbff65908867357647ef04e66f32a00 Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Fri, 19 Oct 2012 23:48:04 +0200 Subject: [PATCH 02/17] Fix compiler warnings These patch fixes 3 compiler warnings introduce by my recent patches. --- lib/connect.c | 1 + lib/init.c | 2 +- lib/socket.c | 58 +++++++++++++++++++++++++-------------------------- 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/lib/connect.c b/lib/connect.c index 2f5b003..c20e4ed 100644 --- a/lib/connect.c +++ b/lib/connect.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "slist.h" #include "iscsi.h" #include "iscsi-private.h" diff --git a/lib/init.c b/lib/init.c index 61852f2..0269e43 100644 --- a/lib/init.c +++ b/lib/init.c @@ -280,7 +280,7 @@ iscsi_set_error(struct iscsi_context *iscsi, const char *error_string, ...) va_end(ap); - DPRINTF(iscsi,1,str); + DPRINTF(iscsi,1,"%s",str); } void diff --git a/lib/socket.c b/lib/socket.c index 930a1eb..1327ee0 100644 --- a/lib/socket.c +++ b/lib/socket.c @@ -58,6 +58,34 @@ static void set_nonblocking(int fd) #endif } +#ifndef TCP_USER_TIMEOUT +#define TCP_USER_TIMEOUT 18 +#endif + +int set_tcp_user_timeout(struct iscsi_context *iscsi) +{ + int level, value; + + #if defined(__FreeBSD__) || defined(__sun) + struct protoent *buf; + + if ((buf = getprotobyname("tcp")) != NULL) + level = buf->p_proto; + else + return -1; + #else + level = SOL_TCP; + #endif + + value = iscsi->tcp_user_timeout; + if (setsockopt(iscsi->fd, level, TCP_USER_TIMEOUT, &value, sizeof(value)) != 0) { + iscsi_set_error(iscsi, "TCP: Failed to set tcp user timeout. Error %s(%d)", strerror(errno), errno); + return -1; + } + DPRINTF(iscsi,3,"TCP_USER_TIMEOUT set to %d",value); + return 0; +} + int iscsi_connect_async(struct iscsi_context *iscsi, const char *portal, iscsi_command_cb cb, void *private_data) @@ -178,7 +206,7 @@ iscsi_connect_async(struct iscsi_context *iscsi, const char *portal, freeaddrinfo(ai); - if (iscsi->connected_portal) free(iscsi->connected_portal); + if (iscsi->connected_portal) free(discard_const(iscsi->connected_portal)); iscsi->connected_portal=strdup(portal); return 0; @@ -529,34 +557,6 @@ iscsi_free_iscsi_inqueue(struct iscsi_in_pdu *inqueue) } } -#ifndef TCP_USER_TIMEOUT -#define TCP_USER_TIMEOUT 18 -#endif - -int set_tcp_user_timeout(struct iscsi_context *iscsi) -{ - int level, value; - - #if defined(__FreeBSD__) || defined(__sun) - struct protoent *buf; - - if ((buf = getprotobyname("tcp")) != NULL) - level = buf->p_proto; - else - return -1; - #else - level = SOL_TCP; - #endif - - value = iscsi->tcp_user_timeout; - if (setsockopt(iscsi->fd, level, TCP_USER_TIMEOUT, &value, sizeof(value)) != 0) { - iscsi_set_error(iscsi, "TCP: Failed to set tcp user timeout. Error %s(%d)", strerror(errno), errno); - return -1; - } - DPRINTF(iscsi,3,"TCP_USER_TIMEOUT set to %d",value); - return 0; -} - void iscsi_set_tcp_user_timeout(struct iscsi_context *iscsi, int timeout_ms) { iscsi->tcp_user_timeout=timeout_ms; From d1110b7515f417aeb640a32e15c8ebd118c89d67 Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Sat, 20 Oct 2012 18:11:35 +0200 Subject: [PATCH 03/17] Add various environment variables This patch allows the following parameters inside libiscsi to be adjusted without code modification: LIBISCSI_DEBUG LIBISCSI_TCP_USER_TIMEOUT LIBISCSI_TCP_KEEPIDLE LIBISCSI_TCP_KEEPCNT LIBISCSI_TCP_KEEPINTVL You can now enable debugging of libiscsi inside e.g. qemu-kvm with LIBISCSI_DEBUG=3 qemu-kvm -hda iscsi://... --- lib/init.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/init.c b/lib/init.c index 0269e43..0cb5cf9 100644 --- a/lib/init.c +++ b/lib/init.c @@ -74,6 +74,26 @@ iscsi_create_context(const char *initiator_name) iscsi->tcp_keepcnt=3; iscsi->tcp_keepintvl=30; iscsi->tcp_keepidle=30; + + if (getenv("LIBISCSI_TCP_USER_TIMEOUT") != NULL) { + iscsi_set_tcp_user_timeout(iscsi,atoi(getenv("LIBISCSI_TCP_USER_TIMEOUT"))); + } + + if (getenv("LIBISCSI_TCP_KEEPCNT") != NULL) { + iscsi_set_tcp_keepcnt(iscsi,atoi(getenv("LIBISCSI_TCP_KEEPCNT"))); + } + + if (getenv("LIBISCSI_TCP_KEEPINTVL") != NULL) { + iscsi_set_tcp_keepintvl(iscsi,atoi(getenv("LIBISCSI_TCP_KEEPINTVL"))); + } + + if (getenv("LIBISCSI_TCP_KEEPIDLE") != NULL) { + iscsi_set_tcp_keepidle(iscsi,atoi(getenv("LIBISCSI_TCP_KEEPIDLE"))); + } + + if (getenv("LIBISCSI_DEBUG") != NULL) { + iscsi_set_debug(iscsi,atoi(getenv("LIBISCSI_DEBUG"))); + } return iscsi; } From 78a31ad4a1286ac192b4f3650242e35ddeb9de23 Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Sat, 20 Oct 2012 18:43:48 +0200 Subject: [PATCH 04/17] Add iscsi_set_tcp_syncnt function This patch adds support for setting TCP_SYNCNT to overwrite the system default values. This allows indirect support for a configurable connect timeout. Linux uses a exponential backoff for SYN retries starting with 1 second. This means for a value n for TCP_SYNCNT, the connect will effectively timeout after 2^(n+1)-1 seconds. --- include/iscsi-private.h | 1 + lib/connect.c | 1 + lib/init.c | 10 +++++++--- lib/libiscsi.def | 1 + lib/libiscsi.syms | 1 + lib/socket.c | 42 ++++++++++++++++++++++++++++++++++++++++- 6 files changed, 52 insertions(+), 4 deletions(-) diff --git a/include/iscsi-private.h b/include/iscsi-private.h index d0a73f6..f523b6e 100644 --- a/include/iscsi-private.h +++ b/include/iscsi-private.h @@ -89,6 +89,7 @@ struct iscsi_context { int tcp_keepcnt; int tcp_keepintvl; int tcp_keepidle; + int tcp_syncnt; int current_phase; int next_phase; diff --git a/lib/connect.c b/lib/connect.c index c20e4ed..650b0a2 100644 --- a/lib/connect.c +++ b/lib/connect.c @@ -238,6 +238,7 @@ try_again: iscsi->tcp_keepidle = old_iscsi->tcp_keepidle; iscsi->tcp_keepcnt = old_iscsi->tcp_keepcnt; iscsi->tcp_keepintvl = old_iscsi->tcp_keepintvl; + iscsi->tcp_syncnt = old_iscsi->tcp_syncnt; if (iscsi_full_connect_sync(iscsi, iscsi->portal, iscsi->lun) != 0) { iscsi_destroy_context(iscsi); diff --git a/lib/init.c b/lib/init.c index 0cb5cf9..41e20f0 100644 --- a/lib/init.c +++ b/lib/init.c @@ -74,6 +74,10 @@ iscsi_create_context(const char *initiator_name) iscsi->tcp_keepcnt=3; iscsi->tcp_keepintvl=30; iscsi->tcp_keepidle=30; + + if (getenv("LIBISCSI_DEBUG") != NULL) { + iscsi_set_debug(iscsi,atoi(getenv("LIBISCSI_DEBUG"))); + } if (getenv("LIBISCSI_TCP_USER_TIMEOUT") != NULL) { iscsi_set_tcp_user_timeout(iscsi,atoi(getenv("LIBISCSI_TCP_USER_TIMEOUT"))); @@ -90,9 +94,9 @@ iscsi_create_context(const char *initiator_name) if (getenv("LIBISCSI_TCP_KEEPIDLE") != NULL) { iscsi_set_tcp_keepidle(iscsi,atoi(getenv("LIBISCSI_TCP_KEEPIDLE"))); } - - if (getenv("LIBISCSI_DEBUG") != NULL) { - iscsi_set_debug(iscsi,atoi(getenv("LIBISCSI_DEBUG"))); + + if (getenv("LIBISCSI_TCP_SYNCNT") != NULL) { + iscsi_set_tcp_syncnt(iscsi,atoi(getenv("LIBISCSI_TCP_SYNCNT"))); } return iscsi; diff --git a/lib/libiscsi.def b/lib/libiscsi.def index 00cef83..6c197f8 100644 --- a/lib/libiscsi.def +++ b/lib/libiscsi.def @@ -76,6 +76,7 @@ iscsi_set_tcp_user_timeout iscsi_set_tcp_keepidle iscsi_set_tcp_keepcnt iscsi_set_tcp_keepintvl +iscsi_set_tcp_syncnt iscsi_startstopunit_sync iscsi_startstopunit_task iscsi_synchronizecache10_sync diff --git a/lib/libiscsi.syms b/lib/libiscsi.syms index 47fb0dc..5a86af8 100644 --- a/lib/libiscsi.syms +++ b/lib/libiscsi.syms @@ -74,6 +74,7 @@ iscsi_set_tcp_user_timeout iscsi_set_tcp_keepidle iscsi_set_tcp_keepcnt iscsi_set_tcp_keepintvl +iscsi_set_tcp_syncnt iscsi_startstopunit_sync iscsi_startstopunit_task iscsi_synchronizecache10_sync diff --git a/lib/socket.c b/lib/socket.c index 1327ee0..f481f60 100644 --- a/lib/socket.c +++ b/lib/socket.c @@ -86,6 +86,34 @@ int set_tcp_user_timeout(struct iscsi_context *iscsi) return 0; } +#ifndef TCP_SYNCNT +#define TCP_SYNCNT 7 +#endif + +int set_tcp_syncnt(struct iscsi_context *iscsi) +{ + int level, value; + + #if defined(__FreeBSD__) || defined(__sun) + struct protoent *buf; + + if ((buf = getprotobyname("tcp")) != NULL) + level = buf->p_proto; + else + return -1; + #else + level = SOL_TCP; + #endif + + value = iscsi->tcp_syncnt; + if (setsockopt(iscsi->fd, level, TCP_SYNCNT, &value, sizeof(value)) != 0) { + iscsi_set_error(iscsi, "TCP: Failed to set tcp syn retries. Error %s(%d)", strerror(errno), errno); + return -1; + } + DPRINTF(iscsi,3,"TCP_SYNCNT set to %d",value); + return 0; +} + int iscsi_connect_async(struct iscsi_context *iscsi, const char *portal, iscsi_command_cb cb, void *private_data) @@ -194,6 +222,10 @@ iscsi_connect_async(struct iscsi_context *iscsi, const char *portal, set_tcp_user_timeout(iscsi); } + if (iscsi->tcp_syncnt > 0) { + set_tcp_syncnt(iscsi); + } + if (connect(iscsi->fd, ai->ai_addr, socksize) != 0 && errno != EINPROGRESS) { iscsi_set_error(iscsi, "Connect failed with errno : " @@ -461,7 +493,7 @@ iscsi_service(struct iscsi_context *iscsi, int revents) if (iscsi->is_connected == 0 && iscsi->fd != -1 && revents&POLLOUT) { int err = 0; socklen_t err_size = sizeof(err); - + DPRINTF(iscsi,3,"inside iscsi service is_connected=0"); if (getsockopt(iscsi->fd, SOL_SOCKET, SO_ERROR, &err, &err_size) != 0 || err != 0) { if (err == 0) { @@ -480,6 +512,8 @@ iscsi_service(struct iscsi_context *iscsi, int revents) return -1; } + DPRINTF(iscsi,2,"connection to %s established",iscsi->connected_portal); + iscsi->is_connected = 1; iscsi->socket_status_cb(iscsi, SCSI_STATUS_GOOD, NULL, iscsi->connect_data); @@ -557,6 +591,12 @@ iscsi_free_iscsi_inqueue(struct iscsi_in_pdu *inqueue) } } +void iscsi_set_tcp_syncnt(struct iscsi_context *iscsi, int value) +{ + iscsi->tcp_syncnt=value; + DPRINTF(iscsi,2,"TCP_SYNCNT will be set to %d on next socket creation",value); +} + void iscsi_set_tcp_user_timeout(struct iscsi_context *iscsi, int timeout_ms) { iscsi->tcp_user_timeout=timeout_ms; From cbae6ae0581d486417c3d8caafc620b352496cbd Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Sat, 20 Oct 2012 18:44:43 +0200 Subject: [PATCH 05/17] Remove accidently left debug message in iscsi_service --- lib/socket.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/socket.c b/lib/socket.c index f481f60..5033269 100644 --- a/lib/socket.c +++ b/lib/socket.c @@ -493,7 +493,6 @@ iscsi_service(struct iscsi_context *iscsi, int revents) if (iscsi->is_connected == 0 && iscsi->fd != -1 && revents&POLLOUT) { int err = 0; socklen_t err_size = sizeof(err); - DPRINTF(iscsi,3,"inside iscsi service is_connected=0"); if (getsockopt(iscsi->fd, SOL_SOCKET, SO_ERROR, &err, &err_size) != 0 || err != 0) { if (err == 0) { From 881fc9743aae33bdad87b3dd18bf90904aa623ce Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Sat, 20 Oct 2012 18:54:46 +0200 Subject: [PATCH 06/17] Use generic tcp_set_sockopt function for tcp setsockopt operations --- lib/socket.c | 73 +++++++++++++++++----------------------------------- 1 file changed, 24 insertions(+), 49 deletions(-) diff --git a/lib/socket.c b/lib/socket.c index 5033269..dbccbfb 100644 --- a/lib/socket.c +++ b/lib/socket.c @@ -58,13 +58,9 @@ static void set_nonblocking(int fd) #endif } -#ifndef TCP_USER_TIMEOUT -#define TCP_USER_TIMEOUT 18 -#endif - -int set_tcp_user_timeout(struct iscsi_context *iscsi) +int set_tcp_sockopt(int sockfd, int optname, int value) { - int level, value; + int level; #if defined(__FreeBSD__) || defined(__sun) struct protoent *buf; @@ -75,14 +71,22 @@ int set_tcp_user_timeout(struct iscsi_context *iscsi) return -1; #else level = SOL_TCP; - #endif + #endif - value = iscsi->tcp_user_timeout; - if (setsockopt(iscsi->fd, level, TCP_USER_TIMEOUT, &value, sizeof(value)) != 0) { + return setsockopt(sockfd, level, optname, &value, sizeof(value)); +} + +#ifndef TCP_USER_TIMEOUT +#define TCP_USER_TIMEOUT 18 +#endif + +int set_tcp_user_timeout(struct iscsi_context *iscsi) +{ + if (set_tcp_sockopt(iscsi->fd, TCP_USER_TIMEOUT, iscsi->tcp_user_timeout) != 0) { iscsi_set_error(iscsi, "TCP: Failed to set tcp user timeout. Error %s(%d)", strerror(errno), errno); return -1; } - DPRINTF(iscsi,3,"TCP_USER_TIMEOUT set to %d",value); + DPRINTF(iscsi,3,"TCP_USER_TIMEOUT set to %d",iscsi->tcp_user_timeout); return 0; } @@ -92,25 +96,11 @@ int set_tcp_user_timeout(struct iscsi_context *iscsi) int set_tcp_syncnt(struct iscsi_context *iscsi) { - int level, value; - - #if defined(__FreeBSD__) || defined(__sun) - struct protoent *buf; - - if ((buf = getprotobyname("tcp")) != NULL) - level = buf->p_proto; - else - return -1; - #else - level = SOL_TCP; - #endif - - value = iscsi->tcp_syncnt; - if (setsockopt(iscsi->fd, level, TCP_SYNCNT, &value, sizeof(value)) != 0) { + if (set_tcp_sockopt(iscsi->fd, TCP_SYNCNT, iscsi->tcp_syncnt) != 0) { iscsi_set_error(iscsi, "TCP: Failed to set tcp syn retries. Error %s(%d)", strerror(errno), errno); return -1; } - DPRINTF(iscsi,3,"TCP_SYNCNT set to %d",value); + DPRINTF(iscsi,3,"TCP_SYNCNT set to %d",iscsi->tcp_syncnt); return 0; } @@ -622,49 +612,34 @@ void iscsi_set_tcp_keepintvl(struct iscsi_context *iscsi, int value) int iscsi_set_tcp_keepalive(struct iscsi_context *iscsi, int idle, int count, int interval) { - int level, value; -#if defined(__FreeBSD__) || defined(__sun) - struct protoent *buf; - - if ((buf = getprotobyname("tcp")) != NULL) - level = buf->p_proto; - else - return -1; -#else - level = SOL_TCP; -#endif - #ifdef SO_KEEPALIVE - value =1; + int value = 1; if (setsockopt(iscsi->fd, SOL_SOCKET, SO_KEEPALIVE, &value, sizeof(value)) != 0) { iscsi_set_error(iscsi, "TCP: Failed to set socket option SO_KEEPALIVE. Error %s(%d)", strerror(errno), errno); return -1; } DPRINTF(iscsi,3,"SO_KEEPALIVE set to %d",value); -#endif #ifdef TCP_KEEPCNT - value = count; - if (setsockopt(iscsi->fd, level, TCP_KEEPCNT, &value, sizeof(value)) != 0) { + if (set_tcp_sockopt(iscsi->fd, TCP_KEEPCNT, count) != 0) { iscsi_set_error(iscsi, "TCP: Failed to set tcp keepalive count. Error %s(%d)", strerror(errno), errno); return -1; } - DPRINTF(iscsi,3,"TCP_KEEPCNT set to %d",value); + DPRINTF(iscsi,3,"TCP_KEEPCNT set to %d",count); #endif #ifdef TCP_KEEPINTVL - value = interval; - if (setsockopt(iscsi->fd, level, TCP_KEEPINTVL, &value, sizeof(value)) != 0) { + if (set_tcp_sockopt(iscsi->fd, TCP_KEEPINTVL, interval) != 0) { iscsi_set_error(iscsi, "TCP: Failed to set tcp keepalive interval. Error %s(%d)", strerror(errno), errno); return -1; } - DPRINTF(iscsi,3,"TCP_KEEPINTVL set to %d",value); + DPRINTF(iscsi,3,"TCP_KEEPINTVL set to %d",interval); #endif #ifdef TCP_KEEPIDLE - value = idle; - if (setsockopt(iscsi->fd, level, TCP_KEEPIDLE, &value, sizeof(value)) != 0) { + if (set_tcp_sockopt(iscsi->fd, TCP_KEEPIDLE, idle) != 0) { iscsi_set_error(iscsi, "TCP: Failed to set tcp keepalive idle. Error %s(%d)", strerror(errno), errno); return -1; } - DPRINTF(iscsi,3,"TCP_KEEPIDLE set to %d",value); + DPRINTF(iscsi,3,"TCP_KEEPIDLE set to %d",idle); +#endif #endif return 0; From 3f5a75ce5ecbff369b2283f20eb941fab6864d63 Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Sat, 20 Oct 2012 19:00:32 +0200 Subject: [PATCH 07/17] Remove redundant code in iscsi_service() --- lib/socket.c | 46 ++++++++++++++++------------------------------ 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/lib/socket.c b/lib/socket.c index dbccbfb..83df934 100644 --- a/lib/socket.c +++ b/lib/socket.c @@ -439,6 +439,17 @@ iscsi_write_to_socket(struct iscsi_context *iscsi) return 0; } +int inline +iscsi_service_reconnect_if_loggedin(struct iscsi_context *iscsi) +{ + if (iscsi->is_loggedin) { + if (iscsi_reconnect(iscsi) == 0) { + return 0; + } + } + return -1; +} + int iscsi_service(struct iscsi_context *iscsi, int revents) { @@ -460,24 +471,14 @@ iscsi_service(struct iscsi_context *iscsi, int revents) } iscsi->socket_status_cb(iscsi, SCSI_STATUS_ERROR, NULL, iscsi->connect_data); - if (iscsi->is_loggedin) { - if (iscsi_reconnect(iscsi) == 0) { - return 0; - } - } - return -1; + return iscsi_service_reconnect_if_loggedin(iscsi); } if (revents & POLLHUP) { iscsi_set_error(iscsi, "iscsi_service: POLLHUP, " "socket error."); iscsi->socket_status_cb(iscsi, SCSI_STATUS_ERROR, NULL, iscsi->connect_data); - if (iscsi->is_loggedin) { - if (iscsi_reconnect(iscsi) == 0) { - return 0; - } - } - return -1; + return iscsi_service_reconnect_if_loggedin(iscsi); } if (iscsi->is_connected == 0 && iscsi->fd != -1 && revents&POLLOUT) { @@ -493,12 +494,7 @@ iscsi_service(struct iscsi_context *iscsi, int revents) strerror(err), err); iscsi->socket_status_cb(iscsi, SCSI_STATUS_ERROR, NULL, iscsi->connect_data); - if (iscsi->is_loggedin) { - if (iscsi_reconnect(iscsi) == 0) { - return 0; - } - } - return -1; + return iscsi_service_reconnect_if_loggedin(iscsi); } DPRINTF(iscsi,2,"connection to %s established",iscsi->connected_portal); @@ -511,22 +507,12 @@ iscsi_service(struct iscsi_context *iscsi, int revents) if (revents & POLLOUT && iscsi->outqueue != NULL) { if (iscsi_write_to_socket(iscsi) != 0) { - if (iscsi->is_loggedin) { - if (iscsi_reconnect(iscsi) == 0) { - return 0; - } - } - return -1; + return iscsi_service_reconnect_if_loggedin(iscsi); } } if (revents & POLLIN) { if (iscsi_read_from_socket(iscsi) != 0) { - if (iscsi->is_loggedin) { - if (iscsi_reconnect(iscsi) == 0) { - return 0; - } - } - return -1; + return iscsi_service_reconnect_if_loggedin(iscsi); } } From 20cf2b279e0def7662c6fed38bb1c63262698804 Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Sat, 20 Oct 2012 19:08:57 +0200 Subject: [PATCH 08/17] Fix incorrect whitespaces At a few places there where spaces where tabulators where appropriate --- lib/connect.c | 10 +++++----- lib/init.c | 5 ++--- lib/login.c | 4 ++-- lib/scsi-lowlevel.c | 12 ++++++------ lib/socket.c | 26 +++++++++++++------------- lib/sync.c | 6 +++--- 6 files changed, 31 insertions(+), 32 deletions(-) diff --git a/lib/connect.c b/lib/connect.c index 650b0a2..8ad2afe 100644 --- a/lib/connect.c +++ b/lib/connect.c @@ -87,13 +87,13 @@ iscsi_login_cb(struct iscsi_context *iscsi, int status, void *command_data _U_, { struct connect_task *ct = private_data; - if (status == SCSI_STATUS_REDIRECT && iscsi->target_address) { + if (status == SCSI_STATUS_REDIRECT && iscsi->target_address) { iscsi_disconnect(iscsi); if (iscsi_connect_async(iscsi, iscsi->target_address, iscsi_connect_cb, iscsi->connect_data) != 0) { return; } return; - } + } if (status != 0) { ct->cb(iscsi, SCSI_STATUS_ERROR, NULL, ct->private_data); @@ -171,7 +171,7 @@ int iscsi_reconnect(struct iscsi_context *old_iscsi) { struct iscsi_context *iscsi = old_iscsi; - DPRINTF(iscsi,2,"reconnect initiated"); + DPRINTF(iscsi,2,"reconnect initiated"); /* This is mainly for tests, where we do not want to automatically reconnect but rather want the commands to fail with an error @@ -210,8 +210,8 @@ int iscsi_reconnect(struct iscsi_context *old_iscsi) return 0; } - int retry = 0; - srand (time(NULL)^getpid()); + int retry = 0; + srand (time(NULL)^getpid()); try_again: diff --git a/lib/init.c b/lib/init.c index 41e20f0..5bd6d0c 100644 --- a/lib/init.c +++ b/lib/init.c @@ -272,7 +272,7 @@ iscsi_destroy_context(struct iscsi_context *iscsi) free(discard_const(iscsi->chap_c)); iscsi->chap_c = NULL; - if (iscsi->connected_portal != NULL) { + if (iscsi->connected_portal != NULL) { free(discard_const(iscsi->connected_portal)); iscsi->connected_portal = NULL; } @@ -587,8 +587,7 @@ iscsi_destroy_url(struct iscsi_url *iscsi_url) int iscsi_set_initiator_username_pwd(struct iscsi_context *iscsi, - const char *user, - const char *passwd) + const char *user, const char *passwd) { free(discard_const(iscsi->user)); iscsi->user = strdup(user); diff --git a/lib/login.c b/lib/login.c index 28b9226..8e57839 100644 --- a/lib/login.c +++ b/lib/login.c @@ -1088,12 +1088,12 @@ iscsi_process_login_reply(struct iscsi_context *iscsi, struct iscsi_pdu *pdu, size -= len + 1; } - if (status == SCSI_STATUS_REDIRECT && iscsi->target_address) { + if (status == SCSI_STATUS_REDIRECT && iscsi->target_address) { DPRINTF(iscsi,2,"target requests redirect to %s",iscsi->target_address); pdu->callback(iscsi, SCSI_STATUS_REDIRECT, NULL, pdu->private_data); return 0; - } + } if (status != 0) { iscsi_set_error(iscsi, "Failed to log in to target. Status: %s(%d)", diff --git a/lib/scsi-lowlevel.c b/lib/scsi-lowlevel.c index 199b4d5..c784af2 100644 --- a/lib/scsi-lowlevel.c +++ b/lib/scsi-lowlevel.c @@ -76,8 +76,8 @@ scsi_malloc(struct scsi_task *task, size_t size) } struct value_string { - int value; - const char *string; + int value; + const char *string; }; static const char * @@ -1285,7 +1285,7 @@ scsi_cdb_compareandwrite(uint64_t lba, uint32_t xferlen, int blocksize, int wrpr task->xfer_dir = SCSI_XFER_WRITE; } else { task->xfer_dir = SCSI_XFER_NONE; - } + } task->expxferlen = xferlen; task->params.compareandwrite.lba = lba; @@ -2390,9 +2390,9 @@ scsi_get_task_private_ptr(struct scsi_task *task) struct scsi_data_buffer { - struct scsi_data_buffer *next; - uint32_t len; - unsigned char *data; + struct scsi_data_buffer *next; + uint32_t len; + unsigned char *data; }; int diff --git a/lib/socket.c b/lib/socket.c index 83df934..bb70083 100644 --- a/lib/socket.c +++ b/lib/socket.c @@ -114,7 +114,7 @@ iscsi_connect_async(struct iscsi_context *iscsi, const char *portal, struct addrinfo *ai = NULL; int socksize; - DPRINTF(iscsi,2,"connecting to portal %s",portal); + DPRINTF(iscsi,2,"connecting to portal %s",portal); if (iscsi->fd != -1) { iscsi_set_error(iscsi, @@ -212,7 +212,7 @@ iscsi_connect_async(struct iscsi_context *iscsi, const char *portal, set_tcp_user_timeout(iscsi); } - if (iscsi->tcp_syncnt > 0) { + if (iscsi->tcp_syncnt > 0) { set_tcp_syncnt(iscsi); } @@ -497,7 +497,7 @@ iscsi_service(struct iscsi_context *iscsi, int revents) return iscsi_service_reconnect_if_loggedin(iscsi); } - DPRINTF(iscsi,2,"connection to %s established",iscsi->connected_portal); + DPRINTF(iscsi,2,"connection to %s established",iscsi->connected_portal); iscsi->is_connected = 1; iscsi->socket_status_cb(iscsi, SCSI_STATUS_GOOD, NULL, @@ -568,32 +568,32 @@ iscsi_free_iscsi_inqueue(struct iscsi_in_pdu *inqueue) void iscsi_set_tcp_syncnt(struct iscsi_context *iscsi, int value) { - iscsi->tcp_syncnt=value; - DPRINTF(iscsi,2,"TCP_SYNCNT will be set to %d on next socket creation",value); + iscsi->tcp_syncnt=value; + DPRINTF(iscsi,2,"TCP_SYNCNT will be set to %d on next socket creation",value); } void iscsi_set_tcp_user_timeout(struct iscsi_context *iscsi, int timeout_ms) { - iscsi->tcp_user_timeout=timeout_ms; - DPRINTF(iscsi,2,"TCP_USER_TIMEOUT will be set to %dms on next socket creation",timeout_ms); + iscsi->tcp_user_timeout=timeout_ms; + DPRINTF(iscsi,2,"TCP_USER_TIMEOUT will be set to %dms on next socket creation",timeout_ms); } void iscsi_set_tcp_keepidle(struct iscsi_context *iscsi, int value) { - iscsi->tcp_keepidle=value; - DPRINTF(iscsi,2,"TCP_KEEPIDLE will be set to %d on next socket creation",value); + iscsi->tcp_keepidle=value; + DPRINTF(iscsi,2,"TCP_KEEPIDLE will be set to %d on next socket creation",value); } void iscsi_set_tcp_keepcnt(struct iscsi_context *iscsi, int value) { - iscsi->tcp_keepcnt=value; - DPRINTF(iscsi,2,"TCP_KEEPCNT will be set to %d on next socket creation",value); + iscsi->tcp_keepcnt=value; + DPRINTF(iscsi,2,"TCP_KEEPCNT will be set to %d on next socket creation",value); } void iscsi_set_tcp_keepintvl(struct iscsi_context *iscsi, int value) { - iscsi->tcp_keepintvl=value; - DPRINTF(iscsi,2,"TCP_KEEPINTVL will be set to %d on next socket creation",value); + iscsi->tcp_keepintvl=value; + DPRINTF(iscsi,2,"TCP_KEEPINTVL will be set to %d on next socket creation",value); } int iscsi_set_tcp_keepalive(struct iscsi_context *iscsi, int idle, int count, int interval) diff --git a/lib/sync.c b/lib/sync.c index f8903f5..e0b8a8d 100644 --- a/lib/sync.c +++ b/lib/sync.c @@ -30,9 +30,9 @@ #include "scsi-lowlevel.h" struct iscsi_sync_state { - int finished; - int status; - struct scsi_task *task; + int finished; + int status; + struct scsi_task *task; }; static void From d30b279474cb273a80e971e300019d6190616bc4 Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Sat, 20 Oct 2012 19:12:13 +0200 Subject: [PATCH 09/17] Unify paramters in tcp_set_user_timeout() Use the same function definition as in the other tcp setters. --- lib/socket.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/socket.c b/lib/socket.c index bb70083..78d8b94 100644 --- a/lib/socket.c +++ b/lib/socket.c @@ -572,9 +572,9 @@ void iscsi_set_tcp_syncnt(struct iscsi_context *iscsi, int value) DPRINTF(iscsi,2,"TCP_SYNCNT will be set to %d on next socket creation",value); } -void iscsi_set_tcp_user_timeout(struct iscsi_context *iscsi, int timeout_ms) +void iscsi_set_tcp_user_timeout(struct iscsi_context *iscsi, int value) { - iscsi->tcp_user_timeout=timeout_ms; + iscsi->tcp_user_timeout=value; DPRINTF(iscsi,2,"TCP_USER_TIMEOUT will be set to %dms on next socket creation",timeout_ms); } From a26a6e12d51f2b54eb6a9d4cc8f67b47cffc497f Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Sat, 20 Oct 2012 19:13:53 +0200 Subject: [PATCH 10/17] Fix paramter to DPRINTF in tcp_set_user_timeout() --- lib/socket.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/socket.c b/lib/socket.c index 78d8b94..5e50954 100644 --- a/lib/socket.c +++ b/lib/socket.c @@ -575,7 +575,7 @@ void iscsi_set_tcp_syncnt(struct iscsi_context *iscsi, int value) void iscsi_set_tcp_user_timeout(struct iscsi_context *iscsi, int value) { iscsi->tcp_user_timeout=value; - DPRINTF(iscsi,2,"TCP_USER_TIMEOUT will be set to %dms on next socket creation",timeout_ms); + DPRINTF(iscsi,2,"TCP_USER_TIMEOUT will be set to %dms on next socket creation",value); } void iscsi_set_tcp_keepidle(struct iscsi_context *iscsi, int value) From 86fa62558e780f76e0221d2937088d22a3f834dd Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Sun, 21 Oct 2012 08:13:16 +0200 Subject: [PATCH 11/17] LD_ISCSI: add debugging support --- src/ld_iscsi.c | 47 ++++++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/src/ld_iscsi.c b/src/ld_iscsi.c index e0e7ad8..b9397a8 100644 --- a/src/ld_iscsi.c +++ b/src/ld_iscsi.c @@ -37,6 +37,10 @@ static const char *initiator = "iqn.2007-10.com.github:sahlberg:libiscsi:ld-iscs #define ISCSI_MAX_FD 255 +static int debug = 0; + +#define LD_ISCSI_DPRINTF(level,fmt,args...) do { if ((debug) >= level) {fprintf(stderr,"ld_iscsi: ");fprintf(stderr, (fmt), ##args); fprintf(stderr,"\n");} } while (0); + struct iscsi_fd_list { int is_iscsi; int dup2fd; @@ -64,15 +68,14 @@ int open(const char *path, int flags, mode_t mode) iscsi = iscsi_create_context(initiator); if (iscsi == NULL) { - fprintf(stderr, "ld-iscsi: Failed to create context\n"); + LD_ISCSI_DPRINTF(0,"Failed to create context"); errno = ENOMEM; return -1; } iscsi_url = iscsi_parse_full_url(iscsi, path); if (iscsi_url == NULL) { - fprintf(stderr, "ld-iscsi: Failed to parse URL: %s\n", - iscsi_get_error(iscsi)); + LD_ISCSI_DPRINTF(0,"Failed to parse URL: %s\n", iscsi_get_error(iscsi)); iscsi_destroy_context(iscsi); errno = EINVAL; return -1; @@ -84,7 +87,7 @@ int open(const char *path, int flags, mode_t mode) if (iscsi_url->user != NULL) { if (iscsi_set_initiator_username_pwd(iscsi, iscsi_url->user, iscsi_url->passwd) != 0) { - fprintf(stderr, "Failed to set initiator username and password\n"); + LD_ISCSI_DPRINTF(0,"Failed to set initiator username and password"); iscsi_destroy_context(iscsi); errno = ENOMEM; return -1; @@ -92,7 +95,7 @@ int open(const char *path, int flags, mode_t mode) } if (iscsi_full_connect_sync(iscsi, iscsi_url->portal, iscsi_url->lun) != 0) { - fprintf(stderr, "ld-iscsi: Login Failed. %s\n", iscsi_get_error(iscsi)); + LD_ISCSI_DPRINTF(0,"Login Failed. %s\n", iscsi_get_error(iscsi)); iscsi_destroy_url(iscsi_url); iscsi_destroy_context(iscsi); errno = EIO; @@ -101,7 +104,7 @@ int open(const char *path, int flags, mode_t mode) task = iscsi_readcapacity10_sync(iscsi, iscsi_url->lun, 0, 0); if (task == NULL || task->status != SCSI_STATUS_GOOD) { - fprintf(stderr, "ld-iscsi: failed to send readcapacity command\n"); + LD_ISCSI_DPRINTF(0,"failed to send readcapacity command"); iscsi_destroy_url(iscsi_url); iscsi_destroy_context(iscsi); errno = EIO; @@ -110,7 +113,7 @@ int open(const char *path, int flags, mode_t mode) rc10 = scsi_datain_unmarshall(task); if (rc10 == NULL) { - fprintf(stderr, "ld-iscsi: failed to unmarshall readcapacity10 data\n"); + LD_ISCSI_DPRINTF(0,"failed to unmarshall readcapacity10 data"); scsi_free_scsi_task(task); iscsi_destroy_url(iscsi_url); iscsi_destroy_context(iscsi); @@ -120,7 +123,7 @@ int open(const char *path, int flags, mode_t mode) fd = iscsi_get_fd(iscsi); if (fd >= ISCSI_MAX_FD) { - fprintf(stderr, "ld-iscsi: Too many files open\n"); + LD_ISCSI_DPRINTF(0,"Too many files open"); iscsi_destroy_url(iscsi_url); iscsi_destroy_context(iscsi); errno = ENFILE; @@ -277,11 +280,13 @@ ssize_t read(int fd, void *buf, size_t count) count = num_blocks * iscsi_fd_list[fd].block_size; } + LD_ISCSI_DPRINTF(4,"read10_sync: lun %d, lba %u, num_blocks: %u, block_size: %d, offset: %lu count: %lu",iscsi_fd_list[fd].lun,lba,num_blocks,iscsi_fd_list[fd].block_size,offset,count); + iscsi_fd_list[fd].in_flight = 1; task = iscsi_read10_sync(iscsi_fd_list[fd].iscsi, iscsi_fd_list[fd].lun, lba, num_blocks * iscsi_fd_list[fd].block_size, iscsi_fd_list[fd].block_size, 0, 0, 0, 0, 0); iscsi_fd_list[fd].in_flight = 0; if (task == NULL || task->status != SCSI_STATUS_GOOD) { - fprintf(stderr, "ld-iscsi: failed to send read10 command\n"); + LD_ISCSI_DPRINTF(0,"failed to send read10 command"); errno = EIO; return -1; } @@ -381,59 +386,63 @@ static void __attribute__((constructor)) _init(void) iscsi_fd_list[i].dup2fd = -1; } + if (getenv("LD_ISCSI_DEBUG") != NULL) { + debug = atoi(getenv("LD_ISCSI_DEBUG")); + } + real_open = dlsym(RTLD_NEXT, "open"); if (real_open == NULL) { - fprintf(stderr, "ld_iscsi: Failed to dlsym(open)\n"); + LD_ISCSI_DPRINTF(0,"Failed to dlsym(open)"); exit(10); } real_close = dlsym(RTLD_NEXT, "close"); if (real_close == NULL) { - fprintf(stderr, "ld_iscsi: Failed to dlsym(close)\n"); + LD_ISCSI_DPRINTF(0,"Failed to dlsym(close)"); exit(10); } real_fxstat = dlsym(RTLD_NEXT, "__fxstat"); if (real_fxstat == NULL) { - fprintf(stderr, "ld_iscsi: Failed to dlsym(__fxstat)\n"); + LD_ISCSI_DPRINTF(0,"Failed to dlsym(__fxstat)"); exit(10); } real_lxstat = dlsym(RTLD_NEXT, "__lxstat"); if (real_lxstat == NULL) { - fprintf(stderr, "ld_iscsi: Failed to dlsym(__lxstat)\n"); + LD_ISCSI_DPRINTF(0,"Failed to dlsym(__lxstat)"); exit(10); } real_xstat = dlsym(RTLD_NEXT, "__xstat"); if (real_xstat == NULL) { - fprintf(stderr, "ld_iscsi: Failed to dlsym(__xstat)\n"); + LD_ISCSI_DPRINTF(0,"Failed to dlsym(__xstat)"); exit(10); } real_read = dlsym(RTLD_NEXT, "read"); if (real_read == NULL) { - fprintf(stderr, "ld_iscsi: Failed to dlsym(read)\n"); + LD_ISCSI_DPRINTF(0,"Failed to dlsym(read)"); exit(10); } real_dup2 = dlsym(RTLD_NEXT, "dup2"); if (real_dup2 == NULL) { - fprintf(stderr, "ld_iscsi: Failed to dlsym(dup2)\n"); + LD_ISCSI_DPRINTF(0,"Failed to dlsym(dup2)"); exit(10); } real_fxstat64 = dlsym(RTLD_NEXT, "__fxstat64"); if (real_fxstat64 == NULL) { - fprintf(stderr, "ld_iscsi: Failed to dlsym(__fxstat64)\n"); + LD_ISCSI_DPRINTF(0,"Failed to dlsym(__fxstat64)"); } real_lxstat64 = dlsym(RTLD_NEXT, "__lxstat64"); if (real_lxstat64 == NULL) { - fprintf(stderr, "ld_iscsi: Failed to dlsym(_lxstat64)\n"); + LD_ISCSI_DPRINTF(0,"Failed to dlsym(_lxstat64)"); } real_xstat64 = dlsym(RTLD_NEXT, "__xstat64"); if (real_xstat64 == NULL) { - fprintf(stderr, "ld_iscsi: Failed to dlsym(__xstat64)\n"); + LD_ISCSI_DPRINTF(0,"Failed to dlsym(__xstat64)"); } } From 54d8aa0ab6758de94e17432ec89ecd8fc0cc101d Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Sun, 21 Oct 2012 08:17:40 +0200 Subject: [PATCH 12/17] LD_ISCSI: use read16 instead of read10 Use READ16 to support for large volumes. READ16 takes a 64bit LBA offset instead of 32bit. --- src/ld_iscsi.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ld_iscsi.c b/src/ld_iscsi.c index b9397a8..118a5d0 100644 --- a/src/ld_iscsi.c +++ b/src/ld_iscsi.c @@ -260,7 +260,7 @@ ssize_t read(int fd, void *buf, size_t count) { if ((iscsi_fd_list[fd].is_iscsi == 1) && (iscsi_fd_list[fd].in_flight == 0)) { uint64_t offset; - uint32_t num_blocks, lba; + uint64_t num_blocks, lba; struct scsi_task *task; if (iscsi_fd_list[fd].dup2fd >= 0) { @@ -280,10 +280,10 @@ ssize_t read(int fd, void *buf, size_t count) count = num_blocks * iscsi_fd_list[fd].block_size; } - LD_ISCSI_DPRINTF(4,"read10_sync: lun %d, lba %u, num_blocks: %u, block_size: %d, offset: %lu count: %lu",iscsi_fd_list[fd].lun,lba,num_blocks,iscsi_fd_list[fd].block_size,offset,count); + LD_ISCSI_DPRINTF(4,"read16_sync: lun %d, lba %lu, num_blocks: %lu, block_size: %d, offset: %lu count: %lu",iscsi_fd_list[fd].lun,lba,num_blocks,iscsi_fd_list[fd].block_size,offset,count); iscsi_fd_list[fd].in_flight = 1; - task = iscsi_read10_sync(iscsi_fd_list[fd].iscsi, iscsi_fd_list[fd].lun, lba, num_blocks * iscsi_fd_list[fd].block_size, iscsi_fd_list[fd].block_size, 0, 0, 0, 0, 0); + task = iscsi_read16_sync(iscsi_fd_list[fd].iscsi, iscsi_fd_list[fd].lun, lba, num_blocks * iscsi_fd_list[fd].block_size, iscsi_fd_list[fd].block_size, 0, 0, 0, 0, 0); iscsi_fd_list[fd].in_flight = 0; if (task == NULL || task->status != SCSI_STATUS_GOOD) { LD_ISCSI_DPRINTF(0,"failed to send read10 command"); From bca635c75da6792c9952a70ccbdc1c7d37b19391 Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Sun, 21 Oct 2012 08:22:17 +0200 Subject: [PATCH 13/17] LD_ISCSI: storage file mode for further usage --- src/ld_iscsi.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ld_iscsi.c b/src/ld_iscsi.c index 118a5d0..cba17c6 100644 --- a/src/ld_iscsi.c +++ b/src/ld_iscsi.c @@ -50,6 +50,7 @@ struct iscsi_fd_list { uint32_t block_size; uint64_t num_blocks; off_t offset; + mode_t mode; }; static struct iscsi_fd_list iscsi_fd_list[ISCSI_MAX_FD]; @@ -137,6 +138,7 @@ int open(const char *path, int flags, mode_t mode) iscsi_fd_list[fd].num_blocks = rc10->lba + 1; iscsi_fd_list[fd].offset = 0; iscsi_fd_list[fd].lun = iscsi_url->lun; + iscsi_fd_list[fd].mode = mode; scsi_free_scsi_task(task); iscsi_destroy_url(iscsi_url); From 8a3e312f06c4dcb6f97dfa1bf654d401705f89bc Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Sun, 21 Oct 2012 08:24:24 +0200 Subject: [PATCH 14/17] LD_ISCSI: Fail open with O_NONBLOCK Non-Blocking I/O is supported nowhere in the code. We should fail until we have support for it. --- src/ld_iscsi.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ld_iscsi.c b/src/ld_iscsi.c index cba17c6..0caf785 100644 --- a/src/ld_iscsi.c +++ b/src/ld_iscsi.c @@ -67,6 +67,12 @@ int open(const char *path, int flags, mode_t mode) struct scsi_task *task; struct scsi_readcapacity10 *rc10; + if (mode & O_NONBLOCK) { + LD_ISCSI_DPRINTF(0,"Non-blocking I/O is currently not supported"); + errno = EINVAL; + return -1; + } + iscsi = iscsi_create_context(initiator); if (iscsi == NULL) { LD_ISCSI_DPRINTF(0,"Failed to create context"); From 08bfabcd366d8745ad0cc0a32c1a3e541645bd28 Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Sun, 21 Oct 2012 08:35:55 +0200 Subject: [PATCH 15/17] LD_ISCSI: Use readcapacity16 instead of readcapacity10 This allows for handling volumes larger than 2TB --- src/ld_iscsi.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/ld_iscsi.c b/src/ld_iscsi.c index 0caf785..8f87ea5 100644 --- a/src/ld_iscsi.c +++ b/src/ld_iscsi.c @@ -65,7 +65,7 @@ int open(const char *path, int flags, mode_t mode) struct iscsi_context *iscsi; struct iscsi_url *iscsi_url; struct scsi_task *task; - struct scsi_readcapacity10 *rc10; + struct scsi_readcapacity16 *rc16; if (mode & O_NONBLOCK) { LD_ISCSI_DPRINTF(0,"Non-blocking I/O is currently not supported"); @@ -109,7 +109,7 @@ int open(const char *path, int flags, mode_t mode) return -1; } - task = iscsi_readcapacity10_sync(iscsi, iscsi_url->lun, 0, 0); + task = iscsi_readcapacity16_sync(iscsi, iscsi_url->lun); if (task == NULL || task->status != SCSI_STATUS_GOOD) { LD_ISCSI_DPRINTF(0,"failed to send readcapacity command"); iscsi_destroy_url(iscsi_url); @@ -118,8 +118,8 @@ int open(const char *path, int flags, mode_t mode) return -1; } - rc10 = scsi_datain_unmarshall(task); - if (rc10 == NULL) { + rc16 = scsi_datain_unmarshall(task); + if (rc16 == NULL) { LD_ISCSI_DPRINTF(0,"failed to unmarshall readcapacity10 data"); scsi_free_scsi_task(task); iscsi_destroy_url(iscsi_url); @@ -127,6 +127,8 @@ int open(const char *path, int flags, mode_t mode) errno = EIO; return -1; } + + LD_ISCSI_DPRINTF(4,"readcapacity16_sync: block_size: %d, num_blocks: %lu",rc16->block_length,rc16->returned_lba + 1); fd = iscsi_get_fd(iscsi); if (fd >= ISCSI_MAX_FD) { @@ -140,8 +142,8 @@ int open(const char *path, int flags, mode_t mode) iscsi_fd_list[fd].is_iscsi = 1; iscsi_fd_list[fd].dup2fd = -1; iscsi_fd_list[fd].iscsi = iscsi; - iscsi_fd_list[fd].block_size = rc10->block_size; - iscsi_fd_list[fd].num_blocks = rc10->lba + 1; + iscsi_fd_list[fd].block_size = rc16->block_length; + iscsi_fd_list[fd].num_blocks = rc16->returned_lba + 1; iscsi_fd_list[fd].offset = 0; iscsi_fd_list[fd].lun = iscsi_url->lun; iscsi_fd_list[fd].mode = mode; From bfac1f85a33d36362e10701fe52ac80d4b7d48cc Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Sun, 21 Oct 2012 09:24:19 +0200 Subject: [PATCH 16/17] LD_SCSI: Introduce get_lba_status in read() For large continous reads I may be benifical to check if the blocks that are going to be read are allocated. If they are not allocated they do not need to be read which massivly speeds up the read. This behaviour is optional and can be turned on with environment variable LD_ISCSI_GET_LBA_STATUS=1 --- src/ld_iscsi.c | 59 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/src/ld_iscsi.c b/src/ld_iscsi.c index 8f87ea5..d3a377b 100644 --- a/src/ld_iscsi.c +++ b/src/ld_iscsi.c @@ -51,6 +51,7 @@ struct iscsi_fd_list { uint64_t num_blocks; off_t offset; mode_t mode; + int get_lba_status; }; static struct iscsi_fd_list iscsi_fd_list[ISCSI_MAX_FD]; @@ -148,6 +149,14 @@ int open(const char *path, int flags, mode_t mode) iscsi_fd_list[fd].lun = iscsi_url->lun; iscsi_fd_list[fd].mode = mode; + if (getenv("LD_ISCSI_GET_LBA_STATUS") != NULL) { + iscsi_fd_list[fd].get_lba_status = atoi(getenv("LD_ISCSI_GET_LBA_STATUS")); + if (rc16->lbpme == 0){ + LD_ISCSI_DPRINTF(1,"Logical unit is fully provisioned. Will skip get_lba_status tasks"); + iscsi_fd_list[fd].get_lba_status = 0; + } + } + scsi_free_scsi_task(task); iscsi_destroy_url(iscsi_url); @@ -272,6 +281,7 @@ ssize_t read(int fd, void *buf, size_t count) uint64_t offset; uint64_t num_blocks, lba; struct scsi_task *task; + struct scsi_get_lba_status *lbas; if (iscsi_fd_list[fd].dup2fd >= 0) { return read(iscsi_fd_list[fd].dup2fd, buf, count); @@ -290,13 +300,58 @@ ssize_t read(int fd, void *buf, size_t count) count = num_blocks * iscsi_fd_list[fd].block_size; } + iscsi_fd_list[fd].in_flight = 1; + if (iscsi_fd_list[fd].get_lba_status != 0) { + LD_ISCSI_DPRINTF(4,"get_lba_status_sync: lun %d, lba %lu, num_blocks: %lu",iscsi_fd_list[fd].lun,lba,num_blocks); + task = iscsi_get_lba_status_sync(iscsi_fd_list[fd].iscsi, iscsi_fd_list[fd].lun, lba, 8+16); + if (task == NULL || task->status != SCSI_STATUS_GOOD) { + LD_ISCSI_DPRINTF(0,"failed to send get_lba_status command"); + iscsi_fd_list[fd].in_flight = 0; + errno = EIO; + return -1; + } + lbas = scsi_datain_unmarshall(task); + if (lbas == NULL) { + LD_ISCSI_DPRINTF(0,"failed to unmarshall get_lba_status data"); + scsi_free_scsi_task(task); + iscsi_fd_list[fd].in_flight = 0; + errno = EIO; + return -1; + } + + u_int32_t i; + LD_ISCSI_DPRINTF(5,"get_lba_status: num_descriptors: %d",lbas->num_descriptors); + u_int32_t _num_allocated=0; + u_int32_t _num_blocks=0; + for (i=0;inum_descriptors;i++) { + struct scsi_lba_status_descriptor *lbasd = &lbas->descriptors[i]; + LD_ISCSI_DPRINTF(5,"get_lba_status_descriptor %d, lba %lu, num_blocks %d, type %d",i,lbasd->lba,lbasd->num_blocks,lbasd->provisioning); + if (lbasd->lba != _num_blocks+lba) { + LD_ISCSI_DPRINTF(0,"get_lba_status response is non-continuous"); + scsi_free_scsi_task(task); + iscsi_fd_list[fd].in_flight = 0; + errno = EIO; + return -1; + } + _num_allocated+=(lbasd->provisioning==0x00)?lbasd->num_blocks:0; + _num_blocks+=lbasd->num_blocks; + } + scsi_free_scsi_task(task); + if (_num_allocated == 0 && _num_blocks >= num_blocks) { + LD_ISCSI_DPRINTF(4,"skipped read16_sync for non-allocated blocks: lun %d, lba %lu, num_blocks: %lu, block_size: %d, offset: %lu count: %lu",iscsi_fd_list[fd].lun,lba,num_blocks,iscsi_fd_list[fd].block_size,offset,count); + memset(buf, 0x00, count); + iscsi_fd_list[fd].offset += count; + iscsi_fd_list[fd].in_flight = 0; + return count; + } + } + LD_ISCSI_DPRINTF(4,"read16_sync: lun %d, lba %lu, num_blocks: %lu, block_size: %d, offset: %lu count: %lu",iscsi_fd_list[fd].lun,lba,num_blocks,iscsi_fd_list[fd].block_size,offset,count); - iscsi_fd_list[fd].in_flight = 1; task = iscsi_read16_sync(iscsi_fd_list[fd].iscsi, iscsi_fd_list[fd].lun, lba, num_blocks * iscsi_fd_list[fd].block_size, iscsi_fd_list[fd].block_size, 0, 0, 0, 0, 0); iscsi_fd_list[fd].in_flight = 0; if (task == NULL || task->status != SCSI_STATUS_GOOD) { - LD_ISCSI_DPRINTF(0,"failed to send read10 command"); + LD_ISCSI_DPRINTF(0,"failed to send read16 command"); errno = EIO; return -1; } From afc963c312acd1278ef1135e011655ab50e04290 Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Sun, 21 Oct 2012 20:29:39 +0200 Subject: [PATCH 17/17] LD_ISCSI: cache last result of get_lba_status get_lba_status returns provisioning for a range of blocks starting from given lba. Especially for sequential reads its likely that the next block read is already covered by the last result. In case there is write support added to ld_iscsi there needs to be an invalidation code for the cache. --- src/ld_iscsi.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/ld_iscsi.c b/src/ld_iscsi.c index d3a377b..915ae26 100644 --- a/src/ld_iscsi.c +++ b/src/ld_iscsi.c @@ -52,6 +52,8 @@ struct iscsi_fd_list { off_t offset; mode_t mode; int get_lba_status; + struct scsi_lba_status_descriptor lbasd_cached; + int lbasd_cache_valid; }; static struct iscsi_fd_list iscsi_fd_list[ISCSI_MAX_FD]; @@ -302,6 +304,17 @@ ssize_t read(int fd, void *buf, size_t count) iscsi_fd_list[fd].in_flight = 1; if (iscsi_fd_list[fd].get_lba_status != 0) { + if (iscsi_fd_list[fd].lbasd_cache_valid==1) { + LD_ISCSI_DPRINTF(5,"cached get_lba_status_descriptor is lba %lu, num_blocks %d, provisioning %d",iscsi_fd_list[fd].lbasd_cached.lba,iscsi_fd_list[fd].lbasd_cached.num_blocks,iscsi_fd_list[fd].lbasd_cached.provisioning); + if (iscsi_fd_list[fd].lbasd_cached.provisioning != 0x00 && lba >= iscsi_fd_list[fd].lbasd_cached.lba && lba+num_blocks < iscsi_fd_list[fd].lbasd_cached.lba+iscsi_fd_list[fd].lbasd_cached.num_blocks) + { + LD_ISCSI_DPRINTF(4,"skipped read16_sync for non-allocated blocks: lun %d, lba %lu, num_blocks: %lu, block_size: %d, offset: %lu count: %lu",iscsi_fd_list[fd].lun,lba,num_blocks,iscsi_fd_list[fd].block_size,offset,count); + memset(buf, 0x00, count); + iscsi_fd_list[fd].offset += count; + iscsi_fd_list[fd].in_flight = 0; + return count; + } + } LD_ISCSI_DPRINTF(4,"get_lba_status_sync: lun %d, lba %lu, num_blocks: %lu",iscsi_fd_list[fd].lun,lba,num_blocks); task = iscsi_get_lba_status_sync(iscsi_fd_list[fd].iscsi, iscsi_fd_list[fd].lun, lba, 8+16); if (task == NULL || task->status != SCSI_STATUS_GOOD) { @@ -325,7 +338,7 @@ ssize_t read(int fd, void *buf, size_t count) u_int32_t _num_blocks=0; for (i=0;inum_descriptors;i++) { struct scsi_lba_status_descriptor *lbasd = &lbas->descriptors[i]; - LD_ISCSI_DPRINTF(5,"get_lba_status_descriptor %d, lba %lu, num_blocks %d, type %d",i,lbasd->lba,lbasd->num_blocks,lbasd->provisioning); + LD_ISCSI_DPRINTF(5,"get_lba_status_descriptor %d, lba %lu, num_blocks %d, provisioning %d",i,lbasd->lba,lbasd->num_blocks,lbasd->provisioning); if (lbasd->lba != _num_blocks+lba) { LD_ISCSI_DPRINTF(0,"get_lba_status response is non-continuous"); scsi_free_scsi_task(task); @@ -335,6 +348,8 @@ ssize_t read(int fd, void *buf, size_t count) } _num_allocated+=(lbasd->provisioning==0x00)?lbasd->num_blocks:0; _num_blocks+=lbasd->num_blocks; + iscsi_fd_list[fd].lbasd_cached=lbas->descriptors[i]; + iscsi_fd_list[fd].lbasd_cache_valid=1; } scsi_free_scsi_task(task); if (_num_allocated == 0 && _num_blocks >= num_blocks) {