From 7bf8091013c2af77c7080391a5cc79fd3d843cc1 Mon Sep 17 00:00:00 2001 From: zhenwei pi Date: Tue, 6 Jun 2023 11:47:25 +0800 Subject: [PATCH] Add iscsi_set_fd_dup_cb libiscsi is widely used by poll driven, include libiscsi itself, QEMU. Using poll works fine after iscsi reconnect, but it does not work in epoll, because the epoll event has been removed during duplicating file descriptor in kernel. Add a new function iscsi_set_fd_dup_cb to set callback, then uplayer gets notified after duplicating. The following codes reproduce this issue, and test this patch by compiling flags -DISCSI_FD_DUP_CB. static void iscsi_epoll_event(struct iscsi_context *iscsi, int epollfd, bool new) { static int epoll_event; struct epoll_event ev = { 0 }; int pevent = iscsi_which_events(iscsi); int event = 0; if (pevent & POLLIN) event |= EPOLLIN; if (pevent & POLLOUT) event |= EPOLLOUT; ev.events = event; ev.data.fd = iscsi_get_fd(iscsi); if (new) epoll_ctl(epollfd, EPOLL_CTL_ADD, ev.data.fd, &ev); else if (epoll_event != event) { epoll_ctl(epollfd, EPOLL_CTL_MOD, ev.data.fd, &ev); } epoll_event = event; } static void iscsi_tsk_cb(struct iscsi_context *iscsi, int status, void *command_data, void *private_data) { printf("iscsi_tsk_cb status %d\n", status); exit(0); } static void iscsi_fd_dup_cb(struct iscsi_context *iscsi, void *opaque) { int epollfd = *(int *)opaque; iscsi_epoll_event(iscsi, epollfd, true); } static void iscsi_on_pollout(struct iscsi_context *iscsi, struct iscsi_url *iscsi_url, int epollfd) { static struct scsi_task *tsk = NULL; iscsi_service(iscsi, POLLOUT); if (!tsk) { tsk = iscsi_readcapacity16_task(iscsi, iscsi_url->lun, iscsi_tsk_cb, NULL); assert(tsk); } } int main(int argc, char *argv[]) { struct iscsi_context *iscsi; struct iscsi_url *iscsi_url; struct epoll_event ev, revent; int epollfd; iscsi = iscsi_create_context("dummy"); assert(iscsi); iscsi_url = iscsi_parse_full_url(iscsi, argv[1]); assert(iscsi_url); iscsi_set_session_type(iscsi, ISCSI_SESSION_NORMAL); iscsi_set_header_digest(iscsi, ISCSI_HEADER_DIGEST_NONE_CRC32C); assert(!iscsi_full_connect_sync(iscsi, iscsi_url->portal, iscsi_url->lun)); epollfd = epoll_create1(0); iscsi_epoll_event(iscsi, epollfd, true); iscsi_set_fd_dup_cb(iscsi, iscsi_fd_dup_cb, &epollfd); iscsi_reconnect(iscsi); while (epoll_wait(epollfd, &revent, 1, 100) >= 0) { if (revent.events & EPOLLIN) iscsi_service(iscsi, POLLIN); if (revent.events & EPOLLOUT) iscsi_on_pollout(iscsi, iscsi_url, epollfd); iscsi_epoll_event(iscsi, epollfd, false); } return 0; } Signed-off-by: zhenwei pi --- include/iscsi-private.h | 12 ++++++++++++ include/iscsi.h | 6 ++++++ lib/connect.c | 2 ++ lib/init.c | 9 +++++++++ lib/iser.c | 4 ++-- lib/libiscsi.def | 1 + lib/socket.c | 2 +- 7 files changed, 33 insertions(+), 3 deletions(-) diff --git a/include/iscsi-private.h b/include/iscsi-private.h index a649254..82a328f 100644 --- a/include/iscsi-private.h +++ b/include/iscsi-private.h @@ -19,6 +19,7 @@ #include #include +#include #if defined(_WIN32) #include @@ -171,6 +172,8 @@ struct iscsi_context { struct iscsi_context *old_iscsi; int retry_cnt; int no_ua_on_reconnect; + void (*fd_dup_cb)(struct iscsi_context *iscsi, void *opaque); + void *fd_dup_opaque; }; #define ISCSI_PDU_IMMEDIATE 0x40 @@ -397,6 +400,15 @@ typedef struct iscsi_transport { int (*which_events)(struct iscsi_context *iscsi); } iscsi_transport; +static inline int iscsi_dup2(struct iscsi_context *iscsi, int oldfd, int newfd) +{ + int ret = dup2(oldfd, newfd); + if ((ret >= 0) && iscsi->fd_dup_cb) + iscsi->fd_dup_cb(iscsi, iscsi->fd_dup_opaque); + + return ret; +} + #ifdef __cplusplus } #endif diff --git a/include/iscsi.h b/include/iscsi.h index 2760fc6..c9f097d 100644 --- a/include/iscsi.h +++ b/include/iscsi.h @@ -1648,6 +1648,12 @@ iscsi_set_reconnect_max_retries(struct iscsi_context *iscsi, int count); EXTERN void iscsi_set_no_ua_on_reconnect(struct iscsi_context *iscsi, int state); +/* Set callback on iscsi file descriptor on duplicating */ +EXTERN void +iscsi_set_fd_dup_cb(struct iscsi_context *iscsi, + void (*cb)(struct iscsi_context *iscsi, void *opaque), + void *opaque); + #ifdef __cplusplus } #endif diff --git a/lib/connect.c b/lib/connect.c index 38b8d19..5234e46 100644 --- a/lib/connect.c +++ b/lib/connect.c @@ -454,6 +454,8 @@ static int reconnect(struct iscsi_context *iscsi, int force) tmp_iscsi->cache_allocations = iscsi->cache_allocations; tmp_iscsi->scsi_timeout = iscsi->scsi_timeout; tmp_iscsi->no_ua_on_reconnect = iscsi->no_ua_on_reconnect; + tmp_iscsi->fd_dup_cb = iscsi->fd_dup_cb; + tmp_iscsi->fd_dup_opaque = iscsi->fd_dup_opaque; tmp_iscsi->reconnect_max_retries = iscsi->reconnect_max_retries; diff --git a/lib/init.c b/lib/init.c index 031299a..6bcc761 100644 --- a/lib/init.c +++ b/lib/init.c @@ -831,3 +831,12 @@ iscsi_set_timeout(struct iscsi_context *iscsi, int timeout) iscsi->scsi_timeout = timeout; return 0; } + +void +iscsi_set_fd_dup_cb(struct iscsi_context *iscsi, + void (*cb)(struct iscsi_context *iscsi, void *opaque), + void *opaque) +{ + iscsi->fd_dup_cb = cb; + iscsi->fd_dup_opaque = opaque; +} diff --git a/lib/iser.c b/lib/iser.c index d5cc8b9..3740564 100644 --- a/lib/iser.c +++ b/lib/iser.c @@ -1525,7 +1525,7 @@ static int iser_connected_handler(struct rdma_cm_id *cma_id) { struct iscsi_context *iscsi = cma_id->context; struct iser_conn *iser_conn = iscsi->opaque; - if (dup2(iser_conn->comp_channel->fd, iscsi->fd) == -1) { + if (iscsi_dup2(iscsi, iser_conn->comp_channel->fd, iscsi->fd) == -1) { return -1; } @@ -1683,7 +1683,7 @@ iscsi_iser_connect(struct iscsi_context *iscsi, union socket_address *sa,__attri goto error; } - if (dup2(iser_conn->cma_channel->fd, iscsi->fd) < 0) { + if (iscsi_dup2(iscsi, iser_conn->cma_channel->fd, iscsi->fd) < 0) { iscsi_set_error(iscsi, "Failed dup event channel fd"); goto error; } diff --git a/lib/libiscsi.def b/lib/libiscsi.def index a32be9d..c56fe59 100644 --- a/lib/libiscsi.def +++ b/lib/libiscsi.def @@ -286,3 +286,4 @@ scsi_task_set_iov_out scsi_version_to_str scsi_version_descriptor_to_str win32_poll +iscsi_set_fd_dup_cb diff --git a/lib/socket.c b/lib/socket.c index 05a3e7d..aaa061a 100644 --- a/lib/socket.c +++ b/lib/socket.c @@ -228,7 +228,7 @@ static int iscsi_tcp_connect(struct iscsi_context *iscsi, union socket_address * } if (iscsi->old_iscsi && iscsi->fd != iscsi->old_iscsi->fd) { - if (dup2(iscsi->fd, iscsi->old_iscsi->fd) == -1) { + if (iscsi_dup2(iscsi, iscsi->fd, iscsi->old_iscsi->fd) == -1) { return -1; } close(iscsi->fd);