From f8542ab5cc2ac16d343ca661ad1c8d384c13da1c Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Fri, 7 Mar 2025 07:04:17 +1000 Subject: [PATCH 01/16] Add checks if pthreads are available and enable HAVE_MULTITHREADING Signed-off-by: Ronnie Sahlberg --- configure.ac | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/configure.ac b/configure.ac index 13ae3bb..033306f 100644 --- a/configure.ac +++ b/configure.ac @@ -216,6 +216,16 @@ AM_CONDITIONAL([HAVE_LINUX_ISER], [test $libiscsi_cv_HAVE_LINUX_ISER = yes]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include ]], [[return RDMA_OPTION_ID_ACK_TIMEOUT;]])],[AC_DEFINE([HAVE_RDMA_ACK_TIMEOUT],[1],[Define to 1 if you have RDMA ack timeout support])],[]) +# check for pthread +AC_CACHE_CHECK([for pthread support],libiscsi_cv_HAVE_PTHREAD,[ +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #include ]], [[pthread_t thread1, thread2;]])],[libiscsi_cv_HAVE_PTHREAD=yes],[libiscsi_cv_HAVE_PTHREAD=no])]) + if test x"$libiscsi_cv_HAVE_PTHREAD" = x"yes"; then + AC_DEFINE(HAVE_PTHREAD,1,[Whether we have pthread support]) + AC_DEFINE(HAVE_MULTITHREADING,1,[Whether we have multithreading support]) + fi +AM_CONDITIONAL([HAVE_PTHREAD], [test x$libiscsi_cv_HAVE_PTHREAD = xyes]) + AC_CACHE_CHECK([whether libcunit is available], [ac_cv_have_cunit], [ac_save_CFLAGS="$CFLAGS" From cb44ad4e267e709d827e16a4dd87812f2facdf3b Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Fri, 7 Mar 2025 08:23:32 +1000 Subject: [PATCH 02/16] Add multithreading helpers Add an abstraction for mutexts and threads that handles both pthread api and native win32 api Signed-off-by: Ronnie Sahlberg --- configure.ac | 4 + include/iscsi-multithreading.h | 73 +++++++ include/iscsi-private.h | 46 +++++ lib/Makefile.am | 1 + lib/init.c | 9 + lib/multithreading.c | 338 +++++++++++++++++++++++++++++++++ 6 files changed, 471 insertions(+) create mode 100644 include/iscsi-multithreading.h create mode 100644 lib/multithreading.c diff --git a/configure.ac b/configure.ac index 033306f..8ab9b6c 100644 --- a/configure.ac +++ b/configure.ac @@ -216,6 +216,10 @@ AM_CONDITIONAL([HAVE_LINUX_ISER], [test $libiscsi_cv_HAVE_LINUX_ISER = yes]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include ]], [[return RDMA_OPTION_ID_ACK_TIMEOUT;]])],[AC_DEFINE([HAVE_RDMA_ACK_TIMEOUT],[1],[Define to 1 if you have RDMA ack timeout support])],[]) +# check for stdatomic.h +dnl Check for stdatomic.h +AC_CHECK_HEADERS([stdatomic.h]) + # check for pthread AC_CACHE_CHECK([for pthread support],libiscsi_cv_HAVE_PTHREAD,[ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ diff --git a/include/iscsi-multithreading.h b/include/iscsi-multithreading.h new file mode 100644 index 0000000..39134b0 --- /dev/null +++ b/include/iscsi-multithreading.h @@ -0,0 +1,73 @@ +/* -*- mode:c; tab-width:8; c-basic-offset:8; indent-tabs-mode:nil; -*- */ +/* + Copyright (C) 2025 by Ronnie Sahlberg + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with this program; if not, see . +*/ + +#ifndef _LIBISCSI_MULTITHREADING_H_ +#define _LIBISCSI_MULTITHREADING_H_ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef HAVE_MULTITHREADING + +#ifdef WIN32 +typedef HANDLE libiscsi_thread_t; +typedef HANDLE libiscsi_mutex_t; +typedef HANDLE libiscsi_sem_t; +typedef DWORD iscsi_tid_t; +#elif defined(HAVE_PTHREAD) +#include +typedef pthread_t libiscsi_thread_t; +typedef pthread_mutex_t libiscsi_mutex_t; + +#if defined(__APPLE__) && defined(HAVE_DISPATCH_DISPATCH_H) +#include +typedef dispatch_semaphore_t libiscsi_sem_t; +#else +#include +typedef sem_t libiscsi_sem_t; +#endif +#ifdef HAVE_PTHREAD_THREADID_NP +typedef uint64_t iscsi_tid_t; +#else +typedef pid_t iscsi_tid_t; +#endif +#endif /* HAVE_PTHREAD */ + +iscsi_tid_t iscsi_mt_get_tid(void); +int iscsi_mt_mutex_init(libiscsi_mutex_t *mutex); +int iscsi_mt_mutex_destroy(libiscsi_mutex_t *mutex); +int iscsi_mt_mutex_lock(libiscsi_mutex_t *mutex); +int iscsi_mt_mutex_unlock(libiscsi_mutex_t *mutex); + +int iscsi_mt_sem_init(libiscsi_sem_t *sem, int value); +int iscsi_mt_sem_destroy(libiscsi_sem_t *sem); +int iscsi_mt_sem_post(libiscsi_sem_t *sem); +int iscsi_mt_sem_wait(libiscsi_sem_t *sem); + +#endif /* HAVE_MULTITHREADING */ + +#ifdef __cplusplus +} +#endif + +#endif /* !_LIBISCSI_MULTITHREADING_H_ */ diff --git a/include/iscsi-private.h b/include/iscsi-private.h index 065f243..8656050 100644 --- a/include/iscsi-private.h +++ b/include/iscsi-private.h @@ -28,6 +28,39 @@ #include "iscsi.h" +#ifdef HAVE_MULTITHREADING +#ifdef HAVE_STDATOMIC_H +#include +#define ATOMIC_INC(rpc, x) \ + atomic_fetch_add_explicit(&x, 1, memory_order_relaxed) +#define ATOMIC_DEC(rpc, x) \ + atomic_fetch_sub_explicit(&x, 1, memory_order_relaxed) +#else /* HAVE_STDATOMIC_H */ +#define ATOMIC_INC(rpc, x) \ + if (rpc->multithreading_enabled) { \ + nfs_mt_mutex_lock(&rpc->atomic_int_mutex); \ + } \ + x++; \ + if (rpc->multithreading_enabled) { \ + nfs_mt_mutex_unlock(&rpc->atomic_int_mutex); \ + } +#define ATOMIC_DEC(rpc, x) \ + if (rpc->multithreading_enabled) { \ + nfs_mt_mutex_lock(&rpc->atomic_int_mutex); \ + } \ + x--; \ + if (rpc->multithreading_enabled) { \ + nfs_mt_mutex_unlock(&rpc->atomic_int_mutex); \ + } +#endif /* HAVE_STDATOMIC_H */ +#else /* HAVE_MULTITHREADING */ +/* no multithreading support, no need to protect the increment */ +#define ATOMIC_INC(rpc, x) x++ +#define ATOMIC_DEC(rpc, x) x-- +#endif /* HAVE_MULTITHREADING */ + +#include "iscsi-multithreading.h" + #ifdef __cplusplus extern "C" { #endif @@ -184,6 +217,19 @@ struct iscsi_context { int no_ua_on_reconnect; void (*fd_dup_cb)(struct iscsi_context *iscsi, void *opaque); void *fd_dup_opaque; + +#ifdef HAVE_MULTITHREADING + int multithreading_enabled; + libiscsi_mutex_t iscsi_mutex; + libiscsi_thread_t service_thread; + int poll_timeout; + //libnfs_mutex_t nfs4_open_counter_mutex; + //libnfs_mutex_t nfs4_open_call_mutex; + //struct nfs_thread_context *thread_ctx; +#ifndef HAVE_STDATOMIC_H + libiscsi_mutex_t atomic_int_mutex; +#endif /* HAVE_STDATOMIC_H */ +#endif /* HAVE_MULTITHREADING */ }; #define ISCSI_PDU_IMMEDIATE 0x40 diff --git a/lib/Makefile.am b/lib/Makefile.am index c277041..5838b10 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -5,6 +5,7 @@ noinst_LTLIBRARIES = libiscsipriv.la libiscsipriv_la_SOURCES = \ connect.c crc32c.c discovery.c init.c \ login.c nop.c pdu.c iscsi-command.c \ + multithreading.c \ scsi-lowlevel.c socket.c sync.c task_mgmt.c \ logging.c utils.c sha1.c sha224-256.c sha3.c diff --git a/lib/init.c b/lib/init.c index 7dc5348..233a5c2 100644 --- a/lib/init.c +++ b/lib/init.c @@ -216,6 +216,11 @@ iscsi_create_context(const char *initiator_name) memset(iscsi, 0, sizeof(struct iscsi_context)); +#ifdef HAVE_MULTITHREADING + iscsi_mt_mutex_init(&iscsi->iscsi_mutex); + iscsi->poll_timeout = 100; +#endif /* HAVE_MULTITHREADING */ + /* initalize transport of context */ if (iscsi_init_transport(iscsi, TCP_TRANSPORT)) { free(iscsi); @@ -431,6 +436,10 @@ iscsi_destroy_context(struct iscsi_context *iscsi) iscsi_destroy_context(iscsi->old_iscsi); } +#ifdef HAVE_MULTITHREADING + iscsi_mt_mutex_destroy(&iscsi->iscsi_mutex); +#endif /* HAVE_MULTITHREADING */ + memset(iscsi, 0, sizeof(struct iscsi_context)); free(iscsi); diff --git a/lib/multithreading.c b/lib/multithreading.c new file mode 100644 index 0000000..138f391 --- /dev/null +++ b/lib/multithreading.c @@ -0,0 +1,338 @@ +/* -*- mode:c; tab-width:8; c-basic-offset:8; indent-tabs-mode:nil; -*- */ +/* + Copyright (C) 2025 by Ronnie Sahlberg + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with this program; if not, see . +*/ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef AROS +#include "aros_compat.h" +#endif + +#ifdef WIN32 +#include "win32/win32_compat.h" +#endif + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#ifdef HAVE_POLL_H +#include +#endif + +#ifdef HAVE_SYS_TIME_H +#include +#endif + +#include +#include "iscsi.h" +#include "iscsi-private.h" + +#ifdef HAVE_MULTITHREADING + +#ifdef WIN32 +iscsi_tid_t iscsi_mt_get_tid(void) +{ + return GetCurrentThreadId(); +} +static void* iscsi_mt_service_thread(void* arg) +{ + struct iscsi_context* iscsi = (struct iscsi_context*)arg; + struct pollfd pfd; + int revents; + int ret; + + iscsi->multithreading_enabled = 1; + + while (iscsi->multithreading_enabled) { + pfd.fd = iscsi_get_fd(iscsi); + pfd.events = iscsi_which_events(iscsi); + pfd.revents = 0; + + ret = poll(&pfd, 1, 0); + if (ret < 0) { + iscsi_set_error(iscsi, "Poll failed"); + revents = -1; + } + else { + revents = pfd.revents; + } + if (iscsi_service(iscsi, revents) < 0) { + if (revents != -1) + iscsi_set_error(iscsi, "iscsi_service failed"); + } + } + return NULL; +} + +static DWORD WINAPI service_thread_init(LPVOID lpParam) +{ + HANDLE hStdout; + struct iscsi_context* iscsi; + + hStdout = GetStdHandle(STD_OUTPUT_HANDLE); + if (hStdout == INVALID_HANDLE_VALUE) { + return 1; + } + iscsi = (struct iscsi_context *)lpParam; + iscsi_mt_service_thread(iscsi); + return 0; +} + +int iscsi_mt_service_thread_start(struct iscsi_context* iscsi) +{ + iscsi->iscsii->service_thread = CreateThread(NULL, 1024*1024, service_thread_init, iscsi, 0, NULL); + if (iscsi->iscsii->service_thread == NULL) { + iscsi_set_error(iscsi, "Failed to start service thread"); + return -1; + } + while (iscsi->multithreading_enabled == 0) { + Sleep(100); + } + return 0; +} + +void iscsi_mt_service_thread_stop(struct iscsi_context* iscsi) +{ + iscsi->multithreading_enabled = 0; + while (WaitForSingleObject(iscsi->iscsii->service_thread, INFINITE) != WAIT_OBJECT_0); +} + +int iscsi_mt_mutex_init(libiscsi_mutex_t* mutex) +{ + *mutex = CreateSemaphoreA(NULL, 1, 1, NULL); + return 0; +} + +int iscsi_mt_mutex_destroy(libiscsi_mutex_t* mutex) +{ + CloseHandle(*mutex); + return 0; +} + +int iscsi_mt_mutex_lock(libiscsi_mutex_t* mutex) +{ + while (WaitForSingleObject(*mutex, INFINITE) != WAIT_OBJECT_0); + return 0; +} + +int iscsi_mt_mutex_unlock(libiscsi_mutex_t* mutex) +{ + ReleaseSemaphore(*mutex, 1, NULL); + return 0; +} + +int iscsi_mt_sem_init(libiscsi_sem_t* sem, int value) +{ + *sem = CreateSemaphoreA(NULL, 0, 16, NULL); + return 0; +} + +int iscsi_mt_sem_destroy(libiscsi_sem_t* sem) +{ + CloseHandle(*sem); + return 0; +} + +int iscsi_mt_sem_post(libiscsi_sem_t* sem) +{ + ReleaseSemaphore(*sem, 1, NULL); + return 0; +} + +int iscsi_mt_sem_wait(libiscsi_sem_t* sem) +{ + while (WaitForSingleObject(*sem, INFINITE) != WAIT_OBJECT_0); + return 0; +} + +#elif defined(HAVE_PTHREAD) /* WIN32 */ + +#include +#include + +iscsi_tid_t iscsi_mt_get_tid(void) +{ +#ifdef HAVE_PTHREAD_THREADID_NP + iscsi_tid_t tid; + pthread_threadid_np(NULL, &tid); + return tid; +#elif defined(SYS_gettid) + pid_t tid = syscall(SYS_gettid); + return tid; +#else +#error "SYS_gettid unavailable on this system" +#endif +} + +static void *iscsi_mt_service_thread(void *arg) +{ + struct iscsi_context *iscsi = (struct iscsi_context *)arg; + struct pollfd pfd; + int revents; + int ret; + + iscsi->multithreading_enabled = 1; + + while (iscsi->multithreading_enabled) { + pfd.fd = iscsi_get_fd(iscsi); + pfd.events = iscsi_which_events(iscsi); + pfd.revents = 0; + + ret = poll(&pfd, 1, iscsi->poll_timeout); + if (ret < 0) { + iscsi_set_error(iscsi, "Poll failed"); + revents = -1; + } else { + revents = pfd.revents; + } + if (iscsi_service(iscsi, revents) < 0) { + if (revents != -1) + iscsi_set_error(iscsi, "iscsi_service failed"); + } + } + return NULL; +} + +int iscsi_mt_service_thread_start(struct iscsi_context *iscsi) +{ + if (pthread_create(&iscsi->service_thread, NULL, + &iscsi_mt_service_thread, iscsi)) { + iscsi_set_error(iscsi, "Failed to start service thread"); + return -1; + } + while (iscsi->multithreading_enabled == 0) { + struct timespec ts = {0, 1000000}; + nanosleep(&ts, NULL); + } + return 0; +} + +void iscsi_mt_service_thread_stop(struct iscsi_context *iscsi) +{ + iscsi->multithreading_enabled = 0; + pthread_join(iscsi->service_thread, NULL); +} + +/* + * If this is enabled we check for the following locking violations, at the + * (slight) cost of performance: + * - Thread holding the lock again tries to lock. + * - Thread not holding the lock tries to unlock. + * + * This is very useful for catching any coding errors. + * The performance hit is not very significant so you can leave it enabled, + * but if you really care then once the code has been vetted, this can be + * undef'ed to get the perf back. + */ +#define DEBUG_PTHREAD_LOCKING_VIOLATIONS + +int iscsi_mt_mutex_init(libiscsi_mutex_t *mutex) +{ + int ret; +#ifdef DEBUG_PTHREAD_LOCKING_VIOLATIONS + pthread_mutexattr_t attr; + + ret = pthread_mutexattr_init(&attr); + if (ret != 0) { + return ret; + } + + ret = pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); + if (ret != 0) { + return ret; + } + + ret = pthread_mutex_init(mutex, &attr); + if (ret != 0) { + return ret; + } +#else + ret = pthread_mutex_init(mutex, NULL); + assert(ret == 0); +#endif + return ret; +} + +int iscsi_mt_mutex_destroy(libiscsi_mutex_t *mutex) +{ + return pthread_mutex_destroy(mutex); +} + +int iscsi_mt_mutex_lock(libiscsi_mutex_t *mutex) +{ + return pthread_mutex_lock(mutex); +} + +int iscsi_mt_mutex_unlock(libiscsi_mutex_t *mutex) +{ + return pthread_mutex_unlock(mutex); +} + +#if defined(__APPLE__) && defined(HAVE_DISPATCH_DISPATCH_H) +int iscsi_mt_sem_init(libiscsi_sem_t *sem, int value) +{ + if ((*sem = dispatch_semaphore_create(value)) != NULL) + return 0; + return -1; +} + +int iscsi_mt_sem_destroy(libiscsi_sem_t *sem) +{ + dispatch_release(*sem); + return 0; +} + +int iscsi_mt_sem_post(libiscsi_sem_t *sem) +{ + dispatch_semaphore_signal(*sem); + return 0; +} + +int iscsi_mt_sem_wait(libiscsi_sem_t *sem) +{ + dispatch_semaphore_wait(*sem, DISPATCH_TIME_FOREVER); + return 0; +} + +#else +int iscsi_mt_sem_init(libiscsi_sem_t *sem, int value) +{ + return sem_init(sem, 0, value); +} + +int iscsi_mt_sem_destroy(libiscsi_sem_t *sem) +{ + return sem_destroy(sem); +} + +int iscsi_mt_sem_post(libiscsi_sem_t *sem) +{ + return sem_post(sem); +} + +int iscsi_mt_sem_wait(libiscsi_sem_t *sem) +{ + return sem_wait(sem); +} +#endif + +#endif /* HAVE_PTHREAD */ + +#endif /* HAVE_MULTITHREADING */ + From 3c48aea225ce14c65a224d1d3fdd489249c6a31d Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Tue, 25 Mar 2025 12:42:31 +1000 Subject: [PATCH 03/16] Add initial multithreading support and example This is the basic support for doing i/o in a separate worker thread. It is still not threads safe but a start. Now we need to protect all variables such as outqueue, waitpdu and friends. Signed-off-by: Ronnie Sahlberg --- Makefile.am | 2 +- examples/Makefile.am | 2 +- examples/iscsi-pthreads-inq.c | 343 ++++++++++++++++++++++++++++++++++ include/iscsi-private.h | 18 +- include/iscsi.h | 12 ++ lib/libiscsi.def | 2 + lib/libiscsi.syms.in | 2 + lib/multithreading.c | 1 + lib/pdu.c | 12 +- lib/sync.c | 193 +++++++++++-------- 10 files changed, 492 insertions(+), 95 deletions(-) create mode 100644 examples/iscsi-pthreads-inq.c diff --git a/Makefile.am b/Makefile.am index abefb53..bc1e08b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -29,5 +29,5 @@ pkgconfig_DATA = libiscsi.pc iscsi_includedir = $(includedir)/iscsi dist_iscsi_include_HEADERS = include/iscsi.h include/scsi-lowlevel.h dist_noinst_HEADERS = include/iscsi-private.h include/md5.h include/slist.h \ - include/iser-private.h include/utils.h + include/iser-private.h include/iscsi-multithreading.h include/utils.h diff --git a/examples/Makefile.am b/examples/Makefile.am index 0d39978..bc00f6c 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -3,4 +3,4 @@ AM_CFLAGS=$(WARN_CFLAGS) AM_LDFLAGS=-no-undefined LIBS=../lib/libiscsi.la -noinst_PROGRAMS = iscsiclient iscsi-dd +noinst_PROGRAMS = iscsiclient iscsi-dd iscsi-pthreads-inq diff --git a/examples/iscsi-pthreads-inq.c b/examples/iscsi-pthreads-inq.c new file mode 100644 index 0000000..64117fd --- /dev/null +++ b/examples/iscsi-pthreads-inq.c @@ -0,0 +1,343 @@ +/* + Copyright (C) 2025 by Ronnie Sahlberg + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . +*/ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_POLL_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include "iscsi.h" +#include "scsi-lowlevel.h" + +const char *initiator = "iqn.2007-10.com.github:sahlberg:libiscsi:iscsi-inq"; + +void inquiry_block_limits(struct scsi_inquiry_block_limits *inq) +{ + printf("wsnz:%d\n", inq->wsnz); + printf("maximum compare and write length:%" PRIu8 "\n", inq->max_cmp); + printf("optimal transfer length granularity:%" PRIu16 "\n", inq->opt_gran); + printf("maximum transfer length:%" PRIu32 "\n", inq->max_xfer_len); + printf("optimal transfer length:%" PRIu32 "\n",inq->opt_xfer_len); + printf("maximum prefetch xdread xdwrite transfer length:%" PRIu32 "\n", inq->max_prefetch); + printf("maximum unmap lba count:%" PRIu32 "\n", inq->max_unmap); + printf("maximum unmap block descriptor count:%" PRIu32 "\n", inq->max_unmap_bdc); + printf("optimal unmap granularity:%" PRIu32 "\n", inq->opt_unmap_gran); + printf("ugavalid:%d\n", inq->ugavalid); + printf("unmap granularity alignment:%" PRIu32 "\n", inq->unmap_gran_align); + printf("maximum write same length:%" PRIu64 "\n", inq->max_ws_len); +} + +void inquiry_logical_block_provisioning(struct scsi_inquiry_logical_block_provisioning *inq) +{ + printf("Threshold Exponent:%d\n", inq->threshold_exponent); + printf("lbpu:%d\n", inq->lbpu); + printf("lbpws:%d\n", inq->lbpws); + printf("lbpws10:%d\n", inq->lbpws10); + printf("lbprz:%d\n", inq->lbprz); + printf("anc_sup:%d\n", inq->anc_sup); + printf("dp:%d\n", inq->dp); + printf("provisioning type:%d\n", inq->provisioning_type); +} + +void inquiry_block_device_characteristics(struct scsi_inquiry_block_device_characteristics *inq) +{ + printf("Medium Rotation Rate:%dRPM\n", inq->medium_rotation_rate); +} + +void inquiry_device_identification(struct scsi_inquiry_device_identification *inq) +{ + struct scsi_inquiry_device_designator *dev; + int i; + + printf("Peripheral Qualifier:%s\n", + scsi_devqualifier_to_str(inq->qualifier)); + printf("Peripheral Device Type:%s\n", + scsi_devtype_to_str(inq->device_type)); + printf("Page Code:(0x%02x) %s\n", + inq->pagecode, scsi_inquiry_pagecode_to_str(inq->pagecode)); + + for (i=0, dev = inq->designators; dev; i++, dev = dev->next) { + printf("DEVICE DESIGNATOR #%d\n", i); + if (dev->piv != 0) { + printf("Device Protocol Identifier:(%d) %s\n", dev->protocol_identifier, scsi_protocol_identifier_to_str(dev->protocol_identifier)); + } + printf("Code Set:(%d) %s\n", dev->code_set, scsi_codeset_to_str(dev->code_set)); + printf("PIV:%d\n", dev->piv); + printf("Association:(%d) %s\n", dev->association, scsi_association_to_str(dev->association)); + printf("Designator Type:(%d) %s\n", dev->designator_type, scsi_designator_type_to_str(dev->designator_type)); + printf("Designator:[%s]\n", dev->designator); + } +} + +void inquiry_unit_serial_number(struct scsi_inquiry_unit_serial_number *inq) +{ + printf("Unit Serial Number:[%s]\n", inq->usn); +} + +void inquiry_supported_pages(struct scsi_inquiry_supported_pages *inq) +{ + int i; + + for (i = 0; i < inq->num_pages; i++) { + printf("Page:0x%02x %s\n", inq->pages[i], scsi_inquiry_pagecode_to_str(inq->pages[i])); + } +} + +void inquiry_standard(struct scsi_inquiry_standard *inq) +{ + int i; + + printf("Peripheral Qualifier:%s\n", + scsi_devqualifier_to_str(inq->qualifier)); + printf("Peripheral Device Type:%s\n", + scsi_devtype_to_str(inq->device_type)); + printf("Removable:%d\n", inq->rmb); + printf("Version:%d %s\n", inq->version, scsi_version_to_str(inq->version)); + printf("NormACA:%d\n", inq->normaca); + printf("HiSup:%d\n", inq->hisup); + printf("ReponseDataFormat:%d\n", inq->response_data_format); + printf("SCCS:%d\n", inq->sccs); + printf("ACC:%d\n", inq->acc); + printf("TPGS:%d\n", inq->tpgs); + printf("3PC:%d\n", inq->threepc); + printf("Protect:%d\n", inq->protect); + printf("EncServ:%d\n", inq->encserv); + printf("MultiP:%d\n", inq->multip); + printf("SYNC:%d\n", inq->sync); + printf("CmdQue:%d\n", inq->cmdque); + printf("Vendor:%s\n", inq->vendor_identification); + printf("Product:%s\n", inq->product_identification); + printf("Revision:%s\n", inq->product_revision_level); + + for (i = 0; i < 8; i++) { + if (inq->version_descriptor[i] == 0) { + continue; + } + + printf("Version Descriptor:%04x %s\n", + inq->version_descriptor[i], + scsi_version_descriptor_to_str( + inq->version_descriptor[i])); + } +} + +void do_inquiry(struct iscsi_context *iscsi, int lun, int evpd, int pc) +{ + struct scsi_task *task; + int full_size; + void *inq; + + /* See how big this inquiry data is */ + task = iscsi_inquiry_sync(iscsi, lun, evpd, pc, 64); + if (task == NULL || task->status != SCSI_STATUS_GOOD) { + fprintf(stderr, "Inquiry command failed : %s\n", iscsi_get_error(iscsi)); + exit(10); + } + + full_size = scsi_datain_getfullsize(task); + if (full_size > task->datain.size) { + scsi_free_scsi_task(task); + + /* we need more data for the full list */ + if ((task = iscsi_inquiry_sync(iscsi, lun, evpd, pc, full_size)) == NULL) { + fprintf(stderr, "Inquiry command failed : %s\n", iscsi_get_error(iscsi)); + exit(10); + } + } + + inq = scsi_datain_unmarshall(task); + if (inq == NULL) { + fprintf(stderr, "failed to unmarshall inquiry datain blob\n"); + exit(10); + } + + if (evpd == 0) { + inquiry_standard(inq); + } else { + switch (pc) { + case SCSI_INQUIRY_PAGECODE_SUPPORTED_VPD_PAGES: + inquiry_supported_pages(inq); + break; + case SCSI_INQUIRY_PAGECODE_UNIT_SERIAL_NUMBER: + inquiry_unit_serial_number(inq); + break; + case SCSI_INQUIRY_PAGECODE_DEVICE_IDENTIFICATION: + inquiry_device_identification(inq); + break; + case SCSI_INQUIRY_PAGECODE_BLOCK_LIMITS: + inquiry_block_limits(inq); + break; + case SCSI_INQUIRY_PAGECODE_BLOCK_DEVICE_CHARACTERISTICS: + inquiry_block_device_characteristics(inq); + break; + case SCSI_INQUIRY_PAGECODE_LOGICAL_BLOCK_PROVISIONING: + inquiry_logical_block_provisioning(inq); + break; + default: + fprintf(stderr, "Usupported pagecode:0x%02x\n", pc); + } + } + scsi_free_scsi_task(task); +} + + +void print_usage(void) +{ + fprintf(stderr, "Usage: iscsi-inq [-?] [-?|--help] [--usage] [-i|--initiator-name=iqn-name]\n" + "\t\t[-e|--evpd=integer] [-c|--pagecode=integer] \n"); +} + +void print_help(void) +{ + fprintf(stderr, "Usage: iscsi-inq [OPTION...] \n"); + fprintf(stderr, " -i, --initiator-name=iqn-name Initiatorname to use\n"); + fprintf(stderr, " -e, --evpd=integer evpd\n"); + fprintf(stderr, " -c, --pagecode=integer page code\n"); + fprintf(stderr, " -d, --debug=integer debug level (0=disabled)\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "Help options:\n"); + fprintf(stderr, " -?, --help Show this help message\n"); + fprintf(stderr, " --usage Display brief usage message\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "iSCSI URL format : %s\n", ISCSI_URL_SYNTAX); + fprintf(stderr, "\n"); + fprintf(stderr, " is either of:\n"); + fprintf(stderr, " \"hostname\" iscsi.example\n"); + fprintf(stderr, " \"ipv4-address\" 10.1.1.27\n"); + fprintf(stderr, " \"ipv6-address\" [fce0::1]\n"); +} + +int main(int argc, char *argv[]) +{ + struct iscsi_context *iscsi; + char *url = NULL; + struct iscsi_url *iscsi_url = NULL; + int evpd = 0, pagecode = 0; + int show_help = 0, show_usage = 0, debug = 0; + int c; + + static struct option long_options[] = { + {"help", no_argument, NULL, 'h'}, + {"usage", no_argument, NULL, 'u'}, + {"debug", required_argument, NULL, 'd'}, + {"initiator-name", required_argument, NULL, 'i'}, + {"evpd", required_argument, NULL, 'e'}, + {"pagecode", required_argument, NULL, 'c'}, + {0, 0, 0, 0} + }; + int option_index; + + while ((c = getopt_long(argc, argv, "h?ud:i:e:c:", long_options, + &option_index)) != -1) { + switch (c) { + case 'h': + case '?': + show_help = 1; + break; + case 'u': + show_usage = 1; + break; + case 'd': + debug = strtol(optarg, NULL, 0); + break; + case 'i': + initiator = optarg; + break; + case 'e': + evpd = strtol(optarg, NULL, 0); + break; + case 'c': + pagecode = strtol(optarg, NULL, 0); + break; + default: + fprintf(stderr, "Unrecognized option '%c'\n\n", c); + print_help(); + exit(0); + } + } + + if (show_help != 0) { + print_help(); + exit(0); + } + + if (show_usage != 0) { + print_usage(); + exit(0); + } + + iscsi = iscsi_create_context(initiator); + if (iscsi == NULL) { + fprintf(stderr, "Failed to create context\n"); + exit(10); + } + + if (debug > 0) { + iscsi_set_log_level(iscsi, debug); + iscsi_set_log_fn(iscsi, iscsi_log_to_stderr); + } + + if (argv[optind] != NULL) { + url = strdup(argv[optind]); + } + if (url == NULL) { + fprintf(stderr, "You must specify the URL\n"); + print_usage(); + exit(10); + } + iscsi_url = iscsi_parse_full_url(iscsi, url); + + free(url); + + if (iscsi_url == NULL) { + fprintf(stderr, "Failed to parse URL: %s\n", + iscsi_get_error(iscsi)); + exit(10); + } + + iscsi_set_session_type(iscsi, ISCSI_SESSION_NORMAL); + iscsi_set_header_digest(iscsi, ISCSI_HEADER_DIGEST_NONE_CRC32C); + + if (iscsi_full_connect_sync(iscsi, iscsi_url->portal, iscsi_url->lun) != 0) { + fprintf(stderr, "Login Failed. %s\n", iscsi_get_error(iscsi)); + iscsi_destroy_url(iscsi_url); + iscsi_destroy_context(iscsi); + exit(10); + } + + if (iscsi_mt_service_thread_start(iscsi)) { + fprintf(stderr, "failed to start service thread\n"); + exit(10); + } + + do_inquiry(iscsi, iscsi_url->lun, evpd, pagecode); + iscsi_destroy_url(iscsi_url); + + iscsi_mt_service_thread_stop(iscsi); + iscsi_logout_sync(iscsi); + iscsi_destroy_context(iscsi); + return 0; +} + diff --git a/include/iscsi-private.h b/include/iscsi-private.h index 8656050..1cd0b5b 100644 --- a/include/iscsi-private.h +++ b/include/iscsi-private.h @@ -37,21 +37,14 @@ atomic_fetch_sub_explicit(&x, 1, memory_order_relaxed) #else /* HAVE_STDATOMIC_H */ #define ATOMIC_INC(rpc, x) \ - if (rpc->multithreading_enabled) { \ - nfs_mt_mutex_lock(&rpc->atomic_int_mutex); \ + iscs_mt_mutex_lock(&iscs->atomic_int_mutex); \ } \ x++; \ - if (rpc->multithreading_enabled) { \ - nfs_mt_mutex_unlock(&rpc->atomic_int_mutex); \ - } + iscsi_mt_mutex_unlock(&iscsi->atomic_int_mutex); #define ATOMIC_DEC(rpc, x) \ - if (rpc->multithreading_enabled) { \ - nfs_mt_mutex_lock(&rpc->atomic_int_mutex); \ - } \ + nfs_mt_mutex_lock(&rpc->atomic_int_mutex); \ x--; \ - if (rpc->multithreading_enabled) { \ - nfs_mt_mutex_unlock(&rpc->atomic_int_mutex); \ - } + nfs_mt_mutex_unlock(&rpc->atomic_int_mutex); #endif /* HAVE_STDATOMIC_H */ #else /* HAVE_MULTITHREADING */ /* no multithreading support, no need to protect the increment */ @@ -223,9 +216,6 @@ struct iscsi_context { libiscsi_mutex_t iscsi_mutex; libiscsi_thread_t service_thread; int poll_timeout; - //libnfs_mutex_t nfs4_open_counter_mutex; - //libnfs_mutex_t nfs4_open_call_mutex; - //struct nfs_thread_context *thread_ctx; #ifndef HAVE_STDATOMIC_H libiscsi_mutex_t atomic_int_mutex; #endif /* HAVE_STDATOMIC_H */ diff --git a/include/iscsi.h b/include/iscsi.h index 375a7d8..d9017bb 100644 --- a/include/iscsi.h +++ b/include/iscsi.h @@ -1702,6 +1702,18 @@ iscsi_set_fd_dup_cb(struct iscsi_context *iscsi, void (*cb)(struct iscsi_context *iscsi, void *opaque), void *opaque); +/* + * MULTITHREADING + */ +/* + * This function starts a separate service thread for multithreading support. + */ +EXTERN int iscsi_mt_service_thread_start(struct iscsi_context *iscsi); +/* + * Shutdown multithreading support. + */ +EXTERN void iscsi_mt_service_thread_stop(struct iscsi_context *iscsi); + #ifdef __cplusplus } #endif diff --git a/lib/libiscsi.def b/lib/libiscsi.def index 65ce2ea..18d7fe3 100644 --- a/lib/libiscsi.def +++ b/lib/libiscsi.def @@ -38,6 +38,8 @@ iscsi_modesense6_sync iscsi_modesense6_task iscsi_modesense10_sync iscsi_modesense10_task +iscsi_mt_service_thread_start +iscsi_mt_service_thread_stop iscsi_nop_out_async iscsi_parse_full_url iscsi_parse_portal_url diff --git a/lib/libiscsi.syms.in b/lib/libiscsi.syms.in index 53fb955..d100968 100644 --- a/lib/libiscsi.syms.in +++ b/lib/libiscsi.syms.in @@ -41,6 +41,8 @@ iscsi_modesense10_sync iscsi_modesense10_task iscsi_modesense6_sync iscsi_modesense6_task +iscsi_mt_service_thread_start +iscsi_mt_service_thread_stop iscsi_nop_out_async iscsi_orwrite_iov_sync iscsi_orwrite_iov_task diff --git a/lib/multithreading.c b/lib/multithreading.c index 138f391..5b57804 100644 --- a/lib/multithreading.c +++ b/lib/multithreading.c @@ -189,6 +189,7 @@ static void *iscsi_mt_service_thread(void *arg) iscsi->multithreading_enabled = 1; + /* TODO: add timeout scanning */ while (iscsi->multithreading_enabled) { pfd.fd = iscsi_get_fd(iscsi); pfd.events = iscsi_which_events(iscsi); diff --git a/lib/pdu.c b/lib/pdu.c index f61e24b..73687d7 100644 --- a/lib/pdu.c +++ b/lib/pdu.c @@ -940,7 +940,17 @@ iscsi_timeout_scan(struct iscsi_context *iscsi) int iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { - return iscsi->drv->queue_pdu(iscsi, pdu); + int ret; + +#ifdef HAVE_MULTITHREADING + iscsi_mt_mutex_lock(&iscsi->iscsi_mutex); +#endif /* HAVE_MULTITHREADING */ + ret = iscsi->drv->queue_pdu(iscsi, pdu); +#ifdef HAVE_MULTITHREADING + iscsi_mt_mutex_unlock(&iscsi->iscsi_mutex); +#endif /* HAVE_MULTITHREADING */ + + return ret; } void diff --git a/lib/sync.c b/lib/sync.c index 562c44f..13013e8 100644 --- a/lib/sync.c +++ b/lib/sync.c @@ -48,6 +48,10 @@ struct iscsi_sync_state { int status; void *ptr; struct scsi_task *task; +#ifdef HAVE_MULTITHREADING + int has_sem; + libiscsi_sem_t wait_sem; +#endif /* HAVE_MULTITHREADING */ }; static void @@ -58,12 +62,21 @@ event_loop(struct iscsi_context *iscsi, struct iscsi_sync_state *state) int ret; time_t t; +#ifdef HAVE_MULTITHREADING + if(iscsi->multithreading_enabled) { + iscsi_mt_sem_wait(&state->wait_sem); + iscsi_mt_sem_destroy(&state->wait_sem); + state->has_sem = 0; + return; + } +#endif + if (iscsi->scsi_timeout) { scsi_timeout = time(NULL) + iscsi->scsi_timeout; } else { scsi_timeout = 0; } - + while (state->finished == 0) { short revents; @@ -118,6 +131,26 @@ iscsi_sync_cb(struct iscsi_context *iscsi, int status, state->status = status; state->finished = 1; +#ifdef HAVE_MULTITHREADING + if (state->has_sem) { + iscsi_mt_sem_post(&state->wait_sem); + } +#endif +} + +static void +iscsi_init_sync_state(struct iscsi_sync_state *state) +{ + memset(state, 0, sizeof(*state)); +#ifdef HAVE_MULTITHREADING + /* + * Create a semaphore and initialize it to zero. So that we + * can wait for it and immetiately block until the service thread + * has received the reply. + */ + iscsi_mt_sem_init(&state->wait_sem, 0); + state->has_sem = 1; +#endif /* HAVE_MULTITHREADING */ } int @@ -125,7 +158,7 @@ iscsi_connect_sync(struct iscsi_context *iscsi, const char *portal) { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_connect_async(iscsi, portal, iscsi_sync_cb, &state) != 0) { @@ -154,7 +187,7 @@ iscsi_full_connect_sync(struct iscsi_context *iscsi, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_full_connect_async(iscsi, portal, lun, iscsi_sync_cb, &state) != 0) { @@ -178,7 +211,7 @@ int iscsi_login_sync(struct iscsi_context *iscsi) { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_login_async(iscsi, iscsi_sync_cb, &state) != 0) { iscsi_set_error(iscsi, "Failed to login. %s", @@ -195,7 +228,7 @@ int iscsi_logout_sync(struct iscsi_context *iscsi) { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_logout_async(iscsi, iscsi_sync_cb, &state) != 0) { iscsi_set_error(iscsi, "Failed to start logout() %s", @@ -243,7 +276,7 @@ int iscsi_reconnect_sync(struct iscsi_context *iscsi) { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_reconnect(iscsi) != 0) { iscsi_set_error(iscsi, "Failed to reconnect. %s", iscsi_get_error(iscsi)); @@ -259,7 +292,7 @@ int iscsi_force_reconnect_sync(struct iscsi_context *iscsi) { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_force_reconnect(iscsi) != 0) { iscsi_set_error(iscsi, "Failed to reconnect. %s", iscsi_get_error(iscsi)); @@ -313,7 +346,7 @@ iscsi_task_mgmt_sync(struct iscsi_context *iscsi, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_task_mgmt_async(iscsi, lun, function, ritt, rcmdsn, @@ -396,6 +429,11 @@ scsi_sync_cb(struct iscsi_context *iscsi, int status, void *command_data, state->status = status; state->finished = 1; state->task = task; +#ifdef HAVE_MULTITHREADING + if (state->has_sem) { + iscsi_mt_sem_post(&state->wait_sem); + } +#endif } struct scsi_task * @@ -404,7 +442,7 @@ iscsi_reportluns_sync(struct iscsi_context *iscsi, int report_type, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_reportluns_task(iscsi, report_type, alloc_len, scsi_sync_cb, &state) == NULL) { @@ -423,7 +461,7 @@ iscsi_testunitready_sync(struct iscsi_context *iscsi, int lun) { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_testunitready_task(iscsi, lun, scsi_sync_cb, &state) == NULL) { @@ -443,7 +481,7 @@ iscsi_inquiry_sync(struct iscsi_context *iscsi, int lun, int evpd, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_inquiry_task(iscsi, lun, evpd, page_code, maxsize, scsi_sync_cb, &state) == NULL) { @@ -462,7 +500,7 @@ iscsi_read6_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_read6_task(iscsi, lun, lba, datalen, blocksize, scsi_sync_cb, &state) == NULL) { @@ -482,7 +520,7 @@ iscsi_read6_iov_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_read6_iov_task(iscsi, lun, lba, datalen, blocksize, scsi_sync_cb, &state, iov, niov) == NULL) { @@ -503,7 +541,7 @@ iscsi_read10_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_read10_task(iscsi, lun, lba, datalen, blocksize, rdprotect, dpo, fua, fua_nv, group_number, @@ -526,7 +564,7 @@ iscsi_read10_iov_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_read10_iov_task(iscsi, lun, lba, datalen, blocksize, rdprotect, dpo, fua, fua_nv, group_number, @@ -548,7 +586,7 @@ iscsi_read12_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_read12_task(iscsi, lun, lba, datalen, blocksize, rdprotect, dpo, fua, fua_nv, group_number, @@ -571,7 +609,7 @@ iscsi_read12_iov_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_read12_iov_task(iscsi, lun, lba, datalen, blocksize, rdprotect, dpo, fua, fua_nv, group_number, @@ -593,7 +631,7 @@ iscsi_read16_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_read16_task(iscsi, lun, lba, datalen, blocksize, rdprotect, dpo, fua, fua_nv, group_number, @@ -616,7 +654,7 @@ iscsi_read16_iov_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_read16_iov_task(iscsi, lun, lba, datalen, blocksize, rdprotect, dpo, fua, fua_nv, group_number, @@ -637,7 +675,7 @@ iscsi_readcapacity10_sync(struct iscsi_context *iscsi, int lun, int lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_readcapacity10_task(iscsi, lun, lba, pmi, scsi_sync_cb, &state) == NULL) { @@ -656,7 +694,7 @@ iscsi_readcapacity16_sync(struct iscsi_context *iscsi, int lun) { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_readcapacity16_task(iscsi, lun, scsi_sync_cb, &state) == NULL) { @@ -677,7 +715,7 @@ iscsi_readdefectdata10_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_readdefectdata10_task(iscsi, lun, req_plist, req_glist, @@ -702,7 +740,7 @@ iscsi_readdefectdata12_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_readdefectdata12_task(iscsi, lun, req_plist, req_glist, @@ -726,7 +764,7 @@ iscsi_sanitize_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_sanitize_task(iscsi, lun, immed, ause, sa, param_len, data, @@ -747,7 +785,7 @@ iscsi_sanitize_block_erase_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_sanitize_block_erase_task(iscsi, lun, immed, ause, @@ -768,7 +806,7 @@ iscsi_sanitize_crypto_erase_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_sanitize_crypto_erase_task(iscsi, lun, immed, ause, @@ -789,7 +827,7 @@ iscsi_sanitize_exit_failure_mode_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_sanitize_exit_failure_mode_task(iscsi, lun, immed, ause, @@ -809,7 +847,7 @@ iscsi_get_lba_status_sync(struct iscsi_context *iscsi, int lun, uint64_t startin { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_get_lba_status_task(iscsi, lun, starting_lba, alloc_len, scsi_sync_cb, &state) == NULL) { @@ -829,7 +867,7 @@ iscsi_synchronizecache10_sync(struct iscsi_context *iscsi, int lun, int lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_synchronizecache10_task(iscsi, lun, lba, num_blocks, syncnv, immed, @@ -851,7 +889,7 @@ iscsi_startstopunit_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_startstopunit_task(iscsi, lun, immed, pcm, pc, no_flush, loej, start, @@ -872,7 +910,7 @@ iscsi_preventallow_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_preventallow_task(iscsi, lun, prevent, scsi_sync_cb, &state) == NULL) { @@ -892,7 +930,7 @@ iscsi_synchronizecache16_sync(struct iscsi_context *iscsi, int lun, uint64_t lba { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_synchronizecache16_task(iscsi, lun, lba, num_blocks, syncnv, immed, @@ -913,7 +951,7 @@ iscsi_prefetch10_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_prefetch10_task(iscsi, lun, lba, num_blocks, immed, group, @@ -934,7 +972,7 @@ iscsi_prefetch16_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_prefetch16_task(iscsi, lun, lba, num_blocks, immed, group, @@ -956,7 +994,7 @@ iscsi_write10_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_write10_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, dpo, fua, fua_nv, group_number, @@ -979,7 +1017,7 @@ iscsi_write10_iov_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_write10_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, dpo, fua, fua_nv, group_number, @@ -1001,7 +1039,7 @@ iscsi_write12_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_write12_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1025,7 +1063,7 @@ iscsi_write12_iov_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_write12_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1048,7 +1086,7 @@ iscsi_write16_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_write16_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1072,7 +1110,7 @@ iscsi_write16_iov_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_write16_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1095,7 +1133,7 @@ iscsi_writeatomic16_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_writeatomic16_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1119,7 +1157,7 @@ iscsi_writeatomic16_iov_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_writeatomic16_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1142,7 +1180,7 @@ iscsi_orwrite_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_orwrite_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1166,7 +1204,7 @@ iscsi_orwrite_iov_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_orwrite_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1189,7 +1227,7 @@ iscsi_compareandwrite_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_compareandwrite_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1213,7 +1251,7 @@ iscsi_compareandwrite_iov_sync(struct iscsi_context *iscsi, int lun, uint64_t lb { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_compareandwrite_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1236,7 +1274,7 @@ iscsi_writeverify10_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_writeverify10_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, dpo, bytchk, group_number, @@ -1259,7 +1297,7 @@ iscsi_writeverify10_iov_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_writeverify10_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, dpo, bytchk, group_number, @@ -1281,7 +1319,7 @@ iscsi_writeverify12_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_writeverify12_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1305,7 +1343,7 @@ iscsi_writeverify12_iov_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_writeverify12_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1328,7 +1366,7 @@ iscsi_writeverify16_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_writeverify16_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1352,7 +1390,7 @@ iscsi_writeverify16_iov_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_writeverify16_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1374,7 +1412,7 @@ iscsi_verify10_sync(struct iscsi_context *iscsi, int lun, unsigned char *data, u { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_verify10_task(iscsi, lun, data, datalen, lba, vprotect, dpo, bytchk, blocksize, scsi_sync_cb, &state) == NULL) { @@ -1394,7 +1432,7 @@ iscsi_verify10_iov_sync(struct iscsi_context *iscsi, int lun, unsigned char *dat { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_verify10_iov_task(iscsi, lun, data, datalen, lba, vprotect, dpo, bytchk, blocksize, scsi_sync_cb, &state, iov, niov) == NULL) { @@ -1415,7 +1453,7 @@ iscsi_verify12_sync(struct iscsi_context *iscsi, int lun, unsigned char *data, u { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_verify12_task(iscsi, lun, data, datalen, lba, vprotect, dpo, bytchk, blocksize, scsi_sync_cb, &state) == NULL) { @@ -1435,7 +1473,7 @@ iscsi_verify12_iov_sync(struct iscsi_context *iscsi, int lun, unsigned char *dat { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_verify12_iov_task(iscsi, lun, data, datalen, lba, vprotect, dpo, bytchk, blocksize, scsi_sync_cb, &state, iov, niov) == NULL) { @@ -1455,7 +1493,7 @@ iscsi_verify16_sync(struct iscsi_context *iscsi, int lun, unsigned char *data, u { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_verify16_task(iscsi, lun, data, datalen, lba, vprotect, dpo, bytchk, blocksize, scsi_sync_cb, &state) == NULL) { @@ -1475,7 +1513,7 @@ iscsi_verify16_iov_sync(struct iscsi_context *iscsi, int lun, unsigned char *dat { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_verify16_iov_task(iscsi, lun, data, datalen, lba, vprotect, dpo, bytchk, blocksize, scsi_sync_cb, &state, iov, niov) == NULL) { @@ -1497,7 +1535,7 @@ iscsi_writesame10_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_writesame10_task(iscsi, lun, lba, data, datalen, num_blocks, @@ -1522,7 +1560,7 @@ iscsi_writesame10_iov_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_writesame10_iov_task(iscsi, lun, lba, data, datalen, num_blocks, @@ -1546,7 +1584,7 @@ iscsi_writesame16_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_writesame16_task(iscsi, lun, lba, data, datalen, num_blocks, @@ -1571,7 +1609,7 @@ iscsi_writesame16_iov_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_writesame16_iov_task(iscsi, lun, lba, data, datalen, num_blocks, @@ -1593,7 +1631,7 @@ iscsi_persistent_reserve_in_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_persistent_reserve_in_task(iscsi, lun, sa, xferlen, scsi_sync_cb, &state) == NULL) { @@ -1613,7 +1651,7 @@ iscsi_persistent_reserve_out_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_persistent_reserve_out_task(iscsi, lun, sa, scope, type, param, @@ -1634,7 +1672,7 @@ iscsi_unmap_sync(struct iscsi_context *iscsi, int lun, int anchor, int group, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_unmap_task(iscsi, lun, anchor, group, list, list_len, scsi_sync_cb, &state) == NULL) { @@ -1654,7 +1692,7 @@ iscsi_readtoc_sync(struct iscsi_context *iscsi, int lun, int msf, int format, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_readtoc_task(iscsi, lun, msf, format, track_session, maxsize, scsi_sync_cb, &state) == NULL) { @@ -1672,7 +1710,7 @@ iscsi_reserve6_sync(struct iscsi_context *iscsi, int lun) { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_reserve6_task(iscsi, lun, scsi_sync_cb, &state) == NULL) { iscsi_set_error(iscsi, "Failed to send RESERVE6 command"); @@ -1689,7 +1727,7 @@ iscsi_release6_sync(struct iscsi_context *iscsi, int lun) { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_release6_task(iscsi, lun, scsi_sync_cb, &state) == NULL) { iscsi_set_error(iscsi, "Failed to send RELEASE6 command"); @@ -1709,7 +1747,7 @@ iscsi_report_supported_opcodes_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_report_supported_opcodes_task(iscsi, lun, rctd, options, opcode, sa, alloc_len, scsi_sync_cb, &state) == NULL) { iscsi_set_error(iscsi, "Failed to send MaintenanceIn:" @@ -1728,7 +1766,7 @@ iscsi_receive_copy_results_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_receive_copy_results_task(iscsi, lun, sa, list_id, alloc_len, scsi_sync_cb, &state) == NULL) { @@ -1748,7 +1786,7 @@ iscsi_extended_copy_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_extended_copy_task(iscsi, lun, param_data, scsi_sync_cb, &state) == NULL) { @@ -1768,7 +1806,7 @@ iscsi_scsi_command_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_scsi_command_async(iscsi, lun, task, scsi_sync_cb, data, &state) != 0) { @@ -1788,7 +1826,7 @@ iscsi_modeselect6_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_modeselect6_task(iscsi, lun, pf, sp, mp, scsi_sync_cb, &state) == NULL) { @@ -1808,7 +1846,7 @@ iscsi_modeselect10_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_modeselect10_task(iscsi, lun, pf, sp, mp, scsi_sync_cb, &state) == NULL) { @@ -1829,7 +1867,7 @@ iscsi_modesense6_sync(struct iscsi_context *iscsi, int lun, int dbd, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_modesense6_task(iscsi, lun, dbd, pc, page_code, sub_page_code, alloc_len, scsi_sync_cb, &state) == NULL) { @@ -1850,7 +1888,7 @@ iscsi_modesense10_sync(struct iscsi_context *iscsi, int lun, int llbaa, int dbd, { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_modesense10_task(iscsi, lun, llbaa, dbd, pc, page_code, sub_page_code, alloc_len, @@ -1924,12 +1962,11 @@ iscsi_discovery_sync(struct iscsi_context *iscsi) { struct iscsi_sync_state state; - memset(&state, 0, sizeof(state)); + iscsi_init_sync_state(&state); if (iscsi_discovery_async(iscsi, iscsi_discovery_cb, &state) != 0) { iscsi_set_error(iscsi, "Failed to run discovery. %s", iscsi_get_error(iscsi)); - printf("async discovery call failed\n"); return NULL; } From 1f91358c8a86d5b170f6de879744c519f198f11d Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Tue, 25 Mar 2025 16:27:50 +1000 Subject: [PATCH 04/16] Flag variables in iscsi_context for multithreading audit These variables may need to be protected by a mutex. Signed-off-by: Ronnie Sahlberg --- include/iscsi-private.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/include/iscsi-private.h b/include/iscsi-private.h index 1cd0b5b..ec1b85e 100644 --- a/include/iscsi-private.h +++ b/include/iscsi-private.h @@ -131,12 +131,12 @@ struct iscsi_context { enum iscsi_session_type session_type; unsigned char isid[6]; uint8_t rdma_ack_timeout; - uint32_t itt; - uint32_t cmdsn; - uint32_t min_cmdsn_waiting; - uint32_t expcmdsn; - uint32_t maxcmdsn; - uint32_t statsn; + uint32_t itt; // multithreading todo: may need mutex + uint32_t cmdsn; // multithreading todo: may need mutex + uint32_t min_cmdsn_waiting; // multithreading todo: may need mutex + uint32_t expcmdsn; // multithreading todo: may need mutex + uint32_t maxcmdsn; // multithreading todo: may need mutex + uint32_t statsn; // multithreading todo: may need mutex enum iscsi_header_digest want_header_digest; enum iscsi_header_digest header_digest; enum iscsi_data_digest want_data_digest; @@ -170,11 +170,11 @@ struct iscsi_context { iscsi_command_cb socket_status_cb; void *connect_data; - struct iscsi_pdu *outqueue; - struct iscsi_pdu *outqueue_current; - struct iscsi_pdu *waitpdu; + struct iscsi_pdu *outqueue; // multithreading todo: may need mutex + struct iscsi_pdu *outqueue_current; // multithreading todo: may need mutex + struct iscsi_pdu *waitpdu; // multithreading todo: may need mutex - struct iscsi_in_pdu *incoming; + struct iscsi_in_pdu *incoming; // multithreading todo: may need mutex uint32_t max_burst_length; uint32_t first_burst_length; From 09ec037e34601d155f8651f7fe4dc1931aa6da56 Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Tue, 25 Mar 2025 16:50:11 +1000 Subject: [PATCH 05/16] Make the mutex symbols always available so we do not need to wrap them inside an ifdef. Signed-off-by: Ronnie Sahlberg --- include/iscsi-multithreading.h | 106 +++++++++++++++++++++++++++++++-- lib/init.c | 4 -- lib/multithreading.c | 79 ------------------------ lib/pdu.c | 4 -- 4 files changed, 100 insertions(+), 93 deletions(-) diff --git a/include/iscsi-multithreading.h b/include/iscsi-multithreading.h index 39134b0..197b08c 100644 --- a/include/iscsi-multithreading.h +++ b/include/iscsi-multithreading.h @@ -31,13 +31,11 @@ extern "C" { #ifdef WIN32 typedef HANDLE libiscsi_thread_t; -typedef HANDLE libiscsi_mutex_t; typedef HANDLE libiscsi_sem_t; typedef DWORD iscsi_tid_t; #elif defined(HAVE_PTHREAD) #include typedef pthread_t libiscsi_thread_t; -typedef pthread_mutex_t libiscsi_mutex_t; #if defined(__APPLE__) && defined(HAVE_DISPATCH_DISPATCH_H) #include @@ -54,10 +52,8 @@ typedef pid_t iscsi_tid_t; #endif /* HAVE_PTHREAD */ iscsi_tid_t iscsi_mt_get_tid(void); -int iscsi_mt_mutex_init(libiscsi_mutex_t *mutex); -int iscsi_mt_mutex_destroy(libiscsi_mutex_t *mutex); -int iscsi_mt_mutex_lock(libiscsi_mutex_t *mutex); -int iscsi_mt_mutex_unlock(libiscsi_mutex_t *mutex); + + int iscsi_mt_sem_init(libiscsi_sem_t *sem, int value); int iscsi_mt_sem_destroy(libiscsi_sem_t *sem); @@ -66,6 +62,104 @@ int iscsi_mt_sem_wait(libiscsi_sem_t *sem); #endif /* HAVE_MULTITHREADING */ +/* + * We always have access to mutex functions even if multithreading + * is not enabled. + */ +#if defined(HAVE_PTHREAD) +typedef pthread_mutex_t libiscsi_mutex_t; +/* + * If this is enabled we check for the following locking violations, at the + * (slight) cost of performance: + * - Thread holding the lock again tries to lock. + * - Thread not holding the lock tries to unlock. + * + * This is very useful for catching any coding errors. + * The performance hit is not very significant so you can leave it enabled, + * but if you really care then once the code has been vetted, this can be + * undef'ed to get the perf back. + */ +#define DEBUG_PTHREAD_LOCKING_VIOLATIONS + +static inline int iscsi_mt_mutex_init(libiscsi_mutex_t *mutex) +{ + int ret; +#ifdef DEBUG_PTHREAD_LOCKING_VIOLATIONS + pthread_mutexattr_t attr; + + ret = pthread_mutexattr_init(&attr); + if (ret != 0) { + return ret; + } + + ret = pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); + if (ret != 0) { + return ret; + } + + ret = pthread_mutex_init(mutex, &attr); + if (ret != 0) { + return ret; + } +#else + ret = pthread_mutex_init(mutex, NULL); + assert(ret == 0); +#endif + return ret; +} + +static inline int iscsi_mt_mutex_destroy(libiscsi_mutex_t *mutex) +{ + return pthread_mutex_destroy(mutex); +} + +static inline int iscsi_mt_mutex_lock(libiscsi_mutex_t *mutex) +{ + return pthread_mutex_lock(mutex); +} + +static inline int iscsi_mt_mutex_unlock(libiscsi_mutex_t *mutex) +{ + return pthread_mutex_unlock(mutex); +} + +#elif defined(WIN32) +typedef HANDLE libiscsi_mutex_t; +static inline int iscsi_mt_mutex_init(libiscsi_mutex_t* mutex) +{ + *mutex = CreateSemaphoreA(NULL, 1, 1, NULL); + return 0; +} + +static inline int iscsi_mt_mutex_destroy(libiscsi_mutex_t* mutex) +{ + CloseHandle(*mutex); + return 0; +} + +static inline int iscsi_mt_mutex_lock(libiscsi_mutex_t* mutex) +{ + while (WaitForSingleObject(*mutex, INFINITE) != WAIT_OBJECT_0); + return 0; +} + +static inline int iscsi_mt_mutex_unlock(libiscsi_mutex_t* mutex) +{ + ReleaseSemaphore(*mutex, 1, NULL); + return 0; +} + +#else + +typedef const int libiscsi_mutex_t; +#define iscsi_mt_mutex_init(x) ; +#define iscsi_mt_mutex_destroy(x) ; +#define iscsi_mt_mutex_lock(x) ; +#define iscsi_mt_mutex_unlock(x) ; + +#endif /* mutex */ + + #ifdef __cplusplus } #endif diff --git a/lib/init.c b/lib/init.c index 233a5c2..817a045 100644 --- a/lib/init.c +++ b/lib/init.c @@ -216,10 +216,8 @@ iscsi_create_context(const char *initiator_name) memset(iscsi, 0, sizeof(struct iscsi_context)); -#ifdef HAVE_MULTITHREADING iscsi_mt_mutex_init(&iscsi->iscsi_mutex); iscsi->poll_timeout = 100; -#endif /* HAVE_MULTITHREADING */ /* initalize transport of context */ if (iscsi_init_transport(iscsi, TCP_TRANSPORT)) { @@ -436,9 +434,7 @@ iscsi_destroy_context(struct iscsi_context *iscsi) iscsi_destroy_context(iscsi->old_iscsi); } -#ifdef HAVE_MULTITHREADING iscsi_mt_mutex_destroy(&iscsi->iscsi_mutex); -#endif /* HAVE_MULTITHREADING */ memset(iscsi, 0, sizeof(struct iscsi_context)); free(iscsi); diff --git a/lib/multithreading.c b/lib/multithreading.c index 5b57804..fd3a5a5 100644 --- a/lib/multithreading.c +++ b/lib/multithreading.c @@ -113,30 +113,6 @@ void iscsi_mt_service_thread_stop(struct iscsi_context* iscsi) while (WaitForSingleObject(iscsi->iscsii->service_thread, INFINITE) != WAIT_OBJECT_0); } -int iscsi_mt_mutex_init(libiscsi_mutex_t* mutex) -{ - *mutex = CreateSemaphoreA(NULL, 1, 1, NULL); - return 0; -} - -int iscsi_mt_mutex_destroy(libiscsi_mutex_t* mutex) -{ - CloseHandle(*mutex); - return 0; -} - -int iscsi_mt_mutex_lock(libiscsi_mutex_t* mutex) -{ - while (WaitForSingleObject(*mutex, INFINITE) != WAIT_OBJECT_0); - return 0; -} - -int iscsi_mt_mutex_unlock(libiscsi_mutex_t* mutex) -{ - ReleaseSemaphore(*mutex, 1, NULL); - return 0; -} - int iscsi_mt_sem_init(libiscsi_sem_t* sem, int value) { *sem = CreateSemaphoreA(NULL, 0, 16, NULL); @@ -230,61 +206,6 @@ void iscsi_mt_service_thread_stop(struct iscsi_context *iscsi) pthread_join(iscsi->service_thread, NULL); } -/* - * If this is enabled we check for the following locking violations, at the - * (slight) cost of performance: - * - Thread holding the lock again tries to lock. - * - Thread not holding the lock tries to unlock. - * - * This is very useful for catching any coding errors. - * The performance hit is not very significant so you can leave it enabled, - * but if you really care then once the code has been vetted, this can be - * undef'ed to get the perf back. - */ -#define DEBUG_PTHREAD_LOCKING_VIOLATIONS - -int iscsi_mt_mutex_init(libiscsi_mutex_t *mutex) -{ - int ret; -#ifdef DEBUG_PTHREAD_LOCKING_VIOLATIONS - pthread_mutexattr_t attr; - - ret = pthread_mutexattr_init(&attr); - if (ret != 0) { - return ret; - } - - ret = pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); - if (ret != 0) { - return ret; - } - - ret = pthread_mutex_init(mutex, &attr); - if (ret != 0) { - return ret; - } -#else - ret = pthread_mutex_init(mutex, NULL); - assert(ret == 0); -#endif - return ret; -} - -int iscsi_mt_mutex_destroy(libiscsi_mutex_t *mutex) -{ - return pthread_mutex_destroy(mutex); -} - -int iscsi_mt_mutex_lock(libiscsi_mutex_t *mutex) -{ - return pthread_mutex_lock(mutex); -} - -int iscsi_mt_mutex_unlock(libiscsi_mutex_t *mutex) -{ - return pthread_mutex_unlock(mutex); -} - #if defined(__APPLE__) && defined(HAVE_DISPATCH_DISPATCH_H) int iscsi_mt_sem_init(libiscsi_sem_t *sem, int value) { diff --git a/lib/pdu.c b/lib/pdu.c index 73687d7..68e8a20 100644 --- a/lib/pdu.c +++ b/lib/pdu.c @@ -942,13 +942,9 @@ iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { int ret; -#ifdef HAVE_MULTITHREADING iscsi_mt_mutex_lock(&iscsi->iscsi_mutex); -#endif /* HAVE_MULTITHREADING */ ret = iscsi->drv->queue_pdu(iscsi, pdu); -#ifdef HAVE_MULTITHREADING iscsi_mt_mutex_unlock(&iscsi->iscsi_mutex); -#endif /* HAVE_MULTITHREADING */ return ret; } From 5b391ad18864eb4b08d607869f857384952527b0 Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Sun, 13 Apr 2025 08:20:37 +1000 Subject: [PATCH 06/16] Pass iscsi to iscsi_init_sync_state Signed-off-by: Ronnie Sahlberg --- lib/sync.c | 176 ++++++++++++++++++++++++++--------------------------- 1 file changed, 88 insertions(+), 88 deletions(-) diff --git a/lib/sync.c b/lib/sync.c index 13013e8..5b5239f 100644 --- a/lib/sync.c +++ b/lib/sync.c @@ -49,7 +49,6 @@ struct iscsi_sync_state { void *ptr; struct scsi_task *task; #ifdef HAVE_MULTITHREADING - int has_sem; libiscsi_sem_t wait_sem; #endif /* HAVE_MULTITHREADING */ }; @@ -64,9 +63,9 @@ event_loop(struct iscsi_context *iscsi, struct iscsi_sync_state *state) #ifdef HAVE_MULTITHREADING if(iscsi->multithreading_enabled) { + /* TODO QQQ Must make sure the service thread event loop handle timeouts properly */ iscsi_mt_sem_wait(&state->wait_sem); iscsi_mt_sem_destroy(&state->wait_sem); - state->has_sem = 0; return; } #endif @@ -132,24 +131,25 @@ iscsi_sync_cb(struct iscsi_context *iscsi, int status, state->status = status; state->finished = 1; #ifdef HAVE_MULTITHREADING - if (state->has_sem) { + if(iscsi->multithreading_enabled) { iscsi_mt_sem_post(&state->wait_sem); } #endif } static void -iscsi_init_sync_state(struct iscsi_sync_state *state) +iscsi_init_sync_state(struct iscsi_context *iscsi, struct iscsi_sync_state *state) { memset(state, 0, sizeof(*state)); #ifdef HAVE_MULTITHREADING - /* - * Create a semaphore and initialize it to zero. So that we - * can wait for it and immetiately block until the service thread - * has received the reply. - */ - iscsi_mt_sem_init(&state->wait_sem, 0); - state->has_sem = 1; + if(iscsi->multithreading_enabled) { + /* + * Create a semaphore and initialize it to zero. So that we + * can wait for it and immetiately block until the service thread + * has received the reply. + */ + iscsi_mt_sem_init(&state->wait_sem, 0); + } #endif /* HAVE_MULTITHREADING */ } @@ -158,7 +158,7 @@ iscsi_connect_sync(struct iscsi_context *iscsi, const char *portal) { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_connect_async(iscsi, portal, iscsi_sync_cb, &state) != 0) { @@ -187,7 +187,7 @@ iscsi_full_connect_sync(struct iscsi_context *iscsi, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_full_connect_async(iscsi, portal, lun, iscsi_sync_cb, &state) != 0) { @@ -211,7 +211,7 @@ int iscsi_login_sync(struct iscsi_context *iscsi) { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_login_async(iscsi, iscsi_sync_cb, &state) != 0) { iscsi_set_error(iscsi, "Failed to login. %s", @@ -228,7 +228,7 @@ int iscsi_logout_sync(struct iscsi_context *iscsi) { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_logout_async(iscsi, iscsi_sync_cb, &state) != 0) { iscsi_set_error(iscsi, "Failed to start logout() %s", @@ -276,7 +276,7 @@ int iscsi_reconnect_sync(struct iscsi_context *iscsi) { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_reconnect(iscsi) != 0) { iscsi_set_error(iscsi, "Failed to reconnect. %s", iscsi_get_error(iscsi)); @@ -292,7 +292,7 @@ int iscsi_force_reconnect_sync(struct iscsi_context *iscsi) { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_force_reconnect(iscsi) != 0) { iscsi_set_error(iscsi, "Failed to reconnect. %s", iscsi_get_error(iscsi)); @@ -346,7 +346,7 @@ iscsi_task_mgmt_sync(struct iscsi_context *iscsi, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_task_mgmt_async(iscsi, lun, function, ritt, rcmdsn, @@ -430,7 +430,7 @@ scsi_sync_cb(struct iscsi_context *iscsi, int status, void *command_data, state->finished = 1; state->task = task; #ifdef HAVE_MULTITHREADING - if (state->has_sem) { + if(iscsi->multithreading_enabled) { iscsi_mt_sem_post(&state->wait_sem); } #endif @@ -442,7 +442,7 @@ iscsi_reportluns_sync(struct iscsi_context *iscsi, int report_type, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_reportluns_task(iscsi, report_type, alloc_len, scsi_sync_cb, &state) == NULL) { @@ -461,7 +461,7 @@ iscsi_testunitready_sync(struct iscsi_context *iscsi, int lun) { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_testunitready_task(iscsi, lun, scsi_sync_cb, &state) == NULL) { @@ -481,7 +481,7 @@ iscsi_inquiry_sync(struct iscsi_context *iscsi, int lun, int evpd, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_inquiry_task(iscsi, lun, evpd, page_code, maxsize, scsi_sync_cb, &state) == NULL) { @@ -500,7 +500,7 @@ iscsi_read6_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_read6_task(iscsi, lun, lba, datalen, blocksize, scsi_sync_cb, &state) == NULL) { @@ -520,7 +520,7 @@ iscsi_read6_iov_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_read6_iov_task(iscsi, lun, lba, datalen, blocksize, scsi_sync_cb, &state, iov, niov) == NULL) { @@ -541,7 +541,7 @@ iscsi_read10_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_read10_task(iscsi, lun, lba, datalen, blocksize, rdprotect, dpo, fua, fua_nv, group_number, @@ -564,7 +564,7 @@ iscsi_read10_iov_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_read10_iov_task(iscsi, lun, lba, datalen, blocksize, rdprotect, dpo, fua, fua_nv, group_number, @@ -586,7 +586,7 @@ iscsi_read12_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_read12_task(iscsi, lun, lba, datalen, blocksize, rdprotect, dpo, fua, fua_nv, group_number, @@ -609,7 +609,7 @@ iscsi_read12_iov_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_read12_iov_task(iscsi, lun, lba, datalen, blocksize, rdprotect, dpo, fua, fua_nv, group_number, @@ -631,7 +631,7 @@ iscsi_read16_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_read16_task(iscsi, lun, lba, datalen, blocksize, rdprotect, dpo, fua, fua_nv, group_number, @@ -654,7 +654,7 @@ iscsi_read16_iov_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_read16_iov_task(iscsi, lun, lba, datalen, blocksize, rdprotect, dpo, fua, fua_nv, group_number, @@ -675,7 +675,7 @@ iscsi_readcapacity10_sync(struct iscsi_context *iscsi, int lun, int lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_readcapacity10_task(iscsi, lun, lba, pmi, scsi_sync_cb, &state) == NULL) { @@ -694,7 +694,7 @@ iscsi_readcapacity16_sync(struct iscsi_context *iscsi, int lun) { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_readcapacity16_task(iscsi, lun, scsi_sync_cb, &state) == NULL) { @@ -715,7 +715,7 @@ iscsi_readdefectdata10_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_readdefectdata10_task(iscsi, lun, req_plist, req_glist, @@ -740,7 +740,7 @@ iscsi_readdefectdata12_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_readdefectdata12_task(iscsi, lun, req_plist, req_glist, @@ -764,7 +764,7 @@ iscsi_sanitize_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_sanitize_task(iscsi, lun, immed, ause, sa, param_len, data, @@ -785,7 +785,7 @@ iscsi_sanitize_block_erase_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_sanitize_block_erase_task(iscsi, lun, immed, ause, @@ -806,7 +806,7 @@ iscsi_sanitize_crypto_erase_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_sanitize_crypto_erase_task(iscsi, lun, immed, ause, @@ -827,7 +827,7 @@ iscsi_sanitize_exit_failure_mode_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_sanitize_exit_failure_mode_task(iscsi, lun, immed, ause, @@ -847,7 +847,7 @@ iscsi_get_lba_status_sync(struct iscsi_context *iscsi, int lun, uint64_t startin { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_get_lba_status_task(iscsi, lun, starting_lba, alloc_len, scsi_sync_cb, &state) == NULL) { @@ -867,7 +867,7 @@ iscsi_synchronizecache10_sync(struct iscsi_context *iscsi, int lun, int lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_synchronizecache10_task(iscsi, lun, lba, num_blocks, syncnv, immed, @@ -889,7 +889,7 @@ iscsi_startstopunit_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_startstopunit_task(iscsi, lun, immed, pcm, pc, no_flush, loej, start, @@ -910,7 +910,7 @@ iscsi_preventallow_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_preventallow_task(iscsi, lun, prevent, scsi_sync_cb, &state) == NULL) { @@ -930,7 +930,7 @@ iscsi_synchronizecache16_sync(struct iscsi_context *iscsi, int lun, uint64_t lba { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_synchronizecache16_task(iscsi, lun, lba, num_blocks, syncnv, immed, @@ -951,7 +951,7 @@ iscsi_prefetch10_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_prefetch10_task(iscsi, lun, lba, num_blocks, immed, group, @@ -972,7 +972,7 @@ iscsi_prefetch16_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_prefetch16_task(iscsi, lun, lba, num_blocks, immed, group, @@ -994,7 +994,7 @@ iscsi_write10_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_write10_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, dpo, fua, fua_nv, group_number, @@ -1017,7 +1017,7 @@ iscsi_write10_iov_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_write10_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, dpo, fua, fua_nv, group_number, @@ -1039,7 +1039,7 @@ iscsi_write12_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_write12_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1063,7 +1063,7 @@ iscsi_write12_iov_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_write12_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1086,7 +1086,7 @@ iscsi_write16_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_write16_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1110,7 +1110,7 @@ iscsi_write16_iov_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_write16_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1133,7 +1133,7 @@ iscsi_writeatomic16_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_writeatomic16_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1157,7 +1157,7 @@ iscsi_writeatomic16_iov_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_writeatomic16_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1180,7 +1180,7 @@ iscsi_orwrite_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_orwrite_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1204,7 +1204,7 @@ iscsi_orwrite_iov_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_orwrite_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1227,7 +1227,7 @@ iscsi_compareandwrite_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_compareandwrite_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1251,7 +1251,7 @@ iscsi_compareandwrite_iov_sync(struct iscsi_context *iscsi, int lun, uint64_t lb { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_compareandwrite_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1274,7 +1274,7 @@ iscsi_writeverify10_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_writeverify10_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, dpo, bytchk, group_number, @@ -1297,7 +1297,7 @@ iscsi_writeverify10_iov_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_writeverify10_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, dpo, bytchk, group_number, @@ -1319,7 +1319,7 @@ iscsi_writeverify12_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_writeverify12_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1343,7 +1343,7 @@ iscsi_writeverify12_iov_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_writeverify12_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1366,7 +1366,7 @@ iscsi_writeverify16_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_writeverify16_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1390,7 +1390,7 @@ iscsi_writeverify16_iov_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_writeverify16_iov_task(iscsi, lun, lba, data, datalen, blocksize, wrprotect, @@ -1412,7 +1412,7 @@ iscsi_verify10_sync(struct iscsi_context *iscsi, int lun, unsigned char *data, u { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_verify10_task(iscsi, lun, data, datalen, lba, vprotect, dpo, bytchk, blocksize, scsi_sync_cb, &state) == NULL) { @@ -1432,7 +1432,7 @@ iscsi_verify10_iov_sync(struct iscsi_context *iscsi, int lun, unsigned char *dat { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_verify10_iov_task(iscsi, lun, data, datalen, lba, vprotect, dpo, bytchk, blocksize, scsi_sync_cb, &state, iov, niov) == NULL) { @@ -1453,7 +1453,7 @@ iscsi_verify12_sync(struct iscsi_context *iscsi, int lun, unsigned char *data, u { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_verify12_task(iscsi, lun, data, datalen, lba, vprotect, dpo, bytchk, blocksize, scsi_sync_cb, &state) == NULL) { @@ -1473,7 +1473,7 @@ iscsi_verify12_iov_sync(struct iscsi_context *iscsi, int lun, unsigned char *dat { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_verify12_iov_task(iscsi, lun, data, datalen, lba, vprotect, dpo, bytchk, blocksize, scsi_sync_cb, &state, iov, niov) == NULL) { @@ -1493,7 +1493,7 @@ iscsi_verify16_sync(struct iscsi_context *iscsi, int lun, unsigned char *data, u { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_verify16_task(iscsi, lun, data, datalen, lba, vprotect, dpo, bytchk, blocksize, scsi_sync_cb, &state) == NULL) { @@ -1513,7 +1513,7 @@ iscsi_verify16_iov_sync(struct iscsi_context *iscsi, int lun, unsigned char *dat { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_verify16_iov_task(iscsi, lun, data, datalen, lba, vprotect, dpo, bytchk, blocksize, scsi_sync_cb, &state, iov, niov) == NULL) { @@ -1535,7 +1535,7 @@ iscsi_writesame10_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_writesame10_task(iscsi, lun, lba, data, datalen, num_blocks, @@ -1560,7 +1560,7 @@ iscsi_writesame10_iov_sync(struct iscsi_context *iscsi, int lun, uint32_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_writesame10_iov_task(iscsi, lun, lba, data, datalen, num_blocks, @@ -1584,7 +1584,7 @@ iscsi_writesame16_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_writesame16_task(iscsi, lun, lba, data, datalen, num_blocks, @@ -1609,7 +1609,7 @@ iscsi_writesame16_iov_sync(struct iscsi_context *iscsi, int lun, uint64_t lba, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_writesame16_iov_task(iscsi, lun, lba, data, datalen, num_blocks, @@ -1631,7 +1631,7 @@ iscsi_persistent_reserve_in_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_persistent_reserve_in_task(iscsi, lun, sa, xferlen, scsi_sync_cb, &state) == NULL) { @@ -1651,7 +1651,7 @@ iscsi_persistent_reserve_out_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_persistent_reserve_out_task(iscsi, lun, sa, scope, type, param, @@ -1672,7 +1672,7 @@ iscsi_unmap_sync(struct iscsi_context *iscsi, int lun, int anchor, int group, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_unmap_task(iscsi, lun, anchor, group, list, list_len, scsi_sync_cb, &state) == NULL) { @@ -1692,7 +1692,7 @@ iscsi_readtoc_sync(struct iscsi_context *iscsi, int lun, int msf, int format, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_readtoc_task(iscsi, lun, msf, format, track_session, maxsize, scsi_sync_cb, &state) == NULL) { @@ -1710,7 +1710,7 @@ iscsi_reserve6_sync(struct iscsi_context *iscsi, int lun) { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_reserve6_task(iscsi, lun, scsi_sync_cb, &state) == NULL) { iscsi_set_error(iscsi, "Failed to send RESERVE6 command"); @@ -1727,7 +1727,7 @@ iscsi_release6_sync(struct iscsi_context *iscsi, int lun) { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_release6_task(iscsi, lun, scsi_sync_cb, &state) == NULL) { iscsi_set_error(iscsi, "Failed to send RELEASE6 command"); @@ -1747,7 +1747,7 @@ iscsi_report_supported_opcodes_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_report_supported_opcodes_task(iscsi, lun, rctd, options, opcode, sa, alloc_len, scsi_sync_cb, &state) == NULL) { iscsi_set_error(iscsi, "Failed to send MaintenanceIn:" @@ -1766,7 +1766,7 @@ iscsi_receive_copy_results_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_receive_copy_results_task(iscsi, lun, sa, list_id, alloc_len, scsi_sync_cb, &state) == NULL) { @@ -1786,7 +1786,7 @@ iscsi_extended_copy_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_extended_copy_task(iscsi, lun, param_data, scsi_sync_cb, &state) == NULL) { @@ -1806,7 +1806,7 @@ iscsi_scsi_command_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_scsi_command_async(iscsi, lun, task, scsi_sync_cb, data, &state) != 0) { @@ -1826,7 +1826,7 @@ iscsi_modeselect6_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_modeselect6_task(iscsi, lun, pf, sp, mp, scsi_sync_cb, &state) == NULL) { @@ -1846,7 +1846,7 @@ iscsi_modeselect10_sync(struct iscsi_context *iscsi, int lun, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_modeselect10_task(iscsi, lun, pf, sp, mp, scsi_sync_cb, &state) == NULL) { @@ -1867,7 +1867,7 @@ iscsi_modesense6_sync(struct iscsi_context *iscsi, int lun, int dbd, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_modesense6_task(iscsi, lun, dbd, pc, page_code, sub_page_code, alloc_len, scsi_sync_cb, &state) == NULL) { @@ -1888,7 +1888,7 @@ iscsi_modesense10_sync(struct iscsi_context *iscsi, int lun, int llbaa, int dbd, { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_modesense10_task(iscsi, lun, llbaa, dbd, pc, page_code, sub_page_code, alloc_len, @@ -1962,7 +1962,7 @@ iscsi_discovery_sync(struct iscsi_context *iscsi) { struct iscsi_sync_state state; - iscsi_init_sync_state(&state); + iscsi_init_sync_state(iscsi, &state); if (iscsi_discovery_async(iscsi, iscsi_discovery_cb, &state) != 0) { iscsi_set_error(iscsi, "Failed to run discovery. %s", From ca0df22ede857e61f6fe057026f8410c27448bf4 Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Sun, 13 Apr 2025 08:38:33 +1000 Subject: [PATCH 07/16] Add iscsi-pthreads-readloop This is a tool to create multiple contexts and threads and run read operations on all of them in a loop. Signed-off-by: Ronnie Sahlberg --- examples/Makefile.am | 2 +- examples/iscsi-pthreads-readloop.c | 248 +++++++++++++++++++++++++++++ 2 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 examples/iscsi-pthreads-readloop.c diff --git a/examples/Makefile.am b/examples/Makefile.am index bc00f6c..4740bd6 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -3,4 +3,4 @@ AM_CFLAGS=$(WARN_CFLAGS) AM_LDFLAGS=-no-undefined LIBS=../lib/libiscsi.la -noinst_PROGRAMS = iscsiclient iscsi-dd iscsi-pthreads-inq +noinst_PROGRAMS = iscsiclient iscsi-dd iscsi-pthreads-inq iscsi-pthreads-readloop diff --git a/examples/iscsi-pthreads-readloop.c b/examples/iscsi-pthreads-readloop.c new file mode 100644 index 0000000..b946c9c --- /dev/null +++ b/examples/iscsi-pthreads-readloop.c @@ -0,0 +1,248 @@ +/* + Copyright (C) 2025 by Ronnie Sahlberg + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . +*/ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_POLL_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iscsi.h" +#include "scsi-lowlevel.h" + +static int finished; + +/* + * Number of iscsi connections to the server. + */ +#define NUM_CONTEXTS 2 + +/* + * Number of threads parallely read device data. + * Usually one thread per context should be sufficient, but here we use more + * threads to demonstrate that multiple threads can very well write to the same + * context. + */ +#define NUM_THREADS 4 + + +const char *initiator = "iqn.2007-10.com.github:sahlberg:libiscsi:iscsi-inq"; + + +void print_usage(void) +{ + fprintf(stderr, "Usage: iscsi-pthreads-readloop [-?] [-?|--help] [--usage] [-i|--initiator-name=iqn-name]\n" + "\t\t\n"); +} + +void print_help(void) +{ + fprintf(stderr, "Usage: iscsi-pthreads-readloop [OPTION...] \n"); + fprintf(stderr, " -i, --initiator-name=iqn-name Initiatorname to use\n"); + fprintf(stderr, " -d, --debug=integer debug level (0=disabled)\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "Help options:\n"); + fprintf(stderr, " -?, --help Show this help message\n"); + fprintf(stderr, " --usage Display brief usage message\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "iSCSI URL format : %s\n", ISCSI_URL_SYNTAX); + fprintf(stderr, "\n"); + fprintf(stderr, " is either of:\n"); + fprintf(stderr, " \"hostname\" iscsi.example\n"); + fprintf(stderr, " \"ipv4-address\" 10.1.1.27\n"); + fprintf(stderr, " \"ipv6-address\" [fce0::1]\n"); +} + +struct read_data { + struct iscsi_context *iscsi; + int lun; + int i; + pthread_t thread; +}; + +static void *iscsi_read_thread(void *arg) +{ + struct read_data *rd = arg; + struct scsi_task *task; + + while (!finished) { + task = iscsi_read10_sync(rd->iscsi, rd->lun, 0, + 512, 512, + 0, 0, 0, 0, 0); + printf("task:%p\n", task); + scsi_free_scsi_task(task); + if (task == NULL) { + fprintf(stderr, "Failed to read from lun in thread #%d\n", rd->i); + exit(10); + } + } + return NULL; +} + +static void sig_alarm(int sig) +{ + finished = 1; +} + +int main(int argc, char *argv[]) +{ + struct iscsi_context *iscsi[NUM_CONTEXTS] = {NULL,}; + char *url = NULL; + struct iscsi_url *iscsi_url = NULL; + int show_help = 0, show_usage = 0, debug = 0; + int c, i; + struct read_data *rd; + + static struct option long_options[] = { + {"help", no_argument, NULL, 'h'}, + {"usage", no_argument, NULL, 'u'}, + {"debug", required_argument, NULL, 'd'}, + {"initiator-name", required_argument, NULL, 'i'}, + {0, 0, 0, 0} + }; + int option_index; + + while ((c = getopt_long(argc, argv, "h?ud:i:e:c:", long_options, + &option_index)) != -1) { + switch (c) { + case 'h': + case '?': + show_help = 1; + break; + case 'u': + show_usage = 1; + break; + case 'd': + debug = strtol(optarg, NULL, 0); + break; + case 'i': + initiator = optarg; + break; + default: + fprintf(stderr, "Unrecognized option '%c'\n\n", c); + print_help(); + exit(0); + } + } + + if (show_help != 0) { + print_help(); + exit(0); + } + + if (show_usage != 0) { + print_usage(); + exit(0); + } + + for (i = 0; i < NUM_CONTEXTS; i++) { + printf("Creating context #%d\n", i); + iscsi[i] = iscsi_create_context(initiator); + if (iscsi[i] == NULL) { + fprintf(stderr, "Failed to create context\n"); + exit(10); + } + if (debug > 0) { + iscsi_set_log_level(iscsi[i], debug); + iscsi_set_log_fn(iscsi[i], iscsi_log_to_stderr); + } + + if (i > 0) { + iscsi_destroy_url(iscsi_url); + } + if (argv[optind] != NULL) { + url = strdup(argv[optind]); + } + if (url == NULL) { + fprintf(stderr, "You must specify the URL\n"); + print_usage(); + exit(10); + } + iscsi_url = iscsi_parse_full_url(iscsi[i], url); + + free(url); + + if (iscsi_url == NULL) { + fprintf(stderr, "Failed to parse URL: %s\n", + iscsi_get_error(iscsi[i])); + exit(10); + } + + iscsi_set_session_type(iscsi[i], ISCSI_SESSION_NORMAL); + iscsi_set_header_digest(iscsi[i], ISCSI_HEADER_DIGEST_NONE_CRC32C); + + if (iscsi_full_connect_sync(iscsi[i], iscsi_url->portal, iscsi_url->lun) != 0) { + fprintf(stderr, "Login Failed. %s\n", iscsi_get_error(iscsi[i])); + iscsi_destroy_url(iscsi_url); + iscsi_destroy_context(iscsi[i]); + exit(10); + } + + if (iscsi_mt_service_thread_start(iscsi[i])) { + fprintf(stderr, "failed to start service thread #%d\n", i); + exit(10); + } + + } + + if ((rd = malloc(sizeof(struct read_data) * NUM_THREADS)) == NULL) { + fprintf(stderr, "Failed to allocated read_data\n"); + exit(10); + } + for (i = 0; i < NUM_THREADS; i++) { + rd[i].iscsi = iscsi[i % NUM_CONTEXTS]; + rd[i].lun = iscsi_url->lun; + rd[i].i = i; + if (pthread_create(&rd[i].thread, NULL, + &iscsi_read_thread, &rd[i])) { + printf("Failed to create read thread #%d\n", i); + exit(10); + } + } + + /* run for 5 seconds */ + signal(SIGALRM, sig_alarm); + alarm(5); + + /* + * Wait for all threads to complete + */ + for (i = 0; i < NUM_THREADS; i++) { + pthread_join(rd[i].thread, NULL); + } + + iscsi_destroy_url(iscsi_url); + + for (i = 0; i < NUM_CONTEXTS; i++) { + iscsi_mt_service_thread_stop(iscsi[i]); + iscsi_logout_sync(iscsi[i]); + iscsi_destroy_context(iscsi[i]); + } + return 0; +} + From f63ed0b76e6c3a70d537f9135957a36a00c476b5 Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Sun, 13 Apr 2025 09:22:15 +1000 Subject: [PATCH 08/16] Remove two global variables used to set the initial seed Signed-off-by: Ronnie Sahlberg --- lib/init.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/init.c b/lib/init.c index 817a045..0f78126 100644 --- a/lib/init.c +++ b/lib/init.c @@ -151,15 +151,14 @@ void iscsi_sfree(struct iscsi_context *iscsi, void* ptr) { } } -static bool rd_set = false; -static pthread_mutex_t rd_mutex = PTHREAD_MUTEX_INITIALIZER; - static void iscsi_srand_init(struct iscsi_context *iscsi) { unsigned int seed; int urand_fd; ssize_t rc; int err; + static bool rd_set = false; + static pthread_mutex_t rd_mutex = PTHREAD_MUTEX_INITIALIZER; if (rd_set) { /* fast case, seed has been set */ From 37bc6fcd81efe8a0edc96b8437d0342e02e2bd23 Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Sun, 13 Apr 2025 10:05:53 +1000 Subject: [PATCH 09/16] TCP: immediately trigger the service thread to write PDU IF qe queue a new PDU to an empty outqueue then the mt service thread will still be stuck in poll() until it timesout or the socket becomes readable. Fix this by sending SIGUSR1 to the service thread when we queue a PDU to an empty queue. This will break out of poll() and we can immediately go and write to the socket. Signed-off-by: Ronnie Sahlberg --- lib/multithreading.c | 23 +++++++++++++++++++++-- lib/socket.c | 32 +++++++++++++++++++++++++++----- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/lib/multithreading.c b/lib/multithreading.c index fd3a5a5..2449f80 100644 --- a/lib/multithreading.c +++ b/lib/multithreading.c @@ -139,6 +139,9 @@ int iscsi_mt_sem_wait(libiscsi_sem_t* sem) #elif defined(HAVE_PTHREAD) /* WIN32 */ +#include +#include +#include #include #include @@ -156,13 +159,20 @@ iscsi_tid_t iscsi_mt_get_tid(void) #endif } +static void on_sigusr1(int _unused) +{ +} + static void *iscsi_mt_service_thread(void *arg) { struct iscsi_context *iscsi = (struct iscsi_context *)arg; struct pollfd pfd; int revents; int ret; - + + /* set signal to break poll when we need to send more data */ + signal(SIGUSR1, on_sigusr1); + iscsi->multithreading_enabled = 1; /* TODO: add timeout scanning */ @@ -172,12 +182,21 @@ static void *iscsi_mt_service_thread(void *arg) pfd.revents = 0; ret = poll(&pfd, 1, iscsi->poll_timeout); - if (ret < 0) { + if (ret < 0 && errno == EINTR) { + /* + * Got a signal. Assume it is because we need to start writing new PDUs + * to the socket. + */ + revents = POLLOUT; + goto call_service; + } + if (ret < 0) { iscsi_set_error(iscsi, "Poll failed"); revents = -1; } else { revents = pfd.revents; } + call_service: if (iscsi_service(iscsi, revents) < 0) { if (revents != -1) iscsi_set_error(iscsi, "iscsi_service failed"); diff --git a/lib/socket.c b/lib/socket.c index f1a1e15..21f4ae3 100644 --- a/lib/socket.c +++ b/lib/socket.c @@ -62,6 +62,10 @@ #include #endif +#ifdef HAVE_PTHREAD +#include +#endif + #include #include #include @@ -88,7 +92,7 @@ union socket_address { void iscsi_add_to_outqueue(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { - struct iscsi_pdu *current = iscsi->outqueue; + struct iscsi_pdu *current; struct iscsi_pdu *last = NULL; if (iscsi->scsi_timeout > 0) { @@ -97,12 +101,15 @@ iscsi_add_to_outqueue(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) pdu->scsi_timeout = 0; } - if (iscsi->outqueue == NULL) { + iscsi_mt_mutex_lock(&iscsi->iscsi_mutex); + + current = iscsi->outqueue; + if (iscsi->outqueue == NULL) { iscsi->outqueue = pdu; pdu->next = NULL; - return; + goto finished; } - + /* queue pdus in ascending order of CmdSN. * ensure that pakets with the same CmdSN are kept in FIFO order. * immediate PDUs are queued in front of queue with the CmdSN @@ -131,7 +138,7 @@ iscsi_add_to_outqueue(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) iscsi->outqueue=pdu; } pdu->next = current; - return; + goto finished; } last=current; current=current->next; @@ -139,9 +146,24 @@ iscsi_add_to_outqueue(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) last->next = pdu; pdu->next = NULL; + + finished: + iscsi_mt_mutex_unlock(&iscsi->iscsi_mutex); + + /* TODO QQQ need to immediately send for the non multithreading case too + * and for the Windows API too */ +#if defined(HAVE_MULTITHREADING) && defined(HAVE_PTHREAD) + if (current == NULL && pdu == iscsi->outqueue) { + if(iscsi->multithreading_enabled) { + pthread_kill(iscsi->service_thread, SIGUSR1); + } + } +#endif + return; } void iscsi_decrement_iface_rr() { + /* TODO QQQ use an atomic here */ iface_rr--; } From 3fc5d2996b10a7a47728029401ef9ccd74d1a87a Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Sun, 20 Apr 2025 13:27:47 +1000 Subject: [PATCH 10/16] iscsi_queue_pdu() can never fail, make it void Signed-off-by: Ronnie Sahlberg --- include/iscsi-private.h | 8 +++---- lib/discovery.c | 7 +----- lib/iscsi-command.c | 14 ++--------- lib/iser.c | 10 ++++---- lib/login.c | 14 ++--------- lib/nop.c | 12 ++-------- lib/pdu.c | 10 ++------ lib/socket.c | 9 +------ lib/task_mgmt.c | 6 +---- test-tool/iscsi-test-cu.c | 2 +- test-tool/iscsi-test-cu.h | 4 ++-- ...est_compareandwrite_invalid_dataout_size.c | 4 ++-- test-tool/test_iscsi_chap.c | 24 +++++++++---------- test-tool/test_iscsi_cmdsn_toohigh.c | 4 ++-- test-tool/test_iscsi_cmdsn_toolow.c | 4 ++-- test-tool/test_iscsi_datasn_invalid.c | 4 ++-- test-tool/test_iscsi_sendtargets.c | 3 +-- .../test_sanitize_block_erase_reserved.c | 4 ++-- .../test_sanitize_crypto_erase_reserved.c | 4 ++-- test-tool/test_sanitize_overwrite_reserved.c | 4 ++-- 20 files changed, 49 insertions(+), 102 deletions(-) diff --git a/include/iscsi-private.h b/include/iscsi-private.h index ec1b85e..bc23b38 100644 --- a/include/iscsi-private.h +++ b/include/iscsi-private.h @@ -344,10 +344,11 @@ void iscsi_cancel_pdus(struct iscsi_context *iscsi); void iscsi_cancel_lun_pdus(struct iscsi_context *iscsi, uint32_t lun); int iscsi_pdu_add_data(struct iscsi_context *iscsi, struct iscsi_pdu *pdu, const unsigned char *dptr, int dsize); -int iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu); +void iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu); int iscsi_add_data(struct iscsi_context *iscsi, struct iscsi_data *data, const unsigned char *dptr, int dsize, int pdualignment); +void iscsi_add_to_outqueue(struct iscsi_context *iscsi, struct iscsi_pdu *pdu); struct scsi_task; void iscsi_pdu_set_cdb(struct iscsi_pdu *pdu, struct scsi_task *task); @@ -417,9 +418,6 @@ void iscsi_decrement_iface_rr(void); void __attribute__((format(printf, 3, 4))) iscsi_log_message(struct iscsi_context *iscsi, int level, const char *format, ...); -void -iscsi_add_to_outqueue(struct iscsi_context *iscsi, struct iscsi_pdu *pdu); - int iscsi_serial32_compare(uint32_t s1, uint32_t s2); uint32_t iscsi_itt_post_increment(struct iscsi_context *iscsi); @@ -443,7 +441,7 @@ union socket_address; typedef struct iscsi_transport { int (*connect)(struct iscsi_context *iscsi, union socket_address *sa, int ai_family); - int (*queue_pdu)(struct iscsi_context *iscsi, struct iscsi_pdu *pdu); + void (*queue_pdu)(struct iscsi_context *iscsi, struct iscsi_pdu *pdu); struct iscsi_pdu* (*new_pdu)(struct iscsi_context *iscsi, size_t size); int (*disconnect)(struct iscsi_context *iscsi); void (*free_pdu)(struct iscsi_context *iscsi, struct iscsi_pdu *pdu); diff --git a/lib/discovery.c b/lib/discovery.c index d785ca8..4518498 100644 --- a/lib/discovery.c +++ b/lib/discovery.c @@ -73,12 +73,7 @@ iscsi_discovery_async(struct iscsi_context *iscsi, iscsi_command_cb cb, pdu->callback = cb; pdu->private_data = private_data; - if (iscsi_queue_pdu(iscsi, pdu) != 0) { - iscsi_set_error(iscsi, "Out-of-memory: failed to queue iscsi " - "text pdu."); - iscsi->drv->free_pdu(iscsi, pdu); - return -1; - } + iscsi_queue_pdu(iscsi, pdu); return 0; } diff --git a/lib/iscsi-command.c b/lib/iscsi-command.c index d218060..047cd8f 100644 --- a/lib/iscsi-command.c +++ b/lib/iscsi-command.c @@ -128,12 +128,7 @@ iscsi_send_data_out(struct iscsi_context *iscsi, struct iscsi_pdu *cmd_pdu, /* update data segment length */ scsi_set_uint32(&pdu->outdata.data[4], pdu->payload_len); - if (iscsi_queue_pdu(iscsi, pdu) != 0) { - iscsi_set_error(iscsi, "Out-of-memory: failed to queue iscsi " - "scsi pdu."); - iscsi->drv->free_pdu(iscsi, pdu); - goto error; - } + iscsi_queue_pdu(iscsi, pdu); tot_len -= len; offset += len; @@ -280,12 +275,7 @@ iscsi_scsi_command_async(struct iscsi_context *iscsi, int lun, pdu->callback = iscsi_scsi_response_cb; pdu->private_data = &pdu->scsi_cbdata; - if (iscsi_queue_pdu(iscsi, pdu) != 0) { - iscsi_set_error(iscsi, "Out-of-memory: failed to queue iscsi " - "scsi pdu."); - iscsi->drv->free_pdu(iscsi, pdu); - return -1; - } + iscsi_queue_pdu(iscsi, pdu); iscsi->cmdsn++; /* The F flag is not set. This means we haven't sent all the unsolicited diff --git a/lib/iser.c b/lib/iser.c index 4e062d8..025bf6b 100644 --- a/lib/iser.c +++ b/lib/iser.c @@ -982,27 +982,27 @@ iscsi_iser_revive_queued_pdus(struct iscsi_context *iscsi) { * Need to be compatible to TCP which has real queue, * in iSER pdus with cmdsn not exceeds maxcmdsn are already sent. */ -static int +static void iscsi_iser_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { if (pdu == NULL) { iscsi_set_error(iscsi, "trying to queue NULL pdu"); - return -1; + return; } if (pdu->outdata.data[0] == ISCSI_PDU_NOP_OUT && iscsi_iser_cm_event(iscsi) != 0) { iscsi_service_reconnect_if_loggedin(iscsi); - return -1; + return; } if (iscsi->outqueue != NULL || (iscsi_serial32_compare(pdu->cmdsn, iscsi->maxcmdsn) > 0 && !(pdu->outdata.data[0] & ISCSI_PDU_IMMEDIATE))) { iscsi_add_to_outqueue(iscsi, pdu); - return 0; + return; } - return iscsi_iser_send_pdu(iscsi, pdu); + iscsi_iser_send_pdu(iscsi, pdu); } diff --git a/lib/login.c b/lib/login.c index cd91429..120c8c2 100644 --- a/lib/login.c +++ b/lib/login.c @@ -1236,12 +1236,7 @@ iscsi_login_async(struct iscsi_context *iscsi, iscsi_command_cb cb, pdu->callback = cb; pdu->private_data = private_data; - if (iscsi_queue_pdu(iscsi, pdu) != 0) { - iscsi_set_error(iscsi, "Out-of-memory: failed to queue iscsi " - "pdu."); - iscsi->drv->free_pdu(iscsi, pdu); - return -1; - } + iscsi_queue_pdu(iscsi, pdu); return 0; } @@ -1587,12 +1582,7 @@ iscsi_logout_async(struct iscsi_context *iscsi, iscsi_command_cb cb, pdu->callback = cb; pdu->private_data = private_data; - if (iscsi_queue_pdu(iscsi, pdu) != 0) { - iscsi_set_error(iscsi, "Out-of-memory: failed to queue iscsi " - "logout pdu."); - iscsi->drv->free_pdu(iscsi, pdu); - return -1; - } + iscsi_queue_pdu(iscsi, pdu); return 0; } diff --git a/lib/nop.c b/lib/nop.c index 2c1391a..e59f169 100644 --- a/lib/nop.c +++ b/lib/nop.c @@ -77,11 +77,7 @@ iscsi_nop_out_async(struct iscsi_context *iscsi, iscsi_command_cb cb, } } - if (iscsi_queue_pdu(iscsi, pdu) != 0) { - iscsi_set_error(iscsi, "failed to queue iscsi nop-out pdu"); - iscsi->drv->free_pdu(iscsi, pdu); - return -1; - } + iscsi_queue_pdu(iscsi, pdu); iscsi->cmdsn++; iscsi->nops_in_flight++; @@ -122,11 +118,7 @@ iscsi_send_target_nop_out(struct iscsi_context *iscsi, uint32_t ttt, uint32_t lu /* cmdsn is not increased if Immediate delivery*/ iscsi_pdu_set_cmdsn(pdu, iscsi->cmdsn); - if (iscsi_queue_pdu(iscsi, pdu) != 0) { - iscsi_set_error(iscsi, "failed to queue iscsi nop-out pdu"); - iscsi->drv->free_pdu(iscsi, pdu); - return -1; - } + iscsi_queue_pdu(iscsi, pdu); ISCSI_LOG(iscsi, (iscsi->nops_in_flight > 1) ? 1 : 6, "NOP Out Send (nops_in_flight: %d, pdu->cmdsn %08x, pdu->itt %08x, pdu->ttt %08x, pdu->lun %8x, iscsi->maxcmdsn %08x, iscsi->expcmdsn %08x)", diff --git a/lib/pdu.c b/lib/pdu.c index 68e8a20..935972a 100644 --- a/lib/pdu.c +++ b/lib/pdu.c @@ -937,16 +937,10 @@ iscsi_timeout_scan(struct iscsi_context *iscsi) } } -int +void iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { - int ret; - - iscsi_mt_mutex_lock(&iscsi->iscsi_mutex); - ret = iscsi->drv->queue_pdu(iscsi, pdu); - iscsi_mt_mutex_unlock(&iscsi->iscsi_mutex); - - return ret; + iscsi->drv->queue_pdu(iscsi, pdu); } void diff --git a/lib/socket.c b/lib/socket.c index 21f4ae3..ce10811 100644 --- a/lib/socket.c +++ b/lib/socket.c @@ -1136,17 +1136,10 @@ iscsi_service(struct iscsi_context *iscsi, int revents) return iscsi->drv->service(iscsi, revents); } -static int iscsi_tcp_queue_pdu(struct iscsi_context *iscsi, +static void iscsi_tcp_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { - if (pdu == NULL) { - iscsi_set_error(iscsi, "trying to queue NULL pdu"); - return -1; - } - iscsi_add_to_outqueue(iscsi, pdu); - - return 0; } void diff --git a/lib/task_mgmt.c b/lib/task_mgmt.c index f3e6824..c63fffb 100644 --- a/lib/task_mgmt.c +++ b/lib/task_mgmt.c @@ -80,11 +80,7 @@ iscsi_task_mgmt_async(struct iscsi_context *iscsi, pdu->callback = cb; pdu->private_data = private_data; - if (iscsi_queue_pdu(iscsi, pdu) != 0) { - iscsi_set_error(iscsi, "failed to queue iscsi taskmgmt pdu"); - iscsi->drv->free_pdu(iscsi, pdu); - return -1; - } + iscsi_queue_pdu(iscsi, pdu); return 0; } diff --git a/test-tool/iscsi-test-cu.c b/test-tool/iscsi-test-cu.c index 4e14913..e8b5aa0 100644 --- a/test-tool/iscsi-test-cu.c +++ b/test-tool/iscsi-test-cu.c @@ -739,7 +739,7 @@ static struct test_family families[] = { */ struct scsi_task *task; unsigned char *read_write_buf; -int (*orig_queue_pdu)(struct iscsi_context *iscsi, struct iscsi_pdu *pdu); +void (*orig_queue_pdu)(struct iscsi_context *iscsi, struct iscsi_pdu *pdu); static void print_usage(void) diff --git a/test-tool/iscsi-test-cu.h b/test-tool/iscsi-test-cu.h index 0b9ad08..fd230cb 100644 --- a/test-tool/iscsi-test-cu.h +++ b/test-tool/iscsi-test-cu.h @@ -35,8 +35,8 @@ /* globals between setup, tests, and teardown */ extern struct scsi_task *task; extern unsigned char *read_write_buf; -extern int (*orig_queue_pdu)(struct iscsi_context *iscsi, - struct iscsi_pdu *pdu); +extern void (*orig_queue_pdu)(struct iscsi_context *iscsi, + struct iscsi_pdu *pdu); #ifndef HAVE_CU_SUITEINFO_PSETUPFUNC /* libcunit version 1 */ diff --git a/test-tool/test_compareandwrite_invalid_dataout_size.c b/test-tool/test_compareandwrite_invalid_dataout_size.c index dfac65d..940cd43 100644 --- a/test-tool/test_compareandwrite_invalid_dataout_size.c +++ b/test-tool/test_compareandwrite_invalid_dataout_size.c @@ -28,13 +28,13 @@ static int new_nlb = -1; -static int my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) +static void my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { if (pdu->outdata.data[0] != ISCSI_PDU_SCSI_REQUEST && new_nlb >= 0) { /* change NUMBER OF LOGICAL BLOCKS to new_nlb */ pdu->outdata.data[32 + 13] = new_nlb; } - return orig_queue_pdu(iscsi, pdu); + orig_queue_pdu(iscsi, pdu); } void diff --git a/test-tool/test_iscsi_chap.c b/test-tool/test_iscsi_chap.c index 57668e0..8dfbd41 100644 --- a/test-tool/test_iscsi_chap.c +++ b/test-tool/test_iscsi_chap.c @@ -77,7 +77,7 @@ test_iscsi_strip_tag(struct iscsi_context *iscsi, struct iscsi_pdu *pdu, return 0; } -static int +static void chap_mod_strip_replace_queue(struct iscsi_context *iscsi, struct iscsi_pdu *pdu, const char *new_chap_a) { @@ -93,41 +93,41 @@ chap_mod_strip_replace_queue(struct iscsi_context *iscsi, struct iscsi_pdu *pdu, goto out; } if (ret < 0) { - return ret; + return; } ret = iscsi_pdu_add_data(iscsi, pdu, (const unsigned char *)new_chap_a, strlen(new_chap_a) + 1); if (ret < 0) { - return ret; + return; } logging(LOG_VERBOSE, "replaced Login PDU CHAP_A setting with %s", new_chap_a); out: - return orig_queue_pdu(iscsi, pdu); + orig_queue_pdu(iscsi, pdu); } -static int +static void chap_mod_many_types_queue(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { - return chap_mod_strip_replace_queue(iscsi, pdu, "CHAP_A=5,6,7,8"); + chap_mod_strip_replace_queue(iscsi, pdu, "CHAP_A=5,6,7,8"); } -static int +static void chap_mod_no_type_queue(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { - return chap_mod_strip_replace_queue(iscsi, pdu, "CHAP_A="); + chap_mod_strip_replace_queue(iscsi, pdu, "CHAP_A="); } -static int +static void chap_mod_bad_type_queue(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { /* value starts with '5', to catch targets that only check one byte */ - return chap_mod_strip_replace_queue(iscsi, pdu, "CHAP_A=56"); + chap_mod_strip_replace_queue(iscsi, pdu, "CHAP_A=56"); } static int -test_iscsi_chap_login(int (*test_queue_pdu)(struct iscsi_context *iscsi, - struct iscsi_pdu *pdu)) +test_iscsi_chap_login(void (*test_queue_pdu)(struct iscsi_context *iscsi, + struct iscsi_pdu *pdu)) { struct iscsi_context *iscsi; struct iscsi_url *iscsi_url; diff --git a/test-tool/test_iscsi_cmdsn_toohigh.c b/test-tool/test_iscsi_cmdsn_toohigh.c index 8d56e1f..7ba094c 100644 --- a/test-tool/test_iscsi_cmdsn_toohigh.c +++ b/test-tool/test_iscsi_cmdsn_toohigh.c @@ -26,7 +26,7 @@ static int change_cmdsn; -static int my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) +static void my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { switch (change_cmdsn) { case 1: @@ -45,7 +45,7 @@ static int my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu } change_cmdsn = 0; - return orig_queue_pdu(iscsi, pdu); + orig_queue_pdu(iscsi, pdu); } void test_iscsi_cmdsn_toohigh(void) diff --git a/test-tool/test_iscsi_cmdsn_toolow.c b/test-tool/test_iscsi_cmdsn_toolow.c index 652f645..75f380b 100644 --- a/test-tool/test_iscsi_cmdsn_toolow.c +++ b/test-tool/test_iscsi_cmdsn_toolow.c @@ -26,7 +26,7 @@ static int change_cmdsn; -static int my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) +static void my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { switch (change_cmdsn) { case 1: @@ -45,7 +45,7 @@ static int my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu } change_cmdsn = 0; - return orig_queue_pdu(iscsi, pdu); + orig_queue_pdu(iscsi, pdu); } void test_iscsi_cmdsn_toolow(void) diff --git a/test-tool/test_iscsi_datasn_invalid.c b/test-tool/test_iscsi_datasn_invalid.c index 9afaf50..b468cc0 100644 --- a/test-tool/test_iscsi_datasn_invalid.c +++ b/test-tool/test_iscsi_datasn_invalid.c @@ -26,7 +26,7 @@ static int change_datasn; -static int my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) +static void my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { uint32_t datasn; @@ -57,7 +57,7 @@ static int my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu break; } out: - return orig_queue_pdu(iscsi, pdu); + orig_queue_pdu(iscsi, pdu); } void test_iscsi_datasn_invalid(void) diff --git a/test-tool/test_iscsi_sendtargets.c b/test-tool/test_iscsi_sendtargets.c index e7a1892..30030e7 100644 --- a/test-tool/test_iscsi_sendtargets.c +++ b/test-tool/test_iscsi_sendtargets.c @@ -148,8 +148,7 @@ test_iscsi_text_req_queue(struct iscsi_context *iscsi, pdu->callback = cb; pdu->private_data = state; - ret = iscsi_queue_pdu(iscsi, pdu); - CU_ASSERT_EQUAL_FATAL(ret, 0); + iscsi_queue_pdu(iscsi, pdu); state->dispatched++; logging(LOG_VERBOSE, "queued Text request %d with %s", state->dispatched, kv_data); diff --git a/test-tool/test_sanitize_block_erase_reserved.c b/test-tool/test_sanitize_block_erase_reserved.c index 43a7c0c..6bff892 100644 --- a/test-tool/test_sanitize_block_erase_reserved.c +++ b/test-tool/test_sanitize_block_erase_reserved.c @@ -26,7 +26,7 @@ static int change_num; -static int my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) +static void my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { switch (change_num) { case 1: @@ -44,7 +44,7 @@ static int my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu } change_num = 0; - return orig_queue_pdu(iscsi, pdu); + orig_queue_pdu(iscsi, pdu); } void test_sanitize_block_erase_reserved(void) diff --git a/test-tool/test_sanitize_crypto_erase_reserved.c b/test-tool/test_sanitize_crypto_erase_reserved.c index 7b71019..c8291b6 100644 --- a/test-tool/test_sanitize_crypto_erase_reserved.c +++ b/test-tool/test_sanitize_crypto_erase_reserved.c @@ -26,7 +26,7 @@ static int change_num; -static int my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) +static void my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { switch (change_num) { case 1: @@ -44,7 +44,7 @@ static int my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu } change_num = 0; - return orig_queue_pdu(iscsi, pdu); + orig_queue_pdu(iscsi, pdu); } void test_sanitize_crypto_erase_reserved(void) diff --git a/test-tool/test_sanitize_overwrite_reserved.c b/test-tool/test_sanitize_overwrite_reserved.c index c45a6f5..c5c36c4 100644 --- a/test-tool/test_sanitize_overwrite_reserved.c +++ b/test-tool/test_sanitize_overwrite_reserved.c @@ -27,7 +27,7 @@ static int change_num; -static int my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) +static void my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { switch (change_num) { case 1: @@ -45,7 +45,7 @@ static int my_iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu } change_num = 0; - return orig_queue_pdu(iscsi, pdu); + orig_queue_pdu(iscsi, pdu); } void test_sanitize_overwrite_reserved(void) From 45937463638dccddb89369881e81e04027cd4e71 Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Sun, 20 Apr 2025 15:19:10 +1000 Subject: [PATCH 11/16] Add spinlock primitives Signed-off-by: Ronnie Sahlberg --- include/iscsi-multithreading.h | 25 ++++++++++++++++++++++++- include/iscsi-private.h | 1 + lib/init.c | 2 ++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/include/iscsi-multithreading.h b/include/iscsi-multithreading.h index 197b08c..a9e3809 100644 --- a/include/iscsi-multithreading.h +++ b/include/iscsi-multithreading.h @@ -123,9 +123,27 @@ static inline int iscsi_mt_mutex_unlock(libiscsi_mutex_t *mutex) return pthread_mutex_unlock(mutex); } +typedef pthread_spinlock_t libiscsi_spinlock_t; + static inline int iscsi_mt_spin_init(libiscsi_spinlock_t *spinlock, int shared) +{ + return pthread_spin_init(spinlock, shared); +} +static inline int iscsi_mt_spin_destroy(libiscsi_spinlock_t *spinlock) +{ + return pthread_spin_destroy(spinlock); +} +static inline int iscsi_mt_spin_lock(libiscsi_spinlock_t *spinlock) +{ + return pthread_spin_lock(spinlock); +} +static inline int iscsi_mt_spin_unlock(libiscsi_spinlock_t *spinlock) +{ + return pthread_spin_unlock(spinlock); +} + #elif defined(WIN32) typedef HANDLE libiscsi_mutex_t; -static inline int iscsi_mt_mutex_init(libiscsi_mutex_t* mutex) + static inline int iscsi_mt_mutex_init(libiscsi_mutex_t* mutex) { *mutex = CreateSemaphoreA(NULL, 1, 1, NULL); return 0; @@ -156,6 +174,11 @@ typedef const int libiscsi_mutex_t; #define iscsi_mt_mutex_destroy(x) ; #define iscsi_mt_mutex_lock(x) ; #define iscsi_mt_mutex_unlock(x) ; +typedef const int libiscsi_spinlock_t; +#define iscsi_mt_spin_init(x) ; +#define iscsi_mt_spin_destroy(x) ; +#define iscsi_mt_spin_lock(x) ; +#define iscsi_mt_spin_unlock(x) ; #endif /* mutex */ diff --git a/include/iscsi-private.h b/include/iscsi-private.h index bc23b38..918b192 100644 --- a/include/iscsi-private.h +++ b/include/iscsi-private.h @@ -213,6 +213,7 @@ struct iscsi_context { #ifdef HAVE_MULTITHREADING int multithreading_enabled; + libiscsi_spinlock_t iscsi_lock; libiscsi_mutex_t iscsi_mutex; libiscsi_thread_t service_thread; int poll_timeout; diff --git a/lib/init.c b/lib/init.c index 0f78126..0c27271 100644 --- a/lib/init.c +++ b/lib/init.c @@ -215,6 +215,7 @@ iscsi_create_context(const char *initiator_name) memset(iscsi, 0, sizeof(struct iscsi_context)); + iscsi_mt_spin_init(&iscsi->iscsi_lock, PTHREAD_PROCESS_PRIVATE); iscsi_mt_mutex_init(&iscsi->iscsi_mutex); iscsi->poll_timeout = 100; @@ -433,6 +434,7 @@ iscsi_destroy_context(struct iscsi_context *iscsi) iscsi_destroy_context(iscsi->old_iscsi); } + iscsi_mt_spin_destroy(&iscsi->iscsi_lock); iscsi_mt_mutex_destroy(&iscsi->iscsi_mutex); memset(iscsi, 0, sizeof(struct iscsi_context)); From 91cc1e4197f3536f53f9bf0fb6ca56502c3540c9 Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Fri, 25 Apr 2025 11:05:37 +1000 Subject: [PATCH 12/16] Protect outqueue and waitpdu with a spinlock Signed-off-by: Ronnie Sahlberg --- include/iscsi-private.h | 9 +- lib/connect.c | 13 +- lib/init.c | 2 + lib/iscsi-command.c | 48 ++++-- lib/iser.c | 20 ++- lib/nop.c | 2 + lib/pdu.c | 307 +++++++++++++++++++++++--------------- lib/socket.c | 28 ++-- test-tool/iscsi-support.c | 2 + 9 files changed, 271 insertions(+), 160 deletions(-) diff --git a/include/iscsi-private.h b/include/iscsi-private.h index 918b192..f68a37a 100644 --- a/include/iscsi-private.h +++ b/include/iscsi-private.h @@ -170,11 +170,10 @@ struct iscsi_context { iscsi_command_cb socket_status_cb; void *connect_data; - struct iscsi_pdu *outqueue; // multithreading todo: may need mutex - struct iscsi_pdu *outqueue_current; // multithreading todo: may need mutex - struct iscsi_pdu *waitpdu; // multithreading todo: may need mutex - - struct iscsi_in_pdu *incoming; // multithreading todo: may need mutex + struct iscsi_pdu *outqueue; /* Protected by iscsi_lock */ + struct iscsi_pdu *outqueue_current; /* Protected by iscsi_lock */ + struct iscsi_pdu *waitpdu; /* Protected by iscsi_lock */ + struct iscsi_in_pdu *incoming; /* Protected by iscsi_lock */ uint32_t max_burst_length; uint32_t first_burst_length; diff --git a/lib/connect.c b/lib/connect.c index 2d3b85d..4dc98f6 100644 --- a/lib/connect.c +++ b/lib/connect.c @@ -280,6 +280,7 @@ void iscsi_reconnect_cb(struct iscsi_context *iscsi, int status, void *command_data, void *private_data) { struct iscsi_context *old_iscsi; + struct iscsi_pdu *tmp = NULL; int i; if (status != SCSI_STATUS_GOOD) { @@ -305,16 +306,20 @@ void iscsi_reconnect_cb(struct iscsi_context *iscsi, int status, old_iscsi = iscsi->old_iscsi; iscsi->old_iscsi = NULL; + iscsi_mt_spin_lock(&iscsi->iscsi_lock); while (old_iscsi->outqueue) { struct iscsi_pdu *pdu = old_iscsi->outqueue; ISCSI_LIST_REMOVE(&old_iscsi->outqueue, pdu); ISCSI_LIST_ADD_END(&old_iscsi->waitpdu, pdu); } + tmp = old_iscsi->waitpdu; + old_iscsi->waitpdu = NULL; + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); - while (old_iscsi->waitpdu) { - struct iscsi_pdu *pdu = old_iscsi->waitpdu; + while (tmp) { + struct iscsi_pdu *pdu = tmp; - ISCSI_LIST_REMOVE(&old_iscsi->waitpdu, pdu); + ISCSI_LIST_REMOVE(&tmp, pdu); if (pdu->itt == 0xffffffff) { iscsi->drv->free_pdu(old_iscsi, pdu); continue; @@ -351,6 +356,7 @@ void iscsi_reconnect_cb(struct iscsi_context *iscsi, int status, iscsi->drv->free_pdu(old_iscsi, pdu); } + iscsi_mt_spin_lock(&iscsi->iscsi_lock); if (old_iscsi->incoming != NULL) { iscsi_free_iscsi_in_pdu(old_iscsi, old_iscsi->incoming); } @@ -358,6 +364,7 @@ void iscsi_reconnect_cb(struct iscsi_context *iscsi, int status, if (old_iscsi->outqueue_current != NULL && old_iscsi->outqueue_current->flags & ISCSI_PDU_DELETE_WHEN_SENT) { iscsi->drv->free_pdu(old_iscsi, old_iscsi->outqueue_current); } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); iscsi_free(old_iscsi, old_iscsi->opaque); diff --git a/lib/init.c b/lib/init.c index 0c27271..fcae7f3 100644 --- a/lib/init.c +++ b/lib/init.c @@ -407,6 +407,7 @@ iscsi_destroy_context(struct iscsi_context *iscsi) iscsi_cancel_pdus(iscsi); + iscsi_mt_spin_lock(&iscsi->iscsi_lock); if (iscsi->outqueue_current != NULL && iscsi->outqueue_current->flags & ISCSI_PDU_DELETE_WHEN_SENT) { iscsi->drv->free_pdu(iscsi, iscsi->outqueue_current); } @@ -414,6 +415,7 @@ iscsi_destroy_context(struct iscsi_context *iscsi) if (iscsi->incoming != NULL) { iscsi_free_iscsi_in_pdu(iscsi, iscsi->incoming); } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); iscsi->connect_data = NULL; diff --git a/lib/iscsi-command.c b/lib/iscsi-command.c index 047cd8f..5fe6135 100644 --- a/lib/iscsi-command.c +++ b/lib/iscsi-command.c @@ -136,8 +136,10 @@ iscsi_send_data_out(struct iscsi_context *iscsi, struct iscsi_pdu *cmd_pdu, return 0; error: + iscsi_mt_spin_lock(&iscsi->iscsi_lock); ISCSI_LIST_REMOVE(&iscsi->outqueue, cmd_pdu); ISCSI_LIST_REMOVE(&iscsi->waitpdu, cmd_pdu); + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); if (cmd_pdu->callback) { cmd_pdu->callback(iscsi, SCSI_STATUS_ERROR, NULL, cmd_pdu->private_data); @@ -2406,11 +2408,13 @@ iscsi_get_scsi_task_iovector_in(struct iscsi_context *iscsi, struct iscsi_in_pdu } itt = scsi_get_uint32(&in->hdr[16]); + iscsi_mt_spin_lock(&iscsi->iscsi_lock); for (pdu = iscsi->waitpdu; pdu; pdu = pdu->next) { if (pdu->itt == itt) { break; } } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); if (pdu == NULL) { return NULL; @@ -2669,11 +2673,14 @@ int iscsi_scsi_is_task_in_outqueue(struct iscsi_context *iscsi, struct scsi_task { struct iscsi_pdu *pdu; + iscsi_mt_spin_lock(&iscsi->iscsi_lock); for (pdu = iscsi->outqueue; pdu; pdu = pdu->next) { if (pdu->itt == task->itt) { + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); return 1; } } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); return 0; } @@ -2682,13 +2689,15 @@ int iscsi_scsi_cancel_task(struct iscsi_context *iscsi, struct scsi_task *task) { - struct iscsi_pdu *pdu; + struct iscsi_pdu *pdu, *tmp; struct iscsi_pdu *next_pdu; uint32_t cmdsn_gap = 0; int ret = -1; + iscsi_mt_spin_lock(&iscsi->iscsi_lock); for (pdu = iscsi->waitpdu; pdu; pdu = pdu->next) { if (pdu->itt == task->itt) { + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); if (pdu->callback) { pdu->callback(iscsi, SCSI_STATUS_CANCELLED, NULL, @@ -2698,6 +2707,8 @@ iscsi_scsi_cancel_task(struct iscsi_context *iscsi, return 0; } } + + tmp = NULL; for (pdu = iscsi->outqueue; pdu; pdu = next_pdu) { next_pdu = pdu->next; @@ -2707,22 +2718,27 @@ iscsi_scsi_cancel_task(struct iscsi_context *iscsi, if (pdu->itt == task->itt) { ISCSI_LIST_REMOVE(&iscsi->outqueue, pdu); - if (pdu->callback) { - pdu->callback(iscsi, SCSI_STATUS_CANCELLED, NULL, + ISCSI_LIST_ADD_END(&tmp, pdu); + } + } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); + for (pdu = tmp; pdu; pdu = next_pdu) { + ISCSI_LIST_REMOVE(&tmp, pdu); + if (pdu->callback) { + pdu->callback(iscsi, SCSI_STATUS_CANCELLED, NULL, pdu->private_data); - } - if (!(pdu->outdata.data[0] & ISCSI_PDU_IMMEDIATE) && - (pdu->outdata.data[0] & 0x3f) != ISCSI_PDU_DATA_OUT) { - iscsi->cmdsn--; - cmdsn_gap++; - } - iscsi->drv->free_pdu(iscsi, pdu); - ret = 0; - if (!cmdsn_gap) { - break; - } - } - } + } + if (!(pdu->outdata.data[0] & ISCSI_PDU_IMMEDIATE) && + (pdu->outdata.data[0] & 0x3f) != ISCSI_PDU_DATA_OUT) { + iscsi->cmdsn--; + cmdsn_gap++; + } + iscsi->drv->free_pdu(iscsi, pdu); + ret = 0; + if (!cmdsn_gap) { + break; + } + } if (iscsi->old_iscsi) { return iscsi_scsi_cancel_task(iscsi->old_iscsi, task); diff --git a/lib/iser.c b/lib/iser.c index 025bf6b..c1a6350 100644 --- a/lib/iser.c +++ b/lib/iser.c @@ -334,6 +334,7 @@ iser_free_queued_pdu_tx_desc(struct iscsi_context *iscsi) struct iscsi_pdu *pdu; struct iser_pdu *iser_pdu; + iscsi_mt_spin_lock(&iscsi->iscsi_lock); for (pdu = iscsi->waitpdu; pdu; pdu = pdu->next) { iser_pdu = container_of(pdu, struct iser_pdu, iscsi_pdu); if (iser_pdu->desc) { @@ -349,6 +350,7 @@ iser_free_queued_pdu_tx_desc(struct iscsi_context *iscsi) iser_pdu->desc = NULL; } } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); } /* @@ -576,9 +578,11 @@ iscsi_iser_free_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) pdu->indata.data = NULL; + iscsi_mt_spin_lock(&iscsi->iscsi_lock); if (iscsi->outqueue_current == pdu) { iscsi->outqueue_current = NULL; } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); iscsi_sfree(iscsi, iser_pdu); } @@ -920,8 +924,10 @@ iscsi_iser_send_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { iser_pdu = container_of(pdu, struct iser_pdu, iscsi_pdu); opcode = pdu->outdata.data[0]; + iscsi_mt_spin_lock(&iscsi->iscsi_lock); iscsi_pdu_set_expstatsn(pdu, iscsi->statsn + 1); ISCSI_LIST_ADD_END(&iscsi->waitpdu, pdu); + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); /* * because of async reconnection, before reconnecting successfully, @@ -954,16 +960,20 @@ static int iscsi_iser_revive_queued_pdus(struct iscsi_context *iscsi) { struct iscsi_pdu *pdu; - while (iscsi->outqueue != NULL) { + while (iscsi->outqueue) { if (iscsi_serial32_compare(iscsi->outqueue->cmdsn, iscsi->maxcmdsn) > 0) { break; } + iscsi_mt_spin_lock(&iscsi->iscsi_lock); pdu = iscsi->outqueue; ISCSI_LIST_REMOVE(&iscsi->outqueue, pdu); + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); if (iscsi_iser_send_pdu(iscsi, pdu) < 0) { + iscsi_mt_spin_lock(&iscsi->iscsi_lock); ISCSI_LIST_ADD(&iscsi->outqueue, pdu); + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); return -1; } } @@ -995,7 +1005,7 @@ iscsi_iser_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) { return; } - if (iscsi->outqueue != NULL || + if (iscsi->outqueue || (iscsi_serial32_compare(pdu->cmdsn, iscsi->maxcmdsn) > 0 && !(pdu->outdata.data[0] & ISCSI_PDU_IMMEDIATE))) { iscsi_add_to_outqueue(iscsi, pdu); @@ -1308,6 +1318,8 @@ iser_rcv_completion(struct iser_rx_desc *rx_desc, struct iscsi_in_pdu in; int empty, err; struct iscsi_context *iscsi = iser_conn->cma_id->context; + struct iscsi_pdu *iscsi_pdu; + struct iser_pdu *iser_pdu; in.hdr = (unsigned char*)rx_desc->iscsi_header; in.data_pos = iscsi_get_pdu_data_size(&in.hdr[0]); @@ -1322,12 +1334,12 @@ iser_rcv_completion(struct iser_rx_desc *rx_desc, if (opcode == ISCSI_PDU_ASYNC_MSG) goto no_waitpdu; - struct iscsi_pdu *iscsi_pdu; - struct iser_pdu *iser_pdu; + iscsi_mt_spin_lock(&iscsi->iscsi_lock); for (iscsi_pdu = iscsi->waitpdu ; iscsi_pdu ; iscsi_pdu = iscsi_pdu->next) { if(iscsi_pdu->itt == itt) break; } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); iser_pdu = container_of(iscsi_pdu, struct iser_pdu, iscsi_pdu); diff --git a/lib/nop.c b/lib/nop.c index e59f169..9e62fae 100644 --- a/lib/nop.c +++ b/lib/nop.c @@ -133,6 +133,7 @@ iscsi_process_nop_out_reply(struct iscsi_context *iscsi, struct iscsi_pdu *pdu, { struct iscsi_data data; + iscsi_mt_spin_lock(&iscsi->iscsi_lock); ISCSI_LOG(iscsi, (iscsi->nops_in_flight > 1) ? 1 : 6, "NOP-In received (pdu->itt %08x, pdu->ttt %08x, iscsi->maxcmdsn %08x, iscsi->expcmdsn %08x, iscsi->statsn %08x)", pdu->itt, 0xffffffff, iscsi->maxcmdsn, iscsi->expcmdsn, iscsi->statsn); @@ -147,6 +148,7 @@ iscsi_process_nop_out_reply(struct iscsi_context *iscsi, struct iscsi_pdu *pdu, iscsi->nops_in_flight = 0; } iscsi->min_cmdsn_waiting = iscsi->waitpdu->cmdsn; + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); if (pdu->callback == NULL) { return 0; diff --git a/lib/pdu.c b/lib/pdu.c index 935972a..3c0aa71 100644 --- a/lib/pdu.c +++ b/lib/pdu.c @@ -458,11 +458,13 @@ int iscsi_process_reject(struct iscsi_context *iscsi, iscsi_dump_pdu_header(iscsi, in->data); + iscsi_mt_spin_lock(&iscsi->iscsi_lock); for (pdu = iscsi->waitpdu; pdu; pdu = pdu->next) { if (pdu->itt == itt) { break; } } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); if (pdu == NULL) { iscsi_set_error(iscsi, "Can not match REJECT with" @@ -476,7 +478,9 @@ int iscsi_process_reject(struct iscsi_context *iscsi, pdu->private_data); } + iscsi_mt_spin_lock(&iscsi->iscsi_lock); ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); iscsi->drv->free_pdu(iscsi, pdu); return 0; } @@ -525,6 +529,8 @@ iscsi_process_pdu(struct iscsi_context *iscsi, struct iscsi_in_pdu *in) enum iscsi_opcode opcode = in->hdr[0] & 0x3f; uint8_t ahslen = in->hdr[4]; struct iscsi_pdu *pdu; + enum iscsi_opcode expected_response; + int is_finished = 1; /* verify header checksum */ if (iscsi->header_digest != ISCSI_HEADER_DIGEST_NONE) { @@ -623,125 +629,148 @@ iscsi_process_pdu(struct iscsi_context *iscsi, struct iscsi_in_pdu *in) return 0; } + iscsi_mt_spin_lock(&iscsi->iscsi_lock); for (pdu = iscsi->waitpdu; pdu; pdu = pdu->next) { - enum iscsi_opcode expected_response = pdu->response_opcode; - int is_finished = 1; - - if (pdu->itt != itt) { - continue; + if (pdu->itt == itt) { + break; } + } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); + if (pdu == NULL) { + iscsi_set_error(iscsi, "Got unsolicited response with " + "itt:%d", + itt); + return -1; + } + + /* we have a special case with scsi-command opcodes, + * they are replied to by either a scsi-response + * or a data-in, or a combination of both. + */ + expected_response = pdu->response_opcode; + if (opcode == ISCSI_PDU_DATA_IN + && expected_response == ISCSI_PDU_SCSI_RESPONSE) { + expected_response = ISCSI_PDU_DATA_IN; + } - /* we have a special case with scsi-command opcodes, - * they are replied to by either a scsi-response - * or a data-in, or a combination of both. - */ - if (opcode == ISCSI_PDU_DATA_IN - && expected_response == ISCSI_PDU_SCSI_RESPONSE) { - expected_response = ISCSI_PDU_DATA_IN; - } + /* Another special case is if we get a R2T. + * In this case we should find the original request and just send an additional + * DATAOUT segment for this task. + */ + if (opcode == ISCSI_PDU_R2T) { + expected_response = ISCSI_PDU_R2T; + } - /* Another special case is if we get a R2T. - * In this case we should find the original request and just send an additional - * DATAOUT segment for this task. - */ - if (opcode == ISCSI_PDU_R2T) { - expected_response = ISCSI_PDU_R2T; - } + if (opcode != expected_response) { + iscsi_set_error(iscsi, "Got wrong opcode back for " + "itt:%d got:%d expected %d", + itt, opcode, pdu->response_opcode); + return -1; + } + switch (opcode) { + case ISCSI_PDU_LOGIN_RESPONSE: + if (iscsi_process_login_reply(iscsi, pdu, in) != 0) { + iscsi_mt_spin_lock(&iscsi->iscsi_lock); + ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); + iscsi->drv->free_pdu(iscsi, pdu); + iscsi_set_error(iscsi, "iscsi login reply " + "failed"); + return -1; + } + break; + case ISCSI_PDU_TEXT_RESPONSE: + if (iscsi_process_text_reply(iscsi, pdu, in) != 0) { + iscsi_mt_spin_lock(&iscsi->iscsi_lock); + ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); + iscsi->drv->free_pdu(iscsi, pdu); + iscsi_set_error(iscsi, "iscsi text reply " + "failed"); + return -1; + } + break; + case ISCSI_PDU_LOGOUT_RESPONSE: + if (iscsi_process_logout_reply(iscsi, pdu, in) != 0) { + iscsi_mt_spin_lock(&iscsi->iscsi_lock); + ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); + iscsi->drv->free_pdu(iscsi, pdu); + iscsi_set_error(iscsi, "iscsi logout reply " + "failed"); + return -1; + } + break; + case ISCSI_PDU_SCSI_RESPONSE: + if (iscsi_process_scsi_reply(iscsi, pdu, in) != 0) { + iscsi_mt_spin_lock(&iscsi->iscsi_lock); + ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); + iscsi->drv->free_pdu(iscsi, pdu); + iscsi_set_error(iscsi, "iscsi response reply " + "failed"); + return -1; + } + break; + case ISCSI_PDU_DATA_IN: + if (iscsi_process_scsi_data_in(iscsi, pdu, in, + &is_finished) != 0) { + iscsi_mt_spin_lock(&iscsi->iscsi_lock); + ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); + iscsi->drv->free_pdu(iscsi, pdu); + iscsi_set_error(iscsi, "iscsi data in " + "failed"); + return -1; + } + break; + case ISCSI_PDU_NOP_IN: + if (iscsi_process_nop_out_reply(iscsi, pdu, in) != 0) { + iscsi_mt_spin_lock(&iscsi->iscsi_lock); + ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); + iscsi->drv->free_pdu(iscsi, pdu); + iscsi_set_error(iscsi, "iscsi nop-in failed"); + return -1; + } + break; + case ISCSI_PDU_SCSI_TASK_MANAGEMENT_RESPONSE: + if (iscsi_process_task_mgmt_reply(iscsi, pdu, + in) != 0) { + iscsi_mt_spin_lock(&iscsi->iscsi_lock); + ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); + iscsi->drv->free_pdu(iscsi, pdu); + iscsi_set_error(iscsi, "iscsi task-mgmt failed"); + return -1; + } + break; + case ISCSI_PDU_R2T: + if (iscsi_process_r2t(iscsi, pdu, in) != 0) { + iscsi_mt_spin_lock(&iscsi->iscsi_lock); + ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); + iscsi->drv->free_pdu(iscsi, pdu); + iscsi_set_error(iscsi, "iscsi r2t " + "failed"); + return -1; + } + is_finished = 0; + break; + default: + iscsi_set_error(iscsi, "Don't know how to handle " + "opcode 0x%02x", opcode); + return -1; + } - if (opcode != expected_response) { - iscsi_set_error(iscsi, "Got wrong opcode back for " - "itt:%d got:%d expected %d", - itt, opcode, pdu->response_opcode); - return -1; - } - switch (opcode) { - case ISCSI_PDU_LOGIN_RESPONSE: - if (iscsi_process_login_reply(iscsi, pdu, in) != 0) { - ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); - iscsi->drv->free_pdu(iscsi, pdu); - iscsi_set_error(iscsi, "iscsi login reply " - "failed"); - return -1; - } - break; - case ISCSI_PDU_TEXT_RESPONSE: - if (iscsi_process_text_reply(iscsi, pdu, in) != 0) { - ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); - iscsi->drv->free_pdu(iscsi, pdu); - iscsi_set_error(iscsi, "iscsi text reply " - "failed"); - return -1; - } - break; - case ISCSI_PDU_LOGOUT_RESPONSE: - if (iscsi_process_logout_reply(iscsi, pdu, in) != 0) { - ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); - iscsi->drv->free_pdu(iscsi, pdu); - iscsi_set_error(iscsi, "iscsi logout reply " - "failed"); - return -1; - } - break; - case ISCSI_PDU_SCSI_RESPONSE: - if (iscsi_process_scsi_reply(iscsi, pdu, in) != 0) { - ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); - iscsi->drv->free_pdu(iscsi, pdu); - iscsi_set_error(iscsi, "iscsi response reply " - "failed"); - return -1; - } - break; - case ISCSI_PDU_DATA_IN: - if (iscsi_process_scsi_data_in(iscsi, pdu, in, - &is_finished) != 0) { - ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); - iscsi->drv->free_pdu(iscsi, pdu); - iscsi_set_error(iscsi, "iscsi data in " - "failed"); - return -1; - } - break; - case ISCSI_PDU_NOP_IN: - if (iscsi_process_nop_out_reply(iscsi, pdu, in) != 0) { - ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); - iscsi->drv->free_pdu(iscsi, pdu); - iscsi_set_error(iscsi, "iscsi nop-in failed"); - return -1; - } - break; - case ISCSI_PDU_SCSI_TASK_MANAGEMENT_RESPONSE: - if (iscsi_process_task_mgmt_reply(iscsi, pdu, - in) != 0) { - ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); - iscsi->drv->free_pdu(iscsi, pdu); - iscsi_set_error(iscsi, "iscsi task-mgmt failed"); - return -1; - } - break; - case ISCSI_PDU_R2T: - if (iscsi_process_r2t(iscsi, pdu, in) != 0) { - ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); - iscsi->drv->free_pdu(iscsi, pdu); - iscsi_set_error(iscsi, "iscsi r2t " - "failed"); - return -1; - } - is_finished = 0; - break; - default: - iscsi_set_error(iscsi, "Don't know how to handle " - "opcode 0x%02x", opcode); - return -1; - } - - if (is_finished && iscsi->waitpdu != NULL) { - ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); - iscsi->drv->free_pdu(iscsi, pdu); - } - return 0; - } - - return 0; + iscsi_mt_spin_lock(&iscsi->iscsi_lock); + if (is_finished && iscsi->waitpdu != NULL) { + ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); + iscsi->drv->free_pdu(iscsi, pdu); + } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); + + return 0; } void @@ -842,7 +871,8 @@ static int iscsi_pdu_data_out_inprocess(struct iscsi_context *iscsi, struct iscs { struct iscsi_pdu *tmp_pdu, *next_pdu; enum scsi_opcode opcode = pdu->outdata.data[32]; - + int ret = 1; + /* only care DATA OUT command here */ if ((pdu->outdata.data[0] & 0x3f) != ISCSI_PDU_SCSI_REQUEST) { return 0; @@ -857,9 +887,10 @@ static int iscsi_pdu_data_out_inprocess(struct iscsi_context *iscsi, struct iscs return 0; }; + iscsi_mt_spin_lock(&iscsi->iscsi_lock); /* current outgoing one is part of the PDU? */ if (iscsi->outqueue_current && (iscsi->outqueue_current->scsi_cbdata.task == pdu->scsi_cbdata.task)) { - return 1; + goto finished; } /* any child DATAOUT PDU in outqueue? */ @@ -867,21 +898,26 @@ static int iscsi_pdu_data_out_inprocess(struct iscsi_context *iscsi, struct iscs next_pdu = tmp_pdu->next; if (tmp_pdu->scsi_cbdata.task == pdu->scsi_cbdata.task) { - return 1; + goto finished; } } - return 0; + ret = 0; + finished: + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); + return ret; } void iscsi_timeout_scan(struct iscsi_context *iscsi) { - struct iscsi_pdu *pdu; + struct iscsi_pdu *pdu, *tmp; struct iscsi_pdu *next_pdu; time_t t = time(NULL); uint32_t cmdsn_gap = 0; + iscsi_mt_spin_lock(&iscsi->iscsi_lock); + tmp = NULL; for (pdu = iscsi->outqueue; pdu; pdu = next_pdu) { next_pdu = pdu->next; @@ -904,6 +940,11 @@ iscsi_timeout_scan(struct iscsi_context *iscsi) continue; } ISCSI_LIST_REMOVE(&iscsi->outqueue, pdu); + ISCSI_LIST_ADD_END(&tmp, pdu); + } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); + for (pdu = tmp; pdu; pdu = next_pdu) { + ISCSI_LIST_REMOVE(&tmp, pdu); iscsi_set_error(iscsi, "command timed out from outqueue"); iscsi_dump_pdu_header(iscsi, pdu->outdata.data); if (pdu->callback) { @@ -912,6 +953,9 @@ iscsi_timeout_scan(struct iscsi_context *iscsi) } iscsi->drv->free_pdu(iscsi, pdu); } + + iscsi_mt_spin_lock(&iscsi->iscsi_lock); + tmp = NULL; for (pdu = iscsi->waitpdu; pdu; pdu = next_pdu) { next_pdu = pdu->next; @@ -927,6 +971,11 @@ iscsi_timeout_scan(struct iscsi_context *iscsi) continue; } ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); + ISCSI_LIST_ADD_END(&tmp, pdu); + } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); + for (pdu = tmp; pdu; pdu = next_pdu) { + ISCSI_LIST_REMOVE(&tmp, pdu); iscsi_set_error(iscsi, "command timed out from waitqueue"); iscsi_dump_pdu_header(iscsi, pdu->outdata.data); if (pdu->callback) { @@ -946,8 +995,9 @@ iscsi_queue_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) void iscsi_cancel_pdus(struct iscsi_context *iscsi) { - struct iscsi_pdu *pdu; + struct iscsi_pdu *pdu, *tmp; + iscsi_mt_spin_lock(&iscsi->iscsi_lock); while ((pdu = iscsi->outqueue)) { ISCSI_LIST_REMOVE(&iscsi->outqueue, pdu); if (pdu->callback) { @@ -960,8 +1010,12 @@ iscsi_cancel_pdus(struct iscsi_context *iscsi) } iscsi->drv->free_pdu(iscsi, pdu); } - while ((pdu = iscsi->waitpdu)) { - ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); + tmp = iscsi->waitpdu; + iscsi->waitpdu = NULL; + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); + + while ((pdu = tmp)) { + ISCSI_LIST_REMOVE(&tmp, pdu); if (pdu->callback) { pdu->callback(iscsi, SCSI_STATUS_CANCELLED, NULL, pdu->private_data); @@ -973,11 +1027,13 @@ iscsi_cancel_pdus(struct iscsi_context *iscsi) void iscsi_cancel_lun_pdus(struct iscsi_context *iscsi, uint32_t lun) { - struct iscsi_pdu *pdu; + struct iscsi_pdu *pdu, *tmp; struct iscsi_pdu *next_pdu; uint32_t cmdsn_gap = 0; struct scsi_task * task = NULL; + iscsi_mt_spin_lock(&iscsi->iscsi_lock); + tmp = NULL; for (pdu = iscsi->outqueue; pdu; pdu = next_pdu) { next_pdu = pdu->next; task = iscsi_scsi_get_task_from_pdu(pdu); @@ -994,6 +1050,11 @@ iscsi_cancel_lun_pdus(struct iscsi_context *iscsi, uint32_t lun) cmdsn_gap++; } ISCSI_LIST_REMOVE(&iscsi->outqueue, pdu); + ISCSI_LIST_ADD_END(&tmp, pdu); + } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); + for (pdu = tmp; pdu; pdu = next_pdu) { + ISCSI_LIST_REMOVE(&tmp, pdu); iscsi_set_error(iscsi, "command cancelled"); if (pdu->callback) { pdu->callback(iscsi, SCSI_STATUS_CANCELLED, diff --git a/lib/socket.c b/lib/socket.c index ce10811..ce7fd0f 100644 --- a/lib/socket.c +++ b/lib/socket.c @@ -101,7 +101,7 @@ iscsi_add_to_outqueue(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) pdu->scsi_timeout = 0; } - iscsi_mt_mutex_lock(&iscsi->iscsi_mutex); + iscsi_mt_spin_lock(&iscsi->iscsi_lock); current = iscsi->outqueue; if (iscsi->outqueue == NULL) { @@ -133,22 +133,22 @@ iscsi_add_to_outqueue(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) (pdu->outdata.data[0] & ISCSI_PDU_IMMEDIATE && !(current->outdata.data[0] & ISCSI_PDU_IMMEDIATE))) { /* insert PDU before the current */ if (last != NULL) { - last->next=pdu; + last->next = pdu; } else { - iscsi->outqueue=pdu; + iscsi->outqueue = pdu; } pdu->next = current; goto finished; } - last=current; - current=current->next; + last = current; + current = current->next; } while (current != NULL); last->next = pdu; pdu->next = NULL; finished: - iscsi_mt_mutex_unlock(&iscsi->iscsi_mutex); + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); /* TODO QQQ need to immediately send for the non multithreading case too * and for the Windows API too */ @@ -491,14 +491,16 @@ iscsi_tcp_which_events(struct iscsi_context *iscsi) return 0; } - if (iscsi->outqueue_current != NULL || - (iscsi->outqueue != NULL && !iscsi->is_corked && + iscsi_mt_spin_lock(&iscsi->iscsi_lock); + if (iscsi->outqueue_current || + (iscsi->outqueue && !iscsi->is_corked && (iscsi_serial32_compare(iscsi->outqueue->cmdsn, iscsi->maxcmdsn) <= 0 || iscsi->outqueue->outdata.data[0] & ISCSI_PDU_IMMEDIATE) ) ) { events |= POLLOUT; } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); return events; } @@ -514,6 +516,7 @@ iscsi_queue_length(struct iscsi_context *iscsi) int i = 0; struct iscsi_pdu *pdu; + iscsi_mt_spin_lock(&iscsi->iscsi_lock); for (pdu = iscsi->outqueue; pdu; pdu = pdu->next) { i++; } @@ -523,6 +526,7 @@ iscsi_queue_length(struct iscsi_context *iscsi) if (iscsi->is_connected == 0) { i++; } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); return i; } @@ -533,9 +537,11 @@ iscsi_out_queue_length(struct iscsi_context *iscsi) int i = 0; struct iscsi_pdu *pdu; + iscsi_mt_spin_lock(&iscsi->iscsi_lock); for (pdu = iscsi->outqueue; pdu; pdu = pdu->next) { i++; } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); return i; } @@ -830,7 +836,7 @@ iscsi_write_to_socket(struct iscsi_context *iscsi) return -1; } - while (iscsi->outqueue != NULL || iscsi->outqueue_current != NULL) { + while (iscsi->outqueue || iscsi->outqueue_current) { if (iscsi->outqueue_current == NULL) { if (iscsi->is_corked) { /* connection is corked we are not allowed to send @@ -855,6 +861,8 @@ iscsi_write_to_socket(struct iscsi_context *iscsi) iscsi->outqueue->cmdsn, iscsi->expcmdsn, iscsi->outqueue->outdata.data[0] & 0x3f); return -1; } + + iscsi_mt_spin_lock(&iscsi->iscsi_lock); iscsi->outqueue_current = iscsi->outqueue; /* set exp statsn */ @@ -863,6 +871,7 @@ iscsi_write_to_socket(struct iscsi_context *iscsi) /* calculate header checksum */ if (iscsi->header_digest != ISCSI_HEADER_DIGEST_NONE && iscsi_pdu_update_headerdigest(iscsi, iscsi->outqueue_current) != 0) { + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); return -1; } @@ -874,6 +883,7 @@ iscsi_write_to_socket(struct iscsi_context *iscsi) cmd PDU the R2T might get lost otherwise. */ ISCSI_LIST_ADD_END(&iscsi->waitpdu, iscsi->outqueue_current); } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); } pdu = iscsi->outqueue_current; diff --git a/test-tool/iscsi-support.c b/test-tool/iscsi-support.c index 9375952..117017e 100644 --- a/test-tool/iscsi-support.c +++ b/test-tool/iscsi-support.c @@ -561,6 +561,7 @@ wait_until_test_finished(struct iscsi_context *iscsi, struct iscsi_async_state * state->finished = 1; state->status = SCSI_STATUS_CANCELLED; state->task->status = SCSI_STATUS_CANCELLED; + iscsi_mt_spin_lock(&iscsi->iscsi_lock); /* this may leak memory since we don't free the pdu */ while ((pdu = iscsi->outqueue)) { ISCSI_LIST_REMOVE(&iscsi->outqueue, pdu); @@ -568,6 +569,7 @@ wait_until_test_finished(struct iscsi_context *iscsi, struct iscsi_async_state * while ((pdu = iscsi->waitpdu)) { ISCSI_LIST_REMOVE(&iscsi->waitpdu, pdu); } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); return; } continue; From 8047421868d3497ba0a4c7f1c22c6e70d6bd346e Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Fri, 25 Apr 2025 11:41:44 +1000 Subject: [PATCH 13/16] Protect some variables in iscsi_context by the spinlock Signed-off-by: Ronnie Sahlberg --- include/iscsi-private.h | 12 ++++++------ lib/iscsi-command.c | 13 ++++++++----- lib/nop.c | 11 +++++++---- lib/pdu.c | 10 ++++++++-- 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/include/iscsi-private.h b/include/iscsi-private.h index f68a37a..99febae 100644 --- a/include/iscsi-private.h +++ b/include/iscsi-private.h @@ -131,12 +131,12 @@ struct iscsi_context { enum iscsi_session_type session_type; unsigned char isid[6]; uint8_t rdma_ack_timeout; - uint32_t itt; // multithreading todo: may need mutex - uint32_t cmdsn; // multithreading todo: may need mutex - uint32_t min_cmdsn_waiting; // multithreading todo: may need mutex - uint32_t expcmdsn; // multithreading todo: may need mutex - uint32_t maxcmdsn; // multithreading todo: may need mutex - uint32_t statsn; // multithreading todo: may need mutex + uint32_t itt; /* Protected by iscsi_lock */ + uint32_t cmdsn; /* Protected by iscsi_lock */ + uint32_t min_cmdsn_waiting; /* Protected by iscsi_lock */ + uint32_t expcmdsn; /* Protected by iscsi_lock */ + uint32_t maxcmdsn; /* Protected by iscsi_lock */ + uint32_t statsn; /* Protected by iscsi_lock */ enum iscsi_header_digest want_header_digest; enum iscsi_header_digest header_digest; enum iscsi_data_digest want_data_digest; diff --git a/lib/iscsi-command.c b/lib/iscsi-command.c index 5fe6135..8b20534 100644 --- a/lib/iscsi-command.c +++ b/lib/iscsi-command.c @@ -261,6 +261,9 @@ iscsi_scsi_command_async(struct iscsi_context *iscsi, int lun, } iscsi_pdu_set_pduflags(pdu, flags); + pdu->callback = iscsi_scsi_response_cb; + pdu->private_data = &pdu->scsi_cbdata; + /* lun */ iscsi_pdu_set_lun(pdu, lun); pdu->lun = lun; @@ -269,16 +272,14 @@ iscsi_scsi_command_async(struct iscsi_context *iscsi, int lun, iscsi_pdu_set_expxferlen(pdu, task->expxferlen); /* cmdsn */ - iscsi_pdu_set_cmdsn(pdu, iscsi->cmdsn); + iscsi_mt_spin_lock(&iscsi->iscsi_lock); + iscsi_pdu_set_cmdsn(pdu, iscsi->cmdsn++); + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); /* cdb */ iscsi_pdu_set_cdb(pdu, task); - pdu->callback = iscsi_scsi_response_cb; - pdu->private_data = &pdu->scsi_cbdata; - iscsi_queue_pdu(iscsi, pdu); - iscsi->cmdsn++; /* The F flag is not set. This means we haven't sent all the unsolicited * data yet. Sent as much as we are allowed as a train of DATA-OUT PDUs. @@ -2730,7 +2731,9 @@ iscsi_scsi_cancel_task(struct iscsi_context *iscsi, } if (!(pdu->outdata.data[0] & ISCSI_PDU_IMMEDIATE) && (pdu->outdata.data[0] & 0x3f) != ISCSI_PDU_DATA_OUT) { + iscsi_mt_spin_lock(&iscsi->iscsi_lock); iscsi->cmdsn--; + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); cmdsn_gap++; } iscsi->drv->free_pdu(iscsi, pdu); diff --git a/lib/nop.c b/lib/nop.c index 9e62fae..e648f46 100644 --- a/lib/nop.c +++ b/lib/nop.c @@ -54,6 +54,9 @@ iscsi_nop_out_async(struct iscsi_context *iscsi, iscsi_command_cb cb, return -1; } + pdu->callback = cb; + pdu->private_data = private_data; + /* flags */ iscsi_pdu_set_pduflags(pdu, 0x80); @@ -64,22 +67,22 @@ iscsi_nop_out_async(struct iscsi_context *iscsi, iscsi_command_cb cb, iscsi_pdu_set_lun(pdu, 0); /* cmdsn */ + iscsi_mt_spin_lock(&iscsi->iscsi_lock); iscsi_pdu_set_cmdsn(pdu, iscsi->cmdsn); - pdu->callback = cb; - pdu->private_data = private_data; - if (data != NULL && len > 0) { if (iscsi_pdu_add_data(iscsi, pdu, data, len) != 0) { + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); iscsi_set_error(iscsi, "Failed to add outdata to nop-out"); iscsi->drv->free_pdu(iscsi, pdu); return -1; } } + iscsi->cmdsn++; + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); iscsi_queue_pdu(iscsi, pdu); - iscsi->cmdsn++; iscsi->nops_in_flight++; ISCSI_LOG(iscsi, (iscsi->nops_in_flight > 1) ? 1 : 6, "NOP Out Send (nops_in_flight: %d, pdu->cmdsn %08x, pdu->itt %08x, pdu->ttt %08x, iscsi->maxcmdsn %08x, iscsi->expcmdsn %08x)", diff --git a/lib/pdu.c b/lib/pdu.c index 3c0aa71..323fc5a 100644 --- a/lib/pdu.c +++ b/lib/pdu.c @@ -70,12 +70,15 @@ iscsi_serial32_compare(uint32_t s1, uint32_t s2) { uint32_t iscsi_itt_post_increment(struct iscsi_context *iscsi) { - uint32_t old_itt = iscsi->itt; - iscsi->itt++; + uint32_t old_itt; + + iscsi_mt_spin_lock(&iscsi->iscsi_lock); + old_itt = iscsi->itt++; /* 0xffffffff is a reserved value */ if (iscsi->itt == 0xffffffff) { iscsi->itt = 0; } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); return old_itt; } @@ -500,6 +503,7 @@ static void iscsi_process_pdu_serials(struct iscsi_context *iscsi, struct iscsi_ return; } + iscsi_mt_spin_lock(&iscsi->iscsi_lock); if (iscsi_serial32_compare(maxcmdsn, iscsi->maxcmdsn) > 0) { iscsi->maxcmdsn = maxcmdsn; } @@ -510,6 +514,7 @@ static void iscsi_process_pdu_serials(struct iscsi_context *iscsi, struct iscsi_ /* RFC3720 10.7.3 (StatSN is invalid if S bit unset in flags) */ if (opcode == ISCSI_PDU_DATA_IN && !(flags & ISCSI_PDU_DATA_CONTAINS_STATUS)) { + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); return; } @@ -520,6 +525,7 @@ static void iscsi_process_pdu_serials(struct iscsi_context *iscsi, struct iscsi_ if (iscsi_serial32_compare(statsn, iscsi->statsn) > 0) { iscsi->statsn = statsn; } + iscsi_mt_spin_unlock(&iscsi->iscsi_lock); } int From edd7d9b8438436765a86065b7fc7410b3240cb5d Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Sat, 26 Apr 2025 08:00:47 +1000 Subject: [PATCH 14/16] Remove the small allocations This is not compatible with multithreading and would require a complete re-design. Signed-off-by: Ronnie Sahlberg --- examples/iscsi-pthreads-readloop.c | 7 +++- include/iscsi-private.h | 15 ++------ lib/connect.c | 9 ----- lib/init.c | 59 +----------------------------- lib/iser.c | 28 +++----------- lib/pdu.c | 28 ++++---------- lib/socket.c | 48 +++++++++++++----------- 7 files changed, 51 insertions(+), 143 deletions(-) diff --git a/examples/iscsi-pthreads-readloop.c b/examples/iscsi-pthreads-readloop.c index b946c9c..9750264 100644 --- a/examples/iscsi-pthreads-readloop.c +++ b/examples/iscsi-pthreads-readloop.c @@ -48,7 +48,7 @@ static int finished; * threads to demonstrate that multiple threads can very well write to the same * context. */ -#define NUM_THREADS 4 +#define NUM_THREADS 8 const char *initiator = "iqn.2007-10.com.github:sahlberg:libiscsi:iscsi-inq"; @@ -90,6 +90,10 @@ static void *iscsi_read_thread(void *arg) struct read_data *rd = arg; struct scsi_task *task; + task = malloc(1024); + printf("iscsi_read_thread %d %p\n", rd->i, task); + free(task); + while (!finished) { task = iscsi_read10_sync(rd->iscsi, rd->lun, 0, 512, 512, @@ -243,6 +247,7 @@ int main(int argc, char *argv[]) iscsi_logout_sync(iscsi[i]); iscsi_destroy_context(iscsi[i]); } + printf("finished\n"); return 0; } diff --git a/include/iscsi-private.h b/include/iscsi-private.h index 99febae..9041b7f 100644 --- a/include/iscsi-private.h +++ b/include/iscsi-private.h @@ -193,14 +193,10 @@ struct iscsi_context { int log_level; iscsi_log_fn log_fn; - int mallocs; - int reallocs; - int frees; - int smallocs; - void* smalloc_ptrs[SMALL_ALLOC_MAX_FREE]; - int smalloc_free; - size_t smalloc_size; - int cache_allocations; + int mallocs; //needs protection? + int reallocs; //needs protection? + int frees; //needs protection? + int cache_allocations; //needs ptotection? time_t next_reconnect; int scsi_timeout; @@ -395,9 +391,6 @@ void* iscsi_zmalloc(struct iscsi_context *iscsi, size_t size); void* iscsi_realloc(struct iscsi_context *iscsi, void* ptr, size_t size); void iscsi_free(struct iscsi_context *iscsi, void* ptr); char* iscsi_strdup(struct iscsi_context *iscsi, const char* str); -void* iscsi_smalloc(struct iscsi_context *iscsi, size_t size); -void* iscsi_szmalloc(struct iscsi_context *iscsi, size_t size); -void iscsi_sfree(struct iscsi_context *iscsi, void* ptr); uint32_t crc32c(uint8_t *buf, int len); void crc32c_init(uint32_t *crc_ptr); diff --git a/lib/connect.c b/lib/connect.c index 4dc98f6..5b944a5 100644 --- a/lib/connect.c +++ b/lib/connect.c @@ -281,7 +281,6 @@ void iscsi_reconnect_cb(struct iscsi_context *iscsi, int status, { struct iscsi_context *old_iscsi; struct iscsi_pdu *tmp = NULL; - int i; if (status != SCSI_STATUS_GOOD) { int backoff = ++iscsi->old_iscsi->retry_cnt; @@ -368,10 +367,6 @@ void iscsi_reconnect_cb(struct iscsi_context *iscsi, int status, iscsi_free(old_iscsi, old_iscsi->opaque); - for (i = 0; i < old_iscsi->smalloc_free; i++) { - iscsi_free(old_iscsi, old_iscsi->smalloc_ptrs[i]); - } - iscsi->mallocs += old_iscsi->mallocs; iscsi->frees += old_iscsi->frees; @@ -468,10 +463,6 @@ static int reconnect(struct iscsi_context *iscsi, int force) tmp_iscsi->reconnect_max_retries = iscsi->reconnect_max_retries; if (iscsi->old_iscsi) { - int i; - for (i = 0; i < iscsi->smalloc_free; i++) { - iscsi_free(iscsi, iscsi->smalloc_ptrs[i]); - } iscsi_free(iscsi, iscsi->opaque); iscsi->old_iscsi->mallocs += iscsi->mallocs; diff --git a/lib/init.c b/lib/init.c index fcae7f3..9d1f6ee 100644 --- a/lib/init.c +++ b/lib/init.c @@ -115,42 +115,6 @@ char* iscsi_strdup(struct iscsi_context *iscsi, const char* str) { return str2; } -void* iscsi_smalloc(struct iscsi_context *iscsi, size_t size) { - void *ptr; - if (size > iscsi->smalloc_size) return NULL; - if (iscsi->smalloc_free > 0) { - ptr = iscsi->smalloc_ptrs[--iscsi->smalloc_free]; - iscsi->smallocs++; - } else { - ptr = iscsi_malloc(iscsi, iscsi->smalloc_size); - } - return ptr; -} - -void* iscsi_szmalloc(struct iscsi_context *iscsi, size_t size) { - void *ptr = iscsi_smalloc(iscsi, size); - if (ptr) { - memset(ptr, 0, size); - } - return ptr; -} - -void iscsi_sfree(struct iscsi_context *iscsi, void* ptr) { - if (ptr == NULL) { - return; - } - if (!iscsi->cache_allocations) { - iscsi_free(iscsi, ptr); - } else if (iscsi->smalloc_free == SMALL_ALLOC_MAX_FREE) { - /* SMALL_ALLOC_MAX_FREE should be adjusted that this */ - /* happens rarely */ - ISCSI_LOG(iscsi, 6, "smalloc free == SMALLOC_MAX_FREE"); - iscsi_free(iscsi, ptr); - } else { - iscsi->smalloc_ptrs[iscsi->smalloc_free++] = ptr; - } -} - static void iscsi_srand_init(struct iscsi_context *iscsi) { unsigned int seed; @@ -201,7 +165,6 @@ struct iscsi_context * iscsi_create_context(const char *initiator_name) { struct iscsi_context *iscsi; - size_t required = ISCSI_RAW_HEADER_SIZE + ISCSI_DIGEST_SIZE; char *ca; if (!initiator_name[0]) { @@ -289,18 +252,6 @@ iscsi_create_context(const char *initiator_name) iscsi->rdma_ack_timeout = atoi(getenv("LIBISCSI_RDMA_ACK_TIMEOUT")); } - /* iscsi->smalloc_size is the size for small allocations. this should be - max(ISCSI_HEADER_SIZE, sizeof(struct iscsi_pdu), sizeof(struct iscsi_in_pdu)) - rounded up to the next power of 2. */ - required = MAX(required, sizeof(struct iscsi_pdu)); - required = MAX(required, sizeof(struct iscsi_in_pdu)); - iscsi->smalloc_size = 1; - while (iscsi->smalloc_size < required) { - iscsi->smalloc_size <<= 1; - } - ISCSI_LOG(iscsi, 5, "small allocation size is %u byte", - (uint32_t)iscsi->smalloc_size); - ca = getenv("LIBISCSI_CACHE_ALLOCATIONS"); if (!ca || atoi(ca) != 0) { iscsi->cache_allocations = 1; @@ -397,8 +348,6 @@ iscsi_set_targetname(struct iscsi_context *iscsi, const char *target_name) int iscsi_destroy_context(struct iscsi_context *iscsi) { - int i; - if (iscsi == NULL) { return 0; } @@ -419,16 +368,12 @@ iscsi_destroy_context(struct iscsi_context *iscsi) iscsi->connect_data = NULL; - for (i=0;ismalloc_free;i++) { - iscsi_free(iscsi, iscsi->smalloc_ptrs[i]); - } - iscsi_free(iscsi, iscsi->opaque); if (iscsi->mallocs != iscsi->frees) { - ISCSI_LOG(iscsi,1,"%d memory blocks lost at iscsi_destroy_context() after %d malloc(s), %d realloc(s), %d free(s) and %d reused small allocations",iscsi->mallocs-iscsi->frees,iscsi->mallocs,iscsi->reallocs,iscsi->frees,iscsi->smallocs); + ISCSI_LOG(iscsi,1,"%d memory blocks lost at iscsi_destroy_context() after %d malloc(s), %d realloc(s), %d free(s)",iscsi->mallocs-iscsi->frees,iscsi->mallocs,iscsi->reallocs,iscsi->frees); } else { - ISCSI_LOG(iscsi,5,"memory is clean at iscsi_destroy_context() after %d mallocs, %d realloc(s), %d free(s) and %d reused small allocations",iscsi->mallocs,iscsi->reallocs,iscsi->frees,iscsi->smallocs); + ISCSI_LOG(iscsi,5,"memory is clean at iscsi_destroy_context() after %d mallocs, %d realloc(s), %d free(s)",iscsi->mallocs,iscsi->reallocs,iscsi->frees); } if (iscsi->old_iscsi) { diff --git a/lib/iser.c b/lib/iser.c index c1a6350..71a0c8f 100644 --- a/lib/iser.c +++ b/lib/iser.c @@ -539,7 +539,7 @@ iscsi_iser_new_pdu(struct iscsi_context *iscsi, __attribute__((unused))size_t si struct iscsi_pdu *pdu; struct iser_pdu *iser_pdu; - iser_pdu = iscsi_szmalloc(iscsi, sizeof(*iser_pdu)); + iser_pdu = iscsi_zmalloc(iscsi, sizeof(*iser_pdu)); pdu = &iser_pdu->iscsi_pdu; pdu->indata.data = NULL; @@ -563,19 +563,10 @@ iscsi_iser_free_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) iser_pdu->desc = NULL; } - if (pdu->outdata.size <= iscsi->smalloc_size) { - iscsi_sfree(iscsi, pdu->outdata.data); - } else { - iscsi_free(iscsi, pdu->outdata.data); - } + iscsi_free(iscsi, pdu->outdata.data); pdu->outdata.data = NULL; - if (pdu->indata.size <= iscsi->smalloc_size) { - iscsi_sfree(iscsi, pdu->indata.data); - } else { - iscsi_free(iscsi, pdu->indata.data); - } - + iscsi_free(iscsi, pdu->indata.data); pdu->indata.data = NULL; iscsi_mt_spin_lock(&iscsi->iscsi_lock); @@ -584,7 +575,7 @@ iscsi_iser_free_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) } iscsi_mt_spin_unlock(&iscsi->iscsi_lock); - iscsi_sfree(iscsi, iser_pdu); + iscsi_free(iscsi, iser_pdu); } /** @@ -774,11 +765,7 @@ iser_prepare_read_cmd(struct iser_conn *iser_conn,struct iser_pdu *iser_pdu) if (data_size > 0) { if (task->iovector_in.iov == NULL) { - if (data_size <= iscsi->smalloc_size) { - iser_pdu->iscsi_pdu.indata.data = iscsi_smalloc(iscsi, data_size); - } else { - iser_pdu->iscsi_pdu.indata.data = iscsi_malloc(iscsi, data_size); - } + iser_pdu->iscsi_pdu.indata.data = iscsi_malloc(iscsi, data_size); if (iser_pdu->iscsi_pdu.indata.data == NULL) { iscsi_set_error(iscsi, "Failed to aloocate data buffer"); return -1; @@ -1743,11 +1730,6 @@ void iscsi_init_iser_transport(struct iscsi_context *iscsi) /* Update iSCSI params as per iSER transport */ iscsi->initiator_max_recv_data_segment_length = ISCSI_DEF_MAX_RECV_SEG_LEN; iscsi->target_max_recv_data_segment_length = ISCSI_DEF_MAX_RECV_SEG_LEN; - - /* ensure smalloc_size is enough for iser_pdu */ - while (iscsi->smalloc_size < sizeof(struct iser_pdu)) { - iscsi->smalloc_size <<= 1; - } } #endif diff --git a/lib/pdu.c b/lib/pdu.c index 323fc5a..c98dd3c 100644 --- a/lib/pdu.c +++ b/lib/pdu.c @@ -187,7 +187,7 @@ void iscsi_dump_pdu_header(struct iscsi_context *iscsi, unsigned char *data) { struct iscsi_pdu* iscsi_tcp_new_pdu(struct iscsi_context *iscsi, size_t size) { - return iscsi_szmalloc(iscsi, size); + return iscsi_zmalloc(iscsi, size); } struct iscsi_pdu * @@ -204,7 +204,7 @@ iscsi_allocate_pdu(struct iscsi_context *iscsi, enum iscsi_opcode opcode, } pdu->outdata.size = ISCSI_HEADER_SIZE(iscsi->header_digest); - pdu->outdata.data = iscsi_szmalloc(iscsi, pdu->outdata.size); + pdu->outdata.data = iscsi_zmalloc(iscsi, pdu->outdata.size); if (pdu->outdata.data == NULL) { iscsi_set_error(iscsi, "failed to allocate pdu header"); @@ -242,25 +242,17 @@ iscsi_tcp_free_pdu(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) return; } - if (pdu->outdata.size <= iscsi->smalloc_size) { - iscsi_sfree(iscsi, pdu->outdata.data); - } else { - iscsi_free(iscsi, pdu->outdata.data); - } + iscsi_free(iscsi, pdu->outdata.data); pdu->outdata.data = NULL; - if (pdu->indata.size <= iscsi->smalloc_size) { - iscsi_sfree(iscsi, pdu->indata.data); - } else { - iscsi_free(iscsi, pdu->indata.data); - } + iscsi_free(iscsi, pdu->indata.data); pdu->indata.data = NULL; if (iscsi->outqueue_current == pdu) { iscsi->outqueue_current = NULL; } - iscsi_sfree(iscsi, pdu); + iscsi_free(iscsi, pdu); } int @@ -283,15 +275,9 @@ iscsi_add_data(struct iscsi_context *iscsi, struct iscsi_data *data, } if (data->size == 0) { - if (aligned <= iscsi->smalloc_size) { - data->data = iscsi_szmalloc(iscsi, aligned); - } else { - data->data = iscsi_malloc(iscsi, aligned); - } + data->data = iscsi_malloc(iscsi, aligned); } else { - if (aligned > iscsi->smalloc_size) { - data->data = iscsi_realloc(iscsi, data->data, aligned); - } + data->data = iscsi_realloc(iscsi, data->data, aligned); } if (data->data == NULL) { iscsi_set_error(iscsi, "failed to allocate buffer for %d " diff --git a/lib/socket.c b/lib/socket.c index ce7fd0f..e10ec9e 100644 --- a/lib/socket.c +++ b/lib/socket.c @@ -153,8 +153,8 @@ iscsi_add_to_outqueue(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) /* TODO QQQ need to immediately send for the non multithreading case too * and for the Windows API too */ #if defined(HAVE_MULTITHREADING) && defined(HAVE_PTHREAD) - if (current == NULL && pdu == iscsi->outqueue) { - if(iscsi->multithreading_enabled) { + if(iscsi->multithreading_enabled) { + if (current == NULL && pdu == iscsi->outqueue) { pthread_kill(iscsi->service_thread, SIGUSR1); } } @@ -666,20 +666,23 @@ iscsi_read_from_socket(struct iscsi_context *iscsi) struct iscsi_in_pdu *in; ssize_t hdr_size, data_size, count, padding_size; bool do_data_digest = (iscsi->data_digest != ISCSI_DATA_DIGEST_NONE); - + int ret = -1; + do { hdr_size = ISCSI_HEADER_SIZE(iscsi->header_digest); if (iscsi->incoming == NULL) { - iscsi->incoming = iscsi_szmalloc(iscsi, sizeof(struct iscsi_in_pdu)); + iscsi->incoming = iscsi_zmalloc(iscsi, sizeof(struct iscsi_in_pdu)); if (iscsi->incoming == NULL) { iscsi_set_error(iscsi, "Out-of-memory: failed to malloc iscsi_in_pdu"); - return -1; + goto finished; } + } + if (iscsi->incoming->hdr == NULL) { crc32c_init(&(iscsi->incoming->calculated_data_digest)); - iscsi->incoming->hdr = iscsi_smalloc(iscsi, hdr_size); + iscsi->incoming->hdr = iscsi_malloc(iscsi, hdr_size); if (iscsi->incoming->hdr == NULL) { iscsi_set_error(iscsi, "Out-of-memory"); - return -1; + goto finished; } } in = iscsi->incoming; @@ -694,7 +697,7 @@ iscsi_read_from_socket(struct iscsi_context *iscsi) count, 0); if (count == 0) { /* remote side has closed the socket. */ - return -1; + goto finished; } if (count < 0) { if (errno == EINTR || errno == EAGAIN) { @@ -702,7 +705,7 @@ iscsi_read_from_socket(struct iscsi_context *iscsi) } iscsi_set_error(iscsi, "read from socket failed, " "errno:%d", errno); - return -1; + goto finished; } in->hdr_pos += count; } @@ -717,7 +720,7 @@ iscsi_read_from_socket(struct iscsi_context *iscsi) if (data_size < 0 || data_size > (ssize_t)iscsi->initiator_max_recv_data_segment_length) { iscsi_set_error(iscsi, "Invalid data size received from target (%d)", (int)data_size); - return -1; + goto finished; } if (data_size != 0) { unsigned char padding_buf[3]; @@ -737,7 +740,7 @@ iscsi_read_from_socket(struct iscsi_context *iscsi) in->data = iscsi_malloc(iscsi, data_size); if (in->data == NULL) { iscsi_set_error(iscsi, "Out-of-memory: failed to malloc iscsi_in_pdu->data(%d)", (int)data_size); - return -1; + goto finished; } } buf = &in->data[in->data_pos]; @@ -748,13 +751,13 @@ iscsi_read_from_socket(struct iscsi_context *iscsi) } if (count == 0) { /* remote side has closed the socket. */ - return -1; + goto finished; } if (count < 0) { if (errno == EINTR || errno == EAGAIN) { break; } - return -1; + goto finished; } in->data_pos += count; } @@ -770,13 +773,13 @@ iscsi_read_from_socket(struct iscsi_context *iscsi) count = recv(iscsi->fd, (void *)(in->data_digest_buf + in->received_data_digest_bytes), ISCSI_DIGEST_SIZE - in->received_data_digest_bytes, 0); if (count == 0) { /* remote side has closed the socket. */ - return -1; + goto finished; } if (count < 0) { if (errno == EINTR || errno == EAGAIN) { break; } - return -1; + goto finished; } in->received_data_digest_bytes += count; @@ -785,15 +788,17 @@ iscsi_read_from_socket(struct iscsi_context *iscsi) } } - iscsi->incoming = NULL; + iscsi->incoming = NULL; if (iscsi_process_pdu(iscsi, in) != 0) { iscsi_free_iscsi_in_pdu(iscsi, in); - return -1; + goto finished; } iscsi_free_iscsi_in_pdu(iscsi, in); - } while (iscsi->tcp_nonblocking && iscsi->waitpdu && iscsi->is_loggedin); + } while (iscsi->tcp_nonblocking && iscsi->waitpdu && iscsi->is_loggedin); //QQQ break the loop - return 0; + ret = 0; + finished: + return ret; } static int iscsi_pdu_update_headerdigest(struct iscsi_context *iscsi, struct iscsi_pdu *pdu) @@ -1123,6 +1128,7 @@ iscsi_tcp_service(struct iscsi_context *iscsi, int revents) return iscsi_service_reconnect_if_loggedin(iscsi); } } + if (revents & POLLOUT) { if (iscsi_write_to_socket(iscsi) != 0) { ISCSI_LOG(iscsi, 1, "%s", iscsi_get_error(iscsi)); @@ -1155,10 +1161,10 @@ static void iscsi_tcp_queue_pdu(struct iscsi_context *iscsi, void iscsi_free_iscsi_in_pdu(struct iscsi_context *iscsi, struct iscsi_in_pdu *in) { - iscsi_sfree(iscsi, in->hdr); + iscsi_free(iscsi, in->hdr); iscsi_free(iscsi, in->data); in->data=NULL; - iscsi_sfree(iscsi, in); + iscsi_free(iscsi, in); in=NULL; } From 31fd95dc8f7e81c201b00cdcc3db949fbd5835d9 Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Sat, 26 Apr 2025 08:59:28 +1000 Subject: [PATCH 15/16] README: add blurb about multithreading Signed-off-by: Ronnie Sahlberg --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 4a3c9be..e091cb7 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,13 @@ Example: iscsi://server/iqn.ronnie.test/1 +MULTITHREADING +============== +Multithreading is supported both on Linux, using pthreads, and Windows, using native API. +By default libicsi will start with multithreading disabled and you will need +to activate once connected to the LUN. +There are examples of multithreading in the examples directory. + CHAP Authentication =================== CHAP authentication can be specified two ways. Either via the URL itself From 7e86ed7962653b6ca55a978edcfe9dac6569b5e3 Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Sat, 26 Apr 2025 09:21:22 +1000 Subject: [PATCH 16/16] Add multithreaded example using the async API Signed-off-by: Ronnie Sahlberg --- examples/Makefile.am | 2 +- examples/iscsi-pthreads-readloop-async.c | 287 +++++++++++++++++++++++ 2 files changed, 288 insertions(+), 1 deletion(-) create mode 100644 examples/iscsi-pthreads-readloop-async.c diff --git a/examples/Makefile.am b/examples/Makefile.am index 4740bd6..b183a33 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -3,4 +3,4 @@ AM_CFLAGS=$(WARN_CFLAGS) AM_LDFLAGS=-no-undefined LIBS=../lib/libiscsi.la -noinst_PROGRAMS = iscsiclient iscsi-dd iscsi-pthreads-inq iscsi-pthreads-readloop +noinst_PROGRAMS = iscsiclient iscsi-dd iscsi-pthreads-inq iscsi-pthreads-readloop iscsi-pthreads-readloop-async diff --git a/examples/iscsi-pthreads-readloop-async.c b/examples/iscsi-pthreads-readloop-async.c new file mode 100644 index 0000000..1437c94 --- /dev/null +++ b/examples/iscsi-pthreads-readloop-async.c @@ -0,0 +1,287 @@ +/* + Copyright (C) 2025 by Ronnie Sahlberg + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . +*/ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_POLL_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iscsi.h" +#include "scsi-lowlevel.h" + +static int finished; + +/* + * Number of iscsi connections to the server. + */ +#define NUM_CONTEXTS 1 + +/* + * Number of threads parallely read device data. + * Usually one thread per context should be sufficient, but here we use more + * threads to demonstrate that multiple threads can very well write to the same + * context. + */ +#define NUM_THREADS 4 + +/* + * Number of tasks per burst in each thread + */ +#define NUM_TASKS 16 + +const char *initiator = "iqn.2007-10.com.github:sahlberg:libiscsi:iscsi-inq"; + + +void print_usage(void) +{ + fprintf(stderr, "Usage: iscsi-pthreads-readloop [-?] [-?|--help] [--usage] [-i|--initiator-name=iqn-name]\n" + "\t\t\n"); +} + +void print_help(void) +{ + fprintf(stderr, "Usage: iscsi-pthreads-readloop [OPTION...] \n"); + fprintf(stderr, " -i, --initiator-name=iqn-name Initiatorname to use\n"); + fprintf(stderr, " -d, --debug=integer debug level (0=disabled)\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "Help options:\n"); + fprintf(stderr, " -?, --help Show this help message\n"); + fprintf(stderr, " --usage Display brief usage message\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "iSCSI URL format : %s\n", ISCSI_URL_SYNTAX); + fprintf(stderr, "\n"); + fprintf(stderr, " is either of:\n"); + fprintf(stderr, " \"hostname\" iscsi.example\n"); + fprintf(stderr, " \"ipv4-address\" 10.1.1.27\n"); + fprintf(stderr, " \"ipv6-address\" [fce0::1]\n"); +} + +struct read_data { + struct iscsi_context *iscsi; + int lun; + int i; + pthread_t thread; + sem_t sem; +}; + +void read_cb(struct iscsi_context *iscsi, int status, void *command_data, void *private_data) +{ + struct read_data *rd = (struct read_data *)private_data; + struct scsi_task *task = command_data; + + if (status == SCSI_STATUS_CHECK_CONDITION) { + fprintf(stderr, "Read10 failed with sense key:%d ascq:%04x\n", task->sense.key, task->sense.ascq); + scsi_free_scsi_task(task); + exit(10); + } + + if (status != SCSI_STATUS_GOOD) { + fprintf(stderr, "Read10/16 failed with %s\n", iscsi_get_error(iscsi)); + scsi_free_scsi_task(task); + exit(10); + } + + sem_post(&rd->sem); + scsi_free_scsi_task(task); +} + +static void *iscsi_read_thread(void *arg) +{ + struct read_data *rd = arg; + struct scsi_task *task; + int i; + + task = malloc(1024); + printf("iscsi_read_thread %d %p\n", rd->i, task); + free(task); + + sem_init(&rd->sem, 0, 0); + + while (!finished) { + for (i = 0; i < NUM_TASKS; i++) { + task = iscsi_read10_task(rd->iscsi, rd->lun, 0, + 512, 512, + 0, 0, 0, 0, 0, + read_cb, rd); + if (task == NULL) { + fprintf(stderr, "Failed to read from lun in thread #%d\n", rd->i); + exit(10); + } + } + for (i = 0; i < NUM_TASKS; i++) { + sem_wait(&rd->sem); + } + } + return NULL; +} + +static void sig_alarm(int sig) +{ + finished = 1; +} + +int main(int argc, char *argv[]) +{ + struct iscsi_context *iscsi[NUM_CONTEXTS] = {NULL,}; + char *url = NULL; + struct iscsi_url *iscsi_url = NULL; + int show_help = 0, show_usage = 0, debug = 0; + int c, i; + struct read_data *rd; + + static struct option long_options[] = { + {"help", no_argument, NULL, 'h'}, + {"usage", no_argument, NULL, 'u'}, + {"debug", required_argument, NULL, 'd'}, + {"initiator-name", required_argument, NULL, 'i'}, + {0, 0, 0, 0} + }; + int option_index; + + while ((c = getopt_long(argc, argv, "h?ud:i:e:c:", long_options, + &option_index)) != -1) { + switch (c) { + case 'h': + case '?': + show_help = 1; + break; + case 'u': + show_usage = 1; + break; + case 'd': + debug = strtol(optarg, NULL, 0); + break; + case 'i': + initiator = optarg; + break; + default: + fprintf(stderr, "Unrecognized option '%c'\n\n", c); + print_help(); + exit(0); + } + } + + if (show_help != 0) { + print_help(); + exit(0); + } + + if (show_usage != 0) { + print_usage(); + exit(0); + } + + for (i = 0; i < NUM_CONTEXTS; i++) { + printf("Creating context #%d\n", i); + iscsi[i] = iscsi_create_context(initiator); + if (iscsi[i] == NULL) { + fprintf(stderr, "Failed to create context\n"); + exit(10); + } + if (debug > 0) { + iscsi_set_log_level(iscsi[i], debug); + iscsi_set_log_fn(iscsi[i], iscsi_log_to_stderr); + } + + if (i > 0) { + iscsi_destroy_url(iscsi_url); + } + if (argv[optind] != NULL) { + url = strdup(argv[optind]); + } + if (url == NULL) { + fprintf(stderr, "You must specify the URL\n"); + print_usage(); + exit(10); + } + iscsi_url = iscsi_parse_full_url(iscsi[i], url); + + free(url); + + if (iscsi_url == NULL) { + fprintf(stderr, "Failed to parse URL: %s\n", + iscsi_get_error(iscsi[i])); + exit(10); + } + + iscsi_set_session_type(iscsi[i], ISCSI_SESSION_NORMAL); + iscsi_set_header_digest(iscsi[i], ISCSI_HEADER_DIGEST_NONE_CRC32C); + + if (iscsi_full_connect_sync(iscsi[i], iscsi_url->portal, iscsi_url->lun) != 0) { + fprintf(stderr, "Login Failed. %s\n", iscsi_get_error(iscsi[i])); + iscsi_destroy_url(iscsi_url); + iscsi_destroy_context(iscsi[i]); + exit(10); + } + + if (iscsi_mt_service_thread_start(iscsi[i])) { + fprintf(stderr, "failed to start service thread #%d\n", i); + exit(10); + } + + } + + if ((rd = malloc(sizeof(struct read_data) * NUM_THREADS)) == NULL) { + fprintf(stderr, "Failed to allocated read_data\n"); + exit(10); + } + for (i = 0; i < NUM_THREADS; i++) { + rd[i].iscsi = iscsi[i % NUM_CONTEXTS]; + rd[i].lun = iscsi_url->lun; + rd[i].i = i; + if (pthread_create(&rd[i].thread, NULL, + &iscsi_read_thread, &rd[i])) { + printf("Failed to create read thread #%d\n", i); + exit(10); + } + } + + /* run for 5 seconds */ + signal(SIGALRM, sig_alarm); + alarm(5); + + /* + * Wait for all threads to complete + */ + for (i = 0; i < NUM_THREADS; i++) { + pthread_join(rd[i].thread, NULL); + } + + iscsi_destroy_url(iscsi_url); + + for (i = 0; i < NUM_CONTEXTS; i++) { + iscsi_mt_service_thread_stop(iscsi[i]); + iscsi_logout_sync(iscsi[i]); + iscsi_destroy_context(iscsi[i]); + } + printf("finished\n"); + return 0; +} +