From ebeca19480b982b89f3259d82f80671644d03b34 Mon Sep 17 00:00:00 2001 From: David Disseldorp Date: Thu, 10 Mar 2016 18:04:31 +0100 Subject: [PATCH 1/3] Tests: Asynchronous iSCSI read test Dispatch a number of READ10 requests simultaneously, and await all responses. The iscsi-support test helper functions are currently all synchronous, so this test mostly uses the bare libiscsi API. As new async tests are added, we can make a more informed decision on which boilerplate code can be split out. Signed-off-by: David Disseldorp --- test-tool/Makefile.am | 3 +- test-tool/iscsi-test-cu.c | 1 + test-tool/iscsi-test-cu.h | 1 + test-tool/test_async_read.c | 119 ++++++++++++++++++++++++++++++++++++ 4 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 test-tool/test_async_read.c diff --git a/test-tool/Makefile.am b/test-tool/Makefile.am index 8bbb8a4..50db484 100644 --- a/test-tool/Makefile.am +++ b/test-tool/Makefile.am @@ -221,7 +221,8 @@ iscsi_test_cu_SOURCES = iscsi-test-cu.c \ test_writeverify16_residuals.c \ test_multipathio_simple.c \ test_multipathio_reset.c \ - test_multipathio_compareandwrite.c + test_multipathio_compareandwrite.c \ + test_async_read.c endif diff --git a/test-tool/iscsi-test-cu.c b/test-tool/iscsi-test-cu.c index a640688..ef27f52 100644 --- a/test-tool/iscsi-test-cu.c +++ b/test-tool/iscsi-test-cu.c @@ -227,6 +227,7 @@ static CU_TestInfo tests_read10[] = { { (char *)"ZeroBlocks", test_read10_0blocks }, { (char *)"ReadProtect", test_read10_rdprotect }, { (char *)"DpoFua", test_read10_dpofua }, + { (char *)"Async", test_async_read }, CU_TEST_INFO_NULL }; diff --git a/test-tool/iscsi-test-cu.h b/test-tool/iscsi-test-cu.h index 38f900f..67ebdef 100644 --- a/test-tool/iscsi-test-cu.h +++ b/test-tool/iscsi-test-cu.h @@ -145,6 +145,7 @@ void test_read10_rdprotect(void); void test_read10_dpofua(void); void test_read10_residuals(void); void test_read10_invalid(void); +void test_async_read(void); void test_read12_simple(void); void test_read12_beyond_eol(void); diff --git a/test-tool/test_async_read.c b/test-tool/test_async_read.c new file mode 100644 index 0000000..d213ee5 --- /dev/null +++ b/test-tool/test_async_read.c @@ -0,0 +1,119 @@ +/* + Copyright (C) SUSE LINUX GmbH 2016 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . +*/ + +#include +#include +#include + +#include + +#include "iscsi.h" +#include "scsi-lowlevel.h" +#include "iscsi-support.h" +#include "iscsi-test-cu.h" +#include "iscsi-multipath.h" + +struct tests_async_read_state { + uint32_t dispatched; + uint32_t completed; + uint32_t prev_cmdsn; +}; + +static void +test_async_read_cb(struct iscsi_context *iscsi __attribute__((unused)), + int status, void *command_data, void *private_data) +{ + struct scsi_task *atask = command_data; + struct tests_async_read_state *state = private_data; + + state->completed++; + logging(LOG_VERBOSE, "READ10 completed: %d of %d (CmdSN=%d)", + state->completed, state->dispatched, atask->cmdsn); + CU_ASSERT_NOT_EQUAL(status, SCSI_STATUS_CHECK_CONDITION); + + if ((state->completed > 1) && (atask->cmdsn != state->prev_cmdsn + 1)) { + logging(LOG_VERBOSE, + "out of order completion (CmdSN=%d, prev=%d)", + atask->cmdsn, state->prev_cmdsn); + } + state->prev_cmdsn = atask->cmdsn; + + scsi_free_scsi_task(atask); +} + +void +test_async_read(void) +{ + int i, ret; + struct tests_async_read_state state = { 0 }; + int blocksize = 512; + int blocks_per_io = 8; + int num_ios = 1000; + /* IOs in flight concurrently, so need a buffer large enough for all */ + unsigned char buf[blocksize * blocks_per_io * num_ios]; + + CHECK_FOR_DATALOSS; + CHECK_FOR_SBC; + if (sd->iscsi_ctx == NULL) { + CU_PASS("[SKIPPED] Non-iSCSI"); + return; + } + + if (maximum_transfer_length + && (maximum_transfer_length < (blocks_per_io * num_ios))) { + CU_PASS("[SKIPPED] device too small for async_read test"); + return; + } + + memset(buf, 0, blocksize * blocks_per_io * num_ios); + + for (i = 0; i < num_ios; i++) { + uint32_t lba = i * blocks_per_io; + struct scsi_task *atask; + + atask = scsi_cdb_read10(lba, blocks_per_io * blocksize, + blocksize, 0, 0, 0, 0, 0); + CU_ASSERT_PTR_NOT_NULL_FATAL(atask); + + ret = scsi_task_add_data_in_buffer(atask, + blocks_per_io * blocksize, + &buf[lba * blocksize]); + CU_ASSERT_EQUAL(ret, 0); + + ret = iscsi_scsi_command_async(sd->iscsi_ctx, sd->iscsi_lun, + atask, test_async_read_cb, NULL, + &state); + CU_ASSERT_EQUAL(ret, 0); + + state.dispatched++; + logging(LOG_VERBOSE, "READ10 dispatched: %d of %d (cmdsn=%d)", + state.dispatched, num_ios, atask->cmdsn); + } + + while (state.completed < state.dispatched) { + struct pollfd pfd; + + pfd.fd = iscsi_get_fd(sd->iscsi_ctx); + pfd.events = iscsi_which_events(sd->iscsi_ctx); + + ret = poll(&pfd, 1, -1); + CU_ASSERT_NOT_EQUAL(ret, -1); + + ret = iscsi_service(sd->iscsi_ctx, pfd.revents); + CU_ASSERT_EQUAL(ret, 0); + } +} From 51da17c41f27923dad22cef59d6b28f61406a9ae Mon Sep 17 00:00:00 2001 From: David Disseldorp Date: Fri, 11 Mar 2016 11:40:37 +0100 Subject: [PATCH 2/3] Tests: Asynchronous iSCSI write test Dispatch a number of WRITE10 requests simultaneously, and await all responses. Signed-off-by: David Disseldorp --- test-tool/Makefile.am | 3 +- test-tool/iscsi-test-cu.c | 1 + test-tool/iscsi-test-cu.h | 1 + test-tool/test_async_write.c | 118 +++++++++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 test-tool/test_async_write.c diff --git a/test-tool/Makefile.am b/test-tool/Makefile.am index 50db484..33ddb41 100644 --- a/test-tool/Makefile.am +++ b/test-tool/Makefile.am @@ -222,7 +222,8 @@ iscsi_test_cu_SOURCES = iscsi-test-cu.c \ test_multipathio_simple.c \ test_multipathio_reset.c \ test_multipathio_compareandwrite.c \ - test_async_read.c + test_async_read.c \ + test_async_write.c endif diff --git a/test-tool/iscsi-test-cu.c b/test-tool/iscsi-test-cu.c index ef27f52..3027e85 100644 --- a/test-tool/iscsi-test-cu.c +++ b/test-tool/iscsi-test-cu.c @@ -378,6 +378,7 @@ static CU_TestInfo tests_write10[] = { { (char *)"ZeroBlocks", test_write10_0blocks }, { (char *)"WriteProtect", test_write10_wrprotect }, { (char *)"DpoFua", test_write10_dpofua }, + { (char *)"Async", test_async_write }, CU_TEST_INFO_NULL }; diff --git a/test-tool/iscsi-test-cu.h b/test-tool/iscsi-test-cu.h index 67ebdef..a8d0ffc 100644 --- a/test-tool/iscsi-test-cu.h +++ b/test-tool/iscsi-test-cu.h @@ -241,6 +241,7 @@ void test_write10_0blocks(void); void test_write10_wrprotect(void); void test_write10_dpofua(void); void test_write10_residuals(void); +void test_async_write(void); void test_write12_simple(void); void test_write12_beyond_eol(void); diff --git a/test-tool/test_async_write.c b/test-tool/test_async_write.c new file mode 100644 index 0000000..bea8152 --- /dev/null +++ b/test-tool/test_async_write.c @@ -0,0 +1,118 @@ +/* + Copyright (C) SUSE LINUX GmbH 2016 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . +*/ + +#include +#include +#include + +#include + +#include "iscsi.h" +#include "scsi-lowlevel.h" +#include "iscsi-support.h" +#include "iscsi-test-cu.h" + +struct tests_async_write_state { + uint32_t dispatched; + uint32_t completed; + uint32_t prev_cmdsn; +}; + +static void +test_async_write_cb(struct iscsi_context *iscsi __attribute__((unused)), + int status, void *command_data, void *private_data) +{ + struct scsi_task *atask = command_data; + struct tests_async_write_state *state = private_data; + + state->completed++; + logging(LOG_VERBOSE, "WRITE10 completed: %d of %d (CmdSN=%d)", + state->completed, state->dispatched, atask->cmdsn); + CU_ASSERT_NOT_EQUAL(status, SCSI_STATUS_CHECK_CONDITION); + + if ((state->completed > 1) && (atask->cmdsn != state->prev_cmdsn + 1)) { + logging(LOG_VERBOSE, + "out of order completion (CmdSN=%d, prev=%d)", + atask->cmdsn, state->prev_cmdsn); + } + state->prev_cmdsn = atask->cmdsn; + + scsi_free_scsi_task(atask); +} + +void +test_async_write(void) +{ + int i, ret; + struct tests_async_write_state state = { 0 }; + int blocksize = 512; + int blocks_per_io = 8; + int num_ios = 1000; + /* IOs in flight concurrently, but all using the same src buffer */ + unsigned char buf[blocksize * blocks_per_io]; + + CHECK_FOR_DATALOSS; + CHECK_FOR_SBC; + if (sd->iscsi_ctx == NULL) { + CU_PASS("[SKIPPED] Non-iSCSI"); + return; + } + + if (maximum_transfer_length + && (maximum_transfer_length < (blocks_per_io * num_ios))) { + CU_PASS("[SKIPPED] device too small for async_write test"); + return; + } + + memset(buf, 0, blocksize * blocks_per_io); + + for (i = 0; i < num_ios; i++) { + uint32_t lba = i * blocks_per_io; + struct scsi_task *atask; + + atask = scsi_cdb_write10(lba, blocks_per_io * blocksize, + blocksize, 0, 0, 0, 0, 0); + CU_ASSERT_PTR_NOT_NULL_FATAL(atask); + + ret = scsi_task_add_data_out_buffer(atask, + blocks_per_io * blocksize, + buf); + CU_ASSERT_EQUAL(ret, 0); + + ret = iscsi_scsi_command_async(sd->iscsi_ctx, sd->iscsi_lun, + atask, test_async_write_cb, NULL, + &state); + CU_ASSERT_EQUAL(ret, 0); + + state.dispatched++; + logging(LOG_VERBOSE, "WRITE10 dispatched: %d of %d (cmdsn=%d)", + state.dispatched, num_ios, atask->cmdsn); + } + + while (state.completed < state.dispatched) { + struct pollfd pfd; + + pfd.fd = iscsi_get_fd(sd->iscsi_ctx); + pfd.events = iscsi_which_events(sd->iscsi_ctx); + + ret = poll(&pfd, 1, -1); + CU_ASSERT_NOT_EQUAL(ret, -1); + + ret = iscsi_service(sd->iscsi_ctx, pfd.revents); + CU_ASSERT_EQUAL(ret, 0); + } +} From 9ae5a646620d732c794f023efc68d53f355b49d5 Mon Sep 17 00:00:00 2001 From: David Disseldorp Date: Fri, 11 Mar 2016 19:11:34 +0100 Subject: [PATCH 3/3] Tests: Asynchronous iSCSI Multipath Compare And Write Dispatch a number of Compare And Write requests simultaneously via separate iSCSI sessions and await all responses. Signed-off-by: David Disseldorp --- test-tool/Makefile.am | 1 + test-tool/iscsi-test-cu.c | 1 + test-tool/iscsi-test-cu.h | 1 + test-tool/test_multipathio_async_caw.c | 184 +++++++++++++++++++++++++ 4 files changed, 187 insertions(+) create mode 100644 test-tool/test_multipathio_async_caw.c diff --git a/test-tool/Makefile.am b/test-tool/Makefile.am index 33ddb41..676a1e1 100644 --- a/test-tool/Makefile.am +++ b/test-tool/Makefile.am @@ -222,6 +222,7 @@ iscsi_test_cu_SOURCES = iscsi-test-cu.c \ test_multipathio_simple.c \ test_multipathio_reset.c \ test_multipathio_compareandwrite.c \ + test_multipathio_async_caw.c \ test_async_read.c \ test_async_write.c diff --git a/test-tool/iscsi-test-cu.c b/test-tool/iscsi-test-cu.c index 3027e85..7f51f95 100644 --- a/test-tool/iscsi-test-cu.c +++ b/test-tool/iscsi-test-cu.c @@ -468,6 +468,7 @@ static CU_TestInfo tests_multipathio[] = { { (char *)"Simple", test_multipathio_simple }, { (char *)"Reset", test_multipathio_reset }, { (char *)"CompareAndWrite", test_multipathio_compareandwrite }, + { (char *)"CompareAndWriteAsync", test_mpio_async_caw }, CU_TEST_INFO_NULL }; diff --git a/test-tool/iscsi-test-cu.h b/test-tool/iscsi-test-cu.h index a8d0ffc..1d85fe6 100644 --- a/test-tool/iscsi-test-cu.h +++ b/test-tool/iscsi-test-cu.h @@ -309,5 +309,6 @@ void test_writeverify16_residuals(void); void test_multipathio_simple(void); void test_multipathio_reset(void); void test_multipathio_compareandwrite(void); +void test_mpio_async_caw(void); #endif /* _ISCSI_TEST_CU_H_ */ diff --git a/test-tool/test_multipathio_async_caw.c b/test-tool/test_multipathio_async_caw.c new file mode 100644 index 0000000..77fda3a --- /dev/null +++ b/test-tool/test_multipathio_async_caw.c @@ -0,0 +1,184 @@ +/* + Copyright (C) SUSE LINUX GmbH 2016 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . +*/ + +#include +#include +#include + +#include + +#include "iscsi.h" +#include "scsi-lowlevel.h" +#include "iscsi-support.h" +#include "iscsi-test-cu.h" +#include "iscsi-multipath.h" + +struct test_mpio_async_caw_state { + uint32_t dispatched; + uint32_t completed; + uint32_t mismatches; +}; + +static void +test_mpio_async_caw_cb(struct iscsi_context *iscsi __attribute__((unused)), + int status, void *command_data, void *private_data) +{ + struct scsi_task *atask = command_data; + struct test_mpio_async_caw_state *state = private_data; + + state->completed++; + if (status == SCSI_STATUS_CHECK_CONDITION) { + CU_ASSERT_EQUAL(atask->sense.key, SCSI_SENSE_MISCOMPARE); + CU_ASSERT_EQUAL(atask->sense.ascq, + SCSI_SENSE_ASCQ_MISCOMPARE_DURING_VERIFY); + state->mismatches++; + logging(LOG_VERBOSE, "COMPARE_AND_WRITE mismatch: %d of %d " + "(CmdSN=%d)", + state->completed, state->dispatched, atask->cmdsn); + + } else { + logging(LOG_VERBOSE, "COMPARE_AND_WRITE success: %d of %d " + "(CmdSN=%d)", + state->completed, state->dispatched, atask->cmdsn); + } + + scsi_free_scsi_task(atask); +} + +static void +test_mpio_async_caw_init_bufs(unsigned char *cmp_buf, unsigned char *wr_buf, + int blocksize, int num_mp_sds) +{ + int sd_i; + + /* + * Each compare and write attempts to modify on-disk data with the + * assumption that the previous operation was successful. E.g. + * + * session 0 session 1 + * --------- --------- + * 0->1 (good) + * 1->0 (good) + * + * This gives us some nice races if the target processes the requests + * out of order. E.g. + * 0->1 (good) + * 1->0 (good) + * 1->0 (mismatch!) + * 0->1 (good) + */ + + for (sd_i = 0; sd_i < num_mp_sds; sd_i++) { + int wr_val; + int cmp_val = sd_i; + + if (sd_i == num_mp_sds - 1) { + wr_val = 0; + } else { + wr_val = sd_i + 1; + } + + memset(&cmp_buf[sd_i * blocksize], cmp_val, blocksize); + memset(&wr_buf[sd_i * blocksize], wr_val, blocksize); + } +} + +void +test_mpio_async_caw(void) +{ + int i, ret; + int sd_i; + struct test_mpio_async_caw_state state = { 0 }; + int blocksize = 512; + int num_ios = 1000; + uint32_t lba = 0; + unsigned char cmp_buf[blocksize * mp_num_sds]; + unsigned char wr_buf[blocksize * mp_num_sds]; + + CHECK_FOR_DATALOSS; + CHECK_FOR_SBC; + MPATH_SKIP_IF_UNAVAILABLE(mp_sds, mp_num_sds); + MPATH_SKIP_UNLESS_ISCSI(mp_sds, mp_num_sds); + + /* synchronously initialise zeros for first CAW */ + memset(wr_buf, 0, block_size); + WRITESAME10(mp_sds[0], 0, block_size, 1, 0, 0, 0, 0, wr_buf, + EXPECT_STATUS_GOOD); + + test_mpio_async_caw_init_bufs(cmp_buf, wr_buf, blocksize, mp_num_sds); + + for (i = 0; i < num_ios; i++) { + /* queue a one-block CAW task on each MPIO sessions */ + for (sd_i = 0; sd_i < mp_num_sds; sd_i++) { + struct scsi_task *atask; + int buf_off = sd_i * blocksize; + + atask = scsi_cdb_compareandwrite(lba, blocksize * 2, + blocksize, + 0, 0, 0, 0, 0); + CU_ASSERT_PTR_NOT_NULL_FATAL(atask); + + /* compare data is first, followed by write data */ + ret = scsi_task_add_data_out_buffer(atask, + blocksize, + &cmp_buf[buf_off]); + CU_ASSERT_EQUAL(ret, 0); + + ret = scsi_task_add_data_out_buffer(atask, + blocksize, + &wr_buf[buf_off]); + CU_ASSERT_EQUAL(ret, 0); + + ret = iscsi_scsi_command_async(mp_sds[sd_i]->iscsi_ctx, + mp_sds[sd_i]->iscsi_lun, + atask, + test_mpio_async_caw_cb, + NULL, &state); + CU_ASSERT_EQUAL(ret, 0); + + state.dispatched++; + logging(LOG_VERBOSE, "COMPARE_AND_WRITE dispatched: " + "%d of %d (cmdsn=%d)", + state.dispatched, num_ios, atask->cmdsn); + } + } + + while (state.completed < state.dispatched) { + struct pollfd pfd[mp_num_sds]; + + for (sd_i = 0; sd_i < mp_num_sds; sd_i++) { + pfd[sd_i].fd = iscsi_get_fd(mp_sds[sd_i]->iscsi_ctx); + pfd[sd_i].events + = iscsi_which_events(mp_sds[sd_i]->iscsi_ctx); + } + + ret = poll(pfd, mp_num_sds, -1); + CU_ASSERT_NOT_EQUAL(ret, -1); + + for (sd_i = 0; sd_i < mp_num_sds; sd_i++) { + if (!pfd[sd_i].revents) { + continue; + } + ret = iscsi_service(mp_sds[sd_i]->iscsi_ctx, + pfd[sd_i].revents); + CU_ASSERT_EQUAL(ret, 0); + } + } + + logging(LOG_VERBOSE, "[OK] All %d COMPARE_AND_WRITE IOs complete, with " + "%d mismatch(es)", state.completed, state.mismatches); +}