76ab15b0df
Add a new backend store that enables iSCSI targets backed by S3-compatible object storage (AWS S3, MinIO, Ceph RGW, etc.). The implementation uses a chunked storage strategy where the virtual block device is divided into fixed-size chunks (default 4 MiB), each stored as an independent S3 object. This enables efficient random read/write access on top of object storage. Key features: - Chunked storage with configurable chunk size - Sparse device support (unwritten chunks treated as zeros) - Concurrent multi-chunk I/O via errgroup - Per-chunk locking for safe read-modify-write - AWS SDK v2 with default credential chain - In-process gofakes3 test server (no Docker needed) - 12 unit tests + 2 integration tests Also updates CI workflow to run S3 backend tests and updates README with S3 backend documentation. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
252 lines
12 KiB
YAML
252 lines
12 KiB
YAML
# This workflow will build a golang project
|
|
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
|
|
|
|
name: Go
|
|
|
|
on:
|
|
push:
|
|
branches: [ "*" ]
|
|
pull_request:
|
|
|
|
env:
|
|
TARGET: 'iqn.2016-09.com.gotgt.gostor:example_tgt_0'
|
|
TGT_CFG: '{"storages":[{"deviceID":1000,"path":"file:/var/tmp/disk.img","online":true}],"iscsiportals":[{"id":0,"portal":"127.0.0.1:3260"}],"iscsitargets":{"iqn.2016-09.com.gotgt.gostor:example_tgt_0":{"tpgts":{"1":[0]},"luns":{"0":1000}}}}'
|
|
|
|
jobs:
|
|
|
|
build:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v5
|
|
with:
|
|
go-version: '1.24'
|
|
check-latest: true
|
|
|
|
- name: Depend install
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install libcunit1 libcunit1-doc libcunit1-dev open-iscsi make libnuma-dev -y
|
|
|
|
- name: Gofmt verify
|
|
run: hack/verify-gofmt.sh
|
|
|
|
- name: Build
|
|
run: make
|
|
|
|
- name: Unit Test
|
|
run: go test -v ./pkg/... ./mock/...
|
|
|
|
- name: S3 Backend Store Test
|
|
run: go test -tags s3integration -v ./pkg/scsi/backingstore/s3store/
|
|
|
|
- name: Function test
|
|
run: |
|
|
dd if=/dev/zero of=/var/tmp/disk.img bs=1024 count=102400
|
|
mkdir $HOME/.gotgt
|
|
echo '${{env.TGT_CFG}}' > $HOME/.gotgt/config.json
|
|
./_output/cmd/bin/gotgt daemon --log debug 1>/dev/null 2>&1 &
|
|
git clone https://github.com/gostor/libiscsi $HOME/libiscsi
|
|
cd $HOME/libiscsi
|
|
sudo ci/install.sh
|
|
export ISCSITEST=yes
|
|
./autogen.sh
|
|
./configure 2>&1 >/dev/null
|
|
make 2>&1 >/dev/null
|
|
|
|
echo "=== Starting Comprehensive libiscsi Tests (60+ tests) ==="
|
|
|
|
# Inquiry Tests
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Inquiry.Standard iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Inquiry.AllocLength iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Inquiry.MandatoryVPDSBC iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Inquiry.SupportedVPD iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Inquiry.VersionDescriptors iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Inquiry.EVPD iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
|
|
# Mandatory Tests
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Mandatory iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.ModeSense6 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.NoMedia iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
|
|
# Read Tests
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Read6 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Read10 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Read12 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Read16 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.ReadOnly iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
|
|
# Read Capacity Tests
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.ReadCapacity10 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.ReadCapacity16 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
|
|
# Write Tests
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Write10 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Write12 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Write16 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.WriteVerify10 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.WriteVerify12 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.WriteVerify16 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
|
|
# Write Same Tests
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.WriteSame10.Simple iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.WriteSame16.Simple iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
|
|
# Verify Tests
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Verify10 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Verify12 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Verify16 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
|
|
# Prefetch Tests
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Prefetch10 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Prefetch16 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
|
|
# Synchronize Cache Tests
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.SynchronizeCache10 iscsi://127.0.0.1:3260/${{env.TARGET}}/0 || true
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.SynchronizeCache16 iscsi://127.0.0.1:3260/${{env.TARGET}}/0 || true
|
|
|
|
# Reserve/Release Tests
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Reserve6.Simple iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
|
|
# Persistent Reservation Tests (known issues with write-path sense data)
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.PrinReadKeys.Simple iscsi://127.0.0.1:3260/${{env.TARGET}}/0 || true
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.PrinReportCapabilities.Simple iscsi://127.0.0.1:3260/${{env.TARGET}}/0 || true
|
|
|
|
# Unmap Tests
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Unmap.Simple iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Unmap.VPD iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.Unmap.ZeroBlocks iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
|
|
# Read Defect Data Tests
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.ReadDefectData10.Simple iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.ReadDefectData12.Simple iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
|
|
# Other SCSI Tests
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.PreventAllow iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.StartStopUnit iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.TestUnitReady iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.ReportSupportedOpcodes.Simple iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
|
|
# iSCSI Protocol Tests
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.iSCSITMF iscsi://127.0.0.1:3260/${{env.TARGET}}/0 || true
|
|
./test-tool/iscsi-test-cu -d -A --test=ALL.iSCSIcmdsn iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
|
|
echo "=== libiscsi Tests Completed ==="
|
|
|
|
# Utility tests
|
|
./utils/iscsi-ls -s iscsi://127.0.0.1:3260/${{env.TARGET}}
|
|
./utils/iscsi-inq iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
./utils/iscsi-readcapacity16 iscsi://127.0.0.1:3260/${{env.TARGET}}/0
|
|
|
|
# iscsi initiator test
|
|
sudo iscsiadm -m discovery -t sendtargets -p 127.0.0.1
|
|
sudo iscsiadm -m node -L all
|
|
sudo iscsiadm -m session
|
|
sudo fdisk -l
|
|
echo -e "n\np\n1\n\n\nt\nc\na\nw" | sudo fdisk /dev/sdb
|
|
sudo mkfs.ext3 /dev/sdb1
|
|
sudo mkdir -p /var/tmp/test
|
|
sudo mount /dev/sdb1 /var/tmp/test
|
|
sudo ls -lh /var/tmp/test/
|
|
|
|
- name: CLI management test
|
|
run: |
|
|
API="http://127.0.0.1:23457"
|
|
|
|
echo "=== CLI Management API Tests ==="
|
|
|
|
# 1. List targets - should show the target from config
|
|
echo "--- list target ---"
|
|
curl -sf "$API/target/list" | tee /tmp/list_target.out
|
|
grep -q "${{env.TARGET}}" /tmp/list_target.out
|
|
echo "PASS: list target shows configured target"
|
|
|
|
# 2. List LUs for the existing target
|
|
echo "--- list lu ---"
|
|
curl -sf "$API/lu/list?target=${{env.TARGET}}" | tee /tmp/list_lu.out
|
|
echo ""
|
|
# Verify it returns a JSON array (may be empty or have LU 0)
|
|
python3 -c "import json,sys; data=json.load(sys.stdin); assert isinstance(data,list)" < /tmp/list_lu.out
|
|
echo "PASS: list lu returns valid JSON array"
|
|
|
|
# 3. List TPGTs for the existing target
|
|
echo "--- list tpgt ---"
|
|
curl -sf "$API/target/tpgt/list?target=${{env.TARGET}}" | tee /tmp/list_tpgt.out
|
|
echo ""
|
|
python3 -c "import json,sys; data=json.load(sys.stdin); assert isinstance(data,list)" < /tmp/list_tpgt.out
|
|
echo "PASS: list tpgt returns valid JSON array"
|
|
|
|
# 4. Create a new target via API
|
|
echo "--- create target ---"
|
|
NEW_TARGET="iqn.2016-09.com.gotgt.gostor:ci_test_tgt"
|
|
curl -sf -X POST "$API/target/create" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"Name\":\"$NEW_TARGET\"}" | tee /tmp/create_target.out
|
|
echo ""
|
|
grep -q "$NEW_TARGET" /tmp/create_target.out
|
|
echo "PASS: create target succeeded"
|
|
|
|
# 5. Verify new target appears in list
|
|
echo "--- verify new target in list ---"
|
|
curl -sf "$API/target/list" | tee /tmp/list_target2.out
|
|
echo ""
|
|
grep -q "$NEW_TARGET" /tmp/list_target2.out
|
|
echo "PASS: new target visible in list"
|
|
|
|
# 6. Create a new LU on the new target
|
|
echo "--- create lu ---"
|
|
dd if=/dev/zero of=/var/tmp/ci_disk.img bs=1024 count=10240 2>/dev/null
|
|
curl -sf -X POST "$API/lu/create" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"targetName\":\"$NEW_TARGET\",\"deviceID\":2000,\"lun\":0,\"path\":\"file:/var/tmp/ci_disk.img\",\"blockShift\":9}" \
|
|
-o /dev/null -w "%{http_code}" | tee /tmp/create_lu_status.out
|
|
echo ""
|
|
grep -q "201" /tmp/create_lu_status.out
|
|
echo "PASS: create lu returned 201"
|
|
|
|
# 7. Verify new LU appears in list
|
|
echo "--- verify new lu in list ---"
|
|
curl -sf "$API/lu/list?target=$NEW_TARGET" | tee /tmp/list_lu2.out
|
|
echo ""
|
|
python3 -c "import json,sys; data=json.load(sys.stdin); assert len(data)==1, f'expected 1 LU, got {len(data)}'" < /tmp/list_lu2.out
|
|
echo "PASS: new LU visible in list"
|
|
|
|
# 8. Remove the LU
|
|
echo "--- remove lu ---"
|
|
curl -sf -X DELETE "$API/lu/delete" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"targetName\":\"$NEW_TARGET\",\"lun\":0}" \
|
|
-o /dev/null -w "%{http_code}" | tee /tmp/rm_lu_status.out
|
|
echo ""
|
|
grep -q "204" /tmp/rm_lu_status.out
|
|
echo "PASS: remove lu returned 204"
|
|
|
|
# 9. Verify LU is gone
|
|
echo "--- verify lu removed ---"
|
|
curl -sf "$API/lu/list?target=$NEW_TARGET" | tee /tmp/list_lu3.out
|
|
echo ""
|
|
python3 -c "import json,sys; data=json.load(sys.stdin); assert data is None or len(data)==0, f'expected 0 LUs, got {data}'" < /tmp/list_lu3.out
|
|
echo "PASS: LU no longer in list"
|
|
|
|
# 10. Remove the target
|
|
echo "--- remove target ---"
|
|
curl -sf -X DELETE "$API/target/$NEW_TARGET?force=1" \
|
|
-o /dev/null -w "%{http_code}" | tee /tmp/rm_target_status.out
|
|
echo ""
|
|
grep -q "204" /tmp/rm_target_status.out
|
|
echo "PASS: remove target returned 204"
|
|
|
|
# 11. Verify target is gone
|
|
echo "--- verify target removed ---"
|
|
curl -sf "$API/target/list" | tee /tmp/list_target3.out
|
|
echo ""
|
|
if grep -q "$NEW_TARGET" /tmp/list_target3.out; then
|
|
echo "FAIL: target still present after removal"
|
|
exit 1
|
|
fi
|
|
echo "PASS: target no longer in list"
|
|
|
|
echo "=== All CLI Management API Tests Passed ==="
|