diff --git a/.github/workflows/gotgt.yml b/.github/workflows/gotgt.yml index 28466f0..5b8c41c 100644 --- a/.github/workflows/gotgt.yml +++ b/.github/workflows/gotgt.yml @@ -39,6 +39,9 @@ jobs: - name: Unit Test run: go test -v ./pkg/... ./mock/... + - name: S3 Backend Store Test + run: go test -tags s3integration -v ./pkg/scsi/backingstore/s3store/ + - name: Function test run: | dd if=/dev/zero of=/var/tmp/disk.img bs=1024 count=102400 diff --git a/README.md b/README.md index a3c6a71..4edf329 100644 --- a/README.md +++ b/README.md @@ -116,15 +116,68 @@ On Linux systems with kernel 5.1 or later, gotgt can use io_uring for high-perfo **Backend Type Options:** - `file` - Standard file I/O (default) - `iouring` - io_uring-based I/O (Linux 5.1+) +- `s3` - S3-compatible object storage (AWS S3, MinIO, etc.) +- `ceph-rbd` - Ceph RADOS Block Device (Linux, requires Ceph libraries) -### 3. Object Pooling +### 3. S3-Compatible Object Storage Backend + +gotgt supports using S3-compatible object storage (AWS S3, MinIO, Ceph RGW, etc.) as backend storage. The virtual block device is divided into fixed-size chunks, each stored as an independent S3 object. This enables iSCSI targets backed by cloud or distributed object storage. + +**Features:** +- Chunked storage strategy with configurable chunk size (default 4 MiB) +- Sparse device support (unwritten chunks are treated as zeros) +- Concurrent multi-chunk reads and writes using goroutines +- Per-chunk locking for safe read-modify-write operations +- Compatible with AWS S3, MinIO, and other S3-compatible services +- AWS SDK v2 default credential chain (env vars, IAM roles, shared config) + +**Path Format:** `s3:bucket/prefix` + +**Configuration:** +```json +{ + "storages": [ + { + "deviceID": 2000, + "path": "s3:my-bucket/iscsi/disk0", + "online": true, + "backendType": "s3", + "blockShift": 9, + "deviceSize": 1073741824, + "s3ChunkSize": 4194304, + "s3Endpoint": "http://localhost:9000", + "s3ForcePathStyle": true + } + ] +} +``` + +**Configuration Fields:** +| Field | Description | Default | +|-------|-------------|---------| +| `deviceSize` | Virtual device size in bytes | Required for new devices | +| `s3ChunkSize` | Chunk size in bytes | 4194304 (4 MiB) | +| `s3Endpoint` | Custom S3 endpoint URL | AWS default | +| `s3Region` | AWS region | From credential chain | +| `s3ForcePathStyle` | Use path-style addressing (required for MinIO) | false | + +**Credentials:** Set via environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`) or AWS shared config files. + +**Performance Considerations:** +- S3 latency (~5-50ms per operation) is higher than local disk. Best suited for archival, disaster recovery, or cloud-native deployments where capacity and durability are prioritized over latency. +- Reads and writes spanning multiple chunks are parallelized automatically. +- Full-chunk writes bypass read-modify-write, so aligning I/O to chunk boundaries improves write throughput. + +For a complete example, see [config-s3.json](./examples/config-s3.json). + +### 4. Object Pooling The iSCSI protocol layer uses sync.Pool for efficient object reuse: - ISCSICommand object pooling to reduce GC pressure - Buffer pooling for protocol header processing - NUMA-aware buffer allocation for data operations -### 4. Combined High-Performance Configuration Example +### 5. Combined High-Performance Configuration Example For maximum performance, combine both NUMA and io_uring: @@ -163,14 +216,14 @@ For maximum performance, combine both NUMA and io_uring: } ``` -### 5. Performance Tuning Tips +### 6. Performance Tuning Tips 1. **NUMA Optimization**: On multi-socket systems, ensure the iSCSI target threads run on the same NUMA node as the storage devices 2. **Queue Depth**: For NVMe or fast SSDs, increase `ioUringQueueDepth` to 4096 or higher 3. **Buffer Sizes**: Match `numaBufferSize` to your typical I/O size (e.g., 64KB, 128KB, 256KB) 4. **CPU Pinning**: Use `numaNode` to pin storage backends to specific NUMA nodes -### 6. Benchmarking +### 7. Benchmarking Use fio to benchmark performance: ```bash diff --git a/cmd/daemon.go b/cmd/daemon.go index 4164a83..ec16bbc 100644 --- a/cmd/daemon.go +++ b/cmd/daemon.go @@ -32,6 +32,7 @@ import ( _ "github.com/gostor/gotgt/pkg/port/iscsit" "github.com/gostor/gotgt/pkg/scsi" _ "github.com/gostor/gotgt/pkg/scsi/backingstore" + _ "github.com/gostor/gotgt/pkg/scsi/backingstore/s3store" ) func newDaemonCommand() *cobra.Command { diff --git a/examples/config-s3.json b/examples/config-s3.json new file mode 100644 index 0000000..462793b --- /dev/null +++ b/examples/config-s3.json @@ -0,0 +1,31 @@ +{ + "storages": [ + { + "deviceID": 2000, + "path": "s3:gotgt-bucket/iscsi/disk0", + "online": true, + "backendType": "s3", + "blockShift": 9, + "deviceSize": 1073741824, + "s3ChunkSize": 4194304, + "s3Endpoint": "http://localhost:9000", + "s3ForcePathStyle": true + } + ], + "iscsiportals": [ + { + "id": 0, + "portal": "127.0.0.1:3260" + } + ], + "iscsitargets": { + "iqn.2016-09.com.gotgt.gostor:s3-target": { + "tpgts": { + "1": [0] + }, + "luns": { + "1": 2000 + } + } + } +} diff --git a/go.mod b/go.mod index 9fd7ce3..8646b53 100644 --- a/go.mod +++ b/go.mod @@ -1,28 +1,50 @@ module github.com/gostor/gotgt -go 1.23 +go 1.25.0 require ( + github.com/aws/aws-sdk-go-v2 v1.41.4 + github.com/aws/aws-sdk-go-v2/config v1.32.12 + github.com/aws/aws-sdk-go-v2/credentials v1.19.12 + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 github.com/ceph/go-ceph v0.30.0 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf github.com/docker/go-connections v0.5.0 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 + github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 github.com/mitchellh/go-homedir v1.1.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 golang.org/x/net v0.24.0 + golang.org/x/sync v0.20.0 ) require ( github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect + github.com/aws/smithy-go v1.24.2 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.2.1 // indirect + github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -30,6 +52,7 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect + go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect diff --git a/go.sum b/go.sum index f9b4643..5daf8b6 100644 --- a/go.sum +++ b/go.sum @@ -1,20 +1,66 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k= +github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 h1:3kGOqnh1pPeddVa/E37XNTaWJ8W6vrbYV9lJEkCnhuY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/config v1.32.12 h1:O3csC7HUGn2895eNrLytOJQdoL2xyJy0iYXhoZ1OmP0= +github.com/aws/aws-sdk-go-v2/config v1.32.12/go.mod h1:96zTvoOFR4FURjI+/5wY1vc1ABceROO4lWgWJuxgy0g= +github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 h1:S61/E3N01oral6B3y9hZ2E1iFDqCZPPOBoBQretCnBI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75/go.mod h1:bDMQbkI1vJbNjnvJYpPTSNYBkI/VIv18ngWb/K84tkk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 h1:SwGMTMLIlvDNyhMteQ6r8IJSBPlRdXX5d4idhIGbkXA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21/go.mod h1:UUxgWxofmOdAMuqEsSppbDtGKLfR04HGsD0HXzvhI1k= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 h1:qtJZ70afD3ISKWnoX3xB0J2otEqu3LqicRcDBqsj0hQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12/go.mod h1:v2pNpJbRNl4vEUWEh5ytQok0zACAKfdmKS51Hotc3pQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 h1:siU1A6xjUZ2N8zjTHSXFhB9L/2OY8Dqs0xXiLjF30jA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20/go.mod h1:4TLZCmVJDM3FOu5P5TJP0zOlu9zWgDWU7aUxWbr+rcw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 h1:csi9NLpFZXb9fxY7rS1xVzgPRGMt7MSNWeQ6eo247kE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1/go.mod h1:qXVal5H0ChqXP63t6jze5LmFalc7+ZE7wOdLtZ0LCP0= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/ceph/go-ceph v0.30.0 h1:p/+rNnn9dUByrDhXfBFilVriRZKJghMJcts8N2wQ+ws= github.com/ceph/go-ceph v0.30.0/go.mod h1:OJFju/Xmtb7ihHo/aXOayw6RhVOUGNke5EwTipwaf6A= +github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc= +github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gofrs/uuid/v5 v5.3.0 h1:m0mUMr+oVYUdxpMLgSYCZiXe7PuVPnI94+OMeVBNedk= +github.com/gofrs/uuid/v5 v5.3.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= @@ -23,8 +69,12 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 h1:0xkWp+RMC2ImuKacheMHEAtrbOTMOa0kYkxyzM1Z/II= +github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73/go.mod h1:S4S9jGBVlLri0OeqrSSbCGG5vsI6he06UJyuz1WT1EE= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -35,8 +85,12 @@ github.com/pelletier/go-toml/v2 v2.2.1 h1:9TA9+T8+8CUCO2+WYnDLCgrYi9+omqKXyjDtos github.com/pelletier/go-toml/v2 v2.2.1/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= @@ -68,6 +122,10 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d h1:Ns9kd1Rwzw7t0BR8XMphenji4SmIoNZPn8zhYmaVKP8= +go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d/go.mod h1:92Uoe3l++MlthCm+koNi0tcUCX3anayogF0Pa/sp24k= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= @@ -78,7 +136,8 @@ golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -88,8 +147,11 @@ golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/api/types.go b/pkg/api/types.go index eba8bc0..c2da2d0 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -407,6 +407,9 @@ type SCSILu struct { PerformCommand CommandFunc FinishCommand func(*SCSITarget, *SCSICommand) + + // BackendConfig holds backend-specific configuration (e.g., *config.BackendStorage) + BackendConfig interface{} `json:"-"` } type LUNMap map[uint64]*SCSILu diff --git a/pkg/config/config.go b/pkg/config/config.go index bda0b53..cc6241a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -114,6 +114,16 @@ type BackendStorage struct { NumaNode int `json:"numaNode,omitempty"` // IoUringQueueDepth specifies the io_uring queue depth (0 for default) IoUringQueueDepth uint32 `json:"ioUringQueueDepth,omitempty"` + // DeviceSize specifies the virtual device size in bytes (used by S3 backend) + DeviceSize uint64 `json:"deviceSize,omitempty"` + // S3ChunkSize specifies the chunk size in bytes for S3 backend (default 4MiB) + S3ChunkSize int64 `json:"s3ChunkSize,omitempty"` + // S3Endpoint specifies a custom S3 endpoint URL (for MinIO, etc.) + S3Endpoint string `json:"s3Endpoint,omitempty"` + // S3Region specifies the AWS region + S3Region string `json:"s3Region,omitempty"` + // S3ForcePathStyle uses path-style addressing (required for MinIO) + S3ForcePathStyle bool `json:"s3ForcePathStyle,omitempty"` } type ISCSIPortalInfo struct { diff --git a/pkg/scsi/backingstore/common.go b/pkg/scsi/backingstore/common.go index 07773f9..08dbeb4 100644 --- a/pkg/scsi/backingstore/common.go +++ b/pkg/scsi/backingstore/common.go @@ -63,7 +63,7 @@ func parseStoragePath(path string) (backendType, filePath string) { possibleType := path[:idx] // Check if it's a known backend type switch possibleType { - case "file", "iouring", "ceph", "null", "RemBs": + case "file", "iouring", "ceph", "null", "RemBs", "s3": return possibleType, path[idx+1:] } } diff --git a/pkg/scsi/backingstore/s3store/integration_test.go b/pkg/scsi/backingstore/s3store/integration_test.go new file mode 100644 index 0000000..662d2dc --- /dev/null +++ b/pkg/scsi/backingstore/s3store/integration_test.go @@ -0,0 +1,242 @@ +//go:build s3integration + +package s3store + +import ( + "bytes" + "context" + "fmt" + "math/rand" + "net/http/httptest" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/johannesboyne/gofakes3" + "github.com/johannesboyne/gofakes3/backend/s3mem" + + "github.com/gostor/gotgt/pkg/api" + "github.com/gostor/gotgt/pkg/config" + "github.com/gostor/gotgt/pkg/scsi" +) + +// startFakeS3Server starts an in-process S3-compatible HTTP server using gofakes3. +func startFakeS3Server(t *testing.T) (*httptest.Server, *s3.Client) { + t.Helper() + + backend := s3mem.New() + faker := gofakes3.New(backend) + ts := httptest.NewServer(faker.Server()) + + cfg, err := awsconfig.LoadDefaultConfig(context.Background(), + awsconfig.WithRegion("us-east-1"), + awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + "accesskey", "secretkey", "", + )), + ) + if err != nil { + t.Fatalf("failed to load AWS config: %v", err) + } + + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(ts.URL) + o.UsePathStyle = true + o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired + }) + + return ts, client +} + +func TestIntegration_FullLifecycle(t *testing.T) { + ts, client := startFakeS3Server(t) + defer ts.Close() + + bucket := fmt.Sprintf("gotgt-test-%d", rand.Int63()) + prefix := "integration/disk0" + + // Create bucket + _, err := client.CreateBucket(context.Background(), &s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + t.Fatalf("failed to create bucket: %v", err) + } + + deviceSize := uint64(1024 * 1024) // 1 MiB + chunkSize := int64(64 * 1024) // 64 KiB + + // Create and open a new S3-backed device + bs := &S3BackingStore{ + BaseBackingStore: scsi.BaseBackingStore{Name: S3BackingStorage}, + client: client, + } + dev := &api.SCSILu{ + BackendConfig: &config.BackendStorage{ + DeviceSize: deviceSize, + S3ChunkSize: chunkSize, + }, + } + + path := fmt.Sprintf("%s/%s", bucket, prefix) + if err := bs.Open(dev, path); err != nil { + t.Fatalf("Open failed: %v", err) + } + if bs.Size(dev) != deviceSize { + t.Fatalf("expected size=%d, got %d", deviceSize, bs.Size(dev)) + } + + // Write data at offset 0 + pattern1 := make([]byte, 1000) + for i := range pattern1 { + pattern1[i] = byte(i % 251) + } + if err := bs.Write(pattern1, 0); err != nil { + t.Fatalf("Write pattern1 failed: %v", err) + } + + // Write across chunk boundary + pattern2 := make([]byte, chunkSize+100) + for i := range pattern2 { + pattern2[i] = byte((i + 37) % 251) + } + offset2 := chunkSize - 50 + if err := bs.Write(pattern2, offset2); err != nil { + t.Fatalf("Write pattern2 failed: %v", err) + } + + // Read back pattern1 + data1, err := bs.Read(0, 1000) + if err != nil { + t.Fatalf("Read pattern1 failed: %v", err) + } + if !bytes.Equal(data1, pattern1) { + t.Fatal("pattern1 data mismatch") + } + + // Read back pattern2 + data2, err := bs.Read(offset2, int64(len(pattern2))) + if err != nil { + t.Fatalf("Read pattern2 failed: %v", err) + } + if !bytes.Equal(data2, pattern2) { + t.Fatal("pattern2 data mismatch") + } + + // Read sparse region (should be zeros) + sparseOffset := int64(deviceSize) - 1000 + dataSparse, err := bs.Read(sparseOffset, 1000) + if err != nil { + t.Fatalf("Read sparse failed: %v", err) + } + for i, b := range dataSparse { + if b != 0 { + t.Fatalf("sparse byte %d: expected 0, got %d", i, b) + } + } + + // Close and reopen to verify persistence + if err := bs.Close(dev); err != nil { + t.Fatalf("Close failed: %v", err) + } + + bs2 := &S3BackingStore{ + BaseBackingStore: scsi.BaseBackingStore{Name: S3BackingStorage}, + client: client, // same in-memory server + } + dev2 := &api.SCSILu{ + BackendConfig: &config.BackendStorage{}, + } + if err := bs2.Open(dev2, path); err != nil { + t.Fatalf("Reopen failed: %v", err) + } + if bs2.Size(dev2) != deviceSize { + t.Fatalf("after reopen: expected size=%d, got %d", deviceSize, bs2.Size(dev2)) + } + + // Verify data persisted + data1Again, err := bs2.Read(0, 1000) + if err != nil { + t.Fatalf("Re-read pattern1 failed: %v", err) + } + if !bytes.Equal(data1Again, pattern1) { + t.Fatal("pattern1 data mismatch after reopen") + } + + // Test unmap (full chunk) + if err := bs2.Unmap([]api.UnmapBlockDescriptor{ + {Offset: 0, TL: uint64(chunkSize)}, + }); err != nil { + t.Fatalf("Unmap failed: %v", err) + } + + // Read unmapped region - should be zeros + dataUnmapped, err := bs2.Read(0, 100) + if err != nil { + t.Fatalf("Read unmapped failed: %v", err) + } + for i, b := range dataUnmapped { + if b != 0 { + t.Fatalf("unmapped byte %d: expected 0, got %d", i, b) + } + } + + bs2.Close(dev2) + t.Log("S3 integration test: full lifecycle passed") +} + +func TestIntegration_LargeWriteRead(t *testing.T) { + ts, client := startFakeS3Server(t) + defer ts.Close() + + bucket := fmt.Sprintf("gotgt-large-%d", rand.Int63()) + _, err := client.CreateBucket(context.Background(), &s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + t.Fatalf("failed to create bucket: %v", err) + } + + deviceSize := uint64(4 * 1024 * 1024) // 4 MiB + chunkSize := int64(256 * 1024) // 256 KiB + + bs := &S3BackingStore{ + BaseBackingStore: scsi.BaseBackingStore{Name: S3BackingStorage}, + client: client, + } + dev := &api.SCSILu{ + BackendConfig: &config.BackendStorage{ + DeviceSize: deviceSize, + S3ChunkSize: chunkSize, + }, + } + + path := fmt.Sprintf("%s/large/disk0", bucket) + if err := bs.Open(dev, path); err != nil { + t.Fatalf("Open failed: %v", err) + } + + // Write 1MiB of data spanning multiple chunks + writeSize := 1024 * 1024 + data := make([]byte, writeSize) + for i := range data { + data[i] = byte(i % 256) + } + + if err := bs.Write(data, 0); err != nil { + t.Fatalf("Large write failed: %v", err) + } + + // Read it back + readBack, err := bs.Read(0, int64(writeSize)) + if err != nil { + t.Fatalf("Large read failed: %v", err) + } + if !bytes.Equal(readBack, data) { + t.Fatal("large write/read data mismatch") + } + + bs.Close(dev) + t.Log("S3 integration test: large write/read passed") +} diff --git a/pkg/scsi/backingstore/s3store/s3store.go b/pkg/scsi/backingstore/s3store/s3store.go new file mode 100644 index 0000000..f67d7dd --- /dev/null +++ b/pkg/scsi/backingstore/s3store/s3store.go @@ -0,0 +1,470 @@ +/* +Copyright 2024 The GoStor Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package s3store + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + "sync" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + log "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" + + "github.com/gostor/gotgt/pkg/api" + "github.com/gostor/gotgt/pkg/config" + "github.com/gostor/gotgt/pkg/scsi" +) + +const ( + S3BackingStorage = "s3" + DefaultChunkSize = 4 * 1024 * 1024 // 4 MiB + metadataKey = "_metadata" +) + +func init() { + scsi.RegisterBackingStore(S3BackingStorage, newS3) +} + +// s3Client is the minimal S3 interface used by this package, enabling test injection. +type s3Client interface { + GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) + PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) + DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) +} + +// deviceMetadata stores device configuration in S3. +type deviceMetadata struct { + DeviceSize uint64 `json:"deviceSize"` + ChunkSize int64 `json:"chunkSize"` +} + +// S3BackingStore implements the BackingStore interface using S3-compatible object storage. +// The virtual block device is divided into fixed-size chunks, each stored as a separate S3 object. +type S3BackingStore struct { + scsi.BaseBackingStore + client s3Client + bucket string + prefix string + chunkSize int64 + chunkLocks sync.Map // map[int64]*sync.Mutex +} + +func newS3() (api.BackingStore, error) { + return &S3BackingStore{ + BaseBackingStore: scsi.BaseBackingStore{ + Name: S3BackingStorage, + DataSize: 0, + OflagsSupported: 0, + }, + }, nil +} + +func (bs *S3BackingStore) lockChunk(idx int64) *sync.Mutex { + val, _ := bs.chunkLocks.LoadOrStore(idx, &sync.Mutex{}) + mu := val.(*sync.Mutex) + mu.Lock() + return mu +} + +func (bs *S3BackingStore) chunkKey(idx int64) string { + return fmt.Sprintf("%s/chunk_%010d", bs.prefix, idx) +} + +func (bs *S3BackingStore) metadataObjKey() string { + return fmt.Sprintf("%s/%s", bs.prefix, metadataKey) +} + +func (bs *S3BackingStore) Open(dev *api.SCSILu, path string) error { + // Parse path: bucket/prefix + idx := strings.Index(path, "/") + if idx <= 0 { + return fmt.Errorf("invalid S3 path %q, expected bucket/prefix", path) + } + bs.bucket = path[:idx] + bs.prefix = path[idx+1:] + if bs.prefix == "" { + return fmt.Errorf("invalid S3 path %q, prefix cannot be empty", path) + } + + // Read backend-specific config + var ( + chunkSize int64 + deviceSize uint64 + endpoint string + region string + forcePathStyle bool + ) + if cfg, ok := dev.BackendConfig.(*config.BackendStorage); ok && cfg != nil { + chunkSize = cfg.S3ChunkSize + deviceSize = cfg.DeviceSize + endpoint = cfg.S3Endpoint + region = cfg.S3Region + forcePathStyle = cfg.S3ForcePathStyle + } + if chunkSize <= 0 { + chunkSize = DefaultChunkSize + } + bs.chunkSize = chunkSize + + // Create S3 client if not already set (e.g., by tests) + if bs.client == nil { + ctx := context.Background() + var opts []func(*awsconfig.LoadOptions) error + if region != "" { + opts = append(opts, awsconfig.WithRegion(region)) + } + cfg, err := awsconfig.LoadDefaultConfig(ctx, opts...) + if err != nil { + return fmt.Errorf("failed to load AWS config: %w", err) + } + + var s3Opts []func(*s3.Options) + if endpoint != "" { + s3Opts = append(s3Opts, func(o *s3.Options) { + o.BaseEndpoint = aws.String(endpoint) + }) + } + if forcePathStyle { + s3Opts = append(s3Opts, func(o *s3.Options) { + o.UsePathStyle = true + }) + } + // Disable response checksum validation for compatibility with + // S3-compatible backends (e.g., MinIO) and range read requests. + s3Opts = append(s3Opts, func(o *s3.Options) { + o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired + }) + bs.client = s3.NewFromConfig(cfg, s3Opts...) + } + + // Try to load existing metadata + ctx := context.Background() + meta, err := bs.loadMetadata(ctx) + if err != nil { + var nsk *types.NoSuchKey + if !errors.As(err, &nsk) { + return fmt.Errorf("failed to load S3 metadata: %w", err) + } + // Metadata does not exist - create new device + if deviceSize == 0 { + return fmt.Errorf("S3 device metadata not found and deviceSize not configured") + } + meta = &deviceMetadata{ + DeviceSize: deviceSize, + ChunkSize: chunkSize, + } + if err := bs.saveMetadata(ctx, meta); err != nil { + return fmt.Errorf("failed to save S3 metadata: %w", err) + } + log.Infof("S3 backing store: created new device %s/%s, size=%d, chunkSize=%d", + bs.bucket, bs.prefix, deviceSize, chunkSize) + } else { + bs.chunkSize = meta.ChunkSize + log.Infof("S3 backing store: opened existing device %s/%s, size=%d, chunkSize=%d", + bs.bucket, bs.prefix, meta.DeviceSize, meta.ChunkSize) + } + + bs.DataSize = meta.DeviceSize + return nil +} + +func (bs *S3BackingStore) loadMetadata(ctx context.Context) (*deviceMetadata, error) { + key := bs.metadataObjKey() + out, err := bs.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bs.bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, err + } + defer out.Body.Close() + + var meta deviceMetadata + if err := json.NewDecoder(out.Body).Decode(&meta); err != nil { + return nil, fmt.Errorf("failed to decode metadata: %w", err) + } + return &meta, nil +} + +func (bs *S3BackingStore) saveMetadata(ctx context.Context, meta *deviceMetadata) error { + data, err := json.Marshal(meta) + if err != nil { + return err + } + key := bs.metadataObjKey() + _, err = bs.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bs.bucket), + Key: aws.String(key), + Body: bytes.NewReader(data), + }) + return err +} + +func (bs *S3BackingStore) Close(dev *api.SCSILu) error { + bs.client = nil + return nil +} + +func (bs *S3BackingStore) Init(dev *api.SCSILu, Opts string) error { + return nil +} + +func (bs *S3BackingStore) Exit(dev *api.SCSILu) error { + return nil +} + +func (bs *S3BackingStore) Size(dev *api.SCSILu) uint64 { + return bs.DataSize +} + +func (bs *S3BackingStore) Read(offset, tl int64) ([]byte, error) { + if bs.client == nil { + return nil, fmt.Errorf("S3 backend store is not open") + } + + result := make([]byte, tl) + startChunk := offset / bs.chunkSize + endChunk := (offset + tl - 1) / bs.chunkSize + + ctx := context.Background() + eg, ctx := errgroup.WithContext(ctx) + + for ci := startChunk; ci <= endChunk; ci++ { + ci := ci + eg.Go(func() error { + chunkStart := ci * bs.chunkSize + readStart := max(offset, chunkStart) - chunkStart + readEnd := min(offset+tl, chunkStart+bs.chunkSize) - chunkStart + + data, err := bs.getChunkRange(ctx, ci, readStart, readEnd) + if err != nil { + return err + } + + destStart := max(offset, chunkStart) - offset + copy(result[destStart:], data) + return nil + }) + } + + if err := eg.Wait(); err != nil { + return nil, err + } + return result, nil +} + +// ReadAt reads directly into the provided buffer to avoid allocation. +func (bs *S3BackingStore) ReadAt(buf []byte, offset int64) (int, error) { + data, err := bs.Read(offset, int64(len(buf))) + if err != nil { + return 0, err + } + copy(buf, data) + return len(buf), nil +} + +// getChunkRange reads a byte range from a chunk. Returns zeros if the chunk does not exist. +func (bs *S3BackingStore) getChunkRange(ctx context.Context, chunkIdx, start, end int64) ([]byte, error) { + key := bs.chunkKey(chunkIdx) + rangeStr := fmt.Sprintf("bytes=%d-%d", start, end-1) + + out, err := bs.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bs.bucket), + Key: aws.String(key), + Range: aws.String(rangeStr), + }) + if err != nil { + var nsk *types.NoSuchKey + if errors.As(err, &nsk) { + // Chunk doesn't exist - return zeros (sparse) + return make([]byte, end-start), nil + } + return nil, fmt.Errorf("failed to read chunk %d: %w", chunkIdx, err) + } + defer out.Body.Close() + + data, err := io.ReadAll(out.Body) + if err != nil { + return nil, fmt.Errorf("failed to read chunk %d body: %w", chunkIdx, err) + } + + // Pad with zeros if chunk is shorter than expected range + expected := int(end - start) + if len(data) < expected { + padded := make([]byte, expected) + copy(padded, data) + return padded, nil + } + return data, nil +} + +func (bs *S3BackingStore) Write(wbuf []byte, offset int64) error { + if bs.client == nil { + return fmt.Errorf("S3 backend store is not open") + } + + tl := int64(len(wbuf)) + startChunk := offset / bs.chunkSize + endChunk := (offset + tl - 1) / bs.chunkSize + + ctx := context.Background() + eg, ctx := errgroup.WithContext(ctx) + + for ci := startChunk; ci <= endChunk; ci++ { + ci := ci + eg.Go(func() error { + chunkStart := ci * bs.chunkSize + writeStart := max(offset, chunkStart) - chunkStart + writeEnd := min(offset+tl, chunkStart+bs.chunkSize) - chunkStart + + srcStart := max(offset, chunkStart) - offset + srcEnd := srcStart + (writeEnd - writeStart) + + if writeStart == 0 && writeEnd == bs.chunkSize { + // Full chunk write - direct upload + return bs.putChunk(ctx, ci, wbuf[srcStart:srcEnd]) + } + // Partial chunk - read-modify-write + return bs.readModifyWriteChunk(ctx, ci, writeStart, writeEnd, wbuf[srcStart:srcEnd]) + }) + } + + return eg.Wait() +} + +func (bs *S3BackingStore) putChunk(ctx context.Context, chunkIdx int64, data []byte) error { + key := bs.chunkKey(chunkIdx) + _, err := bs.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bs.bucket), + Key: aws.String(key), + Body: bytes.NewReader(data), + }) + if err != nil { + return fmt.Errorf("failed to write chunk %d: %w", chunkIdx, err) + } + return nil +} + +func (bs *S3BackingStore) readModifyWriteChunk(ctx context.Context, chunkIdx, writeStart, writeEnd int64, data []byte) error { + mu := bs.lockChunk(chunkIdx) + defer mu.Unlock() + + // Read existing chunk + chunk, err := bs.getFullChunk(ctx, chunkIdx) + if err != nil { + return err + } + + // Modify + copy(chunk[writeStart:writeEnd], data) + + // Write back + return bs.putChunk(ctx, chunkIdx, chunk) +} + +// getFullChunk reads the full chunk, returning a zero-filled buffer if it doesn't exist. +func (bs *S3BackingStore) getFullChunk(ctx context.Context, chunkIdx int64) ([]byte, error) { + key := bs.chunkKey(chunkIdx) + out, err := bs.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bs.bucket), + Key: aws.String(key), + }) + if err != nil { + var nsk *types.NoSuchKey + if errors.As(err, &nsk) { + return make([]byte, bs.chunkSize), nil + } + return nil, fmt.Errorf("failed to read full chunk %d: %w", chunkIdx, err) + } + defer out.Body.Close() + + data, err := io.ReadAll(out.Body) + if err != nil { + return nil, fmt.Errorf("failed to read full chunk %d body: %w", chunkIdx, err) + } + + if int64(len(data)) < bs.chunkSize { + padded := make([]byte, bs.chunkSize) + copy(padded, data) + return padded, nil + } + return data, nil +} + +func (bs *S3BackingStore) DataSync(offset, tl int64) error { + // S3 PutObject is durable once acknowledged + return nil +} + +func (bs *S3BackingStore) DataAdvise(offset, length int64, advise uint32) error { + return nil +} + +func (bs *S3BackingStore) Unmap(descriptors []api.UnmapBlockDescriptor) error { + if bs.client == nil { + return fmt.Errorf("S3 backend store is not open") + } + + ctx := context.Background() + for _, desc := range descriptors { + startChunk := int64(desc.Offset) / bs.chunkSize + endChunk := int64(desc.Offset+desc.TL-1) / bs.chunkSize + + for ci := startChunk; ci <= endChunk; ci++ { + chunkStart := ci * bs.chunkSize + unmapStart := max(int64(desc.Offset), chunkStart) - chunkStart + unmapEnd := min(int64(desc.Offset+desc.TL), chunkStart+bs.chunkSize) - chunkStart + + if unmapStart == 0 && unmapEnd == bs.chunkSize { + // Full chunk unmap - delete the object + key := bs.chunkKey(ci) + if _, err := bs.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bs.bucket), + Key: aws.String(key), + }); err != nil { + return fmt.Errorf("failed to delete chunk %d: %w", ci, err) + } + } + // Partial chunk unmap - ignore (sparse zeros on next read is fine + // since missing chunk data is treated as zeros) + } + } + return nil +} + +func max(a, b int64) int64 { + if a > b { + return a + } + return b +} + +func min(a, b int64) int64 { + if a < b { + return a + } + return b +} diff --git a/pkg/scsi/backingstore/s3store/s3store_test.go b/pkg/scsi/backingstore/s3store/s3store_test.go new file mode 100644 index 0000000..3870df6 --- /dev/null +++ b/pkg/scsi/backingstore/s3store/s3store_test.go @@ -0,0 +1,359 @@ +package s3store + +import ( + "bytes" + "context" + "fmt" + "io" + "sync" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + + "github.com/gostor/gotgt/pkg/api" + "github.com/gostor/gotgt/pkg/config" + "github.com/gostor/gotgt/pkg/scsi" +) + +// fakeS3Client is an in-memory S3 client for testing. +type fakeS3Client struct { + mu sync.Mutex + objects map[string][]byte // key -> data +} + +func newFakeS3Client() *fakeS3Client { + return &fakeS3Client{objects: make(map[string][]byte)} +} + +func (f *fakeS3Client) GetObject(_ context.Context, input *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + f.mu.Lock() + defer f.mu.Unlock() + + key := *input.Key + data, ok := f.objects[key] + if !ok { + return nil, &types.NoSuchKey{} + } + + if input.Range != nil { + var start, end int64 + _, err := fmt.Sscanf(*input.Range, "bytes=%d-%d", &start, &end) + if err == nil && start >= 0 && end < int64(len(data)) { + data = data[start : end+1] + } + } + + return &s3.GetObjectOutput{ + Body: io.NopCloser(bytes.NewReader(data)), + }, nil +} + +func (f *fakeS3Client) PutObject(_ context.Context, input *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + f.mu.Lock() + defer f.mu.Unlock() + + key := *input.Key + data, err := io.ReadAll(input.Body) + if err != nil { + return nil, err + } + f.objects[key] = data + return &s3.PutObjectOutput{}, nil +} + +func (f *fakeS3Client) DeleteObject(_ context.Context, input *s3.DeleteObjectInput, _ ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) { + f.mu.Lock() + defer f.mu.Unlock() + + delete(f.objects, *input.Key) + return &s3.DeleteObjectOutput{}, nil +} + +func newTestStore(client *fakeS3Client, chunkSize int64, deviceSize uint64) *S3BackingStore { + return &S3BackingStore{ + BaseBackingStore: scsi.BaseBackingStore{ + Name: S3BackingStorage, + DataSize: deviceSize, + }, + client: client, + bucket: "test-bucket", + prefix: "test/disk", + chunkSize: chunkSize, + } +} + +func TestOpen_NewDevice(t *testing.T) { + client := newFakeS3Client() + + bs := &S3BackingStore{ + BaseBackingStore: scsi.BaseBackingStore{Name: S3BackingStorage}, + client: client, + } + dev := &api.SCSILu{ + BackendConfig: &config.BackendStorage{ + DeviceSize: 1024 * 1024, // 1MiB + }, + } + + err := bs.Open(dev, "mybucket/myprefix") + if err != nil { + t.Fatalf("Open failed: %v", err) + } + if bs.bucket != "mybucket" { + t.Errorf("expected bucket=mybucket, got %s", bs.bucket) + } + if bs.prefix != "myprefix" { + t.Errorf("expected prefix=myprefix, got %s", bs.prefix) + } + if bs.DataSize != 1024*1024 { + t.Errorf("expected DataSize=1048576, got %d", bs.DataSize) + } + + // Verify metadata was saved + if _, ok := client.objects["myprefix/_metadata"]; !ok { + t.Error("metadata object was not created") + } +} + +func TestOpen_ExistingDevice(t *testing.T) { + client := newFakeS3Client() + // Pre-populate metadata + client.objects["myprefix/_metadata"] = []byte(`{"deviceSize":2097152,"chunkSize":1048576}`) + + bs := &S3BackingStore{ + BaseBackingStore: scsi.BaseBackingStore{Name: S3BackingStorage}, + client: client, + } + dev := &api.SCSILu{ + BackendConfig: &config.BackendStorage{}, + } + + err := bs.Open(dev, "mybucket/myprefix") + if err != nil { + t.Fatalf("Open failed: %v", err) + } + if bs.DataSize != 2097152 { + t.Errorf("expected DataSize=2097152, got %d", bs.DataSize) + } + if bs.chunkSize != 1048576 { + t.Errorf("expected chunkSize=1048576, got %d", bs.chunkSize) + } +} + +func TestOpen_InvalidPath(t *testing.T) { + bs := &S3BackingStore{ + BaseBackingStore: scsi.BaseBackingStore{Name: S3BackingStorage}, + client: newFakeS3Client(), + } + dev := &api.SCSILu{BackendConfig: &config.BackendStorage{}} + + err := bs.Open(dev, "nobucket") + if err == nil { + t.Error("expected error for invalid path") + } +} + +func TestRead_SingleChunk(t *testing.T) { + client := newFakeS3Client() + bs := newTestStore(client, 1024, 4096) + + // Write a chunk + chunkData := make([]byte, 1024) + for i := range chunkData { + chunkData[i] = 0xAB + } + client.objects["test/disk/chunk_0000000000"] = chunkData + + data, err := bs.Read(100, 200) + if err != nil { + t.Fatalf("Read failed: %v", err) + } + if len(data) != 200 { + t.Fatalf("expected 200 bytes, got %d", len(data)) + } + for i, b := range data { + if b != 0xAB { + t.Fatalf("byte %d: expected 0xAB, got 0x%02X", i, b) + } + } +} + +func TestRead_CrossChunk(t *testing.T) { + client := newFakeS3Client() + bs := newTestStore(client, 100, 1000) + + // Chunk 0: bytes 0-99, fill with 0xAA + chunk0 := make([]byte, 100) + for i := range chunk0 { + chunk0[i] = 0xAA + } + client.objects["test/disk/chunk_0000000000"] = chunk0 + + // Chunk 1: bytes 100-199, fill with 0xBB + chunk1 := make([]byte, 100) + for i := range chunk1 { + chunk1[i] = 0xBB + } + client.objects["test/disk/chunk_0000000001"] = chunk1 + + // Read across boundary: 50-149 + data, err := bs.Read(50, 100) + if err != nil { + t.Fatalf("Read failed: %v", err) + } + if len(data) != 100 { + t.Fatalf("expected 100 bytes, got %d", len(data)) + } + + // First 50 bytes from chunk0 (0xAA) + for i := 0; i < 50; i++ { + if data[i] != 0xAA { + t.Fatalf("byte %d: expected 0xAA, got 0x%02X", i, data[i]) + } + } + // Last 50 bytes from chunk1 (0xBB) + for i := 50; i < 100; i++ { + if data[i] != 0xBB { + t.Fatalf("byte %d: expected 0xBB, got 0x%02X", i, data[i]) + } + } +} + +func TestRead_SparseChunk(t *testing.T) { + client := newFakeS3Client() + bs := newTestStore(client, 100, 1000) + + // Don't create any chunks - should return zeros + data, err := bs.Read(0, 100) + if err != nil { + t.Fatalf("Read failed: %v", err) + } + for i, b := range data { + if b != 0 { + t.Fatalf("byte %d: expected 0, got 0x%02X", i, b) + } + } +} + +func TestWrite_SingleChunk(t *testing.T) { + client := newFakeS3Client() + bs := newTestStore(client, 1024, 4096) + + wbuf := []byte{1, 2, 3, 4, 5} + err := bs.Write(wbuf, 10) + if err != nil { + t.Fatalf("Write failed: %v", err) + } + + // The chunk should exist now (read-modify-write since it's partial) + chunk, ok := client.objects["test/disk/chunk_0000000000"] + if !ok { + t.Fatal("chunk was not created") + } + if len(chunk) != 1024 { + t.Fatalf("chunk size: expected 1024, got %d", len(chunk)) + } + // Verify written data + for i, b := range wbuf { + if chunk[10+i] != b { + t.Fatalf("byte %d: expected %d, got %d", 10+i, b, chunk[10+i]) + } + } + // Verify zeros around it + if chunk[9] != 0 || chunk[15] != 0 { + t.Fatal("surrounding bytes should be zero") + } +} + +func TestWrite_FullChunk(t *testing.T) { + client := newFakeS3Client() + bs := newTestStore(client, 8, 64) + + // Write exactly one full chunk + wbuf := []byte{1, 2, 3, 4, 5, 6, 7, 8} + err := bs.Write(wbuf, 0) + if err != nil { + t.Fatalf("Write failed: %v", err) + } + + chunk := client.objects["test/disk/chunk_0000000000"] + if !bytes.Equal(chunk, wbuf) { + t.Fatalf("chunk data mismatch: %v vs %v", chunk, wbuf) + } +} + +func TestWrite_CrossChunk(t *testing.T) { + client := newFakeS3Client() + bs := newTestStore(client, 8, 64) + + // Write across chunk boundary (offset 6, len 4 -> chunks 0 and 1) + wbuf := []byte{0xA1, 0xA2, 0xA3, 0xA4} + err := bs.Write(wbuf, 6) + if err != nil { + t.Fatalf("Write failed: %v", err) + } + + // Verify chunk 0: bytes 6-7 should be 0xA1, 0xA2 + chunk0 := client.objects["test/disk/chunk_0000000000"] + if chunk0[6] != 0xA1 || chunk0[7] != 0xA2 { + t.Fatalf("chunk0 data mismatch at boundary: %v", chunk0[6:8]) + } + + // Verify chunk 1: bytes 0-1 should be 0xA3, 0xA4 + chunk1 := client.objects["test/disk/chunk_0000000001"] + if chunk1[0] != 0xA3 || chunk1[1] != 0xA4 { + t.Fatalf("chunk1 data mismatch at boundary: %v", chunk1[0:2]) + } +} + +func TestWriteThenRead(t *testing.T) { + client := newFakeS3Client() + bs := newTestStore(client, 100, 1000) + + // Write pattern across chunks + wbuf := make([]byte, 250) + for i := range wbuf { + wbuf[i] = byte(i % 256) + } + err := bs.Write(wbuf, 50) + if err != nil { + t.Fatalf("Write failed: %v", err) + } + + // Read back + data, err := bs.Read(50, 250) + if err != nil { + t.Fatalf("Read failed: %v", err) + } + if !bytes.Equal(data, wbuf) { + t.Fatal("read data does not match written data") + } +} + +func TestUnmap_FullChunk(t *testing.T) { + client := newFakeS3Client() + bs := newTestStore(client, 100, 1000) + + // Create chunk + client.objects["test/disk/chunk_0000000001"] = make([]byte, 100) + + err := bs.Unmap([]api.UnmapBlockDescriptor{ + {Offset: 100, TL: 100}, // exactly chunk 1 + }) + if err != nil { + t.Fatalf("Unmap failed: %v", err) + } + + if _, ok := client.objects["test/disk/chunk_0000000001"]; ok { + t.Error("chunk should have been deleted") + } +} + +func TestSize(t *testing.T) { + bs := newTestStore(newFakeS3Client(), 1024, 8192) + dev := &api.SCSILu{} + if bs.Size(dev) != 8192 { + t.Errorf("expected 8192, got %d", bs.Size(dev)) + } +} diff --git a/pkg/scsi/lun.go b/pkg/scsi/lun.go index eda007b..fccdde0 100644 --- a/pkg/scsi/lun.go +++ b/pkg/scsi/lun.go @@ -46,7 +46,7 @@ func NewSCSILu(bs *config.BackendStorage) (*api.SCSILu, error) { // Validate backend type, default to file if unknown switch backendType { - case "file", "iouring", "ceph", "null", "RemBs": + case "file", "iouring", "ceph", "null", "RemBs", "s3": // Valid types default: // Unknown type, treat entire path as file path @@ -67,6 +67,7 @@ func NewSCSILu(bs *config.BackendStorage) (*api.SCSILu, error) { Storage: backing, BlockShift: bs.BlockShift, UUID: bs.DeviceID, + BackendConfig: bs, } err = backing.Open(lu, backendPath)