From 05b6f88de4ae89c5bc458b77ae8a079fa32c6d42 Mon Sep 17 00:00:00 2001 From: Lei Xue Date: Mon, 28 Nov 2016 11:11:24 +0800 Subject: [PATCH] restruct the backend storage command submit --- pkg/api/types.go | 5 +- pkg/scsi/backingstore.go | 158 ++++++++++++++++++++++++---- pkg/scsi/backingstore/common.go | 176 +++++++------------------------- pkg/scsi/backingstore/null.go | 15 ++- pkg/scsi/cmd.go | 33 +++--- pkg/scsi/sbc.go | 2 +- pkg/scsi/scsi.go | 4 +- pkg/util/util.go | 4 +- 8 files changed, 218 insertions(+), 179 deletions(-) diff --git a/pkg/api/types.go b/pkg/api/types.go index 443e619..c97020a 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -335,7 +335,10 @@ type BackingStore interface { Init(dev *SCSILu, Opts string) error Exit(dev *SCSILu) error Size(dev *SCSILu) uint64 - CommandSubmit(cmd *SCSICommand) error + Read(offset, tl int64) ([]byte, error) + Write([]byte, int64) error + DataSync() error + DataAdvise(int64, int64, uint32) error } type SCSIDeviceProtocol interface { diff --git a/pkg/scsi/backingstore.go b/pkg/scsi/backingstore.go index 4286699..ab388c8 100644 --- a/pkg/scsi/backingstore.go +++ b/pkg/scsi/backingstore.go @@ -17,9 +17,13 @@ limitations under the License. package scsi import ( + "bytes" "fmt" + "io" + "github.com/golang/glog" "github.com/gostor/gotgt/pkg/api" + "github.com/gostor/gotgt/pkg/util" ) type BaseBackingStore struct { @@ -42,31 +46,149 @@ func NewBackingStore(name string) (api.BackingStore, error) { } f, ok := registeredBSPlugins[name] if !ok { - return nil, fmt.Errorf("BackingStore %s is not found.", name) + return nil, fmt.Errorf("Backend storage %s is not found.", name) } return f() } -type fakeBackingStore struct { - BaseBackingStore -} +func bsPerformCommand(bs api.BackingStore, cmd *api.SCSICommand) (err error) { + var ( + scb = cmd.SCB.Bytes() + offset = cmd.Offset + opcode = api.SCSICommandType(scb[0]) + lu = cmd.Device + key = ILLEGAL_REQUEST + asc = ASC_INVALID_FIELD_IN_CDB + wbuf []byte = []byte{} + tl int64 = int64(cmd.TL) + rbuf = make([]byte, tl) + length int + doVerify bool = false + doWrite bool = false + ) + switch opcode { + case api.ORWRITE_16: + tmpbuf := []byte{} + tmpbuf, err = bs.Read(int64(offset), tl) + if err != nil { + key = MEDIUM_ERROR + asc = ASC_READ_ERROR + break + } + cmd.InSDBBuffer.Buffer = bytes.NewBuffer(tmpbuf) -func (fake *fakeBackingStore) Open(dev *api.SCSILu, path string) error { - return nil -} + wbuf = cmd.OutSDBBuffer.Buffer.Bytes() + doWrite = true + goto write + case api.COMPARE_AND_WRITE: + // TODO + doWrite = true + goto write + case api.SYNCHRONIZE_CACHE, api.SYNCHRONIZE_CACHE_16: + if err = bs.DataSync(); err != nil { + panic(err) + } + break + case api.WRITE_VERIFY, api.WRITE_VERIFY_12, api.WRITE_VERIFY_16: + doVerify = true + case api.WRITE_6, api.WRITE_10, api.WRITE_12, api.WRITE_16: + wbuf = cmd.OutSDBBuffer.Buffer.Bytes() + doWrite = true + goto write + case api.WRITE_SAME, api.WRITE_SAME_16: + // TODO + break + case api.READ_6, api.READ_10, api.READ_12, api.READ_16: + rbuf, err = bs.Read(int64(offset), tl) + if err != nil && err != io.EOF { + key = MEDIUM_ERROR + asc = ASC_READ_ERROR + break + } + length = len(rbuf) + for i := 0; i < int(tl)-length; i++ { + rbuf = append(rbuf, 0) + } -func (fake *fakeBackingStore) Close(dev *api.SCSILu) error { - return nil -} + if (opcode != api.READ_6) && (scb[1]&0x10 != 0) { + bs.DataAdvise(int64(offset), int64(length), util.POSIX_FADV_NOREUSE) + } + cmd.InSDBBuffer.Buffer = bytes.NewBuffer(rbuf) + case api.PRE_FETCH_10, api.PRE_FETCH_16: + err = bs.DataAdvise(int64(offset), tl, util.POSIX_FADV_WILLNEED) + if err != nil { + key = MEDIUM_ERROR + asc = ASC_READ_ERROR + } + case api.VERIFY_10, api.VERIFY_12, api.VERIFY_16: + doVerify = true + goto verify + case api.UNMAP: + // TODO + default: + break + } +write: + if doWrite { + // hack: wbuf = []byte("hello world!") + err = bs.Write(wbuf, int64(offset)) + if err != nil { + glog.Error(err) + key = MEDIUM_ERROR + asc = ASC_READ_ERROR + goto sense + } + glog.V(2).Infof("write data at %d for length %d", offset, len(wbuf)) + var pg *api.ModePage + for _, p := range lu.ModePages { + if p.PageCode == 0x08 && p.SubPageCode == 0 { + pg = &p + break + } + } + if pg == nil { + key = ILLEGAL_REQUEST + asc = ASC_INVALID_FIELD_IN_CDB + goto sense + } + if ((opcode != api.WRITE_6) && (scb[1]&0x8 != 0)) || (pg.Data[0]&0x04 == 0) { + if err = bs.DataSync(); err != nil { + key = MEDIUM_ERROR + asc = ASC_READ_ERROR + goto sense + } + } -func (fake *fakeBackingStore) Init(dev *api.SCSILu, Opts string) error { + if (opcode != api.WRITE_6) && (scb[1]&0x10 != 0) { + bs.DataAdvise(int64(offset), int64(length), util.POSIX_FADV_NOREUSE) + } + } +verify: + if doVerify { + rbuf, err = bs.Read(int64(offset), tl) + if err != nil { + key = MEDIUM_ERROR + asc = ASC_READ_ERROR + goto sense + } + if !bytes.Equal(cmd.OutSDBBuffer.Buffer.Bytes(), rbuf) { + err = fmt.Errorf("verify fail between out buffer and read buffer") + key = MISCOMPARE + asc = ASC_MISCOMPARE_DURING_VERIFY_OPERATION + goto sense + } + if scb[1]&0x10 != 0 { + bs.DataAdvise(int64(offset), int64(length), util.POSIX_FADV_WILLNEED) + } + } + glog.Infof("io done %s", string(scb)) return nil -} +sense: + if err != nil { + glog.Error(err) + return err + } -func (fake *fakeBackingStore) Exit(dev *api.SCSILu) error { - return nil -} - -func (fake *fakeBackingStore) CommandSubmit(cmd *api.SCSICommand) error { - return nil + err = fmt.Errorf("sense data encounter, key: %v, asc: %v", key, asc) + return err } diff --git a/pkg/scsi/backingstore/common.go b/pkg/scsi/backingstore/common.go index ebf7116..6ed3540 100644 --- a/pkg/scsi/backingstore/common.go +++ b/pkg/scsi/backingstore/common.go @@ -17,9 +17,7 @@ limitations under the License. package backingstore import ( - "bytes" "fmt" - "io" "os" "github.com/golang/glog" @@ -28,19 +26,23 @@ import ( "github.com/gostor/gotgt/pkg/util" ) +const ( + FileBackingStorage = "file" +) + func init() { - scsi.RegisterBackingStore("file", new) + scsi.RegisterBackingStore(FileBackingStorage, new) } type FileBackingStore struct { scsi.BaseBackingStore - File *os.File + file *os.File } func new() (api.BackingStore, error) { return &FileBackingStore{ BaseBackingStore: scsi.BaseBackingStore{ - Name: "file", + Name: FileBackingStorage, DataSize: 0, OflagsSupported: 0, }, @@ -57,12 +59,12 @@ func (bs *FileBackingStore) Open(dev *api.SCSILu, path string) error { f, err := os.OpenFile(path, os.O_RDWR, os.ModePerm) - bs.File = f + bs.file = f return err } func (bs *FileBackingStore) Close(dev *api.SCSILu) error { - return bs.File.Close() + return bs.file.Close() } func (bs *FileBackingStore) Init(dev *api.SCSILu, Opts string) error { @@ -76,142 +78,38 @@ func (bs *FileBackingStore) Exit(dev *api.SCSILu) error { func (bs *FileBackingStore) Size(dev *api.SCSILu) uint64 { return bs.DataSize } -func (bs *FileBackingStore) CommandSubmit(cmd *api.SCSICommand) (err error) { - var ( - scb = cmd.SCB.Bytes() - offset = cmd.Offset - opcode = api.SCSICommandType(scb[0]) - lu = cmd.Device - key = scsi.ILLEGAL_REQUEST - asc = scsi.ASC_INVALID_FIELD_IN_CDB - wbuf []byte = []byte{} - rbuf = make([]byte, cmd.TL) - length int - doVerify bool = false - doWrite bool = false - ) - switch opcode { - case api.ORWRITE_16: - tmpbuf := []byte{} - length, err = bs.File.ReadAt(tmpbuf, int64(offset)) - if length != len(tmpbuf) { - key = scsi.MEDIUM_ERROR - asc = scsi.ASC_READ_ERROR - break - } - cmd.InSDBBuffer.Buffer = bytes.NewBuffer(tmpbuf) - wbuf = cmd.OutSDBBuffer.Buffer.Bytes() - doWrite = true - goto write - case api.COMPARE_AND_WRITE: - // TODO - doWrite = true - goto write - case api.SYNCHRONIZE_CACHE, api.SYNCHRONIZE_CACHE_16: - if err = util.Fdatasync(bs.File); err != nil { - panic(err) - } - break - case api.WRITE_VERIFY, api.WRITE_VERIFY_12, api.WRITE_VERIFY_16: - doVerify = true - case api.WRITE_6, api.WRITE_10, api.WRITE_12, api.WRITE_16: - wbuf = cmd.OutSDBBuffer.Buffer.Bytes() - doWrite = true - goto write - case api.WRITE_SAME, api.WRITE_SAME_16: - // TODO - break - case api.READ_6, api.READ_10, api.READ_12, api.READ_16: - length, err = bs.File.ReadAt(rbuf, int64(offset)) - if err != nil && err != io.EOF { - key = scsi.MEDIUM_ERROR - asc = scsi.ASC_READ_ERROR - break - } - for i := 0; i < int(cmd.TL)-length; i++ { - rbuf = append(rbuf, 0) - } +func (bs *FileBackingStore) Read(offset, tl int64) ([]byte, error) { + if bs.file == nil { + return nil, fmt.Errorf("Backend store is nil") + } + tmpbuf := make([]byte, tl) + length, err := bs.file.ReadAt(tmpbuf, offset) + if err != nil { + return nil, err + } + if length != len(tmpbuf) { + return nil, fmt.Errorf("read is not same length of length") + } + return tmpbuf, nil +} - if (opcode != api.READ_6) && (scb[1]&0x10 != 0) { - util.Fadvise(bs.File, int64(offset), int64(length), util.POSIX_FADV_NOREUSE) - } - cmd.InSDBBuffer.Buffer = bytes.NewBuffer(rbuf) - case api.PRE_FETCH_10, api.PRE_FETCH_16: - err = util.Fadvise(bs.File, int64(offset), int64(cmd.TL), util.POSIX_FADV_WILLNEED) - if err != nil { - key = scsi.MEDIUM_ERROR - asc = scsi.ASC_READ_ERROR - } - case api.VERIFY_10, api.VERIFY_12, api.VERIFY_16: - doVerify = true - goto verify - case api.UNMAP: - // TODO - default: - break - } -write: - if doWrite { - // hack: wbuf = []byte("hello world!") - length, err = bs.File.WriteAt(wbuf, int64(offset)) - if err != nil || length != len(wbuf) { - glog.Error(err) - key = scsi.MEDIUM_ERROR - asc = scsi.ASC_READ_ERROR - goto sense - } - glog.V(2).Infof("write data at %d for length %d", offset, length) - var pg *api.ModePage - for _, p := range lu.ModePages { - if p.PageCode == 0x08 && p.SubPageCode == 0 { - pg = &p - break - } - } - if pg == nil { - key = scsi.ILLEGAL_REQUEST - asc = scsi.ASC_INVALID_FIELD_IN_CDB - goto sense - } - if ((opcode != api.WRITE_6) && (scb[1]&0x8 != 0)) || (pg.Data[0]&0x04 == 0) { - if err = util.Fdatasync(bs.File); err != nil { - key = scsi.MEDIUM_ERROR - asc = scsi.ASC_READ_ERROR - goto sense - } - } - - if (opcode != api.WRITE_6) && (scb[1]&0x10 != 0) { - util.Fadvise(bs.File, int64(offset), int64(length), util.POSIX_FADV_NOREUSE) - } - } -verify: - if doVerify { - length, err = bs.File.ReadAt(rbuf, int64(offset)) - if length != len(rbuf) { - key = scsi.MEDIUM_ERROR - asc = scsi.ASC_READ_ERROR - goto sense - } - if !bytes.Equal(cmd.OutSDBBuffer.Buffer.Bytes(), rbuf) { - err = fmt.Errorf("verify fail between out buffer and read buffer") - key = scsi.MISCOMPARE - asc = scsi.ASC_MISCOMPARE_DURING_VERIFY_OPERATION - goto sense - } - if scb[1]&0x10 != 0 { - util.Fadvise(bs.File, int64(offset), int64(length), util.POSIX_FADV_WILLNEED) - } - } - glog.Infof("io done %s", string(scb)) - return nil -sense: +func (bs *FileBackingStore) Write(wbuf []byte, offset int64) error { + length, err := bs.file.WriteAt(wbuf, offset) if err != nil { glog.Error(err) return err } - - err = fmt.Errorf("sense data encounter, key: %v, asc: %v", key, asc) - return err + if length != len(wbuf) { + return fmt.Errorf("write is not same length of length") + } + return nil +} + +func (bs *FileBackingStore) DataSync() error { + return util.Fdatasync(bs.file) +} + +func (bs *FileBackingStore) DataAdvise(offset, length int64, advise uint32) error { + return util.Fadvise(bs.file, offset, length, advise) } diff --git a/pkg/scsi/backingstore/null.go b/pkg/scsi/backingstore/null.go index 2b99bb6..2cbde02 100644 --- a/pkg/scsi/backingstore/null.go +++ b/pkg/scsi/backingstore/null.go @@ -59,7 +59,18 @@ func (bs *NullBackingStore) Size(dev *api.SCSILu) uint64 { return 0 } -func (bs *NullBackingStore) CommandSubmit(cmd *api.SCSICommand) error { - cmd.Result = api.SAM_STAT_GOOD +func (bs *NullBackingStore) Read(offset, tl int64) ([]byte, error) { + return nil, nil +} + +func (bs *NullBackingStore) Write(wbuf []byte, offset int64) error { + return nil +} + +func (bs *NullBackingStore) DataSync() error { + return nil +} + +func (bs *NullBackingStore) DataAdvise(offset, length int64, advise uint32) error { return nil } diff --git a/pkg/scsi/cmd.go b/pkg/scsi/cmd.go index c2d6206..3ddb462 100644 --- a/pkg/scsi/cmd.go +++ b/pkg/scsi/cmd.go @@ -21,13 +21,13 @@ import ( ) const ( - /* PERSISTENT_RESERVE_IN service action codes */ + // PERSISTENT_RESERVE_IN service action codes PR_IN_READ_KEYS byte = 0x00 PR_IN_READ_RESERVATION byte = 0x01 PR_IN_REPORT_CAPABILITIES byte = 0x02 PR_IN_READ_FULL_STATUS byte = 0x03 - /* PERSISTENT_RESERVE_OUT service action codes */ + // PERSISTENT_RESERVE_OUT service action codes PR_OUT_REGISTER byte = 0x00 PR_OUT_RESERVE byte = 0x01 PR_OUT_RELEASE byte = 0x02 @@ -40,7 +40,7 @@ const ( // Persistent Reservation scope PR_LU_SCOPE byte = 0x00 - /* Persistent Reservation Type Mask format */ + // Persistent Reservation Type Mask format PR_TYPE_WRITE_EXCLUSIVE byte = 0x01 PR_TYPE_EXCLUSIVE_ACCESS byte = 0x03 PR_TYPE_WRITE_EXCLUSIVE_REGONLY byte = 0x05 @@ -50,14 +50,22 @@ const ( ) const ( - CDB_GROUPID_0 = 6 /* 6-byte commands */ - CDB_GROUPID_1 = 10 /* 10-byte commands */ - CDB_GROUPID_2 = 10 /* 10-byte commands */ - CDB_GROUPID_3 = 0 /* reserved */ - CDB_GROUPID_4 = 16 /* 16-byte commands */ - CDB_GROUPID_5 = 12 /* 12-byte commands */ - CDB_GROUPID_6 = 0 /* vendor specific */ - CDB_GROUPID_7 = 0 /* vendor specific */ + // 6-byte commands + CDB_GROUPID_0 = 6 + // 10-byte commands + CDB_GROUPID_1 = 10 + // 10-byte commands + CDB_GROUPID_2 = 10 + // reserved + CDB_GROUPID_3 = 0 + // 16-byte commands + CDB_GROUPID_4 = 16 + // 12-byte commands + CDB_GROUPID_5 = 12 + // vendor specific + CDB_GROUPID_6 = 0 + // vendor specific + CDB_GROUPID_7 = 0 ) func SCSICDBGroupID(opcode byte) byte { @@ -106,7 +114,6 @@ const ( * 2 - Designator field contains ASCII printable chars * 3 - Designaotor field contains UTF-8 */ - const ( INQ_CODE_BIN = byte(1) INQ_CODE_ASCII = byte(2) @@ -121,7 +128,6 @@ const ( * 10b - Associated with SCSI Target device * 11b - Reserved */ - const ( ASS_LU = byte(0x00) ASS_TGT_PORT = byte(0x01) @@ -204,7 +210,6 @@ const ( * 7 - MD5 logical unit identifier - 7.6.3.10 * 8 - SCSI name string - 7.6.3.11 */ - const ( DESG_VENDOR = iota DESG_T10 diff --git a/pkg/scsi/sbc.go b/pkg/scsi/sbc.go index b194125..ce1132e 100644 --- a/pkg/scsi/sbc.go +++ b/pkg/scsi/sbc.go @@ -435,7 +435,7 @@ func SBCReadWrite(host int, cmd *api.SCSICommand) api.SAMStat { */ } - err = dev.Storage.CommandSubmit(cmd) + err = bsPerformCommand(dev.Storage, cmd) if err != nil { glog.Error(err) key = HARDWARE_ERROR diff --git a/pkg/scsi/scsi.go b/pkg/scsi/scsi.go index 201ec72..5222c01 100644 --- a/pkg/scsi/scsi.go +++ b/pkg/scsi/scsi.go @@ -1,5 +1,5 @@ /* -Copyright 2015 The GoStor Authors All rights reserved. +Copyright 2016 The GoStor Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -80,7 +80,7 @@ func (s *SCSITargetService) AddCommandQueue(tid int, scmd *api.SCSICommand) erro lun := *(*uint64)(unsafe.Pointer(&scmd.Lun)) scmd.Device = target.Devices[lun] - glog.V(2).Infof("scsi opcode: 0x%x, LUN: %d:", int(scmd.SCB.Bytes()[0]), binary.LittleEndian.Uint64(scmd.Lun[:])) + glog.V(2).Infof("scsi opcode: 0x%x, LUN: %d", int(scmd.SCB.Bytes()[0]), binary.LittleEndian.Uint64(scmd.Lun[:])) if scmd.Device == nil { scmd.Device = target.LUN0 diff --git a/pkg/util/util.go b/pkg/util/util.go index 8c4042e..4b74570 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -122,9 +122,9 @@ const ( POSIX_FADV_NOREUSE ) -func Fadvise(file *os.File, off, length int64, advice uint32) error { +func Fadvise(file *os.File, off, length int64, advise uint32) error { // syscall.SYS_FADVISE64 = 221 - _, _, err := syscall.Syscall6(221, file.Fd(), uintptr(off), uintptr(length), uintptr(advice), 0, 0) + _, _, err := syscall.Syscall6(221, file.Fd(), uintptr(off), uintptr(length), uintptr(advise), 0, 0) if err != 0 { return err }