From f3ef8c973d0056d2735c1865e146b17263423ef1 Mon Sep 17 00:00:00 2001 From: Utkarsh Mani Tripathi Date: Mon, 18 Nov 2019 14:50:12 +0530 Subject: [PATCH 01/11] add initial implementation of remote backing store This commit implement the BackingStore Interface for remote backing store. Signed-off-by: Utkarsh Mani Tripathi --- pkg/api/types.go | 12 +++ pkg/scsi/backingstore/remote/remote.go | 107 +++++++++++++++++++++++++ pkg/scsi/scsilumap.go | 48 ++++++++++- 3 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 pkg/scsi/backingstore/remote/remote.go diff --git a/pkg/api/types.go b/pkg/api/types.go index 7238de3..3b6a247 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -17,6 +17,7 @@ package api import ( "errors" + "io" "sync" uuid "github.com/satori/go.uuid" @@ -406,3 +407,14 @@ type UnmapBlockDescriptor struct { Offset uint64 TL uint32 } + +type ReaderWriterAt interface { + io.ReaderAt + io.WriterAt +} + +type RemoteBackingStore interface { + ReaderWriterAt + Sync() (int, error) + Unmap(int64, int64) (int, error) +} diff --git a/pkg/scsi/backingstore/remote/remote.go b/pkg/scsi/backingstore/remote/remote.go new file mode 100644 index 0000000..71f15cc --- /dev/null +++ b/pkg/scsi/backingstore/remote/remote.go @@ -0,0 +1,107 @@ +/* +Copyright 2016 openebs authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package remote + +import ( + "fmt" + + "github.com/gostor/gotgt/pkg/api" + "github.com/gostor/gotgt/pkg/scsi" + log "github.com/sirupsen/logrus" +) + +func init() { + scsi.RegisterBackingStore("RemBs", newRemBs) +} + +// RemBackingStore +type RemBackingStore struct { + scsi.BaseBackingStore + // Remote backing store, remote server exposing + // read and write methods. + RemBs api.RemBackingStore +} + +func newRemBs() (api.BackingStore, error) { + return &RemBackingStore{ + BaseBackingStore: scsi.BaseBackingStore{ + Name: "RemBs", + OflagsSupported: 0, + }, + }, nil +} + +func (bs *RemBackingStore) Open(dev *api.SCSILu, path string) error { + bs.DataSize = uint64(dev.Size) + bs.RemBs = scsi.GetTargetBSMap(path) + return nil +} + +func (bs *RemBackingStore) Close(dev *api.SCSILu) error { + /* TODO return bs.File.Close()*/ + return nil +} + +func (bs *RemBackingStore) Init(dev *api.SCSILu, Opts string) error { + return nil +} + +func (bs *RemBackingStore) Exit(dev *api.SCSILu) error { + return nil +} + +func (bs *RemBackingStore) Size(dev *api.SCSILu) uint64 { + return bs.DataSize +} + +func (bs *RemBackingStore) Read(offset, tl int64) ([]byte, error) { + tmpbuf := make([]byte, tl) + length, err := bs.RemBs.ReadAt(tmpbuf, offset) + if err != nil { + return nil, err + } + if length != len(tmpbuf) { + return nil, fmt.Errorf("Incomplete read expected:%d actual:%d", tl, length) + } + return tmpbuf, nil +} + +func (bs *RemBackingStore) Write(wbuf []byte, offset int64) error { + length, err := bs.RemBs.WriteAt(wbuf, offset) + if err != nil { + log.Error(err) + return err + } + if length != len(wbuf) { + return fmt.Errorf("Incomplete write expected:%d actual:%d", len(wbuf), length) + } + return nil +} + +func (bs *RemBackingStore) DataAdvise(offset, length int64, advise uint32) error { + return nil +} + +func (bs *RemBackingStore) DataSync(offset, length int64) (err error) { + _, err = bs.RemBs.Sync() + return +} + +func (bs *RemBackingStore) Unmap(bd []api.UnmapBlockDescriptor) (err error) { + _, err = bs.RemBs.Unmap() + return +} diff --git a/pkg/scsi/scsilumap.go b/pkg/scsi/scsilumap.go index b6106cc..ac5bb27 100644 --- a/pkg/scsi/scsilumap.go +++ b/pkg/scsi/scsilumap.go @@ -17,6 +17,7 @@ limitations under the License. package scsi import ( + "errors" "fmt" "strconv" "sync" @@ -33,9 +34,15 @@ type SCSILUMap struct { AllDevices api.LUNMap // use target name as the key for target's LUN map TargetsLUNMap map[string]api.LUNMap + + TargetsBSMap map[string]api.RemoteBackingStore /* use target name as the key for target's Backing Store (temp) */ } -var globalSCSILUMap = SCSILUMap{AllDevices: make(api.LUNMap), TargetsLUNMap: make(map[string]api.LUNMap)} +var globalSCSILUMap = SCSILUMap{ + AllDevices: make(api.LUNMap), + TargetsLUNMap: make(map[string]api.LUNMap), + TargetsBSMap: make(map[string]api.RemoteBackingStore), +} type LUNMapping struct { TargetName string @@ -71,6 +78,15 @@ func GetTargetLUNMap(tgtName string) api.LUNMap { return lunMap } +func GetTargetBSMap(tgtName string) api.RemoteBackingStore { + /* TODO check for lock held by caller + globalSCSILUMap.mutex.RLock() + defer globalSCSILUMap.mutex.RUnlock()*/ + + lunMap := globalSCSILUMap.TargetsBSMap[tgtName] + return lunMap +} + func AddBackendStorage(bs config.BackendStorage) error { globalSCSILUMap.mutex.Lock() defer globalSCSILUMap.mutex.Unlock() @@ -130,3 +146,33 @@ func InitSCSILUMap(config *config.Config) error { } return nil } + +func InitSCSILUMapEx(tgtName, devpath string, deviceID, lun, size, sectorSize uint64, bs api.RemoteBackingStore) error { + globalSCSILUMap.mutex.Lock() + defer globalSCSILUMap.mutex.Unlock() + + if bs == nil { + return errors.New("Remote backing store is nil") + } + + globalSCSILUMap.TargetsBSMap[tgtName] = bs + + lu, err := NewSCSILu(&config.BackendStorage{ + DeviceID: deviceID, + Path: "RemBs:" + devpath, + Online: true, + }, + ) + if err != nil { + return errors.New("Init SCSI LU map error.") + } + globalSCSILUMap.AllDevices[deviceID] = lu + + mappingLUN(LUNMapping{ + DeviceID: deviceID, + LUN: lun, + TargetName: tgtName, + }, + ) + return nil +} From 1dbc82435f796505925eec4143631e53c8d18ef5 Mon Sep 17 00:00:00 2001 From: Utkarsh Mani Tripathi Date: Mon, 18 Nov 2019 15:16:41 +0530 Subject: [PATCH 02/11] iSCSI target fixes for iSCSIResiduals tests causing libiscsi test suite to hang Signed-off-by: Utkarsh Mani Tripathi --- pkg/api/types.go | 1 + pkg/port/iscsit/cmd.go | 15 +++++++++++++++ pkg/port/iscsit/iscsid.go | 10 +++++++++- pkg/scsi/backingstore.go | 5 +++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/pkg/api/types.go b/pkg/api/types.go index 3b6a247..a638277 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -166,6 +166,7 @@ var ( ) type SCSICommand struct { + OpCode byte Target *SCSITarget DeviceID uint64 Device *SCSILu diff --git a/pkg/port/iscsit/cmd.go b/pkg/port/iscsit/cmd.go index ea8232e..9b55174 100644 --- a/pkg/port/iscsit/cmd.go +++ b/pkg/port/iscsit/cmd.go @@ -20,7 +20,9 @@ import ( "bytes" "fmt" "strings" + "time" + "github.com/gostor/gotgt/pkg/api" "github.com/gostor/gotgt/pkg/util" log "github.com/sirupsen/logrus" ) @@ -93,6 +95,7 @@ type ISCSICommand struct { FinalInSeq bool Immediate bool TaskTag uint32 + StartTime time.Time ExpCmdSN, MaxCmdSN uint32 AHSLen int Resid uint32 @@ -124,6 +127,7 @@ type ISCSICommand struct { StatusDetail uint8 // SCSI commands + SCSIOpCode byte ExpectedDataLen uint32 CDB []byte Status byte @@ -225,6 +229,7 @@ func parseHeader(data []byte) (*ISCSICommand, error) { m.AHSLen = int(data[4]) * 4 m.DataLen = int(ParseUint(data[5:8])) m.TaskTag = uint32(ParseUint(data[16:20])) + m.StartTime = time.Now() switch m.OpCode { case OpSCSICmd: m.LUN = [8]byte{data[9]} @@ -234,6 +239,13 @@ func parseHeader(data []byte) (*ISCSICommand, error) { m.Write = data[1]&0x20 == 0x20 m.CDB = data[32:48] m.ExpStatSN = uint32(ParseUint(data[28:32])) + SCSIOpcode := api.SCSICommandType(m.SCSIOpCode) + switch SCSIOpcode { + case api.READ_6, api.READ_10, api.READ_12, api.READ_16: + m.Read = true + case api.WRITE_6, api.WRITE_10, api.WRITE_12, api.WRITE_16, api.WRITE_VERIFY, api.WRITE_VERIFY_12, api.WRITE_VERIFY_16: + m.Write = true + } fallthrough case OpSCSITaskReq: m.ReferencedTaskTag = uint32(ParseUint(data[20:24])) @@ -359,6 +371,9 @@ func (m *ISCSICommand) dataInBytes() []byte { copy(buf[40:], util.MarshalUint32(m.BufferOffset)) copy(buf[44:], util.MarshalUint32(m.Resid)) copy(buf[48:], m.RawData[m.BufferOffset:m.BufferOffset+uint32(m.DataLen)]) + if m.ExpectedDataLen != 0 { + copy(buf[48:], m.RawData[m.BufferOffset:m.BufferOffset+uint32(m.DataLen)]) + } return buf } diff --git a/pkg/port/iscsit/iscsid.go b/pkg/port/iscsit/iscsid.go index 5a265aa..fbc0720 100644 --- a/pkg/port/iscsit/iscsid.go +++ b/pkg/port/iscsit/iscsid.go @@ -594,6 +594,14 @@ func (s *ISCSITargetDriver) scsiCommandHandler(conn *iscsiConnection) (err error } task := &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, scmd: scmd} + task.scmd.OpCode = conn.req.SCSIOpCode + if scmd.Direction == api.SCSIDataBidirection { + task.scmd.Result = api.SAMStatCheckCondition.Stat + scsi.BuildSenseData(task.scmd, scsi.ILLEGAL_REQUEST, scsi.NO_ADDITIONAL_SENSE) + conn.buildRespPackage(OpSCSIResp, task) + conn.rxTask = nil + break + } if req.Write { task.r2tCount = int(req.ExpectedDataLen) - req.DataLen if !req.Final { @@ -649,7 +657,7 @@ func (s *ISCSITargetDriver) scsiCommandHandler(conn *iscsiConnection) (err error } return } else { - if scmd.Direction == api.SCSIDataRead && scmd.SenseBuffer == nil { + if scmd.Direction == api.SCSIDataRead && scmd.SenseBuffer == nil && req.ExpectedDataLen != 0 { conn.buildRespPackage(OpSCSIIn, task) } else { conn.buildRespPackage(OpSCSIResp, task) diff --git a/pkg/scsi/backingstore.go b/pkg/scsi/backingstore.go index 731b84a..10adff5 100644 --- a/pkg/scsi/backingstore.go +++ b/pkg/scsi/backingstore.go @@ -126,6 +126,11 @@ func bsPerformCommand(bs api.BackingStore, cmd *api.SCSICommand) (err error, key } cmd.InSDBBuffer.Resid = uint32(length) copy(cmd.InSDBBuffer.Buffer, rbuf) + if cmd.InSDBBuffer.Length < uint32(length) { + key = ILLEGAL_REQUEST + asc = ASC_INVALID_FIELD_IN_CDB + goto sense + } case api.PRE_FETCH_10, api.PRE_FETCH_16: err = bs.DataAdvise(int64(offset), tl, util.POSIX_FADV_WILLNEED) if err != nil { From e5339f347a0470ad6a8e521e7e07e115cc1b7809 Mon Sep 17 00:00:00 2001 From: Utkarsh Mani Tripathi Date: Mon, 18 Nov 2019 15:56:34 +0530 Subject: [PATCH 03/11] Add nil pointer check for InSDBBuffer - Gotgt was crashing for few opcodes (sg_inq (sg3-utils)) because InSDBBuffer was not initialized for such opcodes (Need help) Signed-off-by: Utkarsh Mani Tripathi --- pkg/scsi/sbc.go | 8 +++++-- pkg/scsi/scsi.go | 2 +- pkg/scsi/spc.go | 60 ++++++++++++++++++++++++++++++++++++------------ 3 files changed, 52 insertions(+), 18 deletions(-) diff --git a/pkg/scsi/sbc.go b/pkg/scsi/sbc.go index 627234c..1cb2c78 100644 --- a/pkg/scsi/sbc.go +++ b/pkg/scsi/sbc.go @@ -533,7 +533,9 @@ overflow: cmd.InSDBBuffer.Resid = 8 return api.SAMStatGood sense: - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } BuildSenseData(cmd, key, asc) return api.SAMStatCheckCondition } @@ -679,7 +681,9 @@ func SBCGetLbaStatus(host int, cmd *api.SCSICommand) api.SAMStat { } return api.SAMStatGood sense: - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } BuildSenseData(cmd, key, asc) return api.SAMStatCheckCondition } diff --git a/pkg/scsi/scsi.go b/pkg/scsi/scsi.go index 119bfa6..4e52e8e 100644 --- a/pkg/scsi/scsi.go +++ b/pkg/scsi/scsi.go @@ -96,7 +96,7 @@ func (s *SCSITargetService) AddCommandQueue(tid int, scmd *api.SCSICommand) erro result := scmd.Device.PerformCommand(tid, scmd) if result != api.SAMStatGood { scmd.Result = result.Stat - log.Warnf("%v", result.Err) + log.Warnf("opcode: %xh err: %v", scmd.OpCode, result.Err) } return nil } diff --git a/pkg/scsi/spc.go b/pkg/scsi/spc.go index 942ce74..1f3a5cc 100644 --- a/pkg/scsi/spc.go +++ b/pkg/scsi/spc.go @@ -454,7 +454,9 @@ func SPCReportLuns(host int, cmd *api.SCSICommand) api.SAMStat { copy(cmd.InSDBBuffer.Buffer, buf.Bytes()) return api.SAMStatGood sense: - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_FIELD_IN_CDB) return api.SAMStatCheckCondition } @@ -467,7 +469,9 @@ func SPCStartStop(host int, cmd *api.SCSICommand) api.SAMStat { return api.SAMStatReservationConflict } - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } scb := cmd.SCB pwrcnd = scb[4] & 0xf0 if pwrcnd != 0 { @@ -662,7 +666,9 @@ func SPCSendDiagnostics(host int, cmd *api.SCSICommand) api.SAMStat { return api.SAMStatGood sense: - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_FIELD_IN_CDB) return api.SAMStatCheckCondition } @@ -748,7 +754,9 @@ func SPCReportSupportedOperationCodes(host int, cmd *api.SCSICommand) api.SAMSta return api.SAMStatGood sense: - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_FIELD_IN_CDB) return api.SAMStatCheckCondition } @@ -764,7 +772,9 @@ func SPCServiceAction(host int, cmd *api.SCSICommand) api.SAMStat { fnop := serviceAction.(*SCSIServiceAction) return fnop.CommandPerformFunc(host, cmd) } - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_FIELD_IN_CDB) return api.SAMStatCheckCondition } @@ -797,7 +807,9 @@ func SPCPRReadKeys(host int, cmd *api.SCSICommand) api.SAMStat { cmd.InSDBBuffer.Resid = uint32(additionLength) return api.SAMStatGood sense: - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_FIELD_IN_CDB) return api.SAMStatCheckCondition } @@ -853,7 +865,9 @@ func SPCPRReadReservation(host int, cmd *api.SCSICommand) api.SAMStat { return api.SAMStatGood sense: - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_FIELD_IN_CDB) return api.SAMStatCheckCondition } @@ -891,7 +905,9 @@ func SPCPRReportCapabilities(host int, cmd *api.SCSICommand) api.SAMStat { cmd.InSDBBuffer.Resid = uint32(actualLength) return api.SAMStatGood sense: - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_FIELD_IN_CDB) return api.SAMStatCheckCondition } @@ -970,7 +986,9 @@ func SPCPRRegister(host int, cmd *api.SCSICommand) api.SAMStat { return api.SAMStatGood sense: - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_FIELD_IN_CDB) return api.SAMStatCheckCondition } @@ -1032,7 +1050,9 @@ func SPCPRReserve(host int, cmd *api.SCSICommand) api.SAMStat { scsiResOp.Save(tgtName, devUUID) return api.SAMStatGood sense: - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_FIELD_IN_CDB) return api.SAMStatCheckCondition } @@ -1082,7 +1102,9 @@ func SPCPRRelease(host int, cmd *api.SCSICommand) api.SAMStat { } if curRes.Scope != resScope || curRes.Type != resType { - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_RELEASE_OF_PERSISTENT_RESERVATION) return api.SAMStatCheckCondition } @@ -1113,7 +1135,9 @@ func SPCPRRelease(host int, cmd *api.SCSICommand) api.SAMStat { scsiResOp.Save(tgtName, devUUID) return api.SAMStatGood sense: - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_FIELD_IN_CDB) return api.SAMStatCheckCondition } @@ -1167,7 +1191,9 @@ func SPCPRClear(host int, cmd *api.SCSICommand) api.SAMStat { scsiResOp.Save(tgtName, devUUID) return api.SAMStatGood sense: - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_FIELD_IN_CDB) return api.SAMStatCheckCondition } @@ -1267,7 +1293,9 @@ func SPCPRPreempt(host int, cmd *api.SCSICommand) api.SAMStat { scsiResOp.Save(tgtName, devUUID) return api.SAMStatGood sense: - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_FIELD_IN_CDB) return api.SAMStatCheckCondition } @@ -1353,7 +1381,9 @@ found: scsiResOp.Save(tgtName, devUUID) return api.SAMStatGood sense: - cmd.InSDBBuffer.Resid = 0 + if cmd.InSDBBuffer != nil { + cmd.InSDBBuffer.Resid = 0 + } BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_FIELD_IN_CDB) return api.SAMStatCheckCondition } From a8cc3e6db20d53f39f79ac4fbcad84d4139bde3f Mon Sep 17 00:00:00 2001 From: Utkarsh Mani Tripathi Date: Mon, 18 Nov 2019 16:23:42 +0530 Subject: [PATCH 04/11] improve logs and add client (initiator) status Signed-off-by: Utkarsh Mani Tripathi --- pkg/port/iscsit/iscsid.go | 29 +++++++++++++++++++---------- pkg/port/iscsit/session.go | 6 ++++-- pkg/scsi/spc.go | 2 +- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/pkg/port/iscsit/iscsid.go b/pkg/port/iscsit/iscsid.go index fbc0720..75568a7 100644 --- a/pkg/port/iscsit/iscsid.go +++ b/pkg/port/iscsit/iscsid.go @@ -36,11 +36,12 @@ const ( ) type ISCSITargetDriver struct { - SCSI *scsi.SCSITargetService - Name string - iSCSITargets map[string]*ISCSITarget - TSIHPool map[uint16]bool - TSIHPoolMutex sync.Mutex + SCSI *scsi.SCSITargetService + Name string + iSCSITargets map[string]*ISCSITarget + TSIHPool map[uint16]bool + TSIHPoolMutex sync.Mutex + isClientConnected bool mu sync.Mutex l net.Listener @@ -165,9 +166,8 @@ func (s *ISCSITargetDriver) Run() error { s.mu.Lock() s.l = l s.mu.Unlock() - + log.Infof("iSCSI service listening on: %v", s.listen.Addr()) for { - log.Info("Listening ...") conn, err := l.Accept() if err != nil { if err, ok := err.(net.Error); ok { @@ -180,24 +180,30 @@ func (s *ISCSITargetDriver) Run() error { continue } log.Info(conn.LocalAddr().String()) - log.Info("Accepting ...") iscsiConn := &iscsiConnection{conn: conn, loginParam: &iscsiLoginParam{}} iscsiConn.init() iscsiConn.rxIOState = IOSTATE_RX_BHS - - log.Infof("connection is connected from %s...\n", conn.RemoteAddr().String()) + log.Infof("Target is connected to initiator: %s", conn.RemoteAddr().String()) // start a new thread to do with this command go s.handler(DATAIN, iscsiConn) } return nil } +func (s *ISCSITargetDriver) setClientStatus(ok bool) { + s.isClientConnected = ok +} + +func (s *ISCSITargetDriver) isInitiatorConnected() bool { + return s.isClientConnected +} func (s *ISCSITargetDriver) Close() error { s.mu.Lock() l := s.l + s.setClientStatus(false) s.mu.Unlock() if l != nil { return l.Close() @@ -324,6 +330,7 @@ func (s *ISCSITargetDriver) rxHandler(conn *iscsiConnection) { } case OpLogoutReq: log.Debug("OpLogoutReq") + s.setClientStatus(false) if err := iscsiExecLogout(conn); err != nil { log.Warningf("set connection to close") conn.state = CONN_STATE_CLOSE @@ -386,6 +393,7 @@ func (s *ISCSITargetDriver) iscsiExecLogin(conn *iscsiConnection) error { } func iscsiExecLogout(conn *iscsiConnection) error { + log.Infof("Logout request received from initiator: %v", conn.conn.RemoteAddr().String()) cmd := conn.req conn.resp = &ISCSICommand{ OpCode: OpLogoutResp, @@ -718,6 +726,7 @@ func (s *ISCSITargetDriver) scsiCommandHandler(conn *iscsiConnection) (err error case OpNoopOut: iscsiExecNoopOut(conn) case OpLogoutReq: + s.setClientStatus(false) conn.txTask = &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag} conn.txIOState = IOSTATE_TX_BHS iscsiExecLogout(conn) diff --git a/pkg/port/iscsit/session.go b/pkg/port/iscsit/session.go index 4c74463..64d6429 100644 --- a/pkg/port/iscsit/session.go +++ b/pkg/port/iscsit/session.go @@ -392,8 +392,8 @@ func (s *ISCSITargetDriver) BindISCSISession(conn *iscsiConnection) error { } if newSess.SessionType == SESSION_NORMAL { - log.Infof("New Session initiator name:%v,target name:%v,ISID:0x%x", - conn.loginParam.initiator, conn.loginParam.target, conn.loginParam.isid) + log.Infof("Login request received from initiator: %v, Session type: %s, Target name:%v, ISID: 0x%x", + conn.loginParam.initiator, "Normal", conn.loginParam.target, conn.loginParam.isid) //register normal session itnexus := &api.ITNexus{uuid.NewV1(), GeniSCSIITNexusID(newSess)} scsi.AddITNexus(&newSess.Target.SCSITarget, itnexus) @@ -404,6 +404,8 @@ func (s *ISCSITargetDriver) BindISCSISession(conn *iscsiConnection) error { newSess.Target.Sessions[newSess.TSIH] = newSess newSess.Target.SessionsRWMutex.Unlock() } else { + log.Infof("Discovery request received from initiator: %v, Session type: %s, ISID: 0x%x", + conn.loginParam.initiator, "Discovery", conn.loginParam.isid) conn.session = newSess } } else { diff --git a/pkg/scsi/spc.go b/pkg/scsi/spc.go index 1f3a5cc..872f807 100644 --- a/pkg/scsi/spc.go +++ b/pkg/scsi/spc.go @@ -717,7 +717,7 @@ func reportOpcodesAll(cmd *api.SCSICommand, rctd int) error { } func reportOpcodeOne(cmd *api.SCSICommand, rctd int, opcode byte, rsa uint16, serviceAction bool) error { - return fmt.Errorf("non support") + return fmt.Errorf("rsa: %xh, sa:%v not supported", rsa, serviceAction) } func SPCReportSupportedOperationCodes(host int, cmd *api.SCSICommand) api.SAMStat { From 3b8e996a6ca8860538b5e28de29b973efd940f09 Mon Sep 17 00:00:00 2001 From: Utkarsh Mani Tripathi Date: Mon, 18 Nov 2019 16:30:46 +0530 Subject: [PATCH 05/11] Reply with LUN busy when there is a read/write error from backend Signed-off-by: Utkarsh Mani Tripathi --- pkg/scsi/backingstore.go | 4 ++-- pkg/scsi/sbc.go | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/scsi/backingstore.go b/pkg/scsi/backingstore.go index 10adff5..920f97a 100644 --- a/pkg/scsi/backingstore.go +++ b/pkg/scsi/backingstore.go @@ -153,7 +153,7 @@ write: if err != nil { log.Error(err) key = MEDIUM_ERROR - asc = ASC_READ_ERROR + asc = ASC_WRITE_ERROR goto sense } log.Debugf("write data at 0x%x for length %d", offset, len(wbuf)) @@ -172,7 +172,7 @@ write: if ((opcode != api.WRITE_6) && (scb[1]&0x8 != 0)) || (pg.Data[0]&0x04 == 0) { if err = bs.DataSync(int64(offset), tl); err != nil { key = MEDIUM_ERROR - asc = ASC_READ_ERROR + asc = ASC_WRITE_ERROR goto sense } } diff --git a/pkg/scsi/sbc.go b/pkg/scsi/sbc.go index 1cb2c78..13dc242 100644 --- a/pkg/scsi/sbc.go +++ b/pkg/scsi/sbc.go @@ -459,7 +459,9 @@ func SBCReadWrite(host int, cmd *api.SCSICommand) api.SAMStat { err, key, asc = bsPerformCommand(dev.Storage, cmd) if err != nil { - goto sense + log.Errorf("Error from backend: %v", err) + BuildSenseData(cmd, key, asc) + return api.SAMStatBusy } else { return api.SAMStatGood } From 64deda8106a6ceeb48a89726b6ccfeb658c5a56e Mon Sep 17 00:00:00 2001 From: Utkarsh Mani Tripathi Date: Mon, 18 Nov 2019 18:08:58 +0530 Subject: [PATCH 06/11] convert initialR2T to always true for datadigest tttttt-off-by: Utkarsh Mani Tripathi --- pkg/port/iscsit/conn.go | 1 + pkg/port/iscsit/session.go | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/port/iscsit/conn.go b/pkg/port/iscsit/conn.go index d74fd5e..7565f6e 100644 --- a/pkg/port/iscsit/conn.go +++ b/pkg/port/iscsit/conn.go @@ -164,6 +164,7 @@ func (conn *iscsiConnection) buildRespPackage(oc OpCode, task *iscsiTask) error case OpReady: conn.resp.OpCode = OpReady conn.resp.R2TSN = task.r2tSN + conn.resp.Final = true conn.resp.BufferOffset = uint32(task.offset) conn.resp.DesiredLength = uint32(task.r2tCount) if val := conn.loginParam.sessionParam[ISCSI_PARAM_MAX_BURST].Value; task.r2tCount > int(val) { diff --git a/pkg/port/iscsit/session.go b/pkg/port/iscsit/session.go index 64d6429..9d6a70a 100644 --- a/pkg/port/iscsit/session.go +++ b/pkg/port/iscsit/session.go @@ -188,7 +188,9 @@ var sessionKeys map[string]*iscsiSessionKeys = map[string]*iscsiSessionKeys{ // ISCSI_PARAM_HDRDGST_EN "HeaderDigest": {ISCSI_PARAM_HDRDGST_EN, false, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv}, // ISCSI_PARAM_DATADGST_EN - "DataDigest": {ISCSI_PARAM_DATADGST_EN, false, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv}, + // TODO: Not sure why initialR2T was changed to true, need help in + // understanding it's use? + "DataDigest": {ISCSI_PARAM_DATADGST_EN, true, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv}, // ISCSI_PARAM_INITIAL_R2T_EN "InitialR2T": {ISCSI_PARAM_INITIAL_R2T_EN, false, 1, 0, 1, boolKeyConv, boolKeyInConv}, // ISCSI_PARAM_MAX_R2T From 1b02c7897e3fd5ec6ef026b22a0b9043e082f706 Mon Sep 17 00:00:00 2001 From: Utkarsh Mani Tripathi Date: Mon, 18 Nov 2019 18:20:59 +0530 Subject: [PATCH 07/11] fix error handling for unsupported commands - Respond with failure for unsupported task management commands - Increase MaxRecvDataSegmentLength to 65536, to match FirstBurstLength Signed-off-by: Utkarsh Mani Tripathi --- pkg/port/iscsit/iscsid.go | 2 -- pkg/port/iscsit/session.go | 2 +- pkg/scsi/spc.go | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/port/iscsit/iscsid.go b/pkg/port/iscsit/iscsid.go index 75568a7..dc2863c 100644 --- a/pkg/port/iscsit/iscsid.go +++ b/pkg/port/iscsit/iscsid.go @@ -841,10 +841,8 @@ func (s *ISCSITargetDriver) iscsiExecTask(task *iscsiTask) error { fallthrough case ISCSI_TM_FUNC_TARGET_WARM_RESET, ISCSI_TM_FUNC_TARGET_COLD_RESET, ISCSI_TM_FUNC_TASK_REASSIGN: task.result = ISCSI_TMF_RSP_NOT_SUPPORTED - return fmt.Errorf("The task function is not supported") default: task.result = ISCSI_TMF_RSP_REJECTED - return fmt.Errorf("Unknown task function") } // return response to initiator return task.conn.buildRespPackage(OpSCSITaskResp, task) diff --git a/pkg/port/iscsit/session.go b/pkg/port/iscsit/session.go index 9d6a70a..e0d7ba1 100644 --- a/pkg/port/iscsit/session.go +++ b/pkg/port/iscsit/session.go @@ -184,7 +184,7 @@ func boolKeyInConv(value uint) string { var sessionKeys map[string]*iscsiSessionKeys = map[string]*iscsiSessionKeys{ // ISCSI_PARAM_MAX_RECV_DLENGTH - "MaxRecvDataSegmentLength": {ISCSI_PARAM_MAX_RECV_DLENGTH, true, 32768, 512, 16777215, numberKeyConv, numberKeyInConv}, + "MaxRecvDataSegmentLength": {ISCSI_PARAM_MAX_RECV_DLENGTH, true, 65536, 512, 16777215, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_HDRDGST_EN "HeaderDigest": {ISCSI_PARAM_HDRDGST_EN, false, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv}, // ISCSI_PARAM_DATADGST_EN diff --git a/pkg/scsi/spc.go b/pkg/scsi/spc.go index 872f807..5e6a69e 100644 --- a/pkg/scsi/spc.go +++ b/pkg/scsi/spc.go @@ -29,7 +29,7 @@ import ( ) func SPCIllegalOp(host int, cmd *api.SCSICommand) api.SAMStat { - BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_FIELD_IN_CDB) + BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_OP_CODE) return api.SAMStatCheckCondition } From d7caf89610a28d4bf3efcd69cb9c4d833ea98111 Mon Sep 17 00:00:00 2001 From: Utkarsh Mani Tripathi Date: Tue, 19 Nov 2019 10:19:58 +0530 Subject: [PATCH 08/11] Implement InitSCSILuMap func for jiva Signed-off-by: Utkarsh Mani Tripathi --- pkg/api/types.go | 3 +++ pkg/port/iscsit/iscsid.go | 2 +- pkg/scsi/backingstore/remote/remote.go | 14 +++++++++++--- pkg/scsi/cmd.go | 1 + pkg/scsi/sbc.go | 2 +- pkg/scsi/scsilumap.go | 14 ++++---------- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/pkg/api/types.go b/pkg/api/types.go index a638277..4daca00 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -397,6 +397,9 @@ type SCSILu struct { Storage BackingStore DeviceProtocol SCSIDeviceProtocol ModeBlockDescriptor []byte + SCSIVendorID string + SCSIProductID string + SCSIID string PerformCommand CommandFunc FinishCommand func(*SCSITarget, *SCSICommand) diff --git a/pkg/port/iscsit/iscsid.go b/pkg/port/iscsit/iscsid.go index dc2863c..cfb7818 100644 --- a/pkg/port/iscsit/iscsid.go +++ b/pkg/port/iscsit/iscsid.go @@ -166,7 +166,7 @@ func (s *ISCSITargetDriver) Run() error { s.mu.Lock() s.l = l s.mu.Unlock() - log.Infof("iSCSI service listening on: %v", s.listen.Addr()) + log.Infof("iSCSI service listening on: %v", s.l.Addr()) for { conn, err := l.Accept() if err != nil { diff --git a/pkg/scsi/backingstore/remote/remote.go b/pkg/scsi/backingstore/remote/remote.go index 71f15cc..2dcfbf8 100644 --- a/pkg/scsi/backingstore/remote/remote.go +++ b/pkg/scsi/backingstore/remote/remote.go @@ -24,6 +24,10 @@ import ( log "github.com/sirupsen/logrus" ) +var ( + Size uint64 +) + func init() { scsi.RegisterBackingStore("RemBs", newRemBs) } @@ -33,7 +37,7 @@ type RemBackingStore struct { scsi.BaseBackingStore // Remote backing store, remote server exposing // read and write methods. - RemBs api.RemBackingStore + RemBs api.RemoteBackingStore } func newRemBs() (api.BackingStore, error) { @@ -46,7 +50,11 @@ func newRemBs() (api.BackingStore, error) { } func (bs *RemBackingStore) Open(dev *api.SCSILu, path string) error { - bs.DataSize = uint64(dev.Size) + if Size == 0 { + return fmt.Errorf("Size is not initialized") + } + + bs.DataSize = Size bs.RemBs = scsi.GetTargetBSMap(path) return nil } @@ -102,6 +110,6 @@ func (bs *RemBackingStore) DataSync(offset, length int64) (err error) { } func (bs *RemBackingStore) Unmap(bd []api.UnmapBlockDescriptor) (err error) { - _, err = bs.RemBs.Unmap() + _, err = bs.RemBs.Unmap(int64(bd[0].Offset), int64(bd[0].TL)) return } diff --git a/pkg/scsi/cmd.go b/pkg/scsi/cmd.go index 1ad4d4c..8b3cf7d 100644 --- a/pkg/scsi/cmd.go +++ b/pkg/scsi/cmd.go @@ -231,6 +231,7 @@ const ( const ( SCSIVendorID = "GOSTOR" SCSIProductID = "GOTGT" + SCSIID = "iqn.2016-09.com.gotgt.gostor:iscsi-tgt" ) /* diff --git a/pkg/scsi/sbc.go b/pkg/scsi/sbc.go index 13dc242..ee1e333 100644 --- a/pkg/scsi/sbc.go +++ b/pkg/scsi/sbc.go @@ -76,7 +76,7 @@ func (sbc SBCSCSIDeviceProtocol) InitLu(lu *api.SCSILu) error { leave it with a default target name */ - lu.Attrs.SCSIID = "iqn.2016-09.com.gotgt.gostor:iscsi-tgt" + lu.Attrs.SCSIID = SCSIID /* The PRODUCT SERIAL NUMBER field contains right-aligned ASCII data (see 4.3.1) diff --git a/pkg/scsi/scsilumap.go b/pkg/scsi/scsilumap.go index ac5bb27..396138b 100644 --- a/pkg/scsi/scsilumap.go +++ b/pkg/scsi/scsilumap.go @@ -147,7 +147,7 @@ func InitSCSILUMap(config *config.Config) error { return nil } -func InitSCSILUMapEx(tgtName, devpath string, deviceID, lun, size, sectorSize uint64, bs api.RemoteBackingStore) error { +func InitSCSILUMapEx(config *config.BackendStorage, tgtName string, lun uint64, bs api.RemoteBackingStore) error { globalSCSILUMap.mutex.Lock() defer globalSCSILUMap.mutex.Unlock() @@ -156,20 +156,14 @@ func InitSCSILUMapEx(tgtName, devpath string, deviceID, lun, size, sectorSize ui } globalSCSILUMap.TargetsBSMap[tgtName] = bs - - lu, err := NewSCSILu(&config.BackendStorage{ - DeviceID: deviceID, - Path: "RemBs:" + devpath, - Online: true, - }, - ) + lu, err := NewSCSILu(config) if err != nil { return errors.New("Init SCSI LU map error.") } - globalSCSILUMap.AllDevices[deviceID] = lu + globalSCSILUMap.AllDevices[config.DeviceID] = lu mappingLUN(LUNMapping{ - DeviceID: deviceID, + DeviceID: config.DeviceID, LUN: lun, TargetName: tgtName, }, From d7891b1f682e0f8e565d1faa78717c52748b1d07 Mon Sep 17 00:00:00 2001 From: Utkarsh Mani Tripathi Date: Wed, 20 Nov 2019 14:34:35 +0530 Subject: [PATCH 09/11] Implement stats and resize and fix remote backing store apis - Convert constant to var so that it can be configured from backend - Add options to disable persistent reservation and ORWrite16 commands Signed-off-by: Utkarsh Mani Tripathi --- pkg/port/iscsit/cmd.go | 1 + pkg/port/iscsit/conn.go | 16 +++-- pkg/port/iscsit/iscsid.go | 90 ++++++++++++++++++++++++-- pkg/port/iscsit/session.go | 6 +- pkg/scsi/backingstore/remote/remote.go | 11 ++-- pkg/scsi/cmd.go | 2 +- pkg/scsi/drivers.go | 21 ++++++ pkg/scsi/sbc.go | 42 +++++++----- pkg/scsi/scsi.go | 27 +++++++- pkg/scsi/scsilumap.go | 24 ++++--- 10 files changed, 190 insertions(+), 50 deletions(-) diff --git a/pkg/port/iscsit/cmd.go b/pkg/port/iscsit/cmd.go index 9b55174..cc87469 100644 --- a/pkg/port/iscsit/cmd.go +++ b/pkg/port/iscsit/cmd.go @@ -239,6 +239,7 @@ func parseHeader(data []byte) (*ISCSICommand, error) { m.Write = data[1]&0x20 == 0x20 m.CDB = data[32:48] m.ExpStatSN = uint32(ParseUint(data[28:32])) + m.SCSIOpCode = m.CDB[0] SCSIOpcode := api.SCSICommandType(m.SCSIOpCode) switch SCSIOpcode { case api.READ_6, api.READ_10, api.READ_12, api.READ_16: diff --git a/pkg/port/iscsit/conn.go b/pkg/port/iscsit/conn.go index 7565f6e..f72d8e7 100644 --- a/pkg/port/iscsit/conn.go +++ b/pkg/port/iscsit/conn.go @@ -51,6 +51,7 @@ var ( ) type iscsiConnection struct { + ConnNum int state int authState int session *ISCSISession @@ -96,12 +97,13 @@ const ( ) type iscsiTask struct { - tag uint32 - conn *iscsiConnection - cmd *ISCSICommand - scmd *api.SCSICommand - state taskState - result byte + tag uint32 + conn *iscsiConnection + cmd *ISCSICommand + scmd *api.SCSICommand + state taskState + expectedDataLength int64 + result byte offset int r2tCount int @@ -154,6 +156,7 @@ func (conn *iscsiConnection) buildRespPackage(oc OpCode, task *iscsiTask) error task = conn.rxTask } conn.resp = &ISCSICommand{ + StartTime: conn.req.StartTime, StatSN: conn.req.ExpStatSN, TaskTag: conn.req.TaskTag, ExpCmdSN: conn.session.ExpCmdSN, @@ -172,6 +175,7 @@ func (conn *iscsiConnection) buildRespPackage(oc OpCode, task *iscsiTask) error } case OpSCSIIn, OpSCSIResp: conn.resp.OpCode = oc + conn.resp.SCSIOpCode = conn.req.SCSIOpCode conn.resp.Immediate = true conn.resp.Final = true conn.resp.SCSIResponse = 0x00 diff --git a/pkg/port/iscsit/iscsid.go b/pkg/port/iscsit/iscsid.go index cfb7818..b28526b 100644 --- a/pkg/port/iscsit/iscsid.go +++ b/pkg/port/iscsit/iscsid.go @@ -22,6 +22,7 @@ import ( "os" "strconv" "sync" + "time" "github.com/gostor/gotgt/pkg/api" "github.com/gostor/gotgt/pkg/config" @@ -35,6 +36,17 @@ const ( ISCSI_UNSPEC_TSIH = uint16(0) ) +const ( + STATE_INIT = iota + STATE_RUNNING + STATE_SHUTTING_DOWN + STATE_TERMINATE +) + +var ( + EnableStats bool +) + type ISCSITargetDriver struct { SCSI *scsi.SCSITargetService Name string @@ -42,9 +54,13 @@ type ISCSITargetDriver struct { TSIHPool map[uint16]bool TSIHPoolMutex sync.Mutex isClientConnected bool - - mu sync.Mutex - l net.Listener + enableStats bool + SCSIIOCount map[int]int64 + mu *sync.RWMutex + l net.Listener + state uint8 + OpCode int + TargetStats scsi.Stats } func init() { @@ -52,12 +68,19 @@ func init() { } func NewISCSITargetDriver(base *scsi.SCSITargetService) (scsi.SCSITargetDriver, error) { - return &ISCSITargetDriver{ + driver := &ISCSITargetDriver{ Name: iSCSIDriverName, iSCSITargets: map[string]*ISCSITarget{}, SCSI: base, TSIHPool: map[uint16]bool{0: true, 65535: true}, - }, nil + mu: &sync.RWMutex{}, + } + + if EnableStats { + driver.SCSIIOCount = map[int]int64{} + driver.enableStats = true + } + return driver, nil } func (s *ISCSITargetDriver) AllocTSIH() uint16 { @@ -167,19 +190,23 @@ func (s *ISCSITargetDriver) Run() error { s.l = l s.mu.Unlock() log.Infof("iSCSI service listening on: %v", s.l.Addr()) + + s.setState(STATE_RUNNING) for { conn, err := l.Accept() if err != nil { if err, ok := err.(net.Error); ok { if !err.Temporary() { - log.Info("Closing ...") + log.Warning("Closing connection with initiator...") break } } log.Error(err) continue } + log.Info(conn.LocalAddr().String()) + s.setClientStatus(true) iscsiConn := &iscsiConnection{conn: conn, loginParam: &iscsiLoginParam{}} @@ -206,11 +233,28 @@ func (s *ISCSITargetDriver) Close() error { s.setClientStatus(false) s.mu.Unlock() if l != nil { - return l.Close() + s.setState(STATE_SHUTTING_DOWN) + if err := l.Close(); err != nil { + return err + } + s.setState(STATE_TERMINATE) + return nil } return nil } +func (s *ISCSITargetDriver) setState(st uint8) { + s.mu.Lock() + defer s.mu.Unlock() + s.state = st +} + +func (s *ISCSITargetDriver) Resize(size uint64) error { + s.mu.Lock() + defer s.mu.Unlock() + return s.SCSI.Resize(size) +} + func (s *ISCSITargetDriver) handler(events byte, conn *iscsiConnection) { if events&DATAIN != 0 { @@ -481,6 +525,12 @@ func (s *ISCSITargetDriver) txHandler(conn *iscsiConnection) { resp.DataSN = 0 maxCount := conn.maxSeqCount + if s.enableStats { + if resp.OpCode == OpSCSIResp || resp.OpCode == OpSCSIIn { + s.UpdateStats(conn) + } + } + /* send data splitted by segmentLen */ SendRemainingData: if resp.OpCode == OpSCSIIn { @@ -581,6 +631,11 @@ func (s *ISCSITargetDriver) scsiCommandHandler(conn *iscsiConnection) (err error switch req.OpCode { case OpSCSICmd: log.Debugf("SCSI Command processing...") + if _, ok := s.SCSIIOCount[(int)(req.CDB[0])]; ok != false { + s.SCSIIOCount[(int)(req.CDB[0])] += 1 + } else { + s.SCSIIOCount[(int)(req.CDB[0])] = 1 + } scmd := &api.SCSICommand{ ITNexusID: conn.session.ITNexus.ID, SCB: req.CDB, @@ -612,6 +667,7 @@ func (s *ISCSITargetDriver) scsiCommandHandler(conn *iscsiConnection) (err error } if req.Write { task.r2tCount = int(req.ExpectedDataLen) - req.DataLen + task.expectedDataLength = int64(req.ExpectedDataLen) if !req.Final { task.unsolCount = 1 } @@ -849,3 +905,23 @@ func (s *ISCSITargetDriver) iscsiExecTask(task *iscsiTask) error { } return nil } + +func (s *ISCSITargetDriver) Stats() scsi.Stats { + return s.TargetStats +} + +func (s *ISCSITargetDriver) UpdateStats(conn *iscsiConnection) { + s.TargetStats.IsClientConnected = s.isClientConnected + switch api.SCSICommandType(conn.resp.SCSIOpCode) { + case api.READ_6, api.READ_10, api.READ_12, api.READ_16: + s.TargetStats.ReadIOPS += 1 + s.TargetStats.TotalReadTime += int64(time.Since(conn.resp.StartTime)) + s.TargetStats.TotalReadBlockCount += int64(conn.resp.ExpectedDataLen) + break + case api.WRITE_6, api.WRITE_10, api.WRITE_12, api.WRITE_16: + s.TargetStats.WriteIOPS += 1 + s.TargetStats.TotalWriteTime += int64(time.Since(conn.resp.StartTime)) + s.TargetStats.TotalWriteBlockCount += int64(conn.resp.ExpectedDataLen) + break + } +} diff --git a/pkg/port/iscsit/session.go b/pkg/port/iscsit/session.go index e0d7ba1..cd0c072 100644 --- a/pkg/port/iscsit/session.go +++ b/pkg/port/iscsit/session.go @@ -188,11 +188,9 @@ var sessionKeys map[string]*iscsiSessionKeys = map[string]*iscsiSessionKeys{ // ISCSI_PARAM_HDRDGST_EN "HeaderDigest": {ISCSI_PARAM_HDRDGST_EN, false, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv}, // ISCSI_PARAM_DATADGST_EN - // TODO: Not sure why initialR2T was changed to true, need help in - // understanding it's use? - "DataDigest": {ISCSI_PARAM_DATADGST_EN, true, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv}, + "DataDigest": {ISCSI_PARAM_DATADGST_EN, false, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv}, // ISCSI_PARAM_INITIAL_R2T_EN - "InitialR2T": {ISCSI_PARAM_INITIAL_R2T_EN, false, 1, 0, 1, boolKeyConv, boolKeyInConv}, + "InitialR2T": {ISCSI_PARAM_INITIAL_R2T_EN, true, 1, 0, 1, boolKeyConv, boolKeyInConv}, // ISCSI_PARAM_MAX_R2T "MaxOutstandingR2T": {ISCSI_PARAM_MAX_R2T, true, 1, 1, 65535, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_IMM_DATA_EN diff --git a/pkg/scsi/backingstore/remote/remote.go b/pkg/scsi/backingstore/remote/remote.go index 2dcfbf8..38aab1f 100644 --- a/pkg/scsi/backingstore/remote/remote.go +++ b/pkg/scsi/backingstore/remote/remote.go @@ -29,7 +29,7 @@ var ( ) func init() { - scsi.RegisterBackingStore("RemBs", newRemBs) + scsi.RegisterBackingStore("RemBs", NewRemoteBackingStore) } // RemBackingStore @@ -40,7 +40,7 @@ type RemBackingStore struct { RemBs api.RemoteBackingStore } -func newRemBs() (api.BackingStore, error) { +func NewRemoteBackingStore() (api.BackingStore, error) { return &RemBackingStore{ BaseBackingStore: scsi.BaseBackingStore{ Name: "RemBs", @@ -53,9 +53,12 @@ func (bs *RemBackingStore) Open(dev *api.SCSILu, path string) error { if Size == 0 { return fmt.Errorf("Size is not initialized") } - + var err error bs.DataSize = Size - bs.RemBs = scsi.GetTargetBSMap(path) + bs.RemBs, err = scsi.GetTargetBSMap(path) + if err != nil { + return err + } return nil } diff --git a/pkg/scsi/cmd.go b/pkg/scsi/cmd.go index 8b3cf7d..4bb80c4 100644 --- a/pkg/scsi/cmd.go +++ b/pkg/scsi/cmd.go @@ -228,7 +228,7 @@ const ( NAA_IEEE_REGD_EXTD = byte(0x6) ) -const ( +var ( SCSIVendorID = "GOSTOR" SCSIProductID = "GOTGT" SCSIID = "iqn.2016-09.com.gotgt.gostor:iscsi-tgt" diff --git a/pkg/scsi/drivers.go b/pkg/scsi/drivers.go index 0571f91..9c64498 100644 --- a/pkg/scsi/drivers.go +++ b/pkg/scsi/drivers.go @@ -28,6 +28,27 @@ type SCSITargetDriver interface { NewTarget(string, *config.Config) error RereadTargetLUNMap() Close() error + Resize(uint64) error + Stats() Stats +} + +type Stats struct { + IsClientConnected bool + RevisionCounter int64 + ReplicaCounter int64 + SCSIIOCount map[int]int64 + + ReadIOPS int64 + TotalReadTime int64 + TotalReadBlockCount int64 + + WriteIOPS int64 + TotalWriteTime int64 + TotalWriteBlockCount int64 + + UsedLogicalBlocks int64 + UsedBlocks int64 + SectorSize int64 } type TargetDriverFunc func(*SCSITargetService) (SCSITargetDriver, error) diff --git a/pkg/scsi/sbc.go b/pkg/scsi/sbc.go index ee1e333..ec3a4a8 100644 --- a/pkg/scsi/sbc.go +++ b/pkg/scsi/sbc.go @@ -37,6 +37,11 @@ const ( PR_EA_FN = (1 << 0) ) +var ( + EnableORWrite16 = true + EnablePersistentReservation = true +) + type SBCSCSIDeviceProtocol struct { BaseSCSIDeviceProtocol } @@ -176,26 +181,29 @@ func NewSBCDevice(deviceType api.SCSIDeviceType) api.SCSIDeviceProtocol { sbc.SCSIDeviceOps[api.MODE_SELECT_10] = NewSCSIDeviceOperation(SBCModeSelect, nil, PR_WE_FA|PR_EA_FA|PR_EA_FN|PR_WE_FN) sbc.SCSIDeviceOps[api.MODE_SENSE_10] = NewSCSIDeviceOperation(SBCModeSense, nil, PR_WE_FA|PR_WE_FN|PR_EA_FA|PR_EA_FN) - sbc.SCSIDeviceOps[api.PERSISTENT_RESERVE_IN] = NewSCSIDeviceOperation(SPCServiceAction, []*SCSIServiceAction{ - {ServiceAction: PR_IN_READ_KEYS, CommandPerformFunc: SPCPRReadKeys}, - {ServiceAction: PR_IN_READ_RESERVATION, CommandPerformFunc: SPCPRReadReservation}, - {ServiceAction: PR_IN_REPORT_CAPABILITIES, CommandPerformFunc: SPCPRReportCapabilities}, - }, 0) - - sbc.SCSIDeviceOps[api.PERSISTENT_RESERVE_OUT] = NewSCSIDeviceOperation(SPCServiceAction, []*SCSIServiceAction{ - {ServiceAction: PR_OUT_REGISTER, CommandPerformFunc: SPCPRRegister}, - {ServiceAction: PR_OUT_RESERVE, CommandPerformFunc: SPCPRReserve}, - {ServiceAction: PR_OUT_RELEASE, CommandPerformFunc: SPCPRRelease}, - {ServiceAction: PR_OUT_CLEAR, CommandPerformFunc: SPCPRClear}, - {ServiceAction: PR_OUT_PREEMPT, CommandPerformFunc: SPCPRPreempt}, - // {ServiceAction: PR_OUT_PREEMPT_AND_ABORT, CommandPerformFunc: SPCPRPreempt}, - {ServiceAction: PR_OUT_REGISTER_AND_IGNORE_EXISTING_KEY, CommandPerformFunc: SPCPRRegister}, - {ServiceAction: PR_OUT_REGISTER_AND_MOVE, CommandPerformFunc: SPCPRRegisterAndMove}, - }, 0) + if EnablePersistentReservation { + sbc.SCSIDeviceOps[api.PERSISTENT_RESERVE_IN] = NewSCSIDeviceOperation(SPCServiceAction, []*SCSIServiceAction{ + {ServiceAction: PR_IN_READ_KEYS, CommandPerformFunc: SPCPRReadKeys}, + {ServiceAction: PR_IN_READ_RESERVATION, CommandPerformFunc: SPCPRReadReservation}, + {ServiceAction: PR_IN_REPORT_CAPABILITIES, CommandPerformFunc: SPCPRReportCapabilities}, + }, 0) + sbc.SCSIDeviceOps[api.PERSISTENT_RESERVE_OUT] = NewSCSIDeviceOperation(SPCServiceAction, []*SCSIServiceAction{ + {ServiceAction: PR_OUT_REGISTER, CommandPerformFunc: SPCPRRegister}, + {ServiceAction: PR_OUT_RESERVE, CommandPerformFunc: SPCPRReserve}, + {ServiceAction: PR_OUT_RELEASE, CommandPerformFunc: SPCPRRelease}, + {ServiceAction: PR_OUT_CLEAR, CommandPerformFunc: SPCPRClear}, + {ServiceAction: PR_OUT_PREEMPT, CommandPerformFunc: SPCPRPreempt}, + // {ServiceAction: PR_OUT_PREEMPT_AND_ABORT, CommandPerformFunc: SPCPRPreempt}, + {ServiceAction: PR_OUT_REGISTER_AND_IGNORE_EXISTING_KEY, CommandPerformFunc: SPCPRRegister}, + {ServiceAction: PR_OUT_REGISTER_AND_MOVE, CommandPerformFunc: SPCPRRegisterAndMove}, + }, 0) + } sbc.SCSIDeviceOps[api.READ_16] = NewSCSIDeviceOperation(SBCReadWrite, nil, PR_EA_FA|PR_EA_FN) sbc.SCSIDeviceOps[api.WRITE_16] = NewSCSIDeviceOperation(SBCReadWrite, nil, PR_EA_FA|PR_EA_FN|PR_WE_FA|PR_WE_FN) - sbc.SCSIDeviceOps[api.ORWRITE_16] = NewSCSIDeviceOperation(SBCReadWrite, nil, PR_EA_FA|PR_EA_FN) + if EnableORWrite16 { + sbc.SCSIDeviceOps[api.ORWRITE_16] = NewSCSIDeviceOperation(SBCReadWrite, nil, PR_EA_FA|PR_EA_FN) + } sbc.SCSIDeviceOps[api.WRITE_VERIFY_16] = NewSCSIDeviceOperation(SBCReadWrite, nil, PR_EA_FA|PR_EA_FN) sbc.SCSIDeviceOps[api.VERIFY_16] = NewSCSIDeviceOperation(SBCVerify, nil, PR_EA_FA|PR_EA_FN) diff --git a/pkg/scsi/scsi.go b/pkg/scsi/scsi.go index 4e52e8e..dd2aa67 100644 --- a/pkg/scsi/scsi.go +++ b/pkg/scsi/scsi.go @@ -53,6 +53,20 @@ func (s *SCSITargetService) GetTargetList() ([]api.SCSITarget, error) { return result, nil } +func (s *SCSITargetService) Resize(size uint64) error { + s.mutex.Lock() + //TODO for multiple LUNs + for _, t := range s.Targets { + if t.Devices != nil { + for i := range t.Devices { + t.Devices[i].Size = size + } + } + } + s.mutex.Unlock() + return nil +} + func (s *SCSITargetService) AddCommandQueue(tid int, scmd *api.SCSICommand) error { var ( target *api.SCSITarget @@ -128,9 +142,18 @@ func NewSCSIDeviceOperation(fn api.CommandFunc, sa []*SCSIServiceAction, pr uint func BuildSenseData(cmd *api.SCSICommand, key byte, asc SCSISubError) { senseBuffer := &bytes.Buffer{} inBufLen, ok := SCSICDBBufXLength(cmd.SCB) - var length uint32 = 0xa + var ( + length uint32 = 0xa + fixedFormat bool = true + ) - if cmd.Device.Attrs.SenseFormat { + if cmd.Device != nil { + if cmd.Device.Attrs.SenseFormat { + fixedFormat = false + } + } + + if !fixedFormat { // descriptor format // current, not deferred senseBuffer.WriteByte(0x72) diff --git a/pkg/scsi/scsilumap.go b/pkg/scsi/scsilumap.go index 396138b..a6fb823 100644 --- a/pkg/scsi/scsilumap.go +++ b/pkg/scsi/scsilumap.go @@ -78,13 +78,16 @@ func GetTargetLUNMap(tgtName string) api.LUNMap { return lunMap } -func GetTargetBSMap(tgtName string) api.RemoteBackingStore { - /* TODO check for lock held by caller +func GetTargetBSMap(tgtName string) (api.RemoteBackingStore, error) { globalSCSILUMap.mutex.RLock() - defer globalSCSILUMap.mutex.RUnlock()*/ + defer globalSCSILUMap.mutex.RUnlock() - lunMap := globalSCSILUMap.TargetsBSMap[tgtName] - return lunMap + bs, ok := globalSCSILUMap.TargetsBSMap[tgtName] + if !ok { + return nil, errors.New("Remote backing store is not found in globalSCSILUMap") + } + + return bs, nil } func AddBackendStorage(bs config.BackendStorage) error { @@ -148,19 +151,22 @@ func InitSCSILUMap(config *config.Config) error { } func InitSCSILUMapEx(config *config.BackendStorage, tgtName string, lun uint64, bs api.RemoteBackingStore) error { - globalSCSILUMap.mutex.Lock() - defer globalSCSILUMap.mutex.Unlock() - if bs == nil { return errors.New("Remote backing store is nil") } + globalSCSILUMap.mutex.Lock() globalSCSILUMap.TargetsBSMap[tgtName] = bs + globalSCSILUMap.mutex.Unlock() + lu, err := NewSCSILu(config) if err != nil { - return errors.New("Init SCSI LU map error.") + return fmt.Errorf("Init SCSI LU map error, err: %v", err) } + + globalSCSILUMap.mutex.Lock() globalSCSILUMap.AllDevices[config.DeviceID] = lu + globalSCSILUMap.mutex.Unlock() mappingLUN(LUNMapping{ DeviceID: config.DeviceID, From 25f0038456b5cd631bdc1b52fac9a9b8a8d4d0a0 Mon Sep 17 00:00:00 2001 From: Utkarsh Mani Tripathi Date: Wed, 20 Nov 2019 17:03:58 +0530 Subject: [PATCH 10/11] Add tgtNSG to fullfeaturephase and tgtTrans to true Signed-off-by: Utkarsh Mani Tripathi --- pkg/port/iscsit/login.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/port/iscsit/login.go b/pkg/port/iscsit/login.go index 0f819a9..2a8d452 100644 --- a/pkg/port/iscsit/login.go +++ b/pkg/port/iscsit/login.go @@ -152,8 +152,8 @@ func (conn *iscsiConnection) processLoginData() ([]util.KeyValue, error) { conn.loginParam.iniCSG, conn.loginParam.iniNSG, conn.loginParam.iniTrans) } } else { - conn.loginParam.tgtNSG = LoginOperationalNegotiation - conn.loginParam.tgtTrans = false + conn.loginParam.tgtNSG = FullFeaturePhase + conn.loginParam.tgtTrans = true } return negoKV, nil } From 583e9b3a4ab7163217682ec0b84e5454f9f88387 Mon Sep 17 00:00:00 2001 From: Utkarsh Mani Tripathi Date: Thu, 21 Nov 2019 09:57:46 +0530 Subject: [PATCH 11/11] Add option to disable multipath and other fixes Signed-off-by: Utkarsh Mani Tripathi --- pkg/port/iscsit/cmd.go | 1 - pkg/port/iscsit/iscsid.go | 13 +++++++------ pkg/scsi/backingstore.go | 2 +- pkg/scsi/backingstore/remote/remote.go | 2 +- pkg/scsi/spc.go | 10 +++++++++- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/pkg/port/iscsit/cmd.go b/pkg/port/iscsit/cmd.go index cc87469..f592568 100644 --- a/pkg/port/iscsit/cmd.go +++ b/pkg/port/iscsit/cmd.go @@ -371,7 +371,6 @@ func (m *ISCSICommand) dataInBytes() []byte { copy(buf[36:], util.MarshalUint32(m.DataSN)) copy(buf[40:], util.MarshalUint32(m.BufferOffset)) copy(buf[44:], util.MarshalUint32(m.Resid)) - copy(buf[48:], m.RawData[m.BufferOffset:m.BufferOffset+uint32(m.DataLen)]) if m.ExpectedDataLen != 0 { copy(buf[48:], m.RawData[m.BufferOffset:m.BufferOffset+uint32(m.DataLen)]) } diff --git a/pkg/port/iscsit/iscsid.go b/pkg/port/iscsit/iscsid.go index b28526b..ffc14d4 100644 --- a/pkg/port/iscsit/iscsid.go +++ b/pkg/port/iscsit/iscsid.go @@ -55,7 +55,6 @@ type ISCSITargetDriver struct { TSIHPoolMutex sync.Mutex isClientConnected bool enableStats bool - SCSIIOCount map[int]int64 mu *sync.RWMutex l net.Listener state uint8 @@ -77,8 +76,8 @@ func NewISCSITargetDriver(base *scsi.SCSITargetService) (scsi.SCSITargetDriver, } if EnableStats { - driver.SCSIIOCount = map[int]int64{} driver.enableStats = true + driver.TargetStats.SCSIIOCount = map[int]int64{} } return driver, nil } @@ -631,10 +630,12 @@ func (s *ISCSITargetDriver) scsiCommandHandler(conn *iscsiConnection) (err error switch req.OpCode { case OpSCSICmd: log.Debugf("SCSI Command processing...") - if _, ok := s.SCSIIOCount[(int)(req.CDB[0])]; ok != false { - s.SCSIIOCount[(int)(req.CDB[0])] += 1 - } else { - s.SCSIIOCount[(int)(req.CDB[0])] = 1 + if s.enableStats { + if _, ok := s.TargetStats.SCSIIOCount[(int)(req.CDB[0])]; ok != false { + s.TargetStats.SCSIIOCount[(int)(req.CDB[0])] += 1 + } else { + s.TargetStats.SCSIIOCount[(int)(req.CDB[0])] = 1 + } } scmd := &api.SCSICommand{ ITNexusID: conn.session.ITNexus.ID, diff --git a/pkg/scsi/backingstore.go b/pkg/scsi/backingstore.go index 920f97a..9890695 100644 --- a/pkg/scsi/backingstore.go +++ b/pkg/scsi/backingstore.go @@ -125,12 +125,12 @@ func bsPerformCommand(bs api.BackingStore, cmd *api.SCSICommand) (err error, key bs.DataAdvise(int64(offset), int64(length), util.POSIX_FADV_NOREUSE) } cmd.InSDBBuffer.Resid = uint32(length) - copy(cmd.InSDBBuffer.Buffer, rbuf) if cmd.InSDBBuffer.Length < uint32(length) { key = ILLEGAL_REQUEST asc = ASC_INVALID_FIELD_IN_CDB goto sense } + copy(cmd.InSDBBuffer.Buffer, rbuf) case api.PRE_FETCH_10, api.PRE_FETCH_16: err = bs.DataAdvise(int64(offset), tl, util.POSIX_FADV_WILLNEED) if err != nil { diff --git a/pkg/scsi/backingstore/remote/remote.go b/pkg/scsi/backingstore/remote/remote.go index 38aab1f..5d6a12e 100644 --- a/pkg/scsi/backingstore/remote/remote.go +++ b/pkg/scsi/backingstore/remote/remote.go @@ -113,6 +113,6 @@ func (bs *RemBackingStore) DataSync(offset, length int64) (err error) { } func (bs *RemBackingStore) Unmap(bd []api.UnmapBlockDescriptor) (err error) { - _, err = bs.RemBs.Unmap(int64(bd[0].Offset), int64(bd[0].TL)) + //_, err = bs.RemBs.Unmap(int64(bd[0].Offset), int64(bd[0].TL)) return } diff --git a/pkg/scsi/spc.go b/pkg/scsi/spc.go index 5e6a69e..6ed3ee4 100644 --- a/pkg/scsi/spc.go +++ b/pkg/scsi/spc.go @@ -28,6 +28,10 @@ import ( log "github.com/sirupsen/logrus" ) +var ( + EnableMultipath = true +) + func SPCIllegalOp(host int, cmd *api.SCSICommand) api.SAMStat { BuildSenseData(cmd, ILLEGAL_REQUEST, ASC_INVALID_OP_CODE) return api.SAMStatCheckCondition @@ -325,7 +329,11 @@ func SPCInquiry(host int, cmd *api.SCSICommand) api.SAMStat { } //byte 5 //SCCS(0) AAC(0) TPGS(0) 3PC(0) PROTECT(0) - addBuf.WriteByte(INQUIRY_TPGS_IMPLICIT) + if EnableMultipath { + addBuf.WriteByte(INQUIRY_TPGS_IMPLICIT) + } else { + addBuf.WriteByte(INQUIRY_TPGS_NO) + } //byte 6 //ENCSERV(0) VS(0) MULTIP(0) ADDR16(0) addBuf.WriteByte(0x00)