diff --git a/pkg/port/iscsit/cmd.go b/pkg/port/iscsit/cmd.go index 9b55174..cc87469 100644 --- a/pkg/port/iscsit/cmd.go +++ b/pkg/port/iscsit/cmd.go @@ -239,6 +239,7 @@ func parseHeader(data []byte) (*ISCSICommand, error) { m.Write = data[1]&0x20 == 0x20 m.CDB = data[32:48] m.ExpStatSN = uint32(ParseUint(data[28:32])) + m.SCSIOpCode = m.CDB[0] SCSIOpcode := api.SCSICommandType(m.SCSIOpCode) switch SCSIOpcode { case api.READ_6, api.READ_10, api.READ_12, api.READ_16: diff --git a/pkg/port/iscsit/conn.go b/pkg/port/iscsit/conn.go index 7565f6e..f72d8e7 100644 --- a/pkg/port/iscsit/conn.go +++ b/pkg/port/iscsit/conn.go @@ -51,6 +51,7 @@ var ( ) type iscsiConnection struct { + ConnNum int state int authState int session *ISCSISession @@ -96,12 +97,13 @@ const ( ) type iscsiTask struct { - tag uint32 - conn *iscsiConnection - cmd *ISCSICommand - scmd *api.SCSICommand - state taskState - result byte + tag uint32 + conn *iscsiConnection + cmd *ISCSICommand + scmd *api.SCSICommand + state taskState + expectedDataLength int64 + result byte offset int r2tCount int @@ -154,6 +156,7 @@ func (conn *iscsiConnection) buildRespPackage(oc OpCode, task *iscsiTask) error task = conn.rxTask } conn.resp = &ISCSICommand{ + StartTime: conn.req.StartTime, StatSN: conn.req.ExpStatSN, TaskTag: conn.req.TaskTag, ExpCmdSN: conn.session.ExpCmdSN, @@ -172,6 +175,7 @@ func (conn *iscsiConnection) buildRespPackage(oc OpCode, task *iscsiTask) error } case OpSCSIIn, OpSCSIResp: conn.resp.OpCode = oc + conn.resp.SCSIOpCode = conn.req.SCSIOpCode conn.resp.Immediate = true conn.resp.Final = true conn.resp.SCSIResponse = 0x00 diff --git a/pkg/port/iscsit/iscsid.go b/pkg/port/iscsit/iscsid.go index cfb7818..b28526b 100644 --- a/pkg/port/iscsit/iscsid.go +++ b/pkg/port/iscsit/iscsid.go @@ -22,6 +22,7 @@ import ( "os" "strconv" "sync" + "time" "github.com/gostor/gotgt/pkg/api" "github.com/gostor/gotgt/pkg/config" @@ -35,6 +36,17 @@ const ( ISCSI_UNSPEC_TSIH = uint16(0) ) +const ( + STATE_INIT = iota + STATE_RUNNING + STATE_SHUTTING_DOWN + STATE_TERMINATE +) + +var ( + EnableStats bool +) + type ISCSITargetDriver struct { SCSI *scsi.SCSITargetService Name string @@ -42,9 +54,13 @@ type ISCSITargetDriver struct { TSIHPool map[uint16]bool TSIHPoolMutex sync.Mutex isClientConnected bool - - mu sync.Mutex - l net.Listener + enableStats bool + SCSIIOCount map[int]int64 + mu *sync.RWMutex + l net.Listener + state uint8 + OpCode int + TargetStats scsi.Stats } func init() { @@ -52,12 +68,19 @@ func init() { } func NewISCSITargetDriver(base *scsi.SCSITargetService) (scsi.SCSITargetDriver, error) { - return &ISCSITargetDriver{ + driver := &ISCSITargetDriver{ Name: iSCSIDriverName, iSCSITargets: map[string]*ISCSITarget{}, SCSI: base, TSIHPool: map[uint16]bool{0: true, 65535: true}, - }, nil + mu: &sync.RWMutex{}, + } + + if EnableStats { + driver.SCSIIOCount = map[int]int64{} + driver.enableStats = true + } + return driver, nil } func (s *ISCSITargetDriver) AllocTSIH() uint16 { @@ -167,19 +190,23 @@ func (s *ISCSITargetDriver) Run() error { s.l = l s.mu.Unlock() log.Infof("iSCSI service listening on: %v", s.l.Addr()) + + s.setState(STATE_RUNNING) for { conn, err := l.Accept() if err != nil { if err, ok := err.(net.Error); ok { if !err.Temporary() { - log.Info("Closing ...") + log.Warning("Closing connection with initiator...") break } } log.Error(err) continue } + log.Info(conn.LocalAddr().String()) + s.setClientStatus(true) iscsiConn := &iscsiConnection{conn: conn, loginParam: &iscsiLoginParam{}} @@ -206,11 +233,28 @@ func (s *ISCSITargetDriver) Close() error { s.setClientStatus(false) s.mu.Unlock() if l != nil { - return l.Close() + s.setState(STATE_SHUTTING_DOWN) + if err := l.Close(); err != nil { + return err + } + s.setState(STATE_TERMINATE) + return nil } return nil } +func (s *ISCSITargetDriver) setState(st uint8) { + s.mu.Lock() + defer s.mu.Unlock() + s.state = st +} + +func (s *ISCSITargetDriver) Resize(size uint64) error { + s.mu.Lock() + defer s.mu.Unlock() + return s.SCSI.Resize(size) +} + func (s *ISCSITargetDriver) handler(events byte, conn *iscsiConnection) { if events&DATAIN != 0 { @@ -481,6 +525,12 @@ func (s *ISCSITargetDriver) txHandler(conn *iscsiConnection) { resp.DataSN = 0 maxCount := conn.maxSeqCount + if s.enableStats { + if resp.OpCode == OpSCSIResp || resp.OpCode == OpSCSIIn { + s.UpdateStats(conn) + } + } + /* send data splitted by segmentLen */ SendRemainingData: if resp.OpCode == OpSCSIIn { @@ -581,6 +631,11 @@ func (s *ISCSITargetDriver) scsiCommandHandler(conn *iscsiConnection) (err error switch req.OpCode { case OpSCSICmd: log.Debugf("SCSI Command processing...") + if _, ok := s.SCSIIOCount[(int)(req.CDB[0])]; ok != false { + s.SCSIIOCount[(int)(req.CDB[0])] += 1 + } else { + s.SCSIIOCount[(int)(req.CDB[0])] = 1 + } scmd := &api.SCSICommand{ ITNexusID: conn.session.ITNexus.ID, SCB: req.CDB, @@ -612,6 +667,7 @@ func (s *ISCSITargetDriver) scsiCommandHandler(conn *iscsiConnection) (err error } if req.Write { task.r2tCount = int(req.ExpectedDataLen) - req.DataLen + task.expectedDataLength = int64(req.ExpectedDataLen) if !req.Final { task.unsolCount = 1 } @@ -849,3 +905,23 @@ func (s *ISCSITargetDriver) iscsiExecTask(task *iscsiTask) error { } return nil } + +func (s *ISCSITargetDriver) Stats() scsi.Stats { + return s.TargetStats +} + +func (s *ISCSITargetDriver) UpdateStats(conn *iscsiConnection) { + s.TargetStats.IsClientConnected = s.isClientConnected + switch api.SCSICommandType(conn.resp.SCSIOpCode) { + case api.READ_6, api.READ_10, api.READ_12, api.READ_16: + s.TargetStats.ReadIOPS += 1 + s.TargetStats.TotalReadTime += int64(time.Since(conn.resp.StartTime)) + s.TargetStats.TotalReadBlockCount += int64(conn.resp.ExpectedDataLen) + break + case api.WRITE_6, api.WRITE_10, api.WRITE_12, api.WRITE_16: + s.TargetStats.WriteIOPS += 1 + s.TargetStats.TotalWriteTime += int64(time.Since(conn.resp.StartTime)) + s.TargetStats.TotalWriteBlockCount += int64(conn.resp.ExpectedDataLen) + break + } +} diff --git a/pkg/port/iscsit/session.go b/pkg/port/iscsit/session.go index e0d7ba1..cd0c072 100644 --- a/pkg/port/iscsit/session.go +++ b/pkg/port/iscsit/session.go @@ -188,11 +188,9 @@ var sessionKeys map[string]*iscsiSessionKeys = map[string]*iscsiSessionKeys{ // ISCSI_PARAM_HDRDGST_EN "HeaderDigest": {ISCSI_PARAM_HDRDGST_EN, false, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv}, // ISCSI_PARAM_DATADGST_EN - // TODO: Not sure why initialR2T was changed to true, need help in - // understanding it's use? - "DataDigest": {ISCSI_PARAM_DATADGST_EN, true, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv}, + "DataDigest": {ISCSI_PARAM_DATADGST_EN, false, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv}, // ISCSI_PARAM_INITIAL_R2T_EN - "InitialR2T": {ISCSI_PARAM_INITIAL_R2T_EN, false, 1, 0, 1, boolKeyConv, boolKeyInConv}, + "InitialR2T": {ISCSI_PARAM_INITIAL_R2T_EN, true, 1, 0, 1, boolKeyConv, boolKeyInConv}, // ISCSI_PARAM_MAX_R2T "MaxOutstandingR2T": {ISCSI_PARAM_MAX_R2T, true, 1, 1, 65535, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_IMM_DATA_EN diff --git a/pkg/scsi/backingstore/remote/remote.go b/pkg/scsi/backingstore/remote/remote.go index 2dcfbf8..38aab1f 100644 --- a/pkg/scsi/backingstore/remote/remote.go +++ b/pkg/scsi/backingstore/remote/remote.go @@ -29,7 +29,7 @@ var ( ) func init() { - scsi.RegisterBackingStore("RemBs", newRemBs) + scsi.RegisterBackingStore("RemBs", NewRemoteBackingStore) } // RemBackingStore @@ -40,7 +40,7 @@ type RemBackingStore struct { RemBs api.RemoteBackingStore } -func newRemBs() (api.BackingStore, error) { +func NewRemoteBackingStore() (api.BackingStore, error) { return &RemBackingStore{ BaseBackingStore: scsi.BaseBackingStore{ Name: "RemBs", @@ -53,9 +53,12 @@ func (bs *RemBackingStore) Open(dev *api.SCSILu, path string) error { if Size == 0 { return fmt.Errorf("Size is not initialized") } - + var err error bs.DataSize = Size - bs.RemBs = scsi.GetTargetBSMap(path) + bs.RemBs, err = scsi.GetTargetBSMap(path) + if err != nil { + return err + } return nil } diff --git a/pkg/scsi/cmd.go b/pkg/scsi/cmd.go index 8b3cf7d..4bb80c4 100644 --- a/pkg/scsi/cmd.go +++ b/pkg/scsi/cmd.go @@ -228,7 +228,7 @@ const ( NAA_IEEE_REGD_EXTD = byte(0x6) ) -const ( +var ( SCSIVendorID = "GOSTOR" SCSIProductID = "GOTGT" SCSIID = "iqn.2016-09.com.gotgt.gostor:iscsi-tgt" diff --git a/pkg/scsi/drivers.go b/pkg/scsi/drivers.go index 0571f91..9c64498 100644 --- a/pkg/scsi/drivers.go +++ b/pkg/scsi/drivers.go @@ -28,6 +28,27 @@ type SCSITargetDriver interface { NewTarget(string, *config.Config) error RereadTargetLUNMap() Close() error + Resize(uint64) error + Stats() Stats +} + +type Stats struct { + IsClientConnected bool + RevisionCounter int64 + ReplicaCounter int64 + SCSIIOCount map[int]int64 + + ReadIOPS int64 + TotalReadTime int64 + TotalReadBlockCount int64 + + WriteIOPS int64 + TotalWriteTime int64 + TotalWriteBlockCount int64 + + UsedLogicalBlocks int64 + UsedBlocks int64 + SectorSize int64 } type TargetDriverFunc func(*SCSITargetService) (SCSITargetDriver, error) diff --git a/pkg/scsi/sbc.go b/pkg/scsi/sbc.go index ee1e333..ec3a4a8 100644 --- a/pkg/scsi/sbc.go +++ b/pkg/scsi/sbc.go @@ -37,6 +37,11 @@ const ( PR_EA_FN = (1 << 0) ) +var ( + EnableORWrite16 = true + EnablePersistentReservation = true +) + type SBCSCSIDeviceProtocol struct { BaseSCSIDeviceProtocol } @@ -176,26 +181,29 @@ func NewSBCDevice(deviceType api.SCSIDeviceType) api.SCSIDeviceProtocol { sbc.SCSIDeviceOps[api.MODE_SELECT_10] = NewSCSIDeviceOperation(SBCModeSelect, nil, PR_WE_FA|PR_EA_FA|PR_EA_FN|PR_WE_FN) sbc.SCSIDeviceOps[api.MODE_SENSE_10] = NewSCSIDeviceOperation(SBCModeSense, nil, PR_WE_FA|PR_WE_FN|PR_EA_FA|PR_EA_FN) - sbc.SCSIDeviceOps[api.PERSISTENT_RESERVE_IN] = NewSCSIDeviceOperation(SPCServiceAction, []*SCSIServiceAction{ - {ServiceAction: PR_IN_READ_KEYS, CommandPerformFunc: SPCPRReadKeys}, - {ServiceAction: PR_IN_READ_RESERVATION, CommandPerformFunc: SPCPRReadReservation}, - {ServiceAction: PR_IN_REPORT_CAPABILITIES, CommandPerformFunc: SPCPRReportCapabilities}, - }, 0) - - sbc.SCSIDeviceOps[api.PERSISTENT_RESERVE_OUT] = NewSCSIDeviceOperation(SPCServiceAction, []*SCSIServiceAction{ - {ServiceAction: PR_OUT_REGISTER, CommandPerformFunc: SPCPRRegister}, - {ServiceAction: PR_OUT_RESERVE, CommandPerformFunc: SPCPRReserve}, - {ServiceAction: PR_OUT_RELEASE, CommandPerformFunc: SPCPRRelease}, - {ServiceAction: PR_OUT_CLEAR, CommandPerformFunc: SPCPRClear}, - {ServiceAction: PR_OUT_PREEMPT, CommandPerformFunc: SPCPRPreempt}, - // {ServiceAction: PR_OUT_PREEMPT_AND_ABORT, CommandPerformFunc: SPCPRPreempt}, - {ServiceAction: PR_OUT_REGISTER_AND_IGNORE_EXISTING_KEY, CommandPerformFunc: SPCPRRegister}, - {ServiceAction: PR_OUT_REGISTER_AND_MOVE, CommandPerformFunc: SPCPRRegisterAndMove}, - }, 0) + if EnablePersistentReservation { + sbc.SCSIDeviceOps[api.PERSISTENT_RESERVE_IN] = NewSCSIDeviceOperation(SPCServiceAction, []*SCSIServiceAction{ + {ServiceAction: PR_IN_READ_KEYS, CommandPerformFunc: SPCPRReadKeys}, + {ServiceAction: PR_IN_READ_RESERVATION, CommandPerformFunc: SPCPRReadReservation}, + {ServiceAction: PR_IN_REPORT_CAPABILITIES, CommandPerformFunc: SPCPRReportCapabilities}, + }, 0) + sbc.SCSIDeviceOps[api.PERSISTENT_RESERVE_OUT] = NewSCSIDeviceOperation(SPCServiceAction, []*SCSIServiceAction{ + {ServiceAction: PR_OUT_REGISTER, CommandPerformFunc: SPCPRRegister}, + {ServiceAction: PR_OUT_RESERVE, CommandPerformFunc: SPCPRReserve}, + {ServiceAction: PR_OUT_RELEASE, CommandPerformFunc: SPCPRRelease}, + {ServiceAction: PR_OUT_CLEAR, CommandPerformFunc: SPCPRClear}, + {ServiceAction: PR_OUT_PREEMPT, CommandPerformFunc: SPCPRPreempt}, + // {ServiceAction: PR_OUT_PREEMPT_AND_ABORT, CommandPerformFunc: SPCPRPreempt}, + {ServiceAction: PR_OUT_REGISTER_AND_IGNORE_EXISTING_KEY, CommandPerformFunc: SPCPRRegister}, + {ServiceAction: PR_OUT_REGISTER_AND_MOVE, CommandPerformFunc: SPCPRRegisterAndMove}, + }, 0) + } sbc.SCSIDeviceOps[api.READ_16] = NewSCSIDeviceOperation(SBCReadWrite, nil, PR_EA_FA|PR_EA_FN) sbc.SCSIDeviceOps[api.WRITE_16] = NewSCSIDeviceOperation(SBCReadWrite, nil, PR_EA_FA|PR_EA_FN|PR_WE_FA|PR_WE_FN) - sbc.SCSIDeviceOps[api.ORWRITE_16] = NewSCSIDeviceOperation(SBCReadWrite, nil, PR_EA_FA|PR_EA_FN) + if EnableORWrite16 { + sbc.SCSIDeviceOps[api.ORWRITE_16] = NewSCSIDeviceOperation(SBCReadWrite, nil, PR_EA_FA|PR_EA_FN) + } sbc.SCSIDeviceOps[api.WRITE_VERIFY_16] = NewSCSIDeviceOperation(SBCReadWrite, nil, PR_EA_FA|PR_EA_FN) sbc.SCSIDeviceOps[api.VERIFY_16] = NewSCSIDeviceOperation(SBCVerify, nil, PR_EA_FA|PR_EA_FN) diff --git a/pkg/scsi/scsi.go b/pkg/scsi/scsi.go index 4e52e8e..dd2aa67 100644 --- a/pkg/scsi/scsi.go +++ b/pkg/scsi/scsi.go @@ -53,6 +53,20 @@ func (s *SCSITargetService) GetTargetList() ([]api.SCSITarget, error) { return result, nil } +func (s *SCSITargetService) Resize(size uint64) error { + s.mutex.Lock() + //TODO for multiple LUNs + for _, t := range s.Targets { + if t.Devices != nil { + for i := range t.Devices { + t.Devices[i].Size = size + } + } + } + s.mutex.Unlock() + return nil +} + func (s *SCSITargetService) AddCommandQueue(tid int, scmd *api.SCSICommand) error { var ( target *api.SCSITarget @@ -128,9 +142,18 @@ func NewSCSIDeviceOperation(fn api.CommandFunc, sa []*SCSIServiceAction, pr uint func BuildSenseData(cmd *api.SCSICommand, key byte, asc SCSISubError) { senseBuffer := &bytes.Buffer{} inBufLen, ok := SCSICDBBufXLength(cmd.SCB) - var length uint32 = 0xa + var ( + length uint32 = 0xa + fixedFormat bool = true + ) - if cmd.Device.Attrs.SenseFormat { + if cmd.Device != nil { + if cmd.Device.Attrs.SenseFormat { + fixedFormat = false + } + } + + if !fixedFormat { // descriptor format // current, not deferred senseBuffer.WriteByte(0x72) diff --git a/pkg/scsi/scsilumap.go b/pkg/scsi/scsilumap.go index 396138b..a6fb823 100644 --- a/pkg/scsi/scsilumap.go +++ b/pkg/scsi/scsilumap.go @@ -78,13 +78,16 @@ func GetTargetLUNMap(tgtName string) api.LUNMap { return lunMap } -func GetTargetBSMap(tgtName string) api.RemoteBackingStore { - /* TODO check for lock held by caller +func GetTargetBSMap(tgtName string) (api.RemoteBackingStore, error) { globalSCSILUMap.mutex.RLock() - defer globalSCSILUMap.mutex.RUnlock()*/ + defer globalSCSILUMap.mutex.RUnlock() - lunMap := globalSCSILUMap.TargetsBSMap[tgtName] - return lunMap + bs, ok := globalSCSILUMap.TargetsBSMap[tgtName] + if !ok { + return nil, errors.New("Remote backing store is not found in globalSCSILUMap") + } + + return bs, nil } func AddBackendStorage(bs config.BackendStorage) error { @@ -148,19 +151,22 @@ func InitSCSILUMap(config *config.Config) error { } func InitSCSILUMapEx(config *config.BackendStorage, tgtName string, lun uint64, bs api.RemoteBackingStore) error { - globalSCSILUMap.mutex.Lock() - defer globalSCSILUMap.mutex.Unlock() - if bs == nil { return errors.New("Remote backing store is nil") } + globalSCSILUMap.mutex.Lock() globalSCSILUMap.TargetsBSMap[tgtName] = bs + globalSCSILUMap.mutex.Unlock() + lu, err := NewSCSILu(config) if err != nil { - return errors.New("Init SCSI LU map error.") + return fmt.Errorf("Init SCSI LU map error, err: %v", err) } + + globalSCSILUMap.mutex.Lock() globalSCSILUMap.AllDevices[config.DeviceID] = lu + globalSCSILUMap.mutex.Unlock() mappingLUN(LUNMapping{ DeviceID: config.DeviceID,