Implement stats and resize and fix remote backing store apis

- Convert constant to var so that it can be configured from backend
- Add options to disable persistent reservation and ORWrite16 commands

Signed-off-by: Utkarsh Mani Tripathi <utkarsh.tripathi@mayadata.io>

pkg/port/iscsit/cmd.go

@@ -239,6 +239,7 @@ func parseHeader(data []byte) (*ISCSICommand, error) {
 		m.Write = data[1]&0x20 == 0x20
 		m.CDB = data[32:48]
 		m.ExpStatSN = uint32(ParseUint(data[28:32]))
+		m.SCSIOpCode = m.CDB[0]
 		SCSIOpcode := api.SCSICommandType(m.SCSIOpCode)
 		switch SCSIOpcode {
 		case api.READ_6, api.READ_10, api.READ_12, api.READ_16:

pkg/port/iscsit/conn.go

@@ -51,6 +51,7 @@ var (
 )
 
 type iscsiConnection struct {
+	ConnNum   int
 	state     int
 	authState int
 	session   *ISCSISession
@@ -96,12 +97,13 @@ const (
 )
 
 type iscsiTask struct {
-	tag    uint32
+	tag                uint32
-	conn   *iscsiConnection
+	conn               *iscsiConnection
-	cmd    *ISCSICommand
+	cmd                *ISCSICommand
-	scmd   *api.SCSICommand
+	scmd               *api.SCSICommand
-	state  taskState
+	state              taskState
-	result byte
+	expectedDataLength int64
+	result             byte
 
 	offset     int
 	r2tCount   int
@@ -154,6 +156,7 @@ func (conn *iscsiConnection) buildRespPackage(oc OpCode, task *iscsiTask) error
 		task = conn.rxTask
 	}
 	conn.resp = &ISCSICommand{
+		StartTime:       conn.req.StartTime,
 		StatSN:          conn.req.ExpStatSN,
 		TaskTag:         conn.req.TaskTag,
 		ExpCmdSN:        conn.session.ExpCmdSN,
@@ -172,6 +175,7 @@ func (conn *iscsiConnection) buildRespPackage(oc OpCode, task *iscsiTask) error
 		}
 	case OpSCSIIn, OpSCSIResp:
 		conn.resp.OpCode = oc
+		conn.resp.SCSIOpCode = conn.req.SCSIOpCode
 		conn.resp.Immediate = true
 		conn.resp.Final = true
 		conn.resp.SCSIResponse = 0x00

pkg/port/iscsit/iscsid.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"strconv"
 	"sync"
+	"time"
 
 	"github.com/gostor/gotgt/pkg/api"
 	"github.com/gostor/gotgt/pkg/config"
@@ -35,6 +36,17 @@ const (
 	ISCSI_UNSPEC_TSIH = uint16(0)
 )
 
+const (
+	STATE_INIT = iota
+	STATE_RUNNING
+	STATE_SHUTTING_DOWN
+	STATE_TERMINATE
+)
+
+var (
+	EnableStats bool
+)
+
 type ISCSITargetDriver struct {
 	SCSI              *scsi.SCSITargetService
 	Name              string
@@ -42,9 +54,13 @@ type ISCSITargetDriver struct {
 	TSIHPool          map[uint16]bool
 	TSIHPoolMutex     sync.Mutex
 	isClientConnected bool
-
+	enableStats       bool
-	mu sync.Mutex
+	SCSIIOCount       map[int]int64
-	l  net.Listener
+	mu                *sync.RWMutex
+	l                 net.Listener
+	state             uint8
+	OpCode            int
+	TargetStats       scsi.Stats
 }
 
 func init() {
@@ -52,12 +68,19 @@ func init() {
 }
 
 func NewISCSITargetDriver(base *scsi.SCSITargetService) (scsi.SCSITargetDriver, error) {
-	return &ISCSITargetDriver{
+	driver := &ISCSITargetDriver{
 		Name:         iSCSIDriverName,
 		iSCSITargets: map[string]*ISCSITarget{},
 		SCSI:         base,
 		TSIHPool:     map[uint16]bool{0: true, 65535: true},
-	}, nil
+		mu:           &sync.RWMutex{},
+	}
+
+	if EnableStats {
+		driver.SCSIIOCount = map[int]int64{}
+		driver.enableStats = true
+	}
+	return driver, nil
 }
 
 func (s *ISCSITargetDriver) AllocTSIH() uint16 {
@@ -167,19 +190,23 @@ func (s *ISCSITargetDriver) Run() error {
 	s.l = l
 	s.mu.Unlock()
 	log.Infof("iSCSI service listening on: %v", s.l.Addr())
+
+	s.setState(STATE_RUNNING)
 	for {
 		conn, err := l.Accept()
 		if err != nil {
 			if err, ok := err.(net.Error); ok {
 				if !err.Temporary() {
-					log.Info("Closing ...")
+					log.Warning("Closing connection with initiator...")
 					break
 				}
 			}
 			log.Error(err)
 			continue
 		}
+
 		log.Info(conn.LocalAddr().String())
+		s.setClientStatus(true)
 
 		iscsiConn := &iscsiConnection{conn: conn,
 			loginParam: &iscsiLoginParam{}}
@@ -206,11 +233,28 @@ func (s *ISCSITargetDriver) Close() error {
 	s.setClientStatus(false)
 	s.mu.Unlock()
 	if l != nil {
-		return l.Close()
+		s.setState(STATE_SHUTTING_DOWN)
+		if err := l.Close(); err != nil {
+			return err
+		}
+		s.setState(STATE_TERMINATE)
+		return nil
 	}
 	return nil
 }
 
+func (s *ISCSITargetDriver) setState(st uint8) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.state = st
+}
+
+func (s *ISCSITargetDriver) Resize(size uint64) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.SCSI.Resize(size)
+}
+
 func (s *ISCSITargetDriver) handler(events byte, conn *iscsiConnection) {
 
 	if events&DATAIN != 0 {
@@ -481,6 +525,12 @@ func (s *ISCSITargetDriver) txHandler(conn *iscsiConnection) {
 	resp.DataSN = 0
 	maxCount := conn.maxSeqCount
 
+	if s.enableStats {
+		if resp.OpCode == OpSCSIResp || resp.OpCode == OpSCSIIn {
+			s.UpdateStats(conn)
+		}
+	}
+
 	/* send data splitted by segmentLen */
 SendRemainingData:
 	if resp.OpCode == OpSCSIIn {
@@ -581,6 +631,11 @@ func (s *ISCSITargetDriver) scsiCommandHandler(conn *iscsiConnection) (err error
 	switch req.OpCode {
 	case OpSCSICmd:
 		log.Debugf("SCSI Command processing...")
+		if _, ok := s.SCSIIOCount[(int)(req.CDB[0])]; ok != false {
+			s.SCSIIOCount[(int)(req.CDB[0])] += 1
+		} else {
+			s.SCSIIOCount[(int)(req.CDB[0])] = 1
+		}
 		scmd := &api.SCSICommand{
 			ITNexusID:       conn.session.ITNexus.ID,
 			SCB:             req.CDB,
@@ -612,6 +667,7 @@ func (s *ISCSITargetDriver) scsiCommandHandler(conn *iscsiConnection) (err error
 		}
 		if req.Write {
 			task.r2tCount = int(req.ExpectedDataLen) - req.DataLen
+			task.expectedDataLength = int64(req.ExpectedDataLen)
 			if !req.Final {
 				task.unsolCount = 1
 			}
@@ -849,3 +905,23 @@ func (s *ISCSITargetDriver) iscsiExecTask(task *iscsiTask) error {
 	}
 	return nil
 }
+
+func (s *ISCSITargetDriver) Stats() scsi.Stats {
+	return s.TargetStats
+}
+
+func (s *ISCSITargetDriver) UpdateStats(conn *iscsiConnection) {
+	s.TargetStats.IsClientConnected = s.isClientConnected
+	switch api.SCSICommandType(conn.resp.SCSIOpCode) {
+	case api.READ_6, api.READ_10, api.READ_12, api.READ_16:
+		s.TargetStats.ReadIOPS += 1
+		s.TargetStats.TotalReadTime += int64(time.Since(conn.resp.StartTime))
+		s.TargetStats.TotalReadBlockCount += int64(conn.resp.ExpectedDataLen)
+		break
+	case api.WRITE_6, api.WRITE_10, api.WRITE_12, api.WRITE_16:
+		s.TargetStats.WriteIOPS += 1
+		s.TargetStats.TotalWriteTime += int64(time.Since(conn.resp.StartTime))
+		s.TargetStats.TotalWriteBlockCount += int64(conn.resp.ExpectedDataLen)
+		break
+	}
+}

pkg/port/iscsit/session.go

@@ -188,11 +188,9 @@ var sessionKeys map[string]*iscsiSessionKeys = map[string]*iscsiSessionKeys{
 	// ISCSI_PARAM_HDRDGST_EN
 	"HeaderDigest": {ISCSI_PARAM_HDRDGST_EN, false, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv},
 	// ISCSI_PARAM_DATADGST_EN
-	// TODO: Not sure why initialR2T was changed to true, need help in
+	"DataDigest": {ISCSI_PARAM_DATADGST_EN, false, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv},
-	// understanding it's use?
-	"DataDigest": {ISCSI_PARAM_DATADGST_EN, true, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv},
 	// ISCSI_PARAM_INITIAL_R2T_EN
-	"InitialR2T": {ISCSI_PARAM_INITIAL_R2T_EN, false, 1, 0, 1, boolKeyConv, boolKeyInConv},
+	"InitialR2T": {ISCSI_PARAM_INITIAL_R2T_EN, true, 1, 0, 1, boolKeyConv, boolKeyInConv},
 	// ISCSI_PARAM_MAX_R2T
 	"MaxOutstandingR2T": {ISCSI_PARAM_MAX_R2T, true, 1, 1, 65535, numberKeyConv, numberKeyInConv},
 	// ISCSI_PARAM_IMM_DATA_EN

pkg/scsi/backingstore/remote/remote.go

@@ -29,7 +29,7 @@ var (
 )
 
 func init() {
-	scsi.RegisterBackingStore("RemBs", newRemBs)
+	scsi.RegisterBackingStore("RemBs", NewRemoteBackingStore)
 }
 
 // RemBackingStore
@@ -40,7 +40,7 @@ type RemBackingStore struct {
 	RemBs api.RemoteBackingStore
 }
 
-func newRemBs() (api.BackingStore, error) {
+func NewRemoteBackingStore() (api.BackingStore, error) {
 	return &RemBackingStore{
 		BaseBackingStore: scsi.BaseBackingStore{
 			Name:            "RemBs",
@@ -53,9 +53,12 @@ func (bs *RemBackingStore) Open(dev *api.SCSILu, path string) error {
 	if Size == 0 {
 		return fmt.Errorf("Size is not initialized")
 	}
-
+	var err error
 	bs.DataSize = Size
-	bs.RemBs = scsi.GetTargetBSMap(path)
+	bs.RemBs, err = scsi.GetTargetBSMap(path)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 

pkg/scsi/cmd.go

@@ -228,7 +228,7 @@ const (
 	NAA_IEEE_REGD_EXTD = byte(0x6)
 )
 
-const (
+var (
 	SCSIVendorID  = "GOSTOR"
 	SCSIProductID = "GOTGT"
 	SCSIID        = "iqn.2016-09.com.gotgt.gostor:iscsi-tgt"

pkg/scsi/drivers.go

@@ -28,6 +28,27 @@ type SCSITargetDriver interface {
 	NewTarget(string, *config.Config) error
 	RereadTargetLUNMap()
 	Close() error
+	Resize(uint64) error
+	Stats() Stats
+}
+
+type Stats struct {
+	IsClientConnected bool
+	RevisionCounter   int64
+	ReplicaCounter    int64
+	SCSIIOCount       map[int]int64
+
+	ReadIOPS            int64
+	TotalReadTime       int64
+	TotalReadBlockCount int64
+
+	WriteIOPS            int64
+	TotalWriteTime       int64
+	TotalWriteBlockCount int64
+
+	UsedLogicalBlocks int64
+	UsedBlocks        int64
+	SectorSize        int64
 }
 
 type TargetDriverFunc func(*SCSITargetService) (SCSITargetDriver, error)

pkg/scsi/sbc.go

@@ -37,6 +37,11 @@ const (
 	PR_EA_FN   = (1 << 0)
 )
 
+var (
+	EnableORWrite16             = true
+	EnablePersistentReservation = true
+)
+
 type SBCSCSIDeviceProtocol struct {
 	BaseSCSIDeviceProtocol
 }
@@ -176,26 +181,29 @@ func NewSBCDevice(deviceType api.SCSIDeviceType) api.SCSIDeviceProtocol {
 	sbc.SCSIDeviceOps[api.MODE_SELECT_10] = NewSCSIDeviceOperation(SBCModeSelect, nil, PR_WE_FA|PR_EA_FA|PR_EA_FN|PR_WE_FN)
 	sbc.SCSIDeviceOps[api.MODE_SENSE_10] = NewSCSIDeviceOperation(SBCModeSense, nil, PR_WE_FA|PR_WE_FN|PR_EA_FA|PR_EA_FN)
 
-	sbc.SCSIDeviceOps[api.PERSISTENT_RESERVE_IN] = NewSCSIDeviceOperation(SPCServiceAction, []*SCSIServiceAction{
+	if EnablePersistentReservation {
-		{ServiceAction: PR_IN_READ_KEYS, CommandPerformFunc: SPCPRReadKeys},
+		sbc.SCSIDeviceOps[api.PERSISTENT_RESERVE_IN] = NewSCSIDeviceOperation(SPCServiceAction, []*SCSIServiceAction{
-		{ServiceAction: PR_IN_READ_RESERVATION, CommandPerformFunc: SPCPRReadReservation},
+			{ServiceAction: PR_IN_READ_KEYS, CommandPerformFunc: SPCPRReadKeys},
-		{ServiceAction: PR_IN_REPORT_CAPABILITIES, CommandPerformFunc: SPCPRReportCapabilities},
+			{ServiceAction: PR_IN_READ_RESERVATION, CommandPerformFunc: SPCPRReadReservation},
-	}, 0)
+			{ServiceAction: PR_IN_REPORT_CAPABILITIES, CommandPerformFunc: SPCPRReportCapabilities},
-
+		}, 0)
-	sbc.SCSIDeviceOps[api.PERSISTENT_RESERVE_OUT] = NewSCSIDeviceOperation(SPCServiceAction, []*SCSIServiceAction{
-		{ServiceAction: PR_OUT_REGISTER, CommandPerformFunc: SPCPRRegister},
-		{ServiceAction: PR_OUT_RESERVE, CommandPerformFunc: SPCPRReserve},
-		{ServiceAction: PR_OUT_RELEASE, CommandPerformFunc: SPCPRRelease},
-		{ServiceAction: PR_OUT_CLEAR, CommandPerformFunc: SPCPRClear},
-		{ServiceAction: PR_OUT_PREEMPT, CommandPerformFunc: SPCPRPreempt},
-		//		{ServiceAction: PR_OUT_PREEMPT_AND_ABORT, CommandPerformFunc: SPCPRPreempt},
-		{ServiceAction: PR_OUT_REGISTER_AND_IGNORE_EXISTING_KEY, CommandPerformFunc: SPCPRRegister},
-		{ServiceAction: PR_OUT_REGISTER_AND_MOVE, CommandPerformFunc: SPCPRRegisterAndMove},
-	}, 0)
 
+		sbc.SCSIDeviceOps[api.PERSISTENT_RESERVE_OUT] = NewSCSIDeviceOperation(SPCServiceAction, []*SCSIServiceAction{
+			{ServiceAction: PR_OUT_REGISTER, CommandPerformFunc: SPCPRRegister},
+			{ServiceAction: PR_OUT_RESERVE, CommandPerformFunc: SPCPRReserve},
+			{ServiceAction: PR_OUT_RELEASE, CommandPerformFunc: SPCPRRelease},
+			{ServiceAction: PR_OUT_CLEAR, CommandPerformFunc: SPCPRClear},
+			{ServiceAction: PR_OUT_PREEMPT, CommandPerformFunc: SPCPRPreempt},
+			//		{ServiceAction: PR_OUT_PREEMPT_AND_ABORT, CommandPerformFunc: SPCPRPreempt},
+			{ServiceAction: PR_OUT_REGISTER_AND_IGNORE_EXISTING_KEY, CommandPerformFunc: SPCPRRegister},
+			{ServiceAction: PR_OUT_REGISTER_AND_MOVE, CommandPerformFunc: SPCPRRegisterAndMove},
+		}, 0)
+	}
 	sbc.SCSIDeviceOps[api.READ_16] = NewSCSIDeviceOperation(SBCReadWrite, nil, PR_EA_FA|PR_EA_FN)
 	sbc.SCSIDeviceOps[api.WRITE_16] = NewSCSIDeviceOperation(SBCReadWrite, nil, PR_EA_FA|PR_EA_FN|PR_WE_FA|PR_WE_FN)
-	sbc.SCSIDeviceOps[api.ORWRITE_16] = NewSCSIDeviceOperation(SBCReadWrite, nil, PR_EA_FA|PR_EA_FN)
+	if EnableORWrite16 {
+		sbc.SCSIDeviceOps[api.ORWRITE_16] = NewSCSIDeviceOperation(SBCReadWrite, nil, PR_EA_FA|PR_EA_FN)
+	}
 	sbc.SCSIDeviceOps[api.WRITE_VERIFY_16] = NewSCSIDeviceOperation(SBCReadWrite, nil, PR_EA_FA|PR_EA_FN)
 	sbc.SCSIDeviceOps[api.VERIFY_16] = NewSCSIDeviceOperation(SBCVerify, nil, PR_EA_FA|PR_EA_FN)
 

pkg/scsi/scsi.go

@@ -53,6 +53,20 @@ func (s *SCSITargetService) GetTargetList() ([]api.SCSITarget, error) {
 	return result, nil
 }
 
+func (s *SCSITargetService) Resize(size uint64) error {
+	s.mutex.Lock()
+	//TODO for multiple LUNs
+	for _, t := range s.Targets {
+		if t.Devices != nil {
+			for i := range t.Devices {
+				t.Devices[i].Size = size
+			}
+		}
+	}
+	s.mutex.Unlock()
+	return nil
+}
+
 func (s *SCSITargetService) AddCommandQueue(tid int, scmd *api.SCSICommand) error {
 	var (
 		target *api.SCSITarget
@@ -128,9 +142,18 @@ func NewSCSIDeviceOperation(fn api.CommandFunc, sa []*SCSIServiceAction, pr uint
 func BuildSenseData(cmd *api.SCSICommand, key byte, asc SCSISubError) {
 	senseBuffer := &bytes.Buffer{}
 	inBufLen, ok := SCSICDBBufXLength(cmd.SCB)
-	var length uint32 = 0xa
+	var (
+		length      uint32 = 0xa
+		fixedFormat bool   = true
+	)
 
-	if cmd.Device.Attrs.SenseFormat {
+	if cmd.Device != nil {
+		if cmd.Device.Attrs.SenseFormat {
+			fixedFormat = false
+		}
+	}
+
+	if !fixedFormat {
 		// descriptor format
 		// current, not deferred
 		senseBuffer.WriteByte(0x72)

pkg/scsi/scsilumap.go

@@ -78,13 +78,16 @@ func GetTargetLUNMap(tgtName string) api.LUNMap {
 	return lunMap
 }
 
-func GetTargetBSMap(tgtName string) api.RemoteBackingStore {
+func GetTargetBSMap(tgtName string) (api.RemoteBackingStore, error) {
-	/* TODO check for lock held by caller
 	globalSCSILUMap.mutex.RLock()
-	defer globalSCSILUMap.mutex.RUnlock()*/
+	defer globalSCSILUMap.mutex.RUnlock()
 
-	lunMap := globalSCSILUMap.TargetsBSMap[tgtName]
+	bs, ok := globalSCSILUMap.TargetsBSMap[tgtName]
-	return lunMap
+	if !ok {
+		return nil, errors.New("Remote backing store is not found in globalSCSILUMap")
+	}
+
+	return bs, nil
 }
 
 func AddBackendStorage(bs config.BackendStorage) error {
@@ -148,19 +151,22 @@ func InitSCSILUMap(config *config.Config) error {
 }
 
 func InitSCSILUMapEx(config *config.BackendStorage, tgtName string, lun uint64, bs api.RemoteBackingStore) error {
-	globalSCSILUMap.mutex.Lock()
-	defer globalSCSILUMap.mutex.Unlock()
-
 	if bs == nil {
 		return errors.New("Remote backing store is nil")
 	}
 
+	globalSCSILUMap.mutex.Lock()
 	globalSCSILUMap.TargetsBSMap[tgtName] = bs
+	globalSCSILUMap.mutex.Unlock()
+
 	lu, err := NewSCSILu(config)
 	if err != nil {
-		return errors.New("Init SCSI LU map error.")
+		return fmt.Errorf("Init SCSI LU map error, err: %v", err)
 	}
+
+	globalSCSILUMap.mutex.Lock()
 	globalSCSILUMap.AllDevices[config.DeviceID] = lu
+	globalSCSILUMap.mutex.Unlock()
 
 	mappingLUN(LUNMapping{
 		DeviceID:   config.DeviceID,