Merge pull request #31 from orzhang/login_er

login negotiation and session/conn reinstatment

pkg/port/iscsit/cmd.go

@@ -81,7 +81,7 @@ type ISCSICommand struct {
 	// Continue bit.
 	Cont bool
 	// Current Stage, Next Stage.
-	CSG, NSG Stage
+	CSG, NSG iSCSILoginStage
 	// Initiator part of the SSID.
 	ISID uint64
 	// Target-assigned Session Identifying Handle.
@@ -214,8 +214,8 @@ func parseHeader(data []byte) (*ISCSICommand, error) {
 			// rfc7143 11.12.2
 			return nil, fmt.Errorf("transit and continue bits set in same login request")
 		}
-		m.CSG = Stage(data[1]&0xc) >> 2
+		m.CSG = iSCSILoginStage(data[1]&0xc) >> 2
-		m.NSG = Stage(data[1] & 0x3)
+		m.NSG = iSCSILoginStage(data[1] & 0x3)
 		m.ISID = uint64(ParseUint(data[8:14]))
 		m.TSIH = uint16(ParseUint(data[14:16]))
 		m.ConnID = uint16(ParseUint(data[20:22]))
@@ -228,8 +228,8 @@ func parseHeader(data []byte) (*ISCSICommand, error) {
 			// rfc7143 11.12.2
 			return nil, fmt.Errorf("transit and continue bits set in same login request")
 		}
-		m.CSG = Stage(data[1]&0xc) >> 2
+		m.CSG = iSCSILoginStage(data[1]&0xc) >> 2
-		m.NSG = Stage(data[1] & 0x3)
+		m.NSG = iSCSILoginStage(data[1] & 0x3)
 		m.StatSN = uint32(ParseUint(data[24:28]))
 		m.ExpCmdSN = uint32(ParseUint(data[28:32]))
 		m.MaxCmdSN = uint32(ParseUint(data[32:36]))
@@ -421,6 +421,7 @@ func (m *ISCSICommand) scsiTMFRespBytes() []byte {
 }
 
 func (m *ISCSICommand) r2tRespBytes() []byte {
+
 	// rfc7143 11.8
 	buf := &bytes.Buffer{}
 	buf.WriteByte(byte(OpReady))

pkg/port/iscsit/conn.go

@@ -18,6 +18,7 @@ package iscsit
 
 import (
 	"net"
+	"sort"
 	"sync"
 
 	"github.com/gostor/gotgt/pkg/api"
@@ -48,26 +49,23 @@ var (
 )
 
 type iscsiConnection struct {
-	state          int
+	state     int
-	authState      int
+	authState int
-	session        *ISCSISession
+	session   *ISCSISession
-	sessionType    int
+	tid       int
-	sessionParam   []ISCSISessionParam
+	cid       uint16
-	tid            int
+	rxIOState int
-	CID            uint16
+	txIOState int
-	rxIOState      int
+	refcount  int
-	txIOState      int
+	conn      net.Conn
-	refcount       int
-	conn           net.Conn
-	initiator      string
-	initiatorAlias string
-	tpgt           uint16
 
 	rxBuffer []byte
 	txBuffer []byte
 	req      *ISCSICommand
 	resp     *ISCSICommand
 
+	loginParam *iscsiLoginParam
+
 	// StatSN - the status sequence number on this connection
 	statSN uint32
 	// ExpStatSN - the expected status sequence number on this connection
@@ -82,8 +80,6 @@ type iscsiConnection struct {
 	rxTask *iscsiTask
 	txTask *iscsiTask
 
-	authMethod AuthMethod
-
 	readLock *sync.RWMutex
 }
 
@@ -113,10 +109,14 @@ func (c *iscsiConnection) init() {
 	c.state = CONN_STATE_FREE
 	c.refcount = 1
 	c.readLock = new(sync.RWMutex)
-	c.sessionParam = []ISCSISessionParam{}
+	c.loginParam.sessionParam = []ISCSISessionParam{}
+	c.loginParam.tgtCSG = LoginOperationalNegotiation
+	c.loginParam.tgtNSG = LoginOperationalNegotiation
 	for _, param := range sessionKeys {
-		c.sessionParam = append(c.sessionParam, ISCSISessionParam{Value: param.def})
+		c.loginParam.sessionParam = append(c.loginParam.sessionParam,
+			ISCSISessionParam{idx: param.idx, Value: param.def})
 	}
+	sort.Sort(c.loginParam.sessionParam)
 }
 
 func (c *iscsiConnection) readData(size int) ([]byte, int, error) {
@@ -135,3 +135,8 @@ func (c *iscsiConnection) write(resp []byte) (int, error) {
 func (c *iscsiConnection) close() {
 	c.conn.Close()
 }
+
+func (conn *iscsiConnection) ReInstatement(newConn *iscsiConnection) {
+	conn.close()
+	conn.conn = newConn.conn
+}

pkg/port/iscsit/iscsid.go

@@ -22,19 +22,26 @@ import (
 	"net"
 	"os"
 	"strconv"
+	"sync"
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/gostor/gotgt/pkg/api"
 	"github.com/gostor/gotgt/pkg/config"
 	"github.com/gostor/gotgt/pkg/scsi"
 	"github.com/gostor/gotgt/pkg/util"
-	"github.com/satori/go.uuid"
+)
+
+const (
+	ISCSI_MAX_TSIH    = uint16(0xffff)
+	ISCSI_UNSPEC_TSIH = uint16(0)
 )
 
 type ISCSITargetDriver struct {
-	SCSI         *scsi.SCSITargetService
+	SCSI          *scsi.SCSITargetService
-	Name         string
+	Name          string
-	iSCSITargets map[string]*ISCSITarget
+	iSCSITargets  map[string]*ISCSITarget
+	TSIHPool      map[uint16]bool
+	TSIHPoolMutex sync.Mutex
 }
 
 func init() {
@@ -46,9 +53,31 @@ func NewISCSITargetDriver(base *scsi.SCSITargetService) (scsi.SCSITargetDriver,
 		Name:         "iscsi",
 		iSCSITargets: map[string]*ISCSITarget{},
 		SCSI:         base,
+		TSIHPool:     map[uint16]bool{0: true, 65535: true},
 	}, nil
 }
 
+func (s *ISCSITargetDriver) AllocTSIH() uint16 {
+	var i uint16
+	s.TSIHPoolMutex.Lock()
+	for i = uint16(0); i < ISCSI_MAX_TSIH; i++ {
+		exist := s.TSIHPool[i]
+		if !exist {
+			s.TSIHPool[i] = true
+			s.TSIHPoolMutex.Unlock()
+			return i
+		}
+	}
+	s.TSIHPoolMutex.Unlock()
+	return ISCSI_UNSPEC_TSIH
+}
+
+func (s *ISCSITargetDriver) ReleaseTSIH(tsih uint16) {
+	s.TSIHPoolMutex.Lock()
+	delete(s.TSIHPool, tsih)
+	s.TSIHPoolMutex.Unlock()
+}
+
 func (s *ISCSITargetDriver) NewTarget(tgtName string, configInfo *config.Config) error {
 	if _, ok := s.iSCSITargets[tgtName]; ok {
 		return fmt.Errorf("target name has been existed")
@@ -138,7 +167,10 @@ func (s *ISCSITargetDriver) Run() error {
 		}
 		log.Info(conn.LocalAddr().String())
 		log.Info("Accepting ...")
-		iscsiConn := &iscsiConnection{conn: conn}
+
+		iscsiConn := &iscsiConnection{conn: conn,
+			loginParam: &iscsiLoginParam{}}
+
 		iscsiConn.init()
 		iscsiConn.rxIOState = IOSTATE_RX_BHS
 
@@ -160,7 +192,7 @@ func (s *ISCSITargetDriver) handler(events byte, conn *iscsiConnection) {
 		s.txHandler(conn)
 	}
 	if conn.state == CONN_STATE_CLOSE {
-		log.Warningf("iscsi connection[%d] closed", conn.CID)
+		log.Warningf("iscsi connection[%d] closed", conn.cid)
 		conn.close()
 	}
 }
@@ -175,8 +207,8 @@ func (s *ISCSITargetDriver) rxHandler(conn *iscsiConnection) {
 	conn.readLock.Lock()
 	defer conn.readLock.Unlock()
 	if conn.state == CONN_STATE_SCSI {
-		hdigest = conn.sessionParam[ISCSI_PARAM_HDRDGST_EN].Value & DIGEST_CRC32C
+		hdigest = conn.loginParam.sessionParam[ISCSI_PARAM_HDRDGST_EN].Value & DIGEST_CRC32C
-		ddigest = conn.sessionParam[ISCSI_PARAM_DATADGST_EN].Value & DIGEST_CRC32C
+		ddigest = conn.loginParam.sessionParam[ISCSI_PARAM_DATADGST_EN].Value & DIGEST_CRC32C
 	}
 	for {
 		switch conn.rxIOState {
@@ -289,101 +321,62 @@ func (s *ISCSITargetDriver) rxHandler(conn *iscsiConnection) {
 
 func (s *ISCSITargetDriver) iscsiExecLogin(conn *iscsiConnection) error {
 	var (
-		target *ISCSITarget
+		cmd      = conn.req
-		cmd    = conn.req
+		err      error
-		TPGT   uint16
+		negoKeys []util.KeyValue
-		err    error
 	)
-	conn.resp = &ISCSICommand{
-		OpCode:   OpLoginResp,
-		Transit:  true,
-		CSG:      cmd.CSG,
-		NSG:      FullFeaturePhase,
-		StatSN:   cmd.ExpStatSN,
-		TaskTag:  cmd.TaskTag,
-		ExpCmdSN: cmd.CmdSN,
-		MaxCmdSN: cmd.CmdSN,
-		RawData: util.MarshalKVText([]util.KeyValue{
-			{"HeaderDigest", "None"},
-			{"DataDigest", "None"},
-			{"ImmediateData", "Yes"},
-			{"InitialR2T", "Yes"},
-			{"MaxBurstLength", "262144"},
-			{"FirstBurstLength", "65536"},
-			{"DefaultTime2Wait", "2"},
-			{"DefaultTime2Retain", "0"},
-			{"MaxOutstandingR2T", "1"},
-			{"IFMarker", "No"},
-			{"OFMarker", "No"},
-			{"DataPDUInOrder", "Yes"},
-			{"DataSequenceInOrder", "Yes"},
-		}),
-	}
-	conn.CID = cmd.ConnID
-	pairs := util.ParseKVText(cmd.RawData)
-	if initiatorName, ok := pairs["InitiatorName"]; ok {
-		conn.initiator = initiatorName
-	}
-	if alias, ok := pairs["InitiatorAlias"]; ok {
-		conn.initiatorAlias = alias
-	}
-	targetName := pairs["TargetName"]
-	if sessType, ok := pairs["SessionType"]; ok {
-		if sessType == "Normal" {
-			conn.sessionType = SESSION_NORMAL
-		} else {
-			conn.sessionType = SESSION_DISCOVERY
-		}
-	}
-	if conn.sessionType == SESSION_DISCOVERY {
-		conn.tid = 0xffff
-	} else {
-		for _, t := range s.iSCSITargets {
-			if t.SCSITarget.Name == targetName {
-				target = t
-				break
-			}
-		}
-		if target == nil {
-			conn.state = CONN_STATE_EXIT
-			return fmt.Errorf("No target found with name(%s)", targetName)
-		}
 
-		TPGT, err = target.FindTPG(conn.conn.LocalAddr().String())
+	conn.cid = cmd.ConnID
+	conn.loginParam.iniCSG = cmd.CSG
+	conn.loginParam.iniNSG = cmd.NSG
+	conn.loginParam.iniCont = cmd.Cont
+	conn.loginParam.iniTrans = cmd.Transit
+	conn.loginParam.isid = cmd.ISID
+	conn.loginParam.tsih = cmd.TSIH
+	conn.expCmdSN = cmd.CmdSN
+	conn.statSN += 1
+
+	if conn.loginParam.iniCSG == SecurityNegotiation {
+		conn.state = CONN_STATE_EXIT
+		return fmt.Errorf("Doesn't support Auth")
+	}
+
+	pairs := util.ParseKVText(cmd.RawData)
+
+	negoKeys, err = loginKVProcess(conn, pairs)
+	if err != nil {
+		return err
+	}
+	if !conn.loginParam.keyDeclared {
+		negoKeys = loginKVDeclare(conn, negoKeys)
+		conn.loginParam.keyDeclared = true
+	}
+
+	if !conn.loginParam.paramInit {
+		err = s.BindISCSISession(conn)
 		if err != nil {
 			conn.state = CONN_STATE_EXIT
 			return err
 		}
-		conn.tpgt = TPGT
+		conn.loginParam.paramInit = true
-		conn.tid = target.TID
-
 	}
-	switch conn.state {
+	if conn.loginParam.tgtNSG == FullFeaturePhase &&
-	case CONN_STATE_FREE:
+		conn.loginParam.tgtTrans {
-		conn.state = CONN_STATE_SECURITY
+		conn.state = CONN_STATE_LOGIN_FULL
-	case CONN_STATE_SECURITY:
+	} else {
+		conn.state = CONN_STATE_LOGIN
 	}
 
-	conn.state = CONN_STATE_LOGIN_FULL
+	conn.resp = &ISCSICommand{
-	conn.expCmdSN = cmd.CmdSN
+		OpCode:   OpLoginResp,
-	conn.statSN += 1
+		Transit:  conn.loginParam.tgtTrans,
-
+		CSG:      cmd.CSG,
-	switch conn.sessionType {
+		NSG:      conn.loginParam.tgtNSG,
-	case SESSION_NORMAL:
+		StatSN:   cmd.ExpStatSN,
-		if conn.session == nil {
+		TaskTag:  cmd.TaskTag,
-			// create a new session
+		ExpCmdSN: cmd.CmdSN,
-			sess, err := s.NewISCSISession(conn, cmd.ISID)
+		MaxCmdSN: cmd.CmdSN,
-			if err != nil {
+		RawData:  util.MarshalKVText(negoKeys),
-				log.Error(err)
-				return err
-			}
-			itnexus := &api.ITNexus{uuid.NewV1(), GeniSCSIITNexusID(sess)}
-			scsi.AddITNexus(&sess.Target.SCSITarget, itnexus)
-			sess.ITNexusID = itnexus.ID
-			conn.session = sess
-		}
-	case SESSION_DISCOVERY:
-
 	}
 
 	return nil
@@ -476,14 +469,13 @@ func iscsiExecReject(conn *iscsiConnection) error {
 }
 
 func iscsiExecR2T(conn *iscsiConnection) error {
+	var val uint
 	conn.txTask = &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, scmd: &api.SCSICommand{}}
 	conn.txIOState = IOSTATE_TX_BHS
 	conn.statSN += 1
 	task := conn.rxTask
 	resp := &ISCSICommand{
 		OpCode:        OpReady,
-		Immediate:     true,
-		Final:         true,
 		StatSN:        conn.req.ExpStatSN,
 		TaskTag:       conn.req.TaskTag,
 		ExpCmdSN:      conn.session.ExpCmdSN,
@@ -492,7 +484,8 @@ func iscsiExecR2T(conn *iscsiConnection) error {
 		BufferOffset:  uint32(task.offset),
 		DesiredLength: uint32(task.r2tCount),
 	}
-	if val := sessionKeys[ISCSI_PARAM_MAX_BURST].def; task.r2tCount > int(val) {
+
+	if val = conn.loginParam.sessionParam[ISCSI_PARAM_MAX_BURST].Value; task.r2tCount > int(val) {
 		resp.DesiredLength = uint32(val)
 	}
 	conn.resp = resp
@@ -506,8 +499,8 @@ func (s *ISCSITargetDriver) txHandler(conn *iscsiConnection) {
 		final   bool = false
 	)
 	if conn.state == CONN_STATE_SCSI {
-		hdigest = conn.sessionParam[ISCSI_PARAM_HDRDGST_EN].Value & DIGEST_CRC32C
+		hdigest = conn.loginParam.sessionParam[ISCSI_PARAM_HDRDGST_EN].Value & DIGEST_CRC32C
-		ddigest = conn.sessionParam[ISCSI_PARAM_DATADGST_EN].Value & DIGEST_CRC32C
+		ddigest = conn.loginParam.sessionParam[ISCSI_PARAM_DATADGST_EN].Value & DIGEST_CRC32C
 	}
 	if conn.state == CONN_STATE_SCSI && conn.txTask == nil {
 		err := s.scsiCommandHandler(conn)
@@ -564,10 +557,14 @@ func (s *ISCSITargetDriver) txHandler(conn *iscsiConnection) {
 	case CONN_STATE_SECURITY_LOGIN:
 		conn.state = CONN_STATE_LOGIN
 		log.Debugf("CONN_STATE_LOGIN")
+	case CONN_STATE_LOGIN:
+		log.Debugf("CONN_STATE_LOGIN")
+		conn.rxIOState = IOSTATE_RX_BHS
+		s.handler(DATAIN, conn)
 	case CONN_STATE_SECURITY_FULL, CONN_STATE_LOGIN_FULL:
-		if conn.sessionType == SESSION_NORMAL {
+		if conn.session.SessionType == SESSION_NORMAL {
 			conn.state = CONN_STATE_KERNEL
-			log.Infof("CONN_STATE_KERNEL")
+			log.Debugf("CONN_STATE_KERNEL")
 			conn.state = CONN_STATE_SCSI
 			log.Debugf("CONN_STATE_SCSI")
 		} else {
@@ -583,7 +580,6 @@ func (s *ISCSITargetDriver) txHandler(conn *iscsiConnection) {
 		conn.rxIOState = IOSTATE_RX_BHS
 		s.handler(DATAIN, conn)
 	}
-	log.Infof("%d", conn.state)
 }
 
 func (s *ISCSITargetDriver) scsiCommandHandler(conn *iscsiConnection) (err error) {
@@ -604,15 +600,24 @@ func (s *ISCSITargetDriver) scsiCommandHandler(conn *iscsiConnection) (err error
 			if task.scmd.OutSDBBuffer.Buffer == nil {
 				task.scmd.OutSDBBuffer.Buffer = bytes.NewBuffer([]byte{})
 			}
-			task.scmd.OutSDBBuffer.Buffer.Write(conn.req.RawData)
+			if conn.session.SessionParam[ISCSI_PARAM_IMM_DATA_EN].Value == 1 {
+				task.scmd.OutSDBBuffer.Buffer.Write(conn.req.RawData)
+			}
 			if task.r2tCount > 0 {
 				// prepare to receive more data
 				conn.session.ExpCmdSN += 1
 				task.state = taskPending
 				conn.session.PendingTasks.Push(task)
 				conn.rxTask = task
-				iscsiExecR2T(conn)
+				if conn.session.SessionParam[ISCSI_PARAM_INITIAL_R2T_EN].Value == 1 {
-				break
+					iscsiExecR2T(conn)
+					break
+				} else {
+					log.Debugf("Not ready to exec the task")
+					conn.rxIOState = IOSTATE_RX_BHS
+					s.handler(DATAIN, conn)
+					return nil
+				}
 			}
 		}
 		task.offset = 0
@@ -691,7 +696,12 @@ func (s *ISCSITargetDriver) scsiCommandHandler(conn *iscsiConnection) (err error
 			return nil
 		} else if task.r2tCount > 0 {
 			// prepare to receive more data
-			task.r2tSN += 1
+			if task.unsolCount == 0 {
+				task.r2tSN += 1
+			} else {
+				task.r2tSN = 0
+				task.unsolCount = 0
+			}
 			conn.rxTask = task
 			iscsiExecR2T(conn)
 			break
@@ -803,12 +813,12 @@ func (s *ISCSITargetDriver) iscsiExecTask(task *iscsiTask) error {
 				task.scmd.Direction = api.SCSIDataWrite
 			}
 		}
-		task.scmd.ITNexusID = task.conn.session.ITNexusID
+		task.scmd.ITNexusID = task.conn.session.ITNexus.ID
 		task.scmd.SCB = bytes.NewBuffer(cmd.CDB)
 		task.scmd.SCBLength = len(cmd.CDB)
 		task.scmd.Lun = cmd.LUN
 		task.scmd.Tag = uint64(cmd.TaskTag)
-		task.scmd.RelTargetPortID = task.conn.tpgt
+		task.scmd.RelTargetPortID = task.conn.session.TPGT
 		task.state = taskSCSI
 		if task.scmd.OutSDBBuffer.Buffer == nil {
 			task.scmd.OutSDBBuffer.Buffer = bytes.NewBuffer(cmd.RawData)

pkg/port/iscsit/iscsit.go

@@ -20,6 +20,7 @@ package iscsit
 import (
 	"fmt"
 	"strings"
+	"sync"
 
 	"github.com/gostor/gotgt/pkg/api"
 )
@@ -81,15 +82,14 @@ type ISCSITarget struct {
 	// TPGT number is the key
 	TPGTs map[uint16]*iSCSITPGT
 	// TSIH is the key
-	Sessions     map[uint16]*ISCSISession
+	Sessions        map[uint16]*ISCSISession
-	SessionParam []ISCSISessionParam
+	SessionsRWMutex sync.RWMutex
-	Alias        string
+	Alias           string
-	MaxSessions  int
+	MaxSessions     int
-	RedirectInfo ISCSIRedirectInfo
+	RedirectInfo    ISCSIRedirectInfo
-	Rdma         int
+	Rdma            int
-	NopInterval  int
+	NopInterval     int
-	NopCount     int
+	NopCount        int
-	TSIHCounter  uint16
 }
 
 /*
@@ -121,10 +121,9 @@ func (tgt *ISCSITarget) FindTPG(portal string) (uint16, error) {
 
 func newISCSITarget(target *api.SCSITarget) *ISCSITarget {
 	return &ISCSITarget{
-		SCSITarget:  *target,
+		SCSITarget: *target,
-		TPGTs:       make(map[uint16]*iSCSITPGT),
+		TPGTs:      make(map[uint16]*iSCSITPGT),
-		Sessions:    make(map[uint16]*ISCSISession),
+		Sessions:   make(map[uint16]*ISCSISession),
-		TSIHCounter: 1,
 	}
 }
 

pkg/port/iscsit/login.go

@@ -2,10 +2,174 @@ package iscsit
 
 import (
 	"bytes"
+	"fmt"
 
+	//	log "github.com/Sirupsen/logrus"
 	"github.com/gostor/gotgt/pkg/util"
 )
 
+var (
+	iSCSILoginParamTextKV = []util.KeyValue{
+		{"HeaderDigest", "None"},
+		{"DataDigest", "None"},
+		{"ImmediateData", "Yes"},
+		{"InitialR2T", "Yes"},
+		{"MaxBurstLength", "262144"},
+		{"FirstBurstLength", "65536"},
+		{"DefaultTime2Wait", "2"},
+		{"DefaultTime2Retain", "0"},
+		{"MaxOutstandingR2T", "1"},
+		{"IFMarker", "No"},
+		{"OFMarker", "No"},
+		{"DataPDUInOrder", "Yes"},
+		{"DataSequenceInOrder", "Yes"}}
+)
+
+type iSCSILoginStage int
+
+const (
+	SecurityNegotiation         iSCSILoginStage = 0
+	LoginOperationalNegotiation                 = 1
+	FullFeaturePhase                            = 3
+)
+
+func (s iSCSILoginStage) String() string {
+	switch s {
+	case SecurityNegotiation:
+		return "Security Negotiation"
+	case LoginOperationalNegotiation:
+		return "Login Operational Negotiation"
+	case FullFeaturePhase:
+		return "Full Feature Phase"
+	}
+	return "Unknown Stage"
+}
+
+func loginKVDeclare(conn *iscsiConnection, negoKV []util.KeyValue) []util.KeyValue {
+
+	negoKV = append(negoKV, util.KeyValue{"TargetPortalGroupTag",
+		numberKeyInConv(uint(conn.loginParam.tpgt))})
+	negoKV = append(negoKV, util.KeyValue{"MaxRecvDataSegmentLength",
+		numberKeyInConv(sessionKeys["MaxRecvDataSegmentLength"].def)})
+	return negoKV
+
+}
+func loginKVProcess(conn *iscsiConnection, loginKV map[string]string) ([]util.KeyValue, error) {
+	var (
+		uintVal    uint
+		ok         bool
+		defSessKey *iscsiSessionKeys
+		negoKV     []util.KeyValue
+		kvChanges  int
+	)
+
+	for key, val := range loginKV {
+		// The MaxRecvDataSegmentLength of initiator
+		// is the MaxXmitDataSegmentLength of target
+		if key == "MaxRecvDataSegmentLength" {
+			defSessKey, ok = sessionKeys["MaxXmitDataSegmentLength"]
+			uintVal, ok = defSessKey.conv(val)
+			conn.loginParam.sessionParam[defSessKey.idx].Value = uintVal
+			continue
+		}
+
+		if key == "InitiatorName" {
+			conn.loginParam.initiator = val
+			continue
+		} else if key == "InitiatorAlias" {
+			conn.loginParam.initiatorAlias = val
+			continue
+		} else if key == "TargetName" {
+			conn.loginParam.target = val
+			continue
+		} else if key == "SessionType" {
+			if val == "Normal" {
+				conn.loginParam.sessionType = SESSION_NORMAL
+			} else {
+				conn.loginParam.sessionType = SESSION_DISCOVERY
+			}
+			continue
+		}
+
+		defSessKey, ok = sessionKeys[key]
+		if ok {
+			uintVal, ok = defSessKey.conv(val)
+
+			//hack here
+			if key == "HeaderDigest" || key == "DataDigest" {
+				if uintVal == DIGEST_ALL {
+					uintVal = DIGEST_NONE
+				}
+			}
+			if ok {
+				if defSessKey.constValue {
+					//the Negotiation Key cannot be changed! Uses Target default key
+					if uintVal != defSessKey.def {
+						kvChanges++
+					}
+					negoKV = append(negoKV, util.KeyValue{key, defSessKey.inConv(defSessKey.def)})
+				} else {
+					if (uintVal >= defSessKey.min) && (uintVal <= defSessKey.max) {
+						conn.loginParam.sessionParam[defSessKey.idx].Value = uintVal
+						negoKV = append(negoKV, util.KeyValue{key, defSessKey.inConv(uintVal)})
+					} else {
+						// the value out of the acceptable range, Uses target default key
+						negoKV = append(negoKV, util.KeyValue{key, defSessKey.inConv(defSessKey.def)})
+						kvChanges++
+					}
+				}
+			}
+		} else {
+			//Unknown Key, reject it
+			return negoKV, fmt.Errorf("Unknowen Nego KV [%s:%s]", key, val)
+		}
+	}
+
+	if kvChanges == 0 {
+		if (conn.loginParam.iniNSG == FullFeaturePhase) && conn.loginParam.iniTrans {
+			conn.loginParam.tgtNSG = FullFeaturePhase
+			conn.loginParam.tgtTrans = true
+		} else {
+			//Currently, we just reject these kind of cases
+			return negoKV, fmt.Errorf("reject CSG=%d,NSG=%d,trans",
+				conn.loginParam.iniCSG, conn.loginParam.iniNSG, conn.loginParam.iniTrans)
+		}
+	} else {
+		conn.loginParam.tgtNSG = LoginOperationalNegotiation
+		conn.loginParam.tgtTrans = false
+	}
+	return negoKV, nil
+}
+
+type iscsiLoginParam struct {
+	paramInit bool
+
+	iniCSG   iSCSILoginStage
+	iniNSG   iSCSILoginStage
+	iniTrans bool
+	iniCont  bool
+
+	tgtCSG   iSCSILoginStage
+	tgtNSG   iSCSILoginStage
+	tgtTrans bool
+	tgtCont  bool
+
+	sessionType  int
+	sessionParam ISCSISessionParamList
+	keyDeclared  bool
+
+	initiator      string
+	initiatorAlias string
+	target         string
+	targetAlias    string
+
+	tpgt uint16
+	isid uint64
+	tsih uint16
+
+	authMethod AuthMethod
+}
+
 func (m *ISCSICommand) loginRespBytes() []byte {
 	// rfc7143 11.13
 	buf := &bytes.Buffer{}
@@ -51,23 +215,3 @@ func (m *ISCSICommand) loginRespBytes() []byte {
 	buf.Write(rd)
 	return buf.Bytes()
 }
-
-type Stage int
-
-const (
-	SecurityNegotiation         Stage = 0
-	LoginOperationalNegotiation       = 1
-	FullFeaturePhase                  = 3
-)
-
-func (s Stage) String() string {
-	switch s {
-	case SecurityNegotiation:
-		return "Security Negotiation"
-	case LoginOperationalNegotiation:
-		return "Login Operational Negotiation"
-	case FullFeaturePhase:
-		return "Full Feature Phase"
-	}
-	return "Unknown Stage"
-}

pkg/port/iscsit/session.go

@@ -18,9 +18,13 @@ package iscsit
 
 import (
 	"fmt"
-	"math/rand"
+	"strconv"
-	"time"
+	"strings"
+	"sync"
 
+	log "github.com/Sirupsen/logrus"
+	"github.com/gostor/gotgt/pkg/api"
+	"github.com/gostor/gotgt/pkg/scsi"
 	"github.com/satori/go.uuid"
 )
 
@@ -73,9 +77,27 @@ const (
 )
 
 type ISCSISessionParam struct {
+	idx   uint
 	State int
 	Value uint
 }
+type ISCSISessionParamList []ISCSISessionParam
+
+func (list ISCSISessionParamList) Len() int {
+	return len(list)
+}
+
+func (list ISCSISessionParamList) Less(i, j int) bool {
+	if list[i].idx <= list[j].idx {
+		return true
+	} else {
+		return false
+	}
+}
+
+func (list ISCSISessionParamList) Swap(i, j int) {
+	list[i], list[j] = list[j], list[i]
+}
 
 /*
  * The defaults here are according to the spec and must not be changed,
@@ -86,63 +108,130 @@ type ISCSISessionParam struct {
  * to remember the RDSL of the initiator, which defaults to 8k if he has
  * not told us otherwise.
  */
+type KeyConvFunc func(value string) (uint, bool)
+type KeyInConvFunc func(value uint) string
+
 type iscsiSessionKeys struct {
-	name string
+	idx        uint
-	def  uint
+	constValue bool
-	min  uint
+	def        uint
-	max  uint
+	min        uint
+	max        uint
+	conv       KeyConvFunc
+	inConv     KeyInConvFunc
 }
 
-var sessionKeys []iscsiSessionKeys = []iscsiSessionKeys{
+func digestKeyConv(value string) (uint, bool) {
+	var crc uint
+	valueArray := strings.Split(value, ",")
+	if len(valueArray) == 0 {
+		return crc, false
+	}
+	for _, tmpV := range valueArray {
+		if strings.EqualFold(tmpV, "crc32c") {
+			crc |= DIGEST_CRC32C
+		} else if strings.EqualFold(tmpV, "none") {
+			crc |= DIGEST_NONE
+		} else {
+			return crc, false
+		}
+	}
+
+	return crc, true
+}
+
+func digestKeyInConv(value uint) string {
+	str := ""
+	switch value {
+	case DIGEST_NONE:
+		str = "None"
+	case DIGEST_CRC32C:
+		str = "CRC32C"
+	case DIGEST_ALL:
+		str = "None,CRC32C"
+	}
+	return str
+}
+
+func numberKeyConv(value string) (uint, bool) {
+	v, err := strconv.Atoi(value)
+	if err == nil {
+		return uint(v), true
+	}
+	return uint(v), false
+}
+
+func numberKeyInConv(value uint) string {
+	s := strconv.Itoa(int(value))
+	return s
+}
+
+func boolKeyConv(value string) (uint, bool) {
+	if strings.EqualFold(value, "yes") {
+		return 1, true
+	} else if strings.EqualFold(value, "no") {
+		return 0, true
+	}
+	return 0, false
+}
+
+func boolKeyInConv(value uint) string {
+	if value == 0 {
+		return "No"
+	}
+	return "Yes"
+}
+
+var sessionKeys map[string]*iscsiSessionKeys = map[string]*iscsiSessionKeys{
 	// ISCSI_PARAM_MAX_RECV_DLENGTH
-	{"MaxRecvDataSegmentLength", 8192, 512, 16777215},
+	"MaxRecvDataSegmentLength": {ISCSI_PARAM_MAX_RECV_DLENGTH, true, 32768, 512, 16777215, numberKeyConv, numberKeyInConv},
 	// ISCSI_PARAM_HDRDGST_EN
-	{"HeaderDigest", DIGEST_NONE, DIGEST_NONE, DIGEST_ALL},
+	"HeaderDigest": {ISCSI_PARAM_HDRDGST_EN, false, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv},
 	// ISCSI_PARAM_DATADGST_EN
-	{"DataDigest", DIGEST_NONE, DIGEST_NONE, DIGEST_ALL},
+	"DataDigest": {ISCSI_PARAM_DATADGST_EN, false, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv},
 	// ISCSI_PARAM_INITIAL_R2T_EN
-	{"InitialR2T", 1, 0, 1},
+	"InitialR2T": {ISCSI_PARAM_INITIAL_R2T_EN, false, 1, 0, 1, boolKeyConv, boolKeyInConv},
 	// ISCSI_PARAM_MAX_R2T
-	{"MaxOutstandingR2T", 1, 1, 65535},
+	"MaxOutstandingR2T": {ISCSI_PARAM_MAX_R2T, true, 1, 1, 65535, numberKeyConv, numberKeyInConv},
 	// ISCSI_PARAM_IMM_DATA_EN
-	{"ImmediateData", 1, 0, 1},
+	"ImmediateData": {ISCSI_PARAM_IMM_DATA_EN, true, 1, 0, 1, boolKeyConv, boolKeyInConv},
 	// ISCSI_PARAM_FIRST_BURST
-	{"FirstBurstLength", 65536, 512, 16777215},
+	"FirstBurstLength": {ISCSI_PARAM_FIRST_BURST, true, 65536, 512, 16777215, numberKeyConv, numberKeyInConv},
 	// ISCSI_PARAM_MAX_BURST
-	{"MaxBurstLength", 262144, 512, 16777215},
+	"MaxBurstLength": {ISCSI_PARAM_MAX_BURST, true, 262144, 512, 16777215, numberKeyConv, numberKeyInConv},
 	// ISCSI_PARAM_PDU_INORDER_EN
-	{"DataPDUInOrder", 1, 0, 1},
+	"DataPDUInOrder": {ISCSI_PARAM_PDU_INORDER_EN, true, 1, 0, 1, numberKeyConv, numberKeyInConv},
 	// ISCSI_PARAM_DATASEQ_INORDER_EN
-	{"DataSequenceInOrder", 1, 0, 1},
+	"DataSequenceInOrder": {ISCSI_PARAM_DATASEQ_INORDER_EN, true, 1, 0, 1, numberKeyConv, numberKeyInConv},
 	// ISCSI_PARAM_ERL
-	{"ErrorRecoveryLevel", 0, 0, 2},
+	"ErrorRecoveryLevel": {ISCSI_PARAM_ERL, true, 0, 0, 2, numberKeyConv, numberKeyInConv},
 	// ISCSI_PARAM_IFMARKER_EN
-	{"IFMarker", 0, 0, 1},
+	"IFMarker": {ISCSI_PARAM_IFMARKER_EN, true, 0, 0, 1, boolKeyConv, boolKeyInConv},
 	// ISCSI_PARAM_OFMARKER_EN
-	{"OFMarker", 0, 0, 1},
+	"OFMarker": {ISCSI_PARAM_OFMARKER_EN, true, 0, 0, 1, boolKeyConv, boolKeyInConv},
 	// ISCSI_PARAM_DEFAULTTIME2WAIT
-	{"DefaultTime2Wait", 2, 0, 3600},
+	"DefaultTime2Wait": {ISCSI_PARAM_DEFAULTTIME2WAIT, true, 2, 0, 3600, numberKeyConv, numberKeyInConv},
 	// ISCSI_PARAM_DEFAULTTIME2RETAIN
-	{"DefaultTime2Retain", 20, 0, 3600},
+	"DefaultTime2Retain": {ISCSI_PARAM_DEFAULTTIME2RETAIN, false, 20, 0, 3600, numberKeyConv, numberKeyInConv},
 	// ISCSI_PARAM_OFMARKINT
-	{"OFMarkInt", 2048, 1, 65535},
+	"OFMarkInt": {ISCSI_PARAM_OFMARKINT, true, 2048, 1, 65535, numberKeyConv, numberKeyInConv},
 	// ISCSI_PARAM_IFMARKINT
-	{"IFMarkInt", 2048, 1, 65535},
+	"IFMarkInt": {ISCSI_PARAM_IFMARKINT, true, 2048, 1, 65535, numberKeyConv, numberKeyInConv},
 	// ISCSI_PARAM_MAXCONNECTIONS
-	{"MaxConnections", 1, 1, 65535},
+	"MaxConnections": {ISCSI_PARAM_MAXCONNECTIONS, true, 1, 1, 65535, numberKeyConv, numberKeyInConv},
 	// ISCSI_PARAM_RDMA_EXTENSIONS
-	{"RDMAExtensions", 0, 0, 1},
+	"RDMAExtensions": {ISCSI_PARAM_RDMA_EXTENSIONS, true, 0, 0, 1, boolKeyConv, boolKeyInConv},
 	// ISCSI_PARAM_TARGET_RDSL
-	{"TargetRecvDataSegmentLength", 8192, 512, 16777215},
+	"TargetRecvDataSegmentLength": {ISCSI_PARAM_TARGET_RDSL, true, 8192, 512, 16777215, numberKeyConv, numberKeyInConv},
 	// ISCSI_PARAM_INITIATOR_RDSL
-	{"InitiatorRecvDataSegmentLength", 8192, 512, 16777215},
+	"InitiatorRecvDataSegmentLength": {ISCSI_PARAM_INITIATOR_RDSL, true, 8192, 512, 16777215, numberKeyConv, numberKeyInConv},
 	// ISCSI_PARAM_MAX_OUTST_PDU
-	{"MaxOutstandingUnexpectedPDUs", 0, 2, 4294967295},
+	"MaxOutstandingUnexpectedPDUs": {ISCSI_PARAM_MAX_OUTST_PDU, true, 0, 2, 4294967295, numberKeyConv, numberKeyInConv},
 	// "local" parmas, never sent to the initiator
 	// ISCSI_PARAM_MAX_XMIT_DLENGTH
-	{"MaxXmitDataSegmentLength", 8192, 512, 16777215},
+	"MaxXmitDataSegmentLength": {ISCSI_PARAM_MAX_XMIT_DLENGTH, true, 8192, 512, 16777215, numberKeyConv, numberKeyInConv},
 	// ISCSI_PARAM_MAX_QUEUE_CMD
-	{"MaxQueueCmd", MAX_QUEUE_CMD_DEF, MAX_QUEUE_CMD_MIN, MAX_QUEUE_CMD_MAX},
+	"MaxQueueCmd": {ISCSI_PARAM_MAX_QUEUE_CMD, true, MAX_QUEUE_CMD_DEF, MAX_QUEUE_CMD_MIN, MAX_QUEUE_CMD_MAX, numberKeyConv, numberKeyInConv},
 }
 
 // Session is an iSCSI session.
@@ -152,18 +241,22 @@ type ISCSISession struct {
 	InitiatorAlias string
 	Target         *ISCSITarget
 	ISID           uint64
-	TSIH           uint64
+	TSIH           uint16
-	ITNexusID      uuid.UUID
+	TPGT           uint16
+	SessionType    int
+	ITNexus        *api.ITNexus
 
 	ExpCmdSN uint32
-	// only one connection per session
+	MaxCmdSN uint32
-	Connections     []*iscsiConnection
+	// currently, this is only one connection per session
-	Commands        []*ISCSICommand
+	Connections        map[uint16]*iscsiConnection
-	PendingTasks    taskQueue
+	ConnectionsRWMutex sync.RWMutex
-	MaxQueueCommand uint32
+	Commands           []*ISCSICommand
-	SessionParam    []ISCSISessionParam
+	PendingTasks       taskQueue
-	Info            string
+	MaxQueueCommand    uint32
-	Rdma            int
+	SessionParam       ISCSISessionParamList
+	Info               string
+	Rdma               int
 }
 
 type taskQueue []*iscsiTask
@@ -192,11 +285,140 @@ func (tq *taskQueue) Pop() interface{} {
 	return item
 }
 
+func (s *ISCSITargetDriver) LookupISCSISession(tgtName string, iniName string, isid uint64, tsih uint16, tpgt uint16) *ISCSISession {
+	var (
+		tgt  *ISCSITarget
+		sess *ISCSISession
+		ok   bool
+	)
+	tgt, ok = s.iSCSITargets[tgtName]
+	if !ok {
+		return nil
+	}
+	tgt.SessionsRWMutex.RLock()
+	defer tgt.SessionsRWMutex.RUnlock()
+	sess, ok = tgt.Sessions[tsih]
+	if !ok {
+		return nil
+	}
+	if (sess.ISID == isid) && (sess.TPGT == tpgt) {
+		return sess
+	}
+	return nil
+}
+
+func (s *ISCSITargetDriver) UnBindISCSISession(sess *ISCSISession) {
+	target := sess.Target
+	target.SessionsRWMutex.Lock()
+	defer target.SessionsRWMutex.Unlock()
+	delete(target.Sessions, sess.TSIH)
+	scsi.RemoveITNexus(&sess.Target.SCSITarget, sess.ITNexus)
+}
+
+func (s *ISCSITargetDriver) BindISCSISession(conn *iscsiConnection) error {
+	var (
+		target    *ISCSITarget
+		existSess *ISCSISession
+		existConn *iscsiConnection
+		newSess   *ISCSISession
+		tpgt      uint16
+		err       error
+	)
+
+	//Find TPGT and Target ID
+	if conn.loginParam.sessionType == SESSION_DISCOVERY {
+		conn.tid = 0xffff
+	} else {
+		for _, t := range s.iSCSITargets {
+			if t.SCSITarget.Name == conn.loginParam.target {
+				target = t
+				break
+			}
+		}
+		if target == nil {
+			return fmt.Errorf("No target found with name(%s)", conn.loginParam.target)
+		}
+
+		tpgt, err = target.FindTPG(conn.conn.LocalAddr().String())
+		if err != nil {
+			return err
+		}
+		conn.loginParam.tpgt = tpgt
+		conn.tid = target.TID
+	}
+
+	existSess = s.LookupISCSISession(conn.loginParam.target, conn.loginParam.initiator,
+		conn.loginParam.isid, conn.loginParam.tsih, conn.loginParam.tpgt)
+	if existSess != nil {
+		existConn = existSess.LookupConnection(conn.cid)
+	}
+
+	if conn.loginParam.sessionType == SESSION_DISCOVERY &&
+		conn.loginParam.tsih != ISCSI_UNSPEC_TSIH &&
+		existSess != nil {
+		return fmt.Errorf("initiator err, invalid request")
+	}
+
+	if existSess == nil && conn.loginParam.tsih != 0 &&
+		existSess.TSIH != conn.loginParam.tsih {
+		return fmt.Errorf("initiator err, no session")
+	}
+
+	if existSess == nil {
+		newSess, err = s.NewISCSISession(conn)
+		if err != nil {
+			return err
+		}
+
+		if newSess.SessionType == SESSION_NORMAL {
+			log.Infof("New Session initiator name:%v,target name:%v,ISID:0x%x",
+				conn.loginParam.initiator, conn.loginParam.target, conn.loginParam.isid)
+			//register normal session
+			itnexus := &api.ITNexus{uuid.NewV1(), GeniSCSIITNexusID(newSess)}
+			scsi.AddITNexus(&newSess.Target.SCSITarget, itnexus)
+			newSess.ITNexus = itnexus
+			conn.session = newSess
+
+			newSess.Target.SessionsRWMutex.Lock()
+			newSess.Target.Sessions[newSess.TSIH] = newSess
+			newSess.Target.SessionsRWMutex.Unlock()
+		} else {
+			conn.session = newSess
+		}
+	} else {
+		if conn.loginParam.tsih == ISCSI_UNSPEC_TSIH {
+			log.Infof("Session Reinstatement initiator name:%v,target name:%v,ISID:0x%x",
+				conn.loginParam.initiator, conn.loginParam.target, conn.loginParam.isid)
+			newSess, err = s.ReInstatement(existConn.session, conn)
+			if err != nil {
+				return err
+			}
+
+			itnexus := &api.ITNexus{uuid.NewV1(), GeniSCSIITNexusID(newSess)}
+			scsi.AddITNexus(&newSess.Target.SCSITarget, itnexus)
+			newSess.ITNexus = itnexus
+			conn.session = newSess
+
+			newSess.Target.SessionsRWMutex.Lock()
+			newSess.Target.Sessions[newSess.TSIH] = newSess
+			newSess.Target.SessionsRWMutex.Unlock()
+		} else {
+			if existConn != nil {
+				log.Infof("Connection Reinstatement initiator name:%v,target name:%v,ISID:0x%x,CID:%v",
+					conn.loginParam.initiator, conn.loginParam.target, conn.loginParam.isid)
+				existConn.ReInstatement(conn)
+			}
+		}
+	}
+
+	return nil
+}
+
 // New creates a new session.
-func (s *ISCSITargetDriver) NewISCSISession(conn *iscsiConnection, isid uint64) (*ISCSISession, error) {
+func (s *ISCSITargetDriver) NewISCSISession(conn *iscsiConnection) (*ISCSISession, error) {
 	var (
 		target *ISCSITarget
-		tsih   uint64
+		tsih   uint16
 	)
 
 	for _, t := range s.iSCSITargets {
@@ -205,40 +427,54 @@ func (s *ISCSITargetDriver) NewISCSISession(conn *iscsiConnection, isid uint64)
 			break
 		}
 	}
-	if target == nil {
+	if target == nil && conn.tid != 0xffff {
 		return nil, fmt.Errorf("No target found with tid(%d)", conn.tid)
 	}
 
-	for {
+	tsih = s.AllocTSIH()
-		rand.Seed(int64(time.Now().UTC().Nanosecond()))
+	if tsih == ISCSI_UNSPEC_TSIH {
-		tsih = uint64(rand.Uint32())
+		return nil, fmt.Errorf("TSIH Pool exhausted tid(%d)", conn.tid)
-		for _, s := range target.Sessions {
-			if s.TSIH == tsih {
-				tsih = 0
-				break
-			}
-		}
-		if tsih != 0 {
-			break
-		}
 	}
 
 	sess := &ISCSISession{
 		TSIH:            tsih,
-		ISID:            isid,
+		ISID:            conn.loginParam.isid,
-		Initiator:       conn.initiator,
+		TPGT:            conn.loginParam.tpgt,
-		InitiatorAlias:  conn.initiatorAlias,
+		Initiator:       conn.loginParam.initiator,
+		InitiatorAlias:  conn.loginParam.initiatorAlias,
+		SessionType:     conn.loginParam.sessionType,
 		Target:          target,
-		Connections:     []*iscsiConnection{conn},
+		Connections:     map[uint16]*iscsiConnection{conn.cid: conn},
-		SessionParam:    conn.sessionParam,
+		SessionParam:    conn.loginParam.sessionParam,
-		MaxQueueCommand: uint32(conn.sessionParam[ISCSI_PARAM_MAX_QUEUE_CMD].Value),
+		MaxQueueCommand: uint32(conn.loginParam.sessionParam[ISCSI_PARAM_MAX_QUEUE_CMD].Value),
 		Rdma:            0,
 		ExpCmdSN:        conn.expCmdSN,
 	}
-	conn.session = sess
 	return sess, nil
 }
 
+func (sess *ISCSISession) LookupConnection(cid uint16) *iscsiConnection {
+	sess.ConnectionsRWMutex.RLock()
+	defer sess.ConnectionsRWMutex.RUnlock()
+	conn := sess.Connections[cid]
+	return conn
+}
+
+func (s *ISCSITargetDriver) ReInstatement(existSess *ISCSISession, conn *iscsiConnection) (*ISCSISession, error) {
+	newSess, err := s.NewISCSISession(conn)
+	if err != nil {
+		return nil, err
+	}
+	newSess.ExpCmdSN = existSess.ExpCmdSN
+	newSess.MaxCmdSN = existSess.MaxCmdSN + 1
+	s.UnBindISCSISession(existSess)
+	for _, tmpConn := range existSess.Connections {
+		tmpConn.close()
+	}
+	existSess.Connections = map[uint16]*iscsiConnection{}
+	return newSess, nil
+}
+
 /*
  * iSCSI I_T nexus identifer = (iSCSI Initiator Name + 'i' + ISID, iSCSI Target Name + 't' + Portal Group Tag)
  */
@@ -246,6 +482,6 @@ func GeniSCSIITNexusID(sess *ISCSISession) string {
 	strID := fmt.Sprintf("%si0x%12x,%st%d",
 		sess.Initiator, sess.ISID,
 		sess.Target.SCSITarget.Name,
-		sess.Connections[0].tpgt)
+		sess.TPGT)
 	return strID
 }

pkg/scsi/backingstore.go

@@ -181,7 +181,6 @@ verify:
 			bs.DataAdvise(int64(offset), int64(length), util.POSIX_FADV_WILLNEED)
 		}
 	}
-	log.Infof("io done %s", string(scb))
 	return nil
 sense:
 	if err != nil {