diff --git a/pkg/port/iscsit/cmd.go b/pkg/port/iscsit/cmd.go index 3090045..10ce684 100644 --- a/pkg/port/iscsit/cmd.go +++ b/pkg/port/iscsit/cmd.go @@ -81,7 +81,7 @@ type ISCSICommand struct { // Continue bit. Cont bool // Current Stage, Next Stage. - CSG, NSG Stage + CSG, NSG iSCSILoginStage // Initiator part of the SSID. ISID uint64 // Target-assigned Session Identifying Handle. @@ -214,8 +214,8 @@ func parseHeader(data []byte) (*ISCSICommand, error) { // rfc7143 11.12.2 return nil, fmt.Errorf("transit and continue bits set in same login request") } - m.CSG = Stage(data[1]&0xc) >> 2 - m.NSG = Stage(data[1] & 0x3) + m.CSG = iSCSILoginStage(data[1]&0xc) >> 2 + m.NSG = iSCSILoginStage(data[1] & 0x3) m.ISID = uint64(ParseUint(data[8:14])) m.TSIH = uint16(ParseUint(data[14:16])) m.ConnID = uint16(ParseUint(data[20:22])) @@ -228,8 +228,8 @@ func parseHeader(data []byte) (*ISCSICommand, error) { // rfc7143 11.12.2 return nil, fmt.Errorf("transit and continue bits set in same login request") } - m.CSG = Stage(data[1]&0xc) >> 2 - m.NSG = Stage(data[1] & 0x3) + m.CSG = iSCSILoginStage(data[1]&0xc) >> 2 + m.NSG = iSCSILoginStage(data[1] & 0x3) m.StatSN = uint32(ParseUint(data[24:28])) m.ExpCmdSN = uint32(ParseUint(data[28:32])) m.MaxCmdSN = uint32(ParseUint(data[32:36])) @@ -421,6 +421,7 @@ func (m *ISCSICommand) scsiTMFRespBytes() []byte { } func (m *ISCSICommand) r2tRespBytes() []byte { + // rfc7143 11.8 buf := &bytes.Buffer{} buf.WriteByte(byte(OpReady)) diff --git a/pkg/port/iscsit/conn.go b/pkg/port/iscsit/conn.go index e607f15..927e8d6 100644 --- a/pkg/port/iscsit/conn.go +++ b/pkg/port/iscsit/conn.go @@ -18,6 +18,7 @@ package iscsit import ( "net" + "sort" "sync" "github.com/gostor/gotgt/pkg/api" @@ -48,26 +49,23 @@ var ( ) type iscsiConnection struct { - state int - authState int - session *ISCSISession - sessionType int - sessionParam []ISCSISessionParam - tid int - CID uint16 - rxIOState int - txIOState int - refcount int - conn net.Conn - initiator string - initiatorAlias string - tpgt uint16 + state int + authState int + session *ISCSISession + tid int + cid uint16 + rxIOState int + txIOState int + refcount int + conn net.Conn rxBuffer []byte txBuffer []byte req *ISCSICommand resp *ISCSICommand + loginParam *iscsiLoginParam + // StatSN - the status sequence number on this connection statSN uint32 // ExpStatSN - the expected status sequence number on this connection @@ -82,8 +80,6 @@ type iscsiConnection struct { rxTask *iscsiTask txTask *iscsiTask - authMethod AuthMethod - readLock *sync.RWMutex } @@ -113,10 +109,14 @@ func (c *iscsiConnection) init() { c.state = CONN_STATE_FREE c.refcount = 1 c.readLock = new(sync.RWMutex) - c.sessionParam = []ISCSISessionParam{} + c.loginParam.sessionParam = []ISCSISessionParam{} + c.loginParam.tgtCSG = LoginOperationalNegotiation + c.loginParam.tgtNSG = LoginOperationalNegotiation for _, param := range sessionKeys { - c.sessionParam = append(c.sessionParam, ISCSISessionParam{Value: param.def}) + c.loginParam.sessionParam = append(c.loginParam.sessionParam, + ISCSISessionParam{idx: param.idx, Value: param.def}) } + sort.Sort(c.loginParam.sessionParam) } func (c *iscsiConnection) readData(size int) ([]byte, int, error) { @@ -135,3 +135,8 @@ func (c *iscsiConnection) write(resp []byte) (int, error) { func (c *iscsiConnection) close() { c.conn.Close() } + +func (conn *iscsiConnection) ReInstatement(newConn *iscsiConnection) { + conn.close() + conn.conn = newConn.conn +} diff --git a/pkg/port/iscsit/iscsid.go b/pkg/port/iscsit/iscsid.go index d7b58da..c8fb2ad 100644 --- a/pkg/port/iscsit/iscsid.go +++ b/pkg/port/iscsit/iscsid.go @@ -22,19 +22,26 @@ import ( "net" "os" "strconv" + "sync" log "github.com/Sirupsen/logrus" "github.com/gostor/gotgt/pkg/api" "github.com/gostor/gotgt/pkg/config" "github.com/gostor/gotgt/pkg/scsi" "github.com/gostor/gotgt/pkg/util" - "github.com/satori/go.uuid" +) + +const ( + ISCSI_MAX_TSIH = uint16(0xffff) + ISCSI_UNSPEC_TSIH = uint16(0) ) type ISCSITargetDriver struct { - SCSI *scsi.SCSITargetService - Name string - iSCSITargets map[string]*ISCSITarget + SCSI *scsi.SCSITargetService + Name string + iSCSITargets map[string]*ISCSITarget + TSIHPool map[uint16]bool + TSIHPoolMutex sync.Mutex } func init() { @@ -46,9 +53,31 @@ func NewISCSITargetDriver(base *scsi.SCSITargetService) (scsi.SCSITargetDriver, Name: "iscsi", iSCSITargets: map[string]*ISCSITarget{}, SCSI: base, + TSIHPool: map[uint16]bool{0: true, 65535: true}, }, nil } +func (s *ISCSITargetDriver) AllocTSIH() uint16 { + var i uint16 + s.TSIHPoolMutex.Lock() + for i = uint16(0); i < ISCSI_MAX_TSIH; i++ { + exist := s.TSIHPool[i] + if !exist { + s.TSIHPool[i] = true + s.TSIHPoolMutex.Unlock() + return i + } + } + s.TSIHPoolMutex.Unlock() + return ISCSI_UNSPEC_TSIH +} + +func (s *ISCSITargetDriver) ReleaseTSIH(tsih uint16) { + s.TSIHPoolMutex.Lock() + delete(s.TSIHPool, tsih) + s.TSIHPoolMutex.Unlock() +} + func (s *ISCSITargetDriver) NewTarget(tgtName string, configInfo *config.Config) error { if _, ok := s.iSCSITargets[tgtName]; ok { return fmt.Errorf("target name has been existed") @@ -138,7 +167,10 @@ func (s *ISCSITargetDriver) Run() error { } log.Info(conn.LocalAddr().String()) log.Info("Accepting ...") - iscsiConn := &iscsiConnection{conn: conn} + + iscsiConn := &iscsiConnection{conn: conn, + loginParam: &iscsiLoginParam{}} + iscsiConn.init() iscsiConn.rxIOState = IOSTATE_RX_BHS @@ -160,7 +192,7 @@ func (s *ISCSITargetDriver) handler(events byte, conn *iscsiConnection) { s.txHandler(conn) } if conn.state == CONN_STATE_CLOSE { - log.Warningf("iscsi connection[%d] closed", conn.CID) + log.Warningf("iscsi connection[%d] closed", conn.cid) conn.close() } } @@ -175,8 +207,8 @@ func (s *ISCSITargetDriver) rxHandler(conn *iscsiConnection) { conn.readLock.Lock() defer conn.readLock.Unlock() if conn.state == CONN_STATE_SCSI { - hdigest = conn.sessionParam[ISCSI_PARAM_HDRDGST_EN].Value & DIGEST_CRC32C - ddigest = conn.sessionParam[ISCSI_PARAM_DATADGST_EN].Value & DIGEST_CRC32C + hdigest = conn.loginParam.sessionParam[ISCSI_PARAM_HDRDGST_EN].Value & DIGEST_CRC32C + ddigest = conn.loginParam.sessionParam[ISCSI_PARAM_DATADGST_EN].Value & DIGEST_CRC32C } for { switch conn.rxIOState { @@ -289,101 +321,62 @@ func (s *ISCSITargetDriver) rxHandler(conn *iscsiConnection) { func (s *ISCSITargetDriver) iscsiExecLogin(conn *iscsiConnection) error { var ( - target *ISCSITarget - cmd = conn.req - TPGT uint16 - err error + cmd = conn.req + err error + negoKeys []util.KeyValue ) - conn.resp = &ISCSICommand{ - OpCode: OpLoginResp, - Transit: true, - CSG: cmd.CSG, - NSG: FullFeaturePhase, - StatSN: cmd.ExpStatSN, - TaskTag: cmd.TaskTag, - ExpCmdSN: cmd.CmdSN, - MaxCmdSN: cmd.CmdSN, - RawData: util.MarshalKVText([]util.KeyValue{ - {"HeaderDigest", "None"}, - {"DataDigest", "None"}, - {"ImmediateData", "Yes"}, - {"InitialR2T", "Yes"}, - {"MaxBurstLength", "262144"}, - {"FirstBurstLength", "65536"}, - {"DefaultTime2Wait", "2"}, - {"DefaultTime2Retain", "0"}, - {"MaxOutstandingR2T", "1"}, - {"IFMarker", "No"}, - {"OFMarker", "No"}, - {"DataPDUInOrder", "Yes"}, - {"DataSequenceInOrder", "Yes"}, - }), - } - conn.CID = cmd.ConnID - pairs := util.ParseKVText(cmd.RawData) - if initiatorName, ok := pairs["InitiatorName"]; ok { - conn.initiator = initiatorName - } - if alias, ok := pairs["InitiatorAlias"]; ok { - conn.initiatorAlias = alias - } - targetName := pairs["TargetName"] - if sessType, ok := pairs["SessionType"]; ok { - if sessType == "Normal" { - conn.sessionType = SESSION_NORMAL - } else { - conn.sessionType = SESSION_DISCOVERY - } - } - if conn.sessionType == SESSION_DISCOVERY { - conn.tid = 0xffff - } else { - for _, t := range s.iSCSITargets { - if t.SCSITarget.Name == targetName { - target = t - break - } - } - if target == nil { - conn.state = CONN_STATE_EXIT - return fmt.Errorf("No target found with name(%s)", targetName) - } - TPGT, err = target.FindTPG(conn.conn.LocalAddr().String()) + conn.cid = cmd.ConnID + conn.loginParam.iniCSG = cmd.CSG + conn.loginParam.iniNSG = cmd.NSG + conn.loginParam.iniCont = cmd.Cont + conn.loginParam.iniTrans = cmd.Transit + conn.loginParam.isid = cmd.ISID + conn.loginParam.tsih = cmd.TSIH + conn.expCmdSN = cmd.CmdSN + conn.statSN += 1 + + if conn.loginParam.iniCSG == SecurityNegotiation { + conn.state = CONN_STATE_EXIT + return fmt.Errorf("Doesn't support Auth") + } + + pairs := util.ParseKVText(cmd.RawData) + + negoKeys, err = loginKVProcess(conn, pairs) + if err != nil { + return err + } + if !conn.loginParam.keyDeclared { + negoKeys = loginKVDeclare(conn, negoKeys) + conn.loginParam.keyDeclared = true + } + + if !conn.loginParam.paramInit { + err = s.BindISCSISession(conn) if err != nil { conn.state = CONN_STATE_EXIT return err } - conn.tpgt = TPGT - conn.tid = target.TID - + conn.loginParam.paramInit = true } - switch conn.state { - case CONN_STATE_FREE: - conn.state = CONN_STATE_SECURITY - case CONN_STATE_SECURITY: + if conn.loginParam.tgtNSG == FullFeaturePhase && + conn.loginParam.tgtTrans { + conn.state = CONN_STATE_LOGIN_FULL + } else { + conn.state = CONN_STATE_LOGIN } - conn.state = CONN_STATE_LOGIN_FULL - conn.expCmdSN = cmd.CmdSN - conn.statSN += 1 - - switch conn.sessionType { - case SESSION_NORMAL: - if conn.session == nil { - // create a new session - sess, err := s.NewISCSISession(conn, cmd.ISID) - if err != nil { - log.Error(err) - return err - } - itnexus := &api.ITNexus{uuid.NewV1(), GeniSCSIITNexusID(sess)} - scsi.AddITNexus(&sess.Target.SCSITarget, itnexus) - sess.ITNexusID = itnexus.ID - conn.session = sess - } - case SESSION_DISCOVERY: - + conn.resp = &ISCSICommand{ + OpCode: OpLoginResp, + Transit: conn.loginParam.tgtTrans, + CSG: cmd.CSG, + NSG: conn.loginParam.tgtNSG, + StatSN: cmd.ExpStatSN, + TaskTag: cmd.TaskTag, + ExpCmdSN: cmd.CmdSN, + MaxCmdSN: cmd.CmdSN, + RawData: util.MarshalKVText(negoKeys), } return nil @@ -476,14 +469,13 @@ func iscsiExecReject(conn *iscsiConnection) error { } func iscsiExecR2T(conn *iscsiConnection) error { + var val uint conn.txTask = &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, scmd: &api.SCSICommand{}} conn.txIOState = IOSTATE_TX_BHS conn.statSN += 1 task := conn.rxTask resp := &ISCSICommand{ OpCode: OpReady, - Immediate: true, - Final: true, StatSN: conn.req.ExpStatSN, TaskTag: conn.req.TaskTag, ExpCmdSN: conn.session.ExpCmdSN, @@ -492,7 +484,8 @@ func iscsiExecR2T(conn *iscsiConnection) error { BufferOffset: uint32(task.offset), DesiredLength: uint32(task.r2tCount), } - if val := sessionKeys[ISCSI_PARAM_MAX_BURST].def; task.r2tCount > int(val) { + + if val = conn.loginParam.sessionParam[ISCSI_PARAM_MAX_BURST].Value; task.r2tCount > int(val) { resp.DesiredLength = uint32(val) } conn.resp = resp @@ -506,8 +499,8 @@ func (s *ISCSITargetDriver) txHandler(conn *iscsiConnection) { final bool = false ) if conn.state == CONN_STATE_SCSI { - hdigest = conn.sessionParam[ISCSI_PARAM_HDRDGST_EN].Value & DIGEST_CRC32C - ddigest = conn.sessionParam[ISCSI_PARAM_DATADGST_EN].Value & DIGEST_CRC32C + hdigest = conn.loginParam.sessionParam[ISCSI_PARAM_HDRDGST_EN].Value & DIGEST_CRC32C + ddigest = conn.loginParam.sessionParam[ISCSI_PARAM_DATADGST_EN].Value & DIGEST_CRC32C } if conn.state == CONN_STATE_SCSI && conn.txTask == nil { err := s.scsiCommandHandler(conn) @@ -564,10 +557,14 @@ func (s *ISCSITargetDriver) txHandler(conn *iscsiConnection) { case CONN_STATE_SECURITY_LOGIN: conn.state = CONN_STATE_LOGIN log.Debugf("CONN_STATE_LOGIN") + case CONN_STATE_LOGIN: + log.Debugf("CONN_STATE_LOGIN") + conn.rxIOState = IOSTATE_RX_BHS + s.handler(DATAIN, conn) case CONN_STATE_SECURITY_FULL, CONN_STATE_LOGIN_FULL: - if conn.sessionType == SESSION_NORMAL { + if conn.session.SessionType == SESSION_NORMAL { conn.state = CONN_STATE_KERNEL - log.Infof("CONN_STATE_KERNEL") + log.Debugf("CONN_STATE_KERNEL") conn.state = CONN_STATE_SCSI log.Debugf("CONN_STATE_SCSI") } else { @@ -583,7 +580,6 @@ func (s *ISCSITargetDriver) txHandler(conn *iscsiConnection) { conn.rxIOState = IOSTATE_RX_BHS s.handler(DATAIN, conn) } - log.Infof("%d", conn.state) } func (s *ISCSITargetDriver) scsiCommandHandler(conn *iscsiConnection) (err error) { @@ -604,15 +600,24 @@ func (s *ISCSITargetDriver) scsiCommandHandler(conn *iscsiConnection) (err error if task.scmd.OutSDBBuffer.Buffer == nil { task.scmd.OutSDBBuffer.Buffer = bytes.NewBuffer([]byte{}) } - task.scmd.OutSDBBuffer.Buffer.Write(conn.req.RawData) + if conn.session.SessionParam[ISCSI_PARAM_IMM_DATA_EN].Value == 1 { + task.scmd.OutSDBBuffer.Buffer.Write(conn.req.RawData) + } if task.r2tCount > 0 { // prepare to receive more data conn.session.ExpCmdSN += 1 task.state = taskPending conn.session.PendingTasks.Push(task) conn.rxTask = task - iscsiExecR2T(conn) - break + if conn.session.SessionParam[ISCSI_PARAM_INITIAL_R2T_EN].Value == 1 { + iscsiExecR2T(conn) + break + } else { + log.Debugf("Not ready to exec the task") + conn.rxIOState = IOSTATE_RX_BHS + s.handler(DATAIN, conn) + return nil + } } } task.offset = 0 @@ -691,7 +696,12 @@ func (s *ISCSITargetDriver) scsiCommandHandler(conn *iscsiConnection) (err error return nil } else if task.r2tCount > 0 { // prepare to receive more data - task.r2tSN += 1 + if task.unsolCount == 0 { + task.r2tSN += 1 + } else { + task.r2tSN = 0 + task.unsolCount = 0 + } conn.rxTask = task iscsiExecR2T(conn) break @@ -803,12 +813,12 @@ func (s *ISCSITargetDriver) iscsiExecTask(task *iscsiTask) error { task.scmd.Direction = api.SCSIDataWrite } } - task.scmd.ITNexusID = task.conn.session.ITNexusID + task.scmd.ITNexusID = task.conn.session.ITNexus.ID task.scmd.SCB = bytes.NewBuffer(cmd.CDB) task.scmd.SCBLength = len(cmd.CDB) task.scmd.Lun = cmd.LUN task.scmd.Tag = uint64(cmd.TaskTag) - task.scmd.RelTargetPortID = task.conn.tpgt + task.scmd.RelTargetPortID = task.conn.session.TPGT task.state = taskSCSI if task.scmd.OutSDBBuffer.Buffer == nil { task.scmd.OutSDBBuffer.Buffer = bytes.NewBuffer(cmd.RawData) diff --git a/pkg/port/iscsit/iscsit.go b/pkg/port/iscsit/iscsit.go index 9d7568e..0c4714f 100644 --- a/pkg/port/iscsit/iscsit.go +++ b/pkg/port/iscsit/iscsit.go @@ -20,6 +20,7 @@ package iscsit import ( "fmt" "strings" + "sync" "github.com/gostor/gotgt/pkg/api" ) @@ -81,15 +82,14 @@ type ISCSITarget struct { // TPGT number is the key TPGTs map[uint16]*iSCSITPGT // TSIH is the key - Sessions map[uint16]*ISCSISession - SessionParam []ISCSISessionParam - Alias string - MaxSessions int - RedirectInfo ISCSIRedirectInfo - Rdma int - NopInterval int - NopCount int - TSIHCounter uint16 + Sessions map[uint16]*ISCSISession + SessionsRWMutex sync.RWMutex + Alias string + MaxSessions int + RedirectInfo ISCSIRedirectInfo + Rdma int + NopInterval int + NopCount int } /* @@ -121,10 +121,9 @@ func (tgt *ISCSITarget) FindTPG(portal string) (uint16, error) { func newISCSITarget(target *api.SCSITarget) *ISCSITarget { return &ISCSITarget{ - SCSITarget: *target, - TPGTs: make(map[uint16]*iSCSITPGT), - Sessions: make(map[uint16]*ISCSISession), - TSIHCounter: 1, + SCSITarget: *target, + TPGTs: make(map[uint16]*iSCSITPGT), + Sessions: make(map[uint16]*ISCSISession), } } diff --git a/pkg/port/iscsit/login.go b/pkg/port/iscsit/login.go index cad1237..044c8a8 100644 --- a/pkg/port/iscsit/login.go +++ b/pkg/port/iscsit/login.go @@ -2,10 +2,174 @@ package iscsit import ( "bytes" + "fmt" + // log "github.com/Sirupsen/logrus" "github.com/gostor/gotgt/pkg/util" ) +var ( + iSCSILoginParamTextKV = []util.KeyValue{ + {"HeaderDigest", "None"}, + {"DataDigest", "None"}, + {"ImmediateData", "Yes"}, + {"InitialR2T", "Yes"}, + {"MaxBurstLength", "262144"}, + {"FirstBurstLength", "65536"}, + {"DefaultTime2Wait", "2"}, + {"DefaultTime2Retain", "0"}, + {"MaxOutstandingR2T", "1"}, + {"IFMarker", "No"}, + {"OFMarker", "No"}, + {"DataPDUInOrder", "Yes"}, + {"DataSequenceInOrder", "Yes"}} +) + +type iSCSILoginStage int + +const ( + SecurityNegotiation iSCSILoginStage = 0 + LoginOperationalNegotiation = 1 + FullFeaturePhase = 3 +) + +func (s iSCSILoginStage) String() string { + switch s { + case SecurityNegotiation: + return "Security Negotiation" + case LoginOperationalNegotiation: + return "Login Operational Negotiation" + case FullFeaturePhase: + return "Full Feature Phase" + } + return "Unknown Stage" +} + +func loginKVDeclare(conn *iscsiConnection, negoKV []util.KeyValue) []util.KeyValue { + + negoKV = append(negoKV, util.KeyValue{"TargetPortalGroupTag", + numberKeyInConv(uint(conn.loginParam.tpgt))}) + negoKV = append(negoKV, util.KeyValue{"MaxRecvDataSegmentLength", + numberKeyInConv(sessionKeys["MaxRecvDataSegmentLength"].def)}) + return negoKV + +} +func loginKVProcess(conn *iscsiConnection, loginKV map[string]string) ([]util.KeyValue, error) { + var ( + uintVal uint + ok bool + defSessKey *iscsiSessionKeys + negoKV []util.KeyValue + kvChanges int + ) + + for key, val := range loginKV { + // The MaxRecvDataSegmentLength of initiator + // is the MaxXmitDataSegmentLength of target + if key == "MaxRecvDataSegmentLength" { + defSessKey, ok = sessionKeys["MaxXmitDataSegmentLength"] + uintVal, ok = defSessKey.conv(val) + conn.loginParam.sessionParam[defSessKey.idx].Value = uintVal + continue + } + + if key == "InitiatorName" { + conn.loginParam.initiator = val + continue + } else if key == "InitiatorAlias" { + conn.loginParam.initiatorAlias = val + continue + } else if key == "TargetName" { + conn.loginParam.target = val + continue + } else if key == "SessionType" { + if val == "Normal" { + conn.loginParam.sessionType = SESSION_NORMAL + } else { + conn.loginParam.sessionType = SESSION_DISCOVERY + } + continue + } + + defSessKey, ok = sessionKeys[key] + if ok { + uintVal, ok = defSessKey.conv(val) + + //hack here + if key == "HeaderDigest" || key == "DataDigest" { + if uintVal == DIGEST_ALL { + uintVal = DIGEST_NONE + } + } + if ok { + if defSessKey.constValue { + //the Negotiation Key cannot be changed! Uses Target default key + if uintVal != defSessKey.def { + kvChanges++ + } + negoKV = append(negoKV, util.KeyValue{key, defSessKey.inConv(defSessKey.def)}) + } else { + if (uintVal >= defSessKey.min) && (uintVal <= defSessKey.max) { + conn.loginParam.sessionParam[defSessKey.idx].Value = uintVal + negoKV = append(negoKV, util.KeyValue{key, defSessKey.inConv(uintVal)}) + } else { + // the value out of the acceptable range, Uses target default key + negoKV = append(negoKV, util.KeyValue{key, defSessKey.inConv(defSessKey.def)}) + kvChanges++ + } + } + } + } else { + //Unknown Key, reject it + return negoKV, fmt.Errorf("Unknowen Nego KV [%s:%s]", key, val) + } + } + + if kvChanges == 0 { + if (conn.loginParam.iniNSG == FullFeaturePhase) && conn.loginParam.iniTrans { + conn.loginParam.tgtNSG = FullFeaturePhase + conn.loginParam.tgtTrans = true + } else { + //Currently, we just reject these kind of cases + return negoKV, fmt.Errorf("reject CSG=%d,NSG=%d,trans", + conn.loginParam.iniCSG, conn.loginParam.iniNSG, conn.loginParam.iniTrans) + } + } else { + conn.loginParam.tgtNSG = LoginOperationalNegotiation + conn.loginParam.tgtTrans = false + } + return negoKV, nil +} + +type iscsiLoginParam struct { + paramInit bool + + iniCSG iSCSILoginStage + iniNSG iSCSILoginStage + iniTrans bool + iniCont bool + + tgtCSG iSCSILoginStage + tgtNSG iSCSILoginStage + tgtTrans bool + tgtCont bool + + sessionType int + sessionParam ISCSISessionParamList + keyDeclared bool + + initiator string + initiatorAlias string + target string + targetAlias string + + tpgt uint16 + isid uint64 + tsih uint16 + + authMethod AuthMethod +} + func (m *ISCSICommand) loginRespBytes() []byte { // rfc7143 11.13 buf := &bytes.Buffer{} @@ -51,23 +215,3 @@ func (m *ISCSICommand) loginRespBytes() []byte { buf.Write(rd) return buf.Bytes() } - -type Stage int - -const ( - SecurityNegotiation Stage = 0 - LoginOperationalNegotiation = 1 - FullFeaturePhase = 3 -) - -func (s Stage) String() string { - switch s { - case SecurityNegotiation: - return "Security Negotiation" - case LoginOperationalNegotiation: - return "Login Operational Negotiation" - case FullFeaturePhase: - return "Full Feature Phase" - } - return "Unknown Stage" -} diff --git a/pkg/port/iscsit/session.go b/pkg/port/iscsit/session.go index db6f593..1cf285e 100644 --- a/pkg/port/iscsit/session.go +++ b/pkg/port/iscsit/session.go @@ -18,9 +18,13 @@ package iscsit import ( "fmt" - "math/rand" - "time" + "strconv" + "strings" + "sync" + log "github.com/Sirupsen/logrus" + "github.com/gostor/gotgt/pkg/api" + "github.com/gostor/gotgt/pkg/scsi" "github.com/satori/go.uuid" ) @@ -73,9 +77,27 @@ const ( ) type ISCSISessionParam struct { + idx uint State int Value uint } +type ISCSISessionParamList []ISCSISessionParam + +func (list ISCSISessionParamList) Len() int { + return len(list) +} + +func (list ISCSISessionParamList) Less(i, j int) bool { + if list[i].idx <= list[j].idx { + return true + } else { + return false + } +} + +func (list ISCSISessionParamList) Swap(i, j int) { + list[i], list[j] = list[j], list[i] +} /* * The defaults here are according to the spec and must not be changed, @@ -86,63 +108,130 @@ type ISCSISessionParam struct { * to remember the RDSL of the initiator, which defaults to 8k if he has * not told us otherwise. */ +type KeyConvFunc func(value string) (uint, bool) +type KeyInConvFunc func(value uint) string + type iscsiSessionKeys struct { - name string - def uint - min uint - max uint + idx uint + constValue bool + def uint + min uint + max uint + conv KeyConvFunc + inConv KeyInConvFunc } -var sessionKeys []iscsiSessionKeys = []iscsiSessionKeys{ +func digestKeyConv(value string) (uint, bool) { + var crc uint + valueArray := strings.Split(value, ",") + if len(valueArray) == 0 { + return crc, false + } + for _, tmpV := range valueArray { + if strings.EqualFold(tmpV, "crc32c") { + crc |= DIGEST_CRC32C + } else if strings.EqualFold(tmpV, "none") { + crc |= DIGEST_NONE + } else { + return crc, false + } + } + + return crc, true +} + +func digestKeyInConv(value uint) string { + str := "" + switch value { + case DIGEST_NONE: + str = "None" + case DIGEST_CRC32C: + str = "CRC32C" + case DIGEST_ALL: + str = "None,CRC32C" + } + return str +} + +func numberKeyConv(value string) (uint, bool) { + v, err := strconv.Atoi(value) + if err == nil { + return uint(v), true + } + return uint(v), false +} + +func numberKeyInConv(value uint) string { + s := strconv.Itoa(int(value)) + return s +} + +func boolKeyConv(value string) (uint, bool) { + if strings.EqualFold(value, "yes") { + return 1, true + } else if strings.EqualFold(value, "no") { + return 0, true + } + return 0, false +} + +func boolKeyInConv(value uint) string { + if value == 0 { + return "No" + } + return "Yes" +} + +var sessionKeys map[string]*iscsiSessionKeys = map[string]*iscsiSessionKeys{ // ISCSI_PARAM_MAX_RECV_DLENGTH - {"MaxRecvDataSegmentLength", 8192, 512, 16777215}, + "MaxRecvDataSegmentLength": {ISCSI_PARAM_MAX_RECV_DLENGTH, true, 32768, 512, 16777215, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_HDRDGST_EN - {"HeaderDigest", DIGEST_NONE, DIGEST_NONE, DIGEST_ALL}, + "HeaderDigest": {ISCSI_PARAM_HDRDGST_EN, false, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv}, // ISCSI_PARAM_DATADGST_EN - {"DataDigest", DIGEST_NONE, DIGEST_NONE, DIGEST_ALL}, + "DataDigest": {ISCSI_PARAM_DATADGST_EN, false, DIGEST_NONE, DIGEST_NONE, DIGEST_ALL, digestKeyConv, digestKeyInConv}, // ISCSI_PARAM_INITIAL_R2T_EN - {"InitialR2T", 1, 0, 1}, + "InitialR2T": {ISCSI_PARAM_INITIAL_R2T_EN, false, 1, 0, 1, boolKeyConv, boolKeyInConv}, // ISCSI_PARAM_MAX_R2T - {"MaxOutstandingR2T", 1, 1, 65535}, + "MaxOutstandingR2T": {ISCSI_PARAM_MAX_R2T, true, 1, 1, 65535, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_IMM_DATA_EN - {"ImmediateData", 1, 0, 1}, + "ImmediateData": {ISCSI_PARAM_IMM_DATA_EN, true, 1, 0, 1, boolKeyConv, boolKeyInConv}, // ISCSI_PARAM_FIRST_BURST - {"FirstBurstLength", 65536, 512, 16777215}, + "FirstBurstLength": {ISCSI_PARAM_FIRST_BURST, true, 65536, 512, 16777215, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_MAX_BURST - {"MaxBurstLength", 262144, 512, 16777215}, + "MaxBurstLength": {ISCSI_PARAM_MAX_BURST, true, 262144, 512, 16777215, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_PDU_INORDER_EN - {"DataPDUInOrder", 1, 0, 1}, + "DataPDUInOrder": {ISCSI_PARAM_PDU_INORDER_EN, true, 1, 0, 1, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_DATASEQ_INORDER_EN - {"DataSequenceInOrder", 1, 0, 1}, + "DataSequenceInOrder": {ISCSI_PARAM_DATASEQ_INORDER_EN, true, 1, 0, 1, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_ERL - {"ErrorRecoveryLevel", 0, 0, 2}, + "ErrorRecoveryLevel": {ISCSI_PARAM_ERL, true, 0, 0, 2, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_IFMARKER_EN - {"IFMarker", 0, 0, 1}, + "IFMarker": {ISCSI_PARAM_IFMARKER_EN, true, 0, 0, 1, boolKeyConv, boolKeyInConv}, // ISCSI_PARAM_OFMARKER_EN - {"OFMarker", 0, 0, 1}, + "OFMarker": {ISCSI_PARAM_OFMARKER_EN, true, 0, 0, 1, boolKeyConv, boolKeyInConv}, // ISCSI_PARAM_DEFAULTTIME2WAIT - {"DefaultTime2Wait", 2, 0, 3600}, + "DefaultTime2Wait": {ISCSI_PARAM_DEFAULTTIME2WAIT, true, 2, 0, 3600, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_DEFAULTTIME2RETAIN - {"DefaultTime2Retain", 20, 0, 3600}, + "DefaultTime2Retain": {ISCSI_PARAM_DEFAULTTIME2RETAIN, false, 20, 0, 3600, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_OFMARKINT - {"OFMarkInt", 2048, 1, 65535}, + "OFMarkInt": {ISCSI_PARAM_OFMARKINT, true, 2048, 1, 65535, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_IFMARKINT - {"IFMarkInt", 2048, 1, 65535}, + "IFMarkInt": {ISCSI_PARAM_IFMARKINT, true, 2048, 1, 65535, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_MAXCONNECTIONS - {"MaxConnections", 1, 1, 65535}, + "MaxConnections": {ISCSI_PARAM_MAXCONNECTIONS, true, 1, 1, 65535, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_RDMA_EXTENSIONS - {"RDMAExtensions", 0, 0, 1}, + "RDMAExtensions": {ISCSI_PARAM_RDMA_EXTENSIONS, true, 0, 0, 1, boolKeyConv, boolKeyInConv}, // ISCSI_PARAM_TARGET_RDSL - {"TargetRecvDataSegmentLength", 8192, 512, 16777215}, + "TargetRecvDataSegmentLength": {ISCSI_PARAM_TARGET_RDSL, true, 8192, 512, 16777215, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_INITIATOR_RDSL - {"InitiatorRecvDataSegmentLength", 8192, 512, 16777215}, + "InitiatorRecvDataSegmentLength": {ISCSI_PARAM_INITIATOR_RDSL, true, 8192, 512, 16777215, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_MAX_OUTST_PDU - {"MaxOutstandingUnexpectedPDUs", 0, 2, 4294967295}, + "MaxOutstandingUnexpectedPDUs": {ISCSI_PARAM_MAX_OUTST_PDU, true, 0, 2, 4294967295, numberKeyConv, numberKeyInConv}, // "local" parmas, never sent to the initiator // ISCSI_PARAM_MAX_XMIT_DLENGTH - {"MaxXmitDataSegmentLength", 8192, 512, 16777215}, + "MaxXmitDataSegmentLength": {ISCSI_PARAM_MAX_XMIT_DLENGTH, true, 8192, 512, 16777215, numberKeyConv, numberKeyInConv}, // ISCSI_PARAM_MAX_QUEUE_CMD - {"MaxQueueCmd", MAX_QUEUE_CMD_DEF, MAX_QUEUE_CMD_MIN, MAX_QUEUE_CMD_MAX}, + "MaxQueueCmd": {ISCSI_PARAM_MAX_QUEUE_CMD, true, MAX_QUEUE_CMD_DEF, MAX_QUEUE_CMD_MIN, MAX_QUEUE_CMD_MAX, numberKeyConv, numberKeyInConv}, } // Session is an iSCSI session. @@ -152,18 +241,22 @@ type ISCSISession struct { InitiatorAlias string Target *ISCSITarget ISID uint64 - TSIH uint64 - ITNexusID uuid.UUID + TSIH uint16 + TPGT uint16 + SessionType int + ITNexus *api.ITNexus ExpCmdSN uint32 - // only one connection per session - Connections []*iscsiConnection - Commands []*ISCSICommand - PendingTasks taskQueue - MaxQueueCommand uint32 - SessionParam []ISCSISessionParam - Info string - Rdma int + MaxCmdSN uint32 + // currently, this is only one connection per session + Connections map[uint16]*iscsiConnection + ConnectionsRWMutex sync.RWMutex + Commands []*ISCSICommand + PendingTasks taskQueue + MaxQueueCommand uint32 + SessionParam ISCSISessionParamList + Info string + Rdma int } type taskQueue []*iscsiTask @@ -192,11 +285,140 @@ func (tq *taskQueue) Pop() interface{} { return item } +func (s *ISCSITargetDriver) LookupISCSISession(tgtName string, iniName string, isid uint64, tsih uint16, tpgt uint16) *ISCSISession { + var ( + tgt *ISCSITarget + sess *ISCSISession + ok bool + ) + tgt, ok = s.iSCSITargets[tgtName] + if !ok { + return nil + } + tgt.SessionsRWMutex.RLock() + defer tgt.SessionsRWMutex.RUnlock() + sess, ok = tgt.Sessions[tsih] + if !ok { + return nil + } + if (sess.ISID == isid) && (sess.TPGT == tpgt) { + return sess + } + return nil +} + +func (s *ISCSITargetDriver) UnBindISCSISession(sess *ISCSISession) { + target := sess.Target + target.SessionsRWMutex.Lock() + defer target.SessionsRWMutex.Unlock() + delete(target.Sessions, sess.TSIH) + scsi.RemoveITNexus(&sess.Target.SCSITarget, sess.ITNexus) +} + +func (s *ISCSITargetDriver) BindISCSISession(conn *iscsiConnection) error { + var ( + target *ISCSITarget + existSess *ISCSISession + existConn *iscsiConnection + newSess *ISCSISession + tpgt uint16 + err error + ) + + //Find TPGT and Target ID + if conn.loginParam.sessionType == SESSION_DISCOVERY { + conn.tid = 0xffff + } else { + for _, t := range s.iSCSITargets { + if t.SCSITarget.Name == conn.loginParam.target { + target = t + break + } + } + if target == nil { + return fmt.Errorf("No target found with name(%s)", conn.loginParam.target) + } + + tpgt, err = target.FindTPG(conn.conn.LocalAddr().String()) + if err != nil { + return err + } + conn.loginParam.tpgt = tpgt + conn.tid = target.TID + } + + existSess = s.LookupISCSISession(conn.loginParam.target, conn.loginParam.initiator, + conn.loginParam.isid, conn.loginParam.tsih, conn.loginParam.tpgt) + if existSess != nil { + existConn = existSess.LookupConnection(conn.cid) + } + + if conn.loginParam.sessionType == SESSION_DISCOVERY && + conn.loginParam.tsih != ISCSI_UNSPEC_TSIH && + existSess != nil { + return fmt.Errorf("initiator err, invalid request") + } + + if existSess == nil && conn.loginParam.tsih != 0 && + existSess.TSIH != conn.loginParam.tsih { + return fmt.Errorf("initiator err, no session") + } + + if existSess == nil { + newSess, err = s.NewISCSISession(conn) + if err != nil { + return err + } + + if newSess.SessionType == SESSION_NORMAL { + log.Infof("New Session initiator name:%v,target name:%v,ISID:0x%x", + conn.loginParam.initiator, conn.loginParam.target, conn.loginParam.isid) + //register normal session + itnexus := &api.ITNexus{uuid.NewV1(), GeniSCSIITNexusID(newSess)} + scsi.AddITNexus(&newSess.Target.SCSITarget, itnexus) + newSess.ITNexus = itnexus + conn.session = newSess + + newSess.Target.SessionsRWMutex.Lock() + newSess.Target.Sessions[newSess.TSIH] = newSess + newSess.Target.SessionsRWMutex.Unlock() + } else { + conn.session = newSess + } + } else { + if conn.loginParam.tsih == ISCSI_UNSPEC_TSIH { + log.Infof("Session Reinstatement initiator name:%v,target name:%v,ISID:0x%x", + conn.loginParam.initiator, conn.loginParam.target, conn.loginParam.isid) + newSess, err = s.ReInstatement(existConn.session, conn) + if err != nil { + return err + } + + itnexus := &api.ITNexus{uuid.NewV1(), GeniSCSIITNexusID(newSess)} + scsi.AddITNexus(&newSess.Target.SCSITarget, itnexus) + newSess.ITNexus = itnexus + conn.session = newSess + + newSess.Target.SessionsRWMutex.Lock() + newSess.Target.Sessions[newSess.TSIH] = newSess + newSess.Target.SessionsRWMutex.Unlock() + } else { + if existConn != nil { + log.Infof("Connection Reinstatement initiator name:%v,target name:%v,ISID:0x%x,CID:%v", + conn.loginParam.initiator, conn.loginParam.target, conn.loginParam.isid) + existConn.ReInstatement(conn) + } + } + } + + return nil +} + // New creates a new session. -func (s *ISCSITargetDriver) NewISCSISession(conn *iscsiConnection, isid uint64) (*ISCSISession, error) { +func (s *ISCSITargetDriver) NewISCSISession(conn *iscsiConnection) (*ISCSISession, error) { var ( target *ISCSITarget - tsih uint64 + tsih uint16 ) for _, t := range s.iSCSITargets { @@ -205,40 +427,54 @@ func (s *ISCSITargetDriver) NewISCSISession(conn *iscsiConnection, isid uint64) break } } - if target == nil { + if target == nil && conn.tid != 0xffff { return nil, fmt.Errorf("No target found with tid(%d)", conn.tid) } - for { - rand.Seed(int64(time.Now().UTC().Nanosecond())) - tsih = uint64(rand.Uint32()) - for _, s := range target.Sessions { - if s.TSIH == tsih { - tsih = 0 - break - } - } - if tsih != 0 { - break - } + tsih = s.AllocTSIH() + if tsih == ISCSI_UNSPEC_TSIH { + return nil, fmt.Errorf("TSIH Pool exhausted tid(%d)", conn.tid) } sess := &ISCSISession{ TSIH: tsih, - ISID: isid, - Initiator: conn.initiator, - InitiatorAlias: conn.initiatorAlias, + ISID: conn.loginParam.isid, + TPGT: conn.loginParam.tpgt, + Initiator: conn.loginParam.initiator, + InitiatorAlias: conn.loginParam.initiatorAlias, + SessionType: conn.loginParam.sessionType, Target: target, - Connections: []*iscsiConnection{conn}, - SessionParam: conn.sessionParam, - MaxQueueCommand: uint32(conn.sessionParam[ISCSI_PARAM_MAX_QUEUE_CMD].Value), + Connections: map[uint16]*iscsiConnection{conn.cid: conn}, + SessionParam: conn.loginParam.sessionParam, + MaxQueueCommand: uint32(conn.loginParam.sessionParam[ISCSI_PARAM_MAX_QUEUE_CMD].Value), Rdma: 0, ExpCmdSN: conn.expCmdSN, } - conn.session = sess return sess, nil } +func (sess *ISCSISession) LookupConnection(cid uint16) *iscsiConnection { + sess.ConnectionsRWMutex.RLock() + defer sess.ConnectionsRWMutex.RUnlock() + conn := sess.Connections[cid] + return conn +} + +func (s *ISCSITargetDriver) ReInstatement(existSess *ISCSISession, conn *iscsiConnection) (*ISCSISession, error) { + newSess, err := s.NewISCSISession(conn) + if err != nil { + return nil, err + } + newSess.ExpCmdSN = existSess.ExpCmdSN + newSess.MaxCmdSN = existSess.MaxCmdSN + 1 + s.UnBindISCSISession(existSess) + for _, tmpConn := range existSess.Connections { + tmpConn.close() + } + existSess.Connections = map[uint16]*iscsiConnection{} + return newSess, nil +} + /* * iSCSI I_T nexus identifer = (iSCSI Initiator Name + 'i' + ISID, iSCSI Target Name + 't' + Portal Group Tag) */ @@ -246,6 +482,6 @@ func GeniSCSIITNexusID(sess *ISCSISession) string { strID := fmt.Sprintf("%si0x%12x,%st%d", sess.Initiator, sess.ISID, sess.Target.SCSITarget.Name, - sess.Connections[0].tpgt) + sess.TPGT) return strID } diff --git a/pkg/scsi/backingstore.go b/pkg/scsi/backingstore.go index 1178f01..f497e30 100644 --- a/pkg/scsi/backingstore.go +++ b/pkg/scsi/backingstore.go @@ -181,7 +181,6 @@ verify: bs.DataAdvise(int64(offset), int64(length), util.POSIX_FADV_WILLNEED) } } - log.Infof("io done %s", string(scb)) return nil sense: if err != nil {