date: 2016-09-11/21:30

pkg/port/iscsit/cmd.go

@@ -90,6 +90,10 @@ type ISCSICommand struct {
 	Status          byte
 	SCSIResponse    byte
 
+	// R2T
+	R2TSN         uint32
+	DesiredLength uint32
+
 	// Data-In/Out
 	HasStatus    bool
 	DataSN       uint32
@@ -112,6 +116,8 @@ func (cmd *ISCSICommand) Bytes() []byte {
 		return cmd.noopInBytes()
 	case OpSCSITaskResp:
 		return cmd.scsiTMFRespBytes()
+	case OpReady:
+		return cmd.r2tRespBytes()
 	}
 	return nil
 }
@@ -139,7 +145,7 @@ func (m *ISCSICommand) String() string {
 		s = append(s, fmt.Sprintf("Next Stage = %v", m.NSG))
 		s = append(s, fmt.Sprintf("Status Class = %d", m.StatusClass))
 		s = append(s, fmt.Sprintf("Status Detail = %d", m.StatusDetail))
-	case OpSCSICmd:
+	case OpSCSICmd, OpSCSIOut:
 		s = append(s, fmt.Sprintf("LUN = %d", m.LUN))
 		s = append(s, fmt.Sprintf("ExpectedDataLen = %d", m.ExpectedDataLen))
 		s = append(s, fmt.Sprintf("CmdSN = %d", m.CmdSN))
@@ -397,3 +403,33 @@ func (m *ISCSICommand) scsiTMFRespBytes() []byte {
 
 	return buf.Bytes()
 }
+
+func (m *ISCSICommand) r2tRespBytes() []byte {
+	// rfc7143 11.8
+	buf := &bytes.Buffer{}
+	buf.WriteByte(byte(OpReady))
+	var b byte
+	if m.Final {
+		b |= 0x80
+	}
+	buf.WriteByte(b)
+	buf.WriteByte(0x00)
+	buf.WriteByte(0x00)
+
+	// Skip through to byte 16
+	for i := 0; i < 3*4; i++ {
+		buf.WriteByte(0x00)
+	}
+	buf.Write(util.MarshalUint64(uint64(m.TaskTag))[4:])
+	for i := 0; i < 4; i++ {
+		buf.WriteByte(0x00)
+	}
+	buf.Write(util.MarshalUint64(uint64(m.StatSN))[4:])
+	buf.Write(util.MarshalUint64(uint64(m.ExpCmdSN))[4:])
+	buf.Write(util.MarshalUint64(uint64(m.MaxCmdSN))[4:])
+	buf.Write(util.MarshalUint64(uint64(m.R2TSN))[4:])
+	buf.Write(util.MarshalUint64(uint64(m.BufferOffset))[4:])
+	buf.Write(util.MarshalUint64(uint64(m.DesiredLength))[4:])
+
+	return buf.Bytes()
+}

pkg/port/iscsit/conn.go

@@ -89,8 +89,8 @@ type iscsiConnection struct {
 type taskState int
 
 const (
-	taskPending taskState = 1
+	taskPending taskState = 0
-	taskSCSI    taskState = 2
+	taskSCSI    taskState = 1
 )
 
 type iscsiTask struct {
@@ -99,6 +99,13 @@ type iscsiTask struct {
 	cmd   *ISCSICommand
 	scmd  *api.SCSICommand
 	state taskState
+
+	offset     int
+	r2tCount   int
+	unsolCount int
+	expR2TSN   int
+
+	r2tSN uint32
 }
 
 func (c *iscsiConnection) init() {

pkg/port/iscsit/iscsid.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"runtime/debug"
 
 	"github.com/golang/glog"
 	"github.com/gostor/gotgt/pkg/api"
@@ -90,15 +91,15 @@ func (s *ISCSITargetService) Run() error {
 func (s *ISCSITargetService) handler(events byte, conn *iscsiConnection) {
 
 	if events&DATAIN != 0 {
-		glog.Infof("rx handler processing...")
+		glog.V(1).Infof("rx handler processing...")
 		go s.rxHandler(conn)
 	}
 	if conn.state != CONN_STATE_CLOSE && events&DATAOUT != 0 {
-		glog.Infof("tx handler processing...")
+		glog.V(1).Infof("tx handler processing...")
 		s.txHandler(conn)
 	}
 	if conn.state == CONN_STATE_CLOSE {
-		glog.Infof("iscsi connection[%d] closed", conn.cid)
+		glog.Warningf("iscsi connection[%d] closed", conn.cid)
 		conn.close()
 	}
 }
@@ -160,10 +161,16 @@ func (s *ISCSITargetService) rxHandler(conn *iscsiConnection) {
 				return
 			}
 			dl := ((cmd.DataLen + DataPadding - 1) / DataPadding) * DataPadding
-			buf, length, err := conn.readData(dl)
+			buf := []byte{}
-			if err != nil {
+			length := 0
-				glog.Error(err)
+			for length < dl {
-				return
+				b, l, err := conn.readData(dl - length)
+				if err != nil {
+					glog.Error(err)
+					return
+				}
+				length += l
+				buf = append(buf, b...)
 			}
 			if length != dl {
 				glog.V(2).Infof("get length is %d, but expected %d", length, dl)
@@ -213,8 +220,8 @@ func (s *ISCSITargetService) rxHandler(conn *iscsiConnection) {
 		default:
 			iscsiExecReject(conn)
 		}
-		glog.V(1).Infof("connection state is %v", conn.state)
+		glog.V(2).Infof("connection state is %v", conn.state)
-		glog.Infof("%#v", conn.resp.String())
+		glog.V(2).Infof("%#v", conn.resp.String())
 		s.handler(DATAOUT, conn)
 	}
 }
@@ -325,7 +332,8 @@ func (s *ISCSITargetService) iscsiExecText(conn *iscsiConnection) error {
 			}
 			for _, t := range list {
 				result = append(result, util.KeyValue{"TargetName", t.Name})
-				result = append(result, util.KeyValue{"TargetAddress", "127.0.0.1:3260,1"})
+				result = append(result, util.KeyValue{"TargetAddress", "172.16.69.169:3260,1"})
+				//result = append(result, util.KeyValue{"TargetAddress", "127.0.0.1:3260,1"})
 			}
 		}
 	}
@@ -403,7 +411,9 @@ func (s *ISCSITargetService) txHandler(conn *iscsiConnection) {
 			glog.V(2).Infof("length of RawData is %d", len(conn.resp.RawData))
 			glog.V(2).Infof("length of resp is %d", len(conn.resp.Bytes()))
 			if l, err := conn.write(conn.resp.Bytes()); err != nil {
+				debug.PrintStack()
 				glog.Error(err)
+				panic(err)
 				return
 			} else {
 				conn.txIOState = IOSTATE_TX_INIT_AHS
@@ -435,30 +445,30 @@ func (s *ISCSITargetService) txHandler(conn *iscsiConnection) {
 		}
 	}
 
-	glog.Infof("connection state: %d", conn.state)
+	glog.V(3).Infof("connection state: %d", conn.state)
 	switch conn.state {
 	case CONN_STATE_CLOSE, CONN_STATE_EXIT:
 		glog.Warningf("set connection to close")
 		conn.state = CONN_STATE_CLOSE
 	case CONN_STATE_SECURITY_LOGIN:
 		conn.state = CONN_STATE_LOGIN
-		glog.Infof("CONN_STATE_LOGIN")
+		glog.V(3).Infof("CONN_STATE_LOGIN")
 	case CONN_STATE_SECURITY_FULL, CONN_STATE_LOGIN_FULL:
 		if conn.sessionType == SESSION_NORMAL {
 			conn.state = CONN_STATE_KERNEL
 			glog.Infof("CONN_STATE_KERNEL")
 			conn.state = CONN_STATE_SCSI
-			glog.Infof("CONN_STATE_SCSI")
+			glog.V(3).Infof("CONN_STATE_SCSI")
 		} else {
 			conn.state = CONN_STATE_FULL
-			glog.Infof("CONN_STATE_FULL")
+			glog.V(3).Infof("CONN_STATE_FULL")
 		}
 		conn.rxIOState = IOSTATE_RX_BHS
 		s.handler(DATAIN, conn)
 	case CONN_STATE_SCSI:
 		conn.txTask = nil
 	default:
-		glog.Infof("unexpected connection state: %d", conn.state)
+		glog.Warningf("unexpected connection state: %d", conn.state)
 		conn.rxIOState = IOSTATE_RX_BHS
 		s.handler(DATAIN, conn)
 	}
@@ -469,9 +479,49 @@ func (s *ISCSITargetService) scsiCommandHandler(conn *iscsiConnection) (err erro
 	req := conn.req
 	switch req.OpCode {
 	case OpSCSICmd:
-		glog.Infof("SCSI Command processing...")
+		glog.V(2).Infof("SCSI Command processing...")
 		scmd := &api.SCSICommand{}
 		task := &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, scmd: scmd}
+		if req.Write {
+			task.offset = req.DataLen
+			task.r2tCount = int(req.ExpectedDataLen) - req.DataLen
+			if !req.Final {
+				task.unsolCount = 1
+			}
+			glog.V(2).Infof("SCSI write, R2T count: %d, unsol Count: %d, offset: %d", task.r2tCount, task.unsolCount, task.offset)
+
+			if task.scmd.OutSDBBuffer.Buffer == nil {
+				task.scmd.OutSDBBuffer.Buffer = bytes.NewBuffer([]byte{})
+			}
+			task.scmd.OutSDBBuffer.Buffer.Write(conn.req.RawData)
+			if task.r2tCount > 0 {
+				// prepare to receive more data
+				task.state = taskPending
+				conn.session.PendingTasks.Push(task)
+				//conn.rxTask = nil
+				conn.txTask = &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, scmd: &api.SCSICommand{}}
+				conn.txIOState = IOSTATE_TX_BHS
+				conn.statSN += 1
+				resp := &ISCSICommand{
+					OpCode:        OpReady,
+					Immediate:     true,
+					Final:         true,
+					StatSN:        req.ExpStatSN,
+					TaskTag:       req.TaskTag,
+					ExpCmdSN:      conn.session.ExpCmdSN,
+					MaxCmdSN:      conn.session.ExpCmdSN + 10,
+					R2TSN:         task.r2tSN,
+					BufferOffset:  uint32(task.offset),
+					DesiredLength: uint32(task.r2tCount),
+				}
+				if val := sessionKeys[ISCSI_PARAM_MAX_BURST].def; task.r2tCount > int(val) {
+					resp.DesiredLength = uint32(val)
+				}
+				conn.resp = resp
+				break
+			}
+		}
+		task.offset = 0
 		conn.rxTask = task
 		if err = s.iscsiTaskQueueHandler(task); err != nil {
 			return
@@ -516,31 +566,76 @@ func (s *ISCSITargetService) scsiCommandHandler(conn *iscsiConnection) (err erro
 		}
 	case OpSCSITaskReq:
 		// task management function
-		conn.txTask = &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag}
+		conn.txTask = &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, state: taskPending}
 		conn.txIOState = IOSTATE_TX_BHS
 		iscsiExecTMFunction(conn)
 	case OpSCSIOut:
-		glog.Infof("scsi out operation")
+		glog.V(1).Infof("iSCSI Data-out processing...")
-		scmd := &api.SCSICommand{}
+		var task *iscsiTask
-		conn.req.Write = true
+		for _, t := range conn.session.PendingTasks {
-		conn.req.CDB = []byte{api.Write_10}
+			if t.tag == conn.req.TaskTag {
-		task := &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, scmd: scmd}
+				task = t
+			}
+		}
+		if task == nil {
+			err = fmt.Errorf("Cannot find iSCSI task with tag[%v]", conn.req.TaskTag)
+			glog.Error(err)
+			return
+		}
+		task.offset = task.offset + conn.req.DataLen
+		task.r2tCount = task.r2tCount - conn.req.DataLen
+		task.scmd.OutSDBBuffer.Buffer.Write(conn.req.RawData)
+		glog.V(2).Infof("Final: %v", conn.req.Final)
+		glog.V(2).Infof("r2tCount: %v", task.r2tCount)
+		if !conn.req.Final {
+			glog.V(1).Infof("Not ready to exec the task")
+			conn.rxIOState = IOSTATE_RX_BHS
+			s.handler(DATAIN, conn)
+			return nil
+		} else if task.r2tCount > 0 {
+			// prepare to receive more data
+			conn.rxTask = nil
+			conn.txTask = &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, scmd: &api.SCSICommand{}}
+			conn.txIOState = IOSTATE_TX_BHS
+			conn.statSN += 1
+			task.r2tSN += 1
+			resp := &ISCSICommand{
+				OpCode:        OpReady,
+				Immediate:     true,
+				Final:         true,
+				StatSN:        req.ExpStatSN,
+				TaskTag:       req.TaskTag,
+				ExpCmdSN:      conn.session.ExpCmdSN,
+				MaxCmdSN:      conn.session.ExpCmdSN + 10,
+				R2TSN:         task.r2tSN,
+				BufferOffset:  uint32(task.offset),
+				DesiredLength: uint32(task.r2tCount),
+			}
+			if val := sessionKeys[ISCSI_PARAM_MAX_BURST].def; task.r2tCount > int(val) {
+				resp.DesiredLength = uint32(val)
+			}
+			conn.resp = resp
+			break
+		}
+		task.offset = 0
+		glog.V(1).Infof("Process the Data-out package")
 		conn.rxTask = task
 		if err = s.iscsiExecTask(task); err != nil {
 			return
 		} else {
 			conn.rxTask = nil
-			conn.txTask = &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, scmd: &api.SCSICommand{}}
+			conn.txTask = &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, scmd: &api.SCSICommand{}, state: taskSCSI}
 			conn.txIOState = IOSTATE_TX_BHS
 			conn.statSN += 1
 			resp := &ISCSICommand{
+				OpCode:       OpSCSIResp,
 				Immediate:    true,
 				Final:        true,
 				StatSN:       req.ExpStatSN,
 				TaskTag:      req.TaskTag,
 				ExpCmdSN:     conn.session.ExpCmdSN,
 				MaxCmdSN:     conn.session.ExpCmdSN + 10,
-				Status:       scmd.Result,
+				Status:       0,
 				SCSIResponse: 0x00,
 				HasStatus:    true,
 			}
@@ -581,9 +676,9 @@ func (s *ISCSITargetService) iscsiTaskQueueHandler(task *iscsiTask) error {
 	retry:
 		cmdsn += 1
 		sess.ExpCmdSN = cmdsn
-		glog.Infof("session's ExpCmdSN is %d", cmdsn)
+		glog.V(2).Infof("session's ExpCmdSN is %d", cmdsn)
 
-		glog.Infof("process task(%d)", task.cmd.CmdSN)
+		glog.V(2).Infof("process task(%d)", task.cmd.CmdSN)
 		if err := s.iscsiExecTask(task); err != nil {
 			glog.Error(err)
 		}
@@ -604,7 +699,7 @@ func (s *ISCSITargetService) iscsiTaskQueueHandler(task *iscsiTask) error {
 			glog.Error(err)
 			return err
 		}
-		glog.Infof("add task(%d) into task queue", task.cmd.CmdSN)
+		glog.V(1).Infof("add task(%d) into task queue", task.cmd.CmdSN)
 		// add this connection into queue and set this task as pending task
 		task.state = taskPending
 		sess.PendingTasks.Push(task)
@@ -634,7 +729,9 @@ func (s *ISCSITargetService) iscsiExecTask(task *iscsiTask) error {
 		task.scmd.Lun = cmd.LUN
 		task.scmd.Tag = uint64(cmd.TaskTag)
 		task.state = taskSCSI
-		task.scmd.OutSDBBuffer.Buffer = bytes.NewBuffer(cmd.RawData)
+		if task.scmd.OutSDBBuffer.Buffer == nil {
+			task.scmd.OutSDBBuffer.Buffer = bytes.NewBuffer(cmd.RawData)
+		}
 		// add scsi target process queue
 		err := s.SCSI.AddCommandQueue(task.conn.session.Target.SCSITarget.TID, task.scmd)
 		if err != nil {

pkg/scsi/lun.go

@@ -34,7 +34,7 @@ func NewSCSILu(lun uint64, target *api.SCSITarget) (*api.SCSILu, error) {
 		PerformCommand: luPerformCommand,
 		DeviceProtocol: sbc,
 		Storage:        backing,
-		BlockShift:     0,
+		BlockShift:     api.DefaultBlockShift,
 		Size:           1024 * 1024 * 10,
 	}
 	// hack this