diff --git a/pkg/port/iscsit/cmd.go b/pkg/port/iscsit/cmd.go index 049f369..8bd293a 100644 --- a/pkg/port/iscsit/cmd.go +++ b/pkg/port/iscsit/cmd.go @@ -90,6 +90,10 @@ type ISCSICommand struct { Status byte SCSIResponse byte + // R2T + R2TSN uint32 + DesiredLength uint32 + // Data-In/Out HasStatus bool DataSN uint32 @@ -112,6 +116,8 @@ func (cmd *ISCSICommand) Bytes() []byte { return cmd.noopInBytes() case OpSCSITaskResp: return cmd.scsiTMFRespBytes() + case OpReady: + return cmd.r2tRespBytes() } return nil } @@ -139,7 +145,7 @@ func (m *ISCSICommand) String() string { s = append(s, fmt.Sprintf("Next Stage = %v", m.NSG)) s = append(s, fmt.Sprintf("Status Class = %d", m.StatusClass)) s = append(s, fmt.Sprintf("Status Detail = %d", m.StatusDetail)) - case OpSCSICmd: + case OpSCSICmd, OpSCSIOut: s = append(s, fmt.Sprintf("LUN = %d", m.LUN)) s = append(s, fmt.Sprintf("ExpectedDataLen = %d", m.ExpectedDataLen)) s = append(s, fmt.Sprintf("CmdSN = %d", m.CmdSN)) @@ -397,3 +403,33 @@ func (m *ISCSICommand) scsiTMFRespBytes() []byte { return buf.Bytes() } + +func (m *ISCSICommand) r2tRespBytes() []byte { + // rfc7143 11.8 + buf := &bytes.Buffer{} + buf.WriteByte(byte(OpReady)) + var b byte + if m.Final { + b |= 0x80 + } + buf.WriteByte(b) + buf.WriteByte(0x00) + buf.WriteByte(0x00) + + // Skip through to byte 16 + for i := 0; i < 3*4; i++ { + buf.WriteByte(0x00) + } + buf.Write(util.MarshalUint64(uint64(m.TaskTag))[4:]) + for i := 0; i < 4; i++ { + buf.WriteByte(0x00) + } + buf.Write(util.MarshalUint64(uint64(m.StatSN))[4:]) + buf.Write(util.MarshalUint64(uint64(m.ExpCmdSN))[4:]) + buf.Write(util.MarshalUint64(uint64(m.MaxCmdSN))[4:]) + buf.Write(util.MarshalUint64(uint64(m.R2TSN))[4:]) + buf.Write(util.MarshalUint64(uint64(m.BufferOffset))[4:]) + buf.Write(util.MarshalUint64(uint64(m.DesiredLength))[4:]) + + return buf.Bytes() +} diff --git a/pkg/port/iscsit/conn.go b/pkg/port/iscsit/conn.go index 0ac7f91..fd8ddc9 100644 --- a/pkg/port/iscsit/conn.go +++ b/pkg/port/iscsit/conn.go @@ -89,8 +89,8 @@ type iscsiConnection struct { type taskState int const ( - taskPending taskState = 1 - taskSCSI taskState = 2 + taskPending taskState = 0 + taskSCSI taskState = 1 ) type iscsiTask struct { @@ -99,6 +99,13 @@ type iscsiTask struct { cmd *ISCSICommand scmd *api.SCSICommand state taskState + + offset int + r2tCount int + unsolCount int + expR2TSN int + + r2tSN uint32 } func (c *iscsiConnection) init() { diff --git a/pkg/port/iscsit/iscsid.go b/pkg/port/iscsit/iscsid.go index 5be7d10..e715c93 100644 --- a/pkg/port/iscsit/iscsid.go +++ b/pkg/port/iscsit/iscsid.go @@ -21,6 +21,7 @@ import ( "fmt" "net" "os" + "runtime/debug" "github.com/golang/glog" "github.com/gostor/gotgt/pkg/api" @@ -90,15 +91,15 @@ func (s *ISCSITargetService) Run() error { func (s *ISCSITargetService) handler(events byte, conn *iscsiConnection) { if events&DATAIN != 0 { - glog.Infof("rx handler processing...") + glog.V(1).Infof("rx handler processing...") go s.rxHandler(conn) } if conn.state != CONN_STATE_CLOSE && events&DATAOUT != 0 { - glog.Infof("tx handler processing...") + glog.V(1).Infof("tx handler processing...") s.txHandler(conn) } if conn.state == CONN_STATE_CLOSE { - glog.Infof("iscsi connection[%d] closed", conn.cid) + glog.Warningf("iscsi connection[%d] closed", conn.cid) conn.close() } } @@ -160,10 +161,16 @@ func (s *ISCSITargetService) rxHandler(conn *iscsiConnection) { return } dl := ((cmd.DataLen + DataPadding - 1) / DataPadding) * DataPadding - buf, length, err := conn.readData(dl) - if err != nil { - glog.Error(err) - return + buf := []byte{} + length := 0 + for length < dl { + b, l, err := conn.readData(dl - length) + if err != nil { + glog.Error(err) + return + } + length += l + buf = append(buf, b...) } if length != dl { glog.V(2).Infof("get length is %d, but expected %d", length, dl) @@ -213,8 +220,8 @@ func (s *ISCSITargetService) rxHandler(conn *iscsiConnection) { default: iscsiExecReject(conn) } - glog.V(1).Infof("connection state is %v", conn.state) - glog.Infof("%#v", conn.resp.String()) + glog.V(2).Infof("connection state is %v", conn.state) + glog.V(2).Infof("%#v", conn.resp.String()) s.handler(DATAOUT, conn) } } @@ -325,7 +332,8 @@ func (s *ISCSITargetService) iscsiExecText(conn *iscsiConnection) error { } for _, t := range list { result = append(result, util.KeyValue{"TargetName", t.Name}) - result = append(result, util.KeyValue{"TargetAddress", "127.0.0.1:3260,1"}) + result = append(result, util.KeyValue{"TargetAddress", "172.16.69.169:3260,1"}) + //result = append(result, util.KeyValue{"TargetAddress", "127.0.0.1:3260,1"}) } } } @@ -403,7 +411,9 @@ func (s *ISCSITargetService) txHandler(conn *iscsiConnection) { glog.V(2).Infof("length of RawData is %d", len(conn.resp.RawData)) glog.V(2).Infof("length of resp is %d", len(conn.resp.Bytes())) if l, err := conn.write(conn.resp.Bytes()); err != nil { + debug.PrintStack() glog.Error(err) + panic(err) return } else { conn.txIOState = IOSTATE_TX_INIT_AHS @@ -435,30 +445,30 @@ func (s *ISCSITargetService) txHandler(conn *iscsiConnection) { } } - glog.Infof("connection state: %d", conn.state) + glog.V(3).Infof("connection state: %d", conn.state) switch conn.state { case CONN_STATE_CLOSE, CONN_STATE_EXIT: glog.Warningf("set connection to close") conn.state = CONN_STATE_CLOSE case CONN_STATE_SECURITY_LOGIN: conn.state = CONN_STATE_LOGIN - glog.Infof("CONN_STATE_LOGIN") + glog.V(3).Infof("CONN_STATE_LOGIN") case CONN_STATE_SECURITY_FULL, CONN_STATE_LOGIN_FULL: if conn.sessionType == SESSION_NORMAL { conn.state = CONN_STATE_KERNEL glog.Infof("CONN_STATE_KERNEL") conn.state = CONN_STATE_SCSI - glog.Infof("CONN_STATE_SCSI") + glog.V(3).Infof("CONN_STATE_SCSI") } else { conn.state = CONN_STATE_FULL - glog.Infof("CONN_STATE_FULL") + glog.V(3).Infof("CONN_STATE_FULL") } conn.rxIOState = IOSTATE_RX_BHS s.handler(DATAIN, conn) case CONN_STATE_SCSI: conn.txTask = nil default: - glog.Infof("unexpected connection state: %d", conn.state) + glog.Warningf("unexpected connection state: %d", conn.state) conn.rxIOState = IOSTATE_RX_BHS s.handler(DATAIN, conn) } @@ -469,9 +479,49 @@ func (s *ISCSITargetService) scsiCommandHandler(conn *iscsiConnection) (err erro req := conn.req switch req.OpCode { case OpSCSICmd: - glog.Infof("SCSI Command processing...") + glog.V(2).Infof("SCSI Command processing...") scmd := &api.SCSICommand{} task := &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, scmd: scmd} + if req.Write { + task.offset = req.DataLen + task.r2tCount = int(req.ExpectedDataLen) - req.DataLen + if !req.Final { + task.unsolCount = 1 + } + glog.V(2).Infof("SCSI write, R2T count: %d, unsol Count: %d, offset: %d", task.r2tCount, task.unsolCount, task.offset) + + if task.scmd.OutSDBBuffer.Buffer == nil { + task.scmd.OutSDBBuffer.Buffer = bytes.NewBuffer([]byte{}) + } + task.scmd.OutSDBBuffer.Buffer.Write(conn.req.RawData) + if task.r2tCount > 0 { + // prepare to receive more data + task.state = taskPending + conn.session.PendingTasks.Push(task) + //conn.rxTask = nil + conn.txTask = &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, scmd: &api.SCSICommand{}} + conn.txIOState = IOSTATE_TX_BHS + conn.statSN += 1 + resp := &ISCSICommand{ + OpCode: OpReady, + Immediate: true, + Final: true, + StatSN: req.ExpStatSN, + TaskTag: req.TaskTag, + ExpCmdSN: conn.session.ExpCmdSN, + MaxCmdSN: conn.session.ExpCmdSN + 10, + R2TSN: task.r2tSN, + BufferOffset: uint32(task.offset), + DesiredLength: uint32(task.r2tCount), + } + if val := sessionKeys[ISCSI_PARAM_MAX_BURST].def; task.r2tCount > int(val) { + resp.DesiredLength = uint32(val) + } + conn.resp = resp + break + } + } + task.offset = 0 conn.rxTask = task if err = s.iscsiTaskQueueHandler(task); err != nil { return @@ -516,31 +566,76 @@ func (s *ISCSITargetService) scsiCommandHandler(conn *iscsiConnection) (err erro } case OpSCSITaskReq: // task management function - conn.txTask = &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag} + conn.txTask = &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, state: taskPending} conn.txIOState = IOSTATE_TX_BHS iscsiExecTMFunction(conn) case OpSCSIOut: - glog.Infof("scsi out operation") - scmd := &api.SCSICommand{} - conn.req.Write = true - conn.req.CDB = []byte{api.Write_10} - task := &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, scmd: scmd} + glog.V(1).Infof("iSCSI Data-out processing...") + var task *iscsiTask + for _, t := range conn.session.PendingTasks { + if t.tag == conn.req.TaskTag { + task = t + } + } + if task == nil { + err = fmt.Errorf("Cannot find iSCSI task with tag[%v]", conn.req.TaskTag) + glog.Error(err) + return + } + task.offset = task.offset + conn.req.DataLen + task.r2tCount = task.r2tCount - conn.req.DataLen + task.scmd.OutSDBBuffer.Buffer.Write(conn.req.RawData) + glog.V(2).Infof("Final: %v", conn.req.Final) + glog.V(2).Infof("r2tCount: %v", task.r2tCount) + if !conn.req.Final { + glog.V(1).Infof("Not ready to exec the task") + conn.rxIOState = IOSTATE_RX_BHS + s.handler(DATAIN, conn) + return nil + } else if task.r2tCount > 0 { + // prepare to receive more data + conn.rxTask = nil + conn.txTask = &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, scmd: &api.SCSICommand{}} + conn.txIOState = IOSTATE_TX_BHS + conn.statSN += 1 + task.r2tSN += 1 + resp := &ISCSICommand{ + OpCode: OpReady, + Immediate: true, + Final: true, + StatSN: req.ExpStatSN, + TaskTag: req.TaskTag, + ExpCmdSN: conn.session.ExpCmdSN, + MaxCmdSN: conn.session.ExpCmdSN + 10, + R2TSN: task.r2tSN, + BufferOffset: uint32(task.offset), + DesiredLength: uint32(task.r2tCount), + } + if val := sessionKeys[ISCSI_PARAM_MAX_BURST].def; task.r2tCount > int(val) { + resp.DesiredLength = uint32(val) + } + conn.resp = resp + break + } + task.offset = 0 + glog.V(1).Infof("Process the Data-out package") conn.rxTask = task if err = s.iscsiExecTask(task); err != nil { return } else { conn.rxTask = nil - conn.txTask = &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, scmd: &api.SCSICommand{}} + conn.txTask = &iscsiTask{conn: conn, cmd: conn.req, tag: conn.req.TaskTag, scmd: &api.SCSICommand{}, state: taskSCSI} conn.txIOState = IOSTATE_TX_BHS conn.statSN += 1 resp := &ISCSICommand{ + OpCode: OpSCSIResp, Immediate: true, Final: true, StatSN: req.ExpStatSN, TaskTag: req.TaskTag, ExpCmdSN: conn.session.ExpCmdSN, MaxCmdSN: conn.session.ExpCmdSN + 10, - Status: scmd.Result, + Status: 0, SCSIResponse: 0x00, HasStatus: true, } @@ -581,9 +676,9 @@ func (s *ISCSITargetService) iscsiTaskQueueHandler(task *iscsiTask) error { retry: cmdsn += 1 sess.ExpCmdSN = cmdsn - glog.Infof("session's ExpCmdSN is %d", cmdsn) + glog.V(2).Infof("session's ExpCmdSN is %d", cmdsn) - glog.Infof("process task(%d)", task.cmd.CmdSN) + glog.V(2).Infof("process task(%d)", task.cmd.CmdSN) if err := s.iscsiExecTask(task); err != nil { glog.Error(err) } @@ -604,7 +699,7 @@ func (s *ISCSITargetService) iscsiTaskQueueHandler(task *iscsiTask) error { glog.Error(err) return err } - glog.Infof("add task(%d) into task queue", task.cmd.CmdSN) + glog.V(1).Infof("add task(%d) into task queue", task.cmd.CmdSN) // add this connection into queue and set this task as pending task task.state = taskPending sess.PendingTasks.Push(task) @@ -634,7 +729,9 @@ func (s *ISCSITargetService) iscsiExecTask(task *iscsiTask) error { task.scmd.Lun = cmd.LUN task.scmd.Tag = uint64(cmd.TaskTag) task.state = taskSCSI - task.scmd.OutSDBBuffer.Buffer = bytes.NewBuffer(cmd.RawData) + if task.scmd.OutSDBBuffer.Buffer == nil { + task.scmd.OutSDBBuffer.Buffer = bytes.NewBuffer(cmd.RawData) + } // add scsi target process queue err := s.SCSI.AddCommandQueue(task.conn.session.Target.SCSITarget.TID, task.scmd) if err != nil { diff --git a/pkg/scsi/lun.go b/pkg/scsi/lun.go index 983f1e5..53396aa 100644 --- a/pkg/scsi/lun.go +++ b/pkg/scsi/lun.go @@ -34,7 +34,7 @@ func NewSCSILu(lun uint64, target *api.SCSITarget) (*api.SCSILu, error) { PerformCommand: luPerformCommand, DeviceProtocol: sbc, Storage: backing, - BlockShift: 0, + BlockShift: api.DefaultBlockShift, Size: 1024 * 1024 * 10, } // hack this