NVMe over TCP Write/Read命令下发流程梳理

总结

本文对NVMe over TCP的write和read命令下发流程进行了梳理

1. 环境

只针对Linux5.4.0版本的nvme内核模块源代码，使用命令

sudo nvme io-passthru /dev/nvme1n1 --opcode=1 --namespace-id=1 --data-len=4096 --write --cdw10=5120 --cdw11=0 --cdw12=7 -s -i 123.txt
sudo nvme io-passthru /dev/nvme1n1 --opcode=2 --namespace-id=1 --data-len=4096 --write --cdw10=5120 --cdw11=0 --cdw12=7 -s -i 456.txt

下发write和read命令，从nvme_ioctl函数开始对write进行一个梳理过程

2. 一些前置知识

NVMe over TCP传输时，使用的是SGL而不是PRP

3. host端与target端IO调用栈

我将所有函数都打了printk，write命令下发时，host端与target端的函数调用关系如下：

Host端：

[  627.604513] nvme_dev_release
[  709.055115] *****************************nvme_ioctl
[  709.055116] nvme_get_ns_from_disk
[  709.055117] nvme_user_cmd
[  709.055120] nvme_passthru_start
[  709.055120] nvme_to_user_ptr
[  709.055121] nvme_to_user_ptr
[  709.055121] nvme_submit_user_cmd
[  709.055122] nvme_alloc_request
[  709.055128] nvme_tcp_queue_rq
[  709.055128] nvme_tcp_setup_cmd_pdu
[  709.055129] nvme_setup_cmd
[  709.055129] nvme_tcp_map_data
[  709.055129] nvme_tcp_set_sg_inline
[  709.055131] ##### nvme_tcp_queue_request
[  709.055135] *************************************nvme_tcp_io_work
[  709.055136] nvme_tcp_try_send
[  709.055136] nvme_tcp_fetch_request
[  709.055136] nvme_tcp_try_send_cmd_pdu
[  709.055137] nvme_tcp_has_inline_data
[  709.055146] nvme_tcp_init_iter
[  709.055147] nvme_tcp_has_inline_data
[  709.055147] nvme_tcp_try_send_data
[  709.060325] nvme_tcp_data_ready
[  709.060327] nvme_tcp_advance_req
[  709.060328] nvme_tcp_try_recv
[  709.060328] nvme_tcp_recv_skb
[  709.060329] nvme_tcp_recv_pdu
[  709.060329] nvme_tcp_init_recv_ctx
[  709.060329] nvme_tcp_handle_comp
[  709.060330] nvme_tcp_process_nvme_cqe
[  709.060331] nvme_complete_rq
[  709.060331] nvme_error_status
[  709.060339] nvme_tcp_try_send
[  709.060339] nvme_tcp_fetch_request
[  709.060339] nvme_tcp_try_recv
[  709.060340] *************************************nvme_tcp_io_work
[  709.060341] nvme_tcp_try_send
[  709.060341] nvme_tcp_fetch_request
[  709.060341] nvme_tcp_try_recv

target端：

[  587.920537] nvmet_tcp_data_ready
[  587.920666] nvmet_tcp_io_work
[  587.920667] ********************************nvmet_tcp_try_recv
[  587.920667] nvmet_tcp_try_recv_one
[  587.920667] nvmet_tcp_try_recv_pdu
[  587.920672] nvmet_tcp_done_recv_pdu
[  587.920672] nvmet_tcp_get_cmd
[  587.920674] nvmet_req_init
[  587.920674] nvmet_parse_io_cmd
[  587.920675] nvmet_bdev_parse_io_cmd
[  587.920677] nvmet_tcp_map_data
[  587.920681] nvmet_tcp_map_pdu_iovec
[  587.920681] nvmet_tcp_try_recv_data
[  587.920683] nvmet_req_execute
[  587.920684] nvmet_bdev_execute_rw
[  587.920696] nvme_setup_cmd
[  587.920697] nvme_setup_rw
[  587.920711] nvmet_prepare_receive_pdu
[  587.920711] nvmet_tcp_try_recv_one
[  587.920711] nvmet_tcp_try_recv_pdu
[  587.920712] nvmet_tcp_try_send
[  587.920712] nvmet_tcp_try_send_one
[  587.920713] nvmet_tcp_fetch_cmd
[  587.920713] nvmet_tcp_process_resp_list
[  587.922352] nvme_cleanup_cmd
[  587.922353] nvme_complete_rq
[  587.922354] nvme_error_status
[  587.922356] nvmet_bio_done
[  587.922357] blk_to_nvme_status
[  587.922357] nvmet_req_complete
[  587.922358] __nvmet_req_complete
[  587.922359] nvmet_tcp_queue_response
[  587.922370] nvmet_tcp_io_work
[  587.922370] ********************************nvmet_tcp_try_recv
[  587.922371] nvmet_tcp_try_recv_one
[  587.922371] nvmet_tcp_try_recv_pdu
[  587.922373] nvmet_tcp_try_send
[  587.922374] nvmet_tcp_try_send_one
[  587.922374] nvmet_tcp_fetch_cmd
[  587.922374] nvmet_tcp_process_resp_list
[  587.922375] nvmet_setup_response_pdu
[  587.922375] nvmet_try_send_response
[  587.922430] nvmet_tcp_try_send_one
[  587.922431] nvmet_tcp_fetch_cmd
[  587.922431] nvmet_tcp_process_resp_list

read命令下发时，函数调用IO栈如下：

host端：

[10119.754240] *****************************nvme_ioctl
[10119.754241] nvme_get_ns_from_disk
[10119.754246] nvme_user_cmd
[10119.754248] nvme_passthru_start
[10119.754249] nvme_to_user_ptr
[10119.754250] nvme_to_user_ptr
[10119.754250] nvme_submit_user_cmd
[10119.754251] nvme_alloc_request
[10119.754264] nvme_tcp_queue_rq
[10119.754265] nvme_tcp_setup_cmd_pdu
[10119.754265] nvme_setup_cmd
[10119.754268] nvme_tcp_init_iter
[10119.754269] nvme_tcp_map_data
[10119.754269] nvme_tcp_set_sg_host_data
[10119.754271] ##### nvme_tcp_queue_request
[10119.754278] *************************************nvme_tcp_io_work
[10119.754278] nvme_tcp_try_send
[10119.754278] nvme_tcp_fetch_request
[10119.754279] nvme_tcp_try_send_cmd_pdu
[10119.754279] nvme_tcp_has_inline_data
[10119.754363] nvme_tcp_done_send_req
[10119.754364] nvme_tcp_has_inline_data
[10119.754364] nvme_tcp_try_recv
[10119.754365] nvme_tcp_try_send
[10119.754365] nvme_tcp_fetch_request
[10119.754366] nvme_tcp_try_recv
[10119.760851] nvme_tcp_data_ready
[10119.760860] nvme_tcp_data_ready
[10119.760864] *************************************nvme_tcp_io_work
[10119.760865] nvme_tcp_try_send
[10119.760865] nvme_tcp_fetch_request
[10119.760866] nvme_tcp_try_recv
[10119.762412] nvme_tcp_recv_skb
[10119.762412] nvme_tcp_recv_pdu
[10119.762413] nvme_tcp_handle_c2h_data
[10119.762415] nvme_tcp_recv_data
[10119.762417] nvme_tcp_init_recv_ctx
[10119.762417] nvme_tcp_recv_pdu
[10119.762417] nvme_tcp_init_recv_ctx
[10119.762418] nvme_tcp_handle_comp
[10119.762418] nvme_tcp_process_nvme_cqe
[10119.762419] nvme_complete_rq
[10119.762419] nvme_error_status
[10119.762426] nvme_tcp_try_send
[10119.762426] nvme_tcp_fetch_request
[10119.762427] nvme_tcp_try_recv

target端：

[10113.557551] nvmet_tcp_io_work
[10113.557552] ********************************nvmet_tcp_try_recv
[10113.557552] nvmet_tcp_try_recv_one
[10113.557553] nvmet_tcp_try_recv_pdu
[10113.557562] nvmet_tcp_done_recv_pdu
[10113.557562] nvmet_tcp_get_cmd
[10113.557567] nvmet_req_init
[10113.557567] nvmet_parse_io_cmd
[10113.557576] nvmet_bdev_parse_io_cmd
[10113.557579] nvmet_tcp_map_data
[10113.557583] nvmet_req_execute
[10113.557583] nvmet_bdev_execute_rw
[10113.557607] nvme_setup_cmd
[10113.557607] nvme_setup_rw
[10113.557635] nvme_cleanup_cmd
[10113.557636] nvme_complete_rq
[10113.557637] nvme_error_status
[10113.557637] blk_mq_end_request
[10113.557638] nvmet_bio_done
[10113.557638] blk_to_nvme_status
[10113.557639] nvmet_req_complete
[10113.557639] __nvmet_req_complete
[10113.557640] nvmet_tcp_queue_response
[10113.557645] nvmet_prepare_receive_pdu
[10113.557645] nvmet_tcp_try_recv_one
[10113.557646] nvmet_tcp_try_recv_pdu
[10113.557646] nvmet_tcp_try_send
[10113.557647] nvmet_tcp_try_send_one
[10113.557647] nvmet_tcp_fetch_cmd
[10113.557647] nvmet_tcp_process_resp_list
[10113.557648] nvmet_setup_c2h_data_pdu
[10113.557649] nvmet_try_send_data_pdu
[10113.557653] nvmet_try_send_data
[10113.557685] nvmet_setup_response_pdu
[10113.557685] nvmet_try_send_response
[10113.557686] nvmet_tcp_put_cmd
[10113.557687] nvmet_tcp_try_send_one
[10113.557687] nvmet_tcp_fetch_cmd
[10113.557687] nvmet_tcp_process_resp_list
[10113.557687] llist_del_all=NULL
[10113.557688] nvmet_tcp_io_work
[10113.557688] ********************************nvmet_tcp_try_recv
[10113.557689] nvmet_tcp_try_recv_one
[10113.557689] nvmet_tcp_try_recv_pdu
[10113.557690] nvmet_tcp_try_send
[10113.557690] nvmet_tcp_try_send_one
[10113.557690] nvmet_tcp_fetch_cmd
[10113.557691] nvmet_tcp_process_resp_list

4. Write命令函数调用关系图

5. Read命令函数调用关系图

Write和Read命令下发的IO栈函数关系图如图所示，自己画的不是特别美观，看不清楚的可以下载下来然后放大

下面以Write命令为例将函数按照顺序从上到下，从左到右，按箭头的顺序进行分析，Read命令类似（写的不是太好，可能得仔细看才能跟得上逻辑）

6. 函数具体分析

host端

nvme_ioctl

进入nvme_ioctl函数，传递由nvme-cli下发下来的写命令

static int nvme_ioctl(struct block_device *bdev, fmode_t mode,unsigned int cmd, unsigned long arg)
{struct nvme_ns_head *head = NULL;void __user *argp = (void __user *)arg;struct nvme_ns *ns;int srcu_idx, ret;// 读取nvme设备的namespacens = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);if (unlikely(!ns))return -EWOULDBLOCK;/** Handle ioctls that apply to the controller instead of the namespace* seperately and drop the ns SRCU reference early.  This avoids a* deadlock when deleting namespaces using the passthrough interface.*/if (is_ctrl_ioctl(cmd)){// 如果是admin命令，进入该分支，提前释放nsreturn nvme_handle_ctrl_ioctl(ns, cmd, argp, head, srcu_idx);}switch (cmd) {case NVME_IOCTL_ID:force_successful_syscall_return();ret = ns->head->ns_id;break;case NVME_IOCTL_IO_CMD:// 如果是IO命令进入该分支ret = nvme_user_cmd(ns->ctrl, ns, argp);break;case NVME_IOCTL_SUBMIT_IO:ret = nvme_submit_io(ns, argp);break;case NVME_IOCTL_IO64_CMD:ret = nvme_user_cmd64(ns->ctrl, ns, argp);break;default:if (ns->ndev)ret = nvme_nvm_ioctl(ns, cmd, arg);elseret = -ENOTTY;}nvme_put_ns_from_disk(head, srcu_idx);return ret;
}

由于是写命令，所以会进入nvme_user_cmd函数中进行命令下发

nvme_user_cmd

static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,struct nvme_passthru_cmd __user *ucmd)
{struct nvme_passthru_cmd cmd;struct nvme_command c;unsigned timeout = 0;u32 effects;u64 result;int status;if (!capable(CAP_SYS_ADMIN))return -EACCES;/*复制用户态已经设定好的cmd到cmd结构体中*/if (copy_from_user(&cmd, ucmd, sizeof(cmd)))return -EFAULT;if (cmd.flags)return -EINVAL;/*对nvme_command做一些初始化*/memset(&c, 0, sizeof(c));c.common.opcode = cmd.opcode;c.common.flags = cmd.flags;c.common.nsid = cpu_to_le32(cmd.nsid);c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);c.common.cdw10 = cpu_to_le32(cmd.cdw10);c.common.cdw11 = cpu_to_le32(cmd.cdw11);c.common.cdw12 = cpu_to_le32(cmd.cdw12);c.common.cdw13 = cpu_to_le32(cmd.cdw13);c.common.cdw14 = cpu_to_le32(cmd.cdw14);c.common.cdw15 = cpu_to_le32(cmd.cdw15);/*设置超时时间*/if (cmd.timeout_ms)timeout = msecs_to_jiffies(cmd.timeout_ms);effects = nvme_passthru_start(ctrl, ns, cmd.opcode);/*下发nvme_command,因为是使用nvme_over_TCP，所以使用的是tcp queue，而不是通过pci queue*/status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,nvme_to_user_ptr(cmd.addr), cmd.data_len,nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,0, &result, timeout);nvme_passthru_end(ctrl, effects);if (status >= 0) {/*用于admin命令或者nvme over fabric命令回传结果信息*/if (put_user(result, &ucmd->result)){return -EFAULT;}}return status;
}

nvme_submit_user_cmd

static int nvme_submit_user_cmd(struct request_queue *q,struct nvme_command *cmd, void __user *ubuffer,unsigned bufflen, void __user *meta_buffer, unsigned meta_len,u32 meta_seed, u64 *result, unsigned timeout)
{/*NVMe 命令中，如果最低位为1 代表这是一个写数据命令，如果不为0 则是一个读数据命令*/bool write = nvme_is_write(cmd);struct nvme_ns *ns = q->queuedata;struct gendisk *disk = ns ? ns->disk : NULL;struct request *req;struct bio *bio = NULL;void *meta = NULL;int ret;/*创建一个request结构体供后续执行*/req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY);if (IS_ERR(req)){return PTR_ERR(req);}req->timeout = timeout ? timeout : ADMIN_TIMEOUT;nvme_req(req)->flags |= NVME_REQ_USERCMD;if (ubuffer && bufflen) {/*完成用户态内存ubuffer和req->bio的映射，这样下发数据可以直接根据req->bio取得用户态的数据或者读取数据时，直接通过该bio将数据读取到用户态的buffer中*/ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,GFP_KERNEL);if (ret)goto out;bio = req->bio;bio->bi_disk = disk;if (disk && meta_buffer && meta_len) {/*加入元数据*/meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,meta_seed, write);if (IS_ERR(meta)) {ret = PTR_ERR(meta);goto out_unmap;}req->cmd_flags |= REQ_INTEGRITY;}}/*驱动层调用blk_execute_rq让块设备层执行request，由于是tcp queue，所以调用的是nvme_tcp_queue_rq函数*//*insert a request into queue for execution*/blk_execute_rq(req->q, disk, req, 0);if (nvme_req(req)->flags & NVME_REQ_CANCELLED)ret = -EINTR;elseret = nvme_req(req)->status;if (result)*result = le64_to_cpu(nvme_req(req)->result.u64);if (meta && !ret && !write) {/*如果是读命令且由metadata，将metadata写入到metabuffer中*/if (copy_to_user(meta_buffer, meta, meta_len))ret = -EFAULT;}kfree(meta);out_unmap:if (bio)blk_rq_unmap_user(bio); /*取消用户态bio的映射关系*/out:blk_mq_free_request(req);return ret;
}

nvme_tcp_queue_rq

static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx,const struct blk_mq_queue_data *bd)
{struct nvme_ns *ns = hctx->queue->queuedata;struct nvme_tcp_queue *queue = hctx->driver_data;struct request *rq = bd->rq;struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags);blk_status_t ret;if (!nvmf_check_ready(&queue->ctrl->ctrl, rq, queue_ready))return nvmf_fail_nonready_command(&queue->ctrl->ctrl, rq);/*对将要发送的cmd pdu做一些初始化操作 包括分配cmd结构体，分配sgl*/ret = nvme_tcp_setup_cmd_pdu(ns, rq);if (unlikely(ret))return ret;/*为request的执行做一些前置工作*/blk_mq_start_request(rq);/*准备开始执行request*/nvme_tcp_queue_request(req);return BLK_STS_OK;
}

nvme_tcp_setup_cmd_pdu

static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns,struct request *rq)
{struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);struct nvme_tcp_cmd_pdu *pdu = req->pdu;struct nvme_tcp_queue *queue = req->queue;u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0;blk_status_t ret;ret = nvme_setup_cmd(ns, rq, &pdu->cmd);if (ret)return ret;/*初始化nvme_tcp_request的一些状态，上面初始化的是request，而根据blk_mq_rq_to_pdu(rq)可知，两者是在物理空间上相邻的*/req->state = NVME_TCP_SEND_CMD_PDU;req->offset = 0;req->data_sent = 0;req->pdu_len = 0;req->pdu_sent = 0;req->data_len = blk_rq_nr_phys_segments(rq) ?blk_rq_payload_bytes(rq) : 0; /*要发送数据的长度，这里的长度由*/req->curr_bio = rq->bio;/*如果发送的是写命令而且req的data_len <= 要发送的数据大小，则pdu_len赋值为data_len 这里可能是说，有data_len < 要发送数据的情况，可能是用户在自定义参数赋值时出错了啥的*/if (rq_data_dir(rq) == WRITE &&req->data_len <= nvme_tcp_inline_data_size(queue))req->pdu_len = req->data_len;/*如果curr_bio不为空，说明是读命令，则初始化iter，这个是用于后续接收到sock数据写到这里*/else if (req->curr_bio)nvme_tcp_init_iter(req, READ);/*对将要发送的pdu的hdr进行一些初始化操作*/pdu->hdr.type = nvme_tcp_cmd;pdu->hdr.flags = 0;if (queue->hdr_digest)pdu->hdr.flags |= NVME_TCP_F_HDGST;if (queue->data_digest && req->pdu_len) {pdu->hdr.flags |= NVME_TCP_F_DDGST;ddgst = nvme_tcp_ddgst_len(queue);}pdu->hdr.hlen = sizeof(*pdu);pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0;pdu->hdr.plen =cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst);/*初始化SGL，但不知道里面的addr中0和icdoff的意义*/ret = nvme_tcp_map_data(queue, rq);if (unlikely(ret)) {nvme_cleanup_cmd(rq);dev_err(queue->ctrl->ctrl.device,"Failed to map data (%d)\n", ret);return ret;}return 0;
}

nvme_tcp_queue_request

static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req)
{printk("##### nvme_tcp_queue_request");struct nvme_tcp_queue *queue = req->queue;spin_lock(&queue->lock);list_add_tail(&req->entry, &queue->send_list);spin_unlock(&queue->lock);/*执行io_work函数*/queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
}

nvme_tcp_io_work

static void nvme_tcp_io_work(struct work_struct *w)
{struct nvme_tcp_queue *queue =container_of(w, struct nvme_tcp_queue, io_work);unsigned long deadline = jiffies + msecs_to_jiffies(1);do {bool pending = false;int result;/*利用TCP/IP发送PDU*/result = nvme_tcp_try_send(queue);if (result > 0) {pending = true;} else if (unlikely(result < 0)) {dev_err(queue->ctrl->ctrl.device,"failed to send request %d\n", result);/** Fail the request unless peer closed the connection,* in which case error recovery flow will complete all.*/if ((result != -EPIPE) && (result != -ECONNRESET)){nvme_tcp_fail_request(queue->request);}nvme_tcp_done_send_req(queue);return;}/*接收cqe*/result = nvme_tcp_try_recv(queue);if (result > 0)pending = true;if (!pending){return;}} while (!time_after(jiffies, deadline)); /* quota is exhausted */queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
}

nvme_tcp_try_send

static int nvme_tcp_try_send(struct nvme_tcp_queue *queue)
{struct nvme_tcp_request *req;int ret = 1;if (!queue->request) {/*从queue中取出request执行，如果queue中没有request，则不执行，直接返回*/queue->request = nvme_tcp_fetch_request(queue);if (!queue->request){return 0;}}req = queue->request;/*注意这里不是if else，而是4个if，req_>state的值会在各个send函数中改变，write命令在这里的执行顺序是，nvme_tcp_try_send_cmd_pdu->nvme_tcp_try_send_data，无DDGST*/if (req->state == NVME_TCP_SEND_CMD_PDU) {ret = nvme_tcp_try_send_cmd_pdu(req);if (ret <= 0)goto done;if (!nvme_tcp_has_inline_data(req)){return ret;}}if (req->state == NVME_TCP_SEND_H2C_PDU) {ret = nvme_tcp_try_send_data_pdu(req);if (ret <= 0)goto done;}if (req->state == NVME_TCP_SEND_DATA) {ret = nvme_tcp_try_send_data(req);if (ret <= 0)goto done;}if (req->state == NVME_TCP_SEND_DDGST)ret = nvme_tcp_try_send_ddgst(req);
done:if (ret == -EAGAIN)ret = 0;return ret;
}

nvme_tcp_try_send_cmd_pdu

static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
{struct nvme_tcp_queue *queue = req->queue;struct nvme_tcp_cmd_pdu *pdu = req->pdu;bool inline_data = nvme_tcp_has_inline_data(req);int flags = MSG_DONTWAIT | (inline_data ? MSG_MORE : MSG_EOR);u8 hdgst = nvme_tcp_hdgst_len(queue);int len = sizeof(*pdu) + hdgst - req->offset;int ret;/*如果有摘要，则先处理摘要*/if (queue->hdr_digest && !req->offset)nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));/*发送PDU*/ret = kernel_sendpage(queue->sock, virt_to_page(pdu),offset_in_page(pdu) + req->offset, len,  flags);if (unlikely(ret <= 0))return ret;len -= ret;if (!len) {if (inline_data) {/*如果有data要发送，则将state置为NVME_TCP_SEND_DATA，这样就可以进入nvme_tcp_try_send_data函数了*/req->state = NVME_TCP_SEND_DATA;if (queue->data_digest)crypto_ahash_init(queue->snd_hash);/*对iter进行初始化，不过不知道这个iter和SGL什么关系*/nvme_tcp_init_iter(req, WRITE);} else {nvme_tcp_done_send_req(queue);}return 1;} /*累加offset*/req->offset += ret;return -EAGAIN;
}

nvme_tcp_try_send_data

static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
{struct nvme_tcp_queue *queue = req->queue;while (true) {struct page *page = nvme_tcp_req_cur_page(req);size_t offset = nvme_tcp_req_cur_offset(req);size_t len = nvme_tcp_req_cur_length(req);bool last = nvme_tcp_pdu_last_send(req, len);int ret, flags = MSG_DONTWAIT;/*如果是最后的数据发送了而且没有data摘要，flag加上EOR，否则是MORE*/if (last && !queue->data_digest)flags |= MSG_EOR;elseflags |= MSG_MORE;if (sendpage_ok(page)) {ret = kernel_sendpage(queue->sock, page, offset, len,flags);} else {ret = sock_no_sendpage(queue->sock, page, offset, len,flags);}if (ret <= 0)return ret;/*计算偏移值，便于继续发送下一个BIO的数据，所以，senddata其实发送的是BIO的数据？如果不连续的话，则发送完一个BIO再发送下一个BIO的数据*/nvme_tcp_advance_req(req, ret);if (queue->data_digest)nvme_tcp_ddgst_update(queue->snd_hash, page,offset, ret);/* fully successful last write*/if (last && ret == len) {if (queue->data_digest) {nvme_tcp_ddgst_final(queue->snd_hash,&req->ddgst);req->state = NVME_TCP_SEND_DDGST;req->offset = 0;} else {/*发送全部结束*/nvme_tcp_done_send_req(queue);}return 1;}}return -EAGAIN;
}

至此，Host端的数据发送已经结束（无DIGIST），之后按时间顺序转到Target端执行相关函数

target端

nvmet_tcp_io_work

static void nvmet_tcp_io_work(struct work_struct *w)
{struct nvmet_tcp_queue *queue =container_of(w, struct nvmet_tcp_queue, io_work);bool pending;int ret, ops = 0;do {pending = false;/*与host端的io_work不一样，target端的io_work最开始是尝试接收*/ret = nvmet_tcp_try_recv(queue, NVMET_TCP_RECV_BUDGET, &ops);if (ret > 0) {pending = true;} else if (ret < 0) {if (ret == -EPIPE || ret == -ECONNRESET)kernel_sock_shutdown(queue->sock, SHUT_RDWR);elsenvmet_tcp_fatal_error(queue);return;}/*接收完毕后，开始根据接收到的数据向主机端发送cmd，可能是R2T，可能是cqe*/ret = nvmet_tcp_try_send(queue, NVMET_TCP_SEND_BUDGET, &ops);if (ret > 0) {/* transmitted message/data */pending = true;} else if (ret < 0) {if (ret == -EPIPE || ret == -ECONNRESET)kernel_sock_shutdown(queue->sock, SHUT_RDWR);elsenvmet_tcp_fatal_error(queue);return;}} while (pending && ops < NVMET_TCP_IO_WORK_BUDGET);/** We exahusted our budget, requeue our selves*/if (pending)queue_work_on(queue->cpu, nvmet_tcp_wq, &queue->io_work);
}

nvmet_tcp_try_recv

static int nvmet_tcp_try_recv(struct nvmet_tcp_queue *queue,int budget, int *recvs)
{int i, ret = 0;for (i = 0; i < budget; i++) {/*尝试从sock接收数据*/ret = nvmet_tcp_try_recv_one(queue);if (ret <= 0)break;(*recvs)++;}return ret;
}

nvmet_tcp_try_recv_one

static int nvmet_tcp_try_recv_one(struct nvmet_tcp_queue *queue)
{int result = 0;if (unlikely(queue->rcv_state == NVMET_TCP_RECV_ERR))return 0;/*根据queue->rcv_state来进入不同的分支，write的state变化是，NVMET_TCP_RECV_PDU->nvmet_tcp_try_recv_dataqueue->rcv_state在队列最开始初始化时，状态是NVMET_TCP_RECV_PDU*/if (queue->rcv_state == NVMET_TCP_RECV_PDU) {result = nvmet_tcp_try_recv_pdu(queue);if (result != 0)goto done_recv;}if (queue->rcv_state == NVMET_TCP_RECV_DATA) {result = nvmet_tcp_try_recv_data(queue);if (result != 0)goto done_recv;}if (queue->rcv_state == NVMET_TCP_RECV_DDGST) {result = nvmet_tcp_try_recv_ddgst(queue);if (result != 0)goto done_recv;}done_recv:if (result < 0) {if (result == -EAGAIN)return 0;return result;}return 1;
}

nvmet_tcp_try_recv_pdu

static int nvmet_tcp_try_recv_pdu(struct nvmet_tcp_queue *queue)
{struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr;int len;struct kvec iov;struct msghdr msg = { .msg_flags = MSG_DONTWAIT };recv:/*建立iov与queue->pdu的映射关系，这样sock接收到的数据可以直接写到queue->pdu中进行访问queue->left初始值为sizeof(struct nvme_tcp_hdr)*/iov.iov_base = (void *)&queue->pdu + queue->offset;iov.iov_len = queue->left初始值为;len = kernel_recvmsg(queue->sock, &msg, &iov, 1,iov.iov_len, msg.msg_flags);if (unlikely(len < 0))return len;queue->offset += len;queue->left -= len;if (queue->left)return -EAGAIN;if (queue->offset == sizeof(struct nvme_tcp_hdr)) {u8 hdgst = nvmet_tcp_hdgst_len(queue);if (unlikely(!nvmet_tcp_pdu_valid(hdr->type))) {pr_err("unexpected pdu type %d\n", hdr->type);nvmet_tcp_fatal_error(queue);return -EIO;}if (unlikely(hdr->hlen != nvmet_tcp_pdu_size(hdr->type))) {pr_err("pdu %d bad hlen %d\n", hdr->type, hdr->hlen);return -EIO;}queue->left = hdr->hlen - queue->offset + hdgst;goto recv;}if (queue->hdr_digest &&nvmet_tcp_verify_hdgst(queue, &queue->pdu, queue->offset)) {nvmet_tcp_fatal_error(queue); /* fatal */return -EPROTO;}if (queue->data_digest &&nvmet_tcp_check_ddgst(queue, &queue->pdu)) {nvmet_tcp_fatal_error(queue); /* fatal */return -EPROTO;}/*结束接收PDU的工作，接下来根据接受到的PDU进行处理*/return nvmet_tcp_done_recv_pdu(queue);
}

nvmet_tcp_done_recv_pdu

static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue)
{struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr;struct nvme_command *nvme_cmd = &queue->pdu.cmd.cmd;struct nvmet_req *req;int ret;if (unlikely(queue->state == NVMET_TCP_Q_CONNECTING)) {if (hdr->type != nvme_tcp_icreq) {pr_err("unexpected pdu type (%d) before icreq\n",hdr->type);nvmet_tcp_fatal_error(queue);return -EPROTO;}return nvmet_tcp_handle_icreq(queue);}if (hdr->type == nvme_tcp_h2c_data) {ret = nvmet_tcp_handle_h2c_data_pdu(queue);if (unlikely(ret))return ret;return 0;}/*新建nvmet_tcp_cmd结构体并做一些初始化操作*/queue->cmd = nvmet_tcp_get_cmd(queue);if (unlikely(!queue->cmd)) {/* This should never happen */pr_err("queue %d: out of commands (%d) send_list_len: %d, opcode: %d",queue->idx, queue->nr_cmds, queue->send_list_len,nvme_cmd->common.opcode);nvmet_tcp_fatal_error(queue);return -ENOMEM;}req = &queue->cmd->req;/*复制传递过来的nvme_cmd到queue->cmd->req->cmd中去*/memcpy(req->cmd, nvme_cmd, sizeof(*nvme_cmd));/*对nvmet_req做一些初始化操作，包括根据opcode绑定执行函数，绑定sqe，cqe等*/if (unlikely(!nvmet_req_init(req, &queue->nvme_cq,&queue->nvme_sq, &nvmet_tcp_ops))) {pr_err("failed cmd %p id %d opcode %d, data_len: %d\n",req->cmd, req->cmd->common.command_id,req->cmd->common.opcode,le32_to_cpu(req->cmd->common.dptr.sgl.length));nvmet_tcp_handle_req_failure(queue, queue->cmd, req);return -EAGAIN;}/*申请SGL数组，申请iov数组*/ret = nvmet_tcp_map_data(queue->cmd);if (unlikely(ret)) {pr_err("queue %d: failed to map data\n", queue->idx);if (nvmet_tcp_has_inline_data(queue->cmd))nvmet_tcp_fatal_error(queue);elsenvmet_req_complete(req, ret);ret = -EAGAIN;goto out;}/*对上面申请的iov和sgl做映射，然后初始化cmd->recv_msg.msg_iter，将状态变为NVMET_TCP_RECV_DATA，准备接收host端发送过来的数据，然后直接返回到nvmet_tcp_try_recv_one函数中去*/if (nvmet_tcp_need_data_in(queue->cmd)) {if (nvmet_tcp_has_inline_data(queue->cmd)) {queue->rcv_state = NVMET_TCP_RECV_DATA;nvmet_tcp_map_pdu_iovec(queue->cmd);return 0;}/* send back R2T */nvmet_tcp_queue_response(&queue->cmd->req);goto out;}nvmet_req_execute(&queue->cmd->req);
out:nvmet_prepare_receive_pdu(queue);return ret;
}

nvmet_req_init

bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq,struct nvmet_sq *sq, const struct nvmet_fabrics_ops *ops)
{u8 flags = req->cmd->common.flags;u16 status;req->cq = cq;req->sq = sq;req->ops = ops;req->sg = NULL;req->sg_cnt = 0;req->transfer_len = 0;req->cqe->status = 0;req->cqe->sq_head = 0;req->ns = NULL;req->error_loc = NVMET_NO_ERROR_LOC;req->error_slba = 0;/* no support for fused commands yet */if (unlikely(flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND)) || req->cmd->common.opcode == nvme_admin_ndp_execute) {req->error_loc = offsetof(struct nvme_common_command, flags);status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;goto fail;}/** For fabrics, PSDT field shall describe metadata pointer (MPTR) that* contains an address of a single contiguous physical buffer that is* byte aligned.*/if (unlikely((flags & NVME_CMD_SGL_ALL) != NVME_CMD_SGL_METABUF)) {req->error_loc = offsetof(struct nvme_common_command, flags);status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;printk("For fabrics, PSDT field shall describe metadata pointer (MPTR) status=%d",status);goto fail;}if (unlikely(!req->sq->ctrl)){/* will return an error for any Non-connect command: */status = nvmet_parse_connect_cmd(req);}/*IO命令进入，做相关初始化操作*/else if (likely(req->sq->qid != 0)){status = nvmet_parse_io_cmd(req);}else if (nvme_is_fabrics(req->cmd)){status = nvmet_parse_fabrics_cmd(req);}else if (req->sq->ctrl->subsys->type == NVME_NQN_DISC){status = nvmet_parse_discovery_cmd(req);}/*admin命令进入，做相关初始化操作*/else{status = nvmet_parse_admin_cmd(req);}if (status)goto fail;trace_nvmet_req_init(req, req->cmd);if (unlikely(!percpu_ref_tryget_live(&sq->ref))) {status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;printk("unlikely(!percpu_ref_tryget_live(&sq->ref)) status=%d",status);goto fail;}if (sq->ctrl)sq->ctrl->cmd_seen = true;return true;fail:__nvmet_req_complete(req, status);return false;
}
EXPORT_SYMBOL_GPL(nvmet_req_init);

nvmet_parse_io_cmd

static u16 nvmet_parse_io_cmd(struct nvmet_req *req)
{struct nvme_command *cmd = req->cmd;u16 ret;ret = nvmet_check_ctrl_status(req, cmd);if (unlikely(ret))return ret;/*检查各种状态后，绑定执行函数*/req->ns = nvmet_find_namespace(req->sq->ctrl, cmd->rw.nsid);if (unlikely(!req->ns)) {req->error_loc = offsetof(struct nvme_common_command, nsid);return NVME_SC_INVALID_NS | NVME_SC_DNR;}ret = nvmet_check_ana_state(req->port, req->ns);if (unlikely(ret)) {req->error_loc = offsetof(struct nvme_common_command, nsid);return ret;}ret = nvmet_io_cmd_check_access(req);if (unlikely(ret)) {req->error_loc = offsetof(struct nvme_common_command, nsid);return ret;}if (req->ns->file){return nvmet_file_parse_io_cmd(req);}else{/*Write操作进入这个函数绑定执行操作以及计算data_len*/return nvmet_bdev_parse_io_cmd(req);}
}

nvmet_bdev_parse_io_cmd

u16 nvmet_bdev_parse_io_cmd(struct nvmet_req *req)
{struct nvme_command *cmd = req->cmd;switch (cmd->common.opcode) {case nvme_cmd_read:/*绑定request的执行函数，根据传输的参数length计算该request需要处理的数据长度*/case nvme_cmd_write:req->execute = nvmet_bdev_execute_rw;req->data_len = nvmet_rw_len(req);return 0;case nvme_cmd_flush:req->execute = nvmet_bdev_execute_flush;req->data_len = 0;return 0;case nvme_cmd_dsm:req->execute = nvmet_bdev_execute_dsm;req->data_len = (le32_to_cpu(cmd->dsm.nr) + 1) *sizeof(struct nvme_dsm_range);return 0;case nvme_cmd_write_zeroes:req->execute = nvmet_bdev_execute_write_zeroes;req->data_len = 0;return 0;default:pr_err("unhandled cmd %d on qid %d\n", cmd->common.opcode,req->sq->qid);req->error_loc = offsetof(struct nvme_common_command, opcode);return NVME_SC_INVALID_OPCODE | NVME_SC_DNR;}
}

nvmet_tcp_map_data

static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd)
{printk("nvmet_tcp_map_data");struct nvme_sgl_desc *sgl = &cmd->req.cmd->common.dptr.sgl;u32 len = le32_to_cpu(sgl->length);if (!cmd->req.data_len)return 0;if (sgl->type == ((NVME_SGL_FMT_DATA_DESC << 4) |NVME_SGL_FMT_OFFSET)) {if (!nvme_is_write(cmd->req.cmd))return NVME_SC_INVALID_FIELD | NVME_SC_DNR;if (len > cmd->req.port->inline_data_size)return NVME_SC_SGL_INVALID_OFFSET | NVME_SC_DNR;cmd->pdu_len = len;}/*这里transfer_len等于sgl->length*/cmd->req.transfer_len += len;/*申请sgl，准备好将数据放在这里*/cmd->req.sg = sgl_alloc(len, GFP_KERNEL, &cmd->req.sg_cnt);if (!cmd->req.sg)return NVME_SC_INTERNAL;cmd->cur_sg = cmd->req.sg;/*如果随着pdu一起过来的还有data，申请cmd->iov数组*/if (nvmet_tcp_has_data_in(cmd)) {cmd->iov = kmalloc_array(cmd->req.sg_cnt,sizeof(*cmd->iov), GFP_KERNEL);if (!cmd->iov)goto err;}return 0;
err:sgl_free(cmd->req.sg);return NVME_SC_INTERNAL;
}

nvmet_tcp_try_recv_data

static int nvmet_tcp_try_recv_data(struct nvmet_tcp_queue *queue)
{struct nvmet_tcp_cmd  *cmd = queue->cmd;int ret;/*host端将非连续的page分次send，那么target端也要同步进行recv*/while (msg_data_left(&cmd->recv_msg)) {ret = sock_recvmsg(cmd->queue->sock, &cmd->recv_msg,cmd->recv_msg.msg_flags);if (ret <= 0)return ret;cmd->pdu_recv += ret;cmd->rbytes_done += ret;}/*解除SGL和iov的映射关系 此时，数据已经全部被接收*/nvmet_tcp_unmap_pdu_iovec(cmd);if (!(cmd->flags & NVMET_TCP_F_INIT_FAILED) &&cmd->rbytes_done == cmd->req.transfer_len) {if (queue->data_digest) {nvmet_tcp_prep_recv_ddgst(cmd);return 0;}/*执行之前绑定的处理函数，这里是nvmet_bdev_execute_rw*/nvmet_req_execute(&cmd->req);}nvmet_prepare_receive_pdu(queue);return 0;
}

nvmet_bdev_execute_rw

static void nvmet_bdev_execute_rw(struct nvmet_req *req)
{int sg_cnt = req->sg_cnt;struct bio *bio;struct scatterlist *sg;sector_t sector;int op, op_flags = 0, i;if (!req->sg_cnt) {nvmet_req_complete(req, 0);return;}/*设置用于target端通过PCI接口下发sgl的op*/if (req->cmd->rw.opcode == nvme_cmd_write) {op = REQ_OP_WRITE;op_flags = REQ_SYNC | REQ_IDLE;if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))op_flags |= REQ_FUA;} else {op = REQ_OP_READ;}if (is_pci_p2pdma_page(sg_page(req->sg)))op_flags |= REQ_NOMERGE;sector = le64_to_cpu(req->cmd->rw.slba);sector <<= (req->ns->blksize_shift - 9);/*初始化或者申请BIO*/if (req->data_len <= NVMET_MAX_INLINE_DATA_LEN) {bio = &req->b.inline_bio;bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec));} else {bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES));}bio_set_dev(bio, req->ns->bdev);bio->bi_iter.bi_sector = sector;bio->bi_private = req;bio->bi_end_io = nvmet_bio_done;bio_set_op_attrs(bio, op, op_flags);/*遍历所有的sgl，对可以进行合并的sgl合并到同一个BIO中，不连续的则申请新的BIO，将上一个BIO通过submit_bio下发*/for_each_sg(req->sg, sg, req->sg_cnt, i) {while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)!= sg->length) {struct bio *prev = bio;bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES));bio_set_dev(bio, req->ns->bdev);bio->bi_iter.bi_sector = sector;bio_set_op_attrs(bio, op, op_flags);bio_chain(bio, prev);submit_bio(prev);}sector += sg->length >> 9;sg_cnt--;}submit_bio(bio);
}

nvmet_bio_done

BIO全部下发结束后，触发bi_end_io，也就是nvmet_bio_done函数，函数一路到最后会触发nvmet_tcp_queue_response函数，表示该nvme命令处理完毕，需要回传结果了

static void nvmet_bio_done(struct bio *bio)
{struct nvmet_req *req = bio->bi_private;/*调用绑定好的complete函数，准备cqe*/nvmet_req_complete(req, blk_to_nvme_status(req, bio->bi_status));if (bio != &req->b.inline_bio)bio_put(bio);
}

void nvmet_req_complete(struct nvmet_req *req, u16 status)
{__nvmet_req_complete(req, status);percpu_ref_put(&req->sq->ref);
}
EXPORT_SYMBOL_GPL(nvmet_req_complete);

static void __nvmet_req_complete(struct nvmet_req *req, u16 status)
{if (!req->sq->sqhd_disabled)nvmet_update_sq_head(req);/*填充cqe，等待后续发送给host端*/req->cqe->sq_id = cpu_to_le16(req->sq->qid);req->cqe->command_id = req->cmd->common.command_id;if (unlikely(status))nvmet_set_error(req, status);trace_nvmet_req_complete(req);if (req->ns)nvmet_put_namespace(req->ns);req->ops->queue_response(req);
}

static void nvmet_tcp_queue_response(struct nvmet_req *req)
{struct nvmet_tcp_cmd *cmd =container_of(req, struct nvmet_tcp_cmd, req);struct nvmet_tcp_queue    *queue = cmd->queue;llist_add(&cmd->lentry, &queue->resp_list);/*触发新的io_work函数 发送cqe*/queue_work_on(cmd->queue->cpu, nvmet_tcp_wq, &cmd->queue->io_work);
}

nvmet_tcp_try_send_one

static int nvmet_tcp_try_send_one(struct nvmet_tcp_queue *queue,bool last_in_batch)
{struct nvmet_tcp_cmd *cmd = queue->snd_cmd;int ret = 0;/*此时还没有构造要发送的cmd，所以进入fetch_cmd构造cmd*/if (!cmd || queue->state == NVMET_TCP_Q_DISCONNECTING) {cmd = nvmet_tcp_fetch_cmd(queue);if (unlikely(!cmd))return 0;}if (cmd->state == NVMET_TCP_SEND_DATA_PDU) {ret = nvmet_try_send_data_pdu(cmd);if (ret <= 0)goto done_send;}if (cmd->state == NVMET_TCP_SEND_DATA) {ret = nvmet_try_send_data(cmd, last_in_batch);if (ret <= 0)goto done_send;}if (cmd->state == NVMET_TCP_SEND_DDGST) {ret = nvmet_try_send_ddgst(cmd);if (ret <= 0)goto done_send;}if (cmd->state == NVMET_TCP_SEND_R2T) {ret = nvmet_try_send_r2t(cmd, last_in_batch);if (ret <= 0)goto done_send;}/*构造完成后，进入nvmet_try_send_response函数，发送cqe*/if (cmd->state == NVMET_TCP_SEND_RESPONSE)ret = nvmet_try_send_response(cmd, last_in_batch);done_send:if (ret < 0) {if (ret == -EAGAIN)return 0;return ret;}return 1;
}

static struct nvmet_tcp_cmd *nvmet_tcp_fetch_cmd(struct nvmet_tcp_queue *queue)
{/*从response_send_list申请结构体*/queue->snd_cmd = list_first_entry_or_null(&queue->resp_send_list,struct nvmet_tcp_cmd, entry);if (!queue->snd_cmd) {/*如果无法申请说明list未初始化，初始化后重新申请*/nvmet_tcp_process_resp_list(queue);queue->snd_cmd =list_first_entry_or_null(&queue->resp_send_list,struct nvmet_tcp_cmd, entry);if (unlikely(!queue->snd_cmd))return NULL;}/*queue的send_list状态变化*/list_del_init(&queue->snd_cmd->entry);queue->send_list_len--;if (nvmet_tcp_need_data_out(queue->snd_cmd))nvmet_setup_c2h_data_pdu(queue->snd_cmd);else if (nvmet_tcp_need_data_in(queue->snd_cmd))nvmet_setup_r2t_pdu(queue->snd_cmd);else/*由于是write命令，不需要数据，所以调用nvmet_setup_response_pdu函数构造response_pdu*/nvmet_setup_response_pdu(queue->snd_cmd);return queue->snd_cmd;
}

nvmet_setup_response_pdu

static void nvmet_setup_response_pdu(struct nvmet_tcp_cmd *cmd)
{struct nvme_tcp_rsp_pdu *pdu = cmd->rsp_pdu;struct nvmet_tcp_queue *queue = cmd->queue;u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);cmd->offset = 0;cmd->state = NVMET_TCP_SEND_RESPONSE;/*初始化pdu*/pdu->hdr.type = nvme_tcp_rsp;pdu->hdr.flags = 0;pdu->hdr.hlen = sizeof(*pdu);pdu->hdr.pdo = 0;pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);if (cmd->queue->hdr_digest) {pdu->hdr.flags |= NVME_TCP_F_HDGST;nvmet_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));}
}

static int nvmet_try_send_response(struct nvmet_tcp_cmd *cmd,bool last_in_batch)
{u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);int left = sizeof(*cmd->rsp_pdu) - cmd->offset + hdgst;int flags = MSG_DONTWAIT;int ret;if (!last_in_batch && cmd->queue->send_list_len)flags |= MSG_MORE;elseflags |= MSG_EOR;/*发送response_pdu给host端*/ret = kernel_sendpage(cmd->queue->sock, virt_to_page(cmd->rsp_pdu),offset_in_page(cmd->rsp_pdu) + cmd->offset, left, flags);if (ret <= 0)return ret;cmd->offset += ret;left -= ret;if (left)return -EAGAIN;kfree(cmd->iov);sgl_free(cmd->req.sg);cmd->queue->snd_cmd = NULL;nvmet_tcp_put_cmd(cmd);return 1;
}

至此，target端的工作结束，转回host端看host端try_recv的执行流程

host端

nvme_tcp_try_recv

static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)
{struct socket *sock = queue->sock;struct sock *sk = sock->sk;read_descriptor_t rd_desc;int consumed;rd_desc.arg.data = queue;rd_desc.count = 1;lock_sock(sk);queue->nr_cqe = 0;/*调用nvme_tcp_recv_skb对接收到的数据进行处理*/consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb);release_sock(sk);return consumed;
}

static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,unsigned int offset, size_t len)
{struct nvme_tcp_queue *queue = desc->arg.data;size_t consumed = len;int result;while (len) {switch (nvme_tcp_recv_state(queue)) {case NVME_TCP_RECV_PDU:/*进入分支，调用nvme_tcp_recv_pdu函数*/result = nvme_tcp_recv_pdu(queue, skb, &offset, &len);break;case NVME_TCP_RECV_DATA:result = nvme_tcp_recv_data(queue, skb, &offset, &len);break;case NVME_TCP_RECV_DDGST:result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len);break;default:result = -EFAULT;}if (result) {dev_err(queue->ctrl->ctrl.device,"receive failed:  %d\n", result);queue->rd_enabled = false;nvme_tcp_error_recovery(&queue->ctrl->ctrl);return result;}}return consumed;
}

static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb,unsigned int *offset, size_t *len)
{struct nvme_tcp_hdr *hdr;char *pdu = queue->pdu;size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining);int ret;/*将接收到的pdu数据写入queue的pdu中*/ret = skb_copy_bits(skb, *offset,&pdu[queue->pdu_offset], rcv_len);if (unlikely(ret))return ret;queue->pdu_remaining -= rcv_len;queue->pdu_offset += rcv_len;*offset += rcv_len;*len -= rcv_len;if (queue->pdu_remaining){return 0;}hdr = queue->pdu;if (queue->hdr_digest) {ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen);if (unlikely(ret))return ret;}if (queue->data_digest) {ret = nvme_tcp_check_ddgst(queue, queue->pdu);if (unlikely(ret))return ret;}switch (hdr->type) {case nvme_tcp_c2h_data:return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu);case nvme_tcp_rsp:/*write命令进入一下分支，初始化recv ctx，准备处理cqe*/nvme_tcp_init_recv_ctx(queue);return nvme_tcp_handle_comp(queue, (void *)queue->pdu);case nvme_tcp_r2t:nvme_tcp_init_recv_ctx(queue);return nvme_tcp_handle_r2t(queue, (void *)queue->pdu);default:dev_err(queue->ctrl->ctrl.device,"unsupported pdu type (%d)\n", hdr->type);return -EINVAL;}
}

static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue)
{queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) +nvme_tcp_hdgst_len(queue);queue->pdu_offset = 0;queue->data_remaining = -1;queue->ddgst_remaining = 0;
}

static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue,struct nvme_tcp_rsp_pdu *pdu)
{struct nvme_completion *cqe = &pdu->cqe;int ret = 0;/** AEN requests are special as they don't time out and can* survive any kind of queue freeze and often don't respond to* aborts.  We don't even bother to allocate a struct request* for them but rather special case them here.*/if (unlikely(nvme_tcp_queue_id(queue) == 0 &&cqe->command_id >= NVME_AQ_BLK_MQ_DEPTH))nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,&cqe->result);else/*处理接受到的cqe*/ret = nvme_tcp_process_nvme_cqe(queue, cqe);return ret;
}

nvme_tcp_process_nvme_cqe

static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,struct nvme_completion *cqe)
{struct request *rq;rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), cqe->command_id);if (!rq) {dev_err(queue->ctrl->ctrl.device,"queue %d tag 0x%x not found\n",nvme_tcp_queue_id(queue), cqe->command_id);nvme_tcp_error_recovery(&queue->ctrl->ctrl);return -EINVAL;}nvme_end_request(rq, cqe->status, cqe->result);queue->nr_cqe++;return 0;
}

static inline void nvme_end_request(struct request *req, __le16 status,union nvme_result result)
{struct nvme_request *rq = nvme_req(req);rq->status = le16_to_cpu(status) >> 1;rq->result = result;/* inject error when permitted by fault injection framework */nvme_should_fail(req);/*执行TCP queue绑定好的complete函数*/blk_mq_complete_request(req);
}

void nvme_complete_rq(struct request *req)
{blk_status_t status = nvme_error_status(nvme_req(req)->status);trace_nvme_complete_rq(req);if (nvme_req(req)->ctrl->kas)nvme_req(req)->ctrl->comp_seen = true;if (unlikely(status != BLK_STS_OK && nvme_req_needs_retry(req))) {if ((req->cmd_flags & REQ_NVME_MPATH) && nvme_failover_req(req))return;if (!blk_queue_dying(req->q)) {nvme_retry_req(req);return;}}nvme_trace_bio_complete(req, status);/*结束request命令*/blk_mq_end_request(req, status);
}
EXPORT_SYMBOL_GPL(nvme_complete_rq);

返回到最初的nvme_user_cmd函数中

nvme_passthru_end

static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
{/** Revalidate LBA changes prior to unfreezing. This is necessary to* prevent memory corruption if a logical block size was changed by* this command.*/if (effects & NVME_CMD_EFFECTS_LBCC)nvme_update_formats(ctrl);if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {nvme_unfreeze(ctrl);nvme_mpath_unfreeze(ctrl->subsys);mutex_unlock(&ctrl->subsys->lock);nvme_remove_invalid_namespaces(ctrl, NVME_NSID_ALL);mutex_unlock(&ctrl->scan_lock);}if (effects & NVME_CMD_EFFECTS_CCC)nvme_init_identify(ctrl);if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC))nvme_queue_scan(ctrl);
}

至此一条ioctl write命令结束