readv、io_uring、liburing and command cat

使用多种方式实现 cat 命令的等效程序。

文章目录

readv version of cat
io_uring version of cat
liburing version of cat

readv version of cat

一次最大数据量，4K x 1024 是不会存在问题的，iovecs 最大块数有内核限制 1024 块，每块大小 4K，大于时程序运行出错。

#include <stdio.h>
#include <sys/uio.h>
#include <sys/stat.h>
#include <linux/fs.h>
#include <sys/ioctl.h>
#include <fcntl.h>
#include <stdlib.h>
#include <unistd.h>#define BLOCK_SZ    4096/** Returns the size of the file whose open file descriptor is passed in.* Properly handles regular file and block devices as well. Pretty.* */off_t get_file_size(int fd) {struct stat st;if(fstat(fd, &st) < 0) {perror("fstat");return -1;}if (S_ISBLK(st.st_mode)) {unsigned long long bytes;if (ioctl(fd, BLKGETSIZE64, &bytes) != 0) {perror("ioctl");return -1;}return bytes;} else if (S_ISREG(st.st_mode)) // Regular filereturn st.st_size;return -1;
}/** Output a string of characters of len length to stdout.* We use buffered output here to be efficient,* since we need to output character-by-character.* */
void output_to_console(char *buf, int len) {while (len--) {fputc(*buf++, stdout);}
}int read_and_print_file(char *file_name) {struct iovec *iovecs;int file_fd = open(file_name, O_RDONLY);if (file_fd < 0) {perror("open");return 1;}off_t file_sz = get_file_size(file_fd);off_t bytes_remaining = file_sz;int blocks = (int) file_sz / BLOCK_SZ;if (file_sz % BLOCK_SZ) blocks++;iovecs = malloc(sizeof(struct iovec) * blocks);int current_block = 0;/* 申请并初始化 iovecs 块以满足存储文件要求，每块 4K 大小* For the file we're reading, allocate enough blocks to be able to hold* the file data. Each block is described in an iovec structure, which is* passed to readv as part of the array of iovecs.* */while (bytes_remaining) {off_t bytes_to_read = bytes_remaining;if (bytes_to_read > BLOCK_SZ)bytes_to_read = BLOCK_SZ;void *buf;if( posix_memalign(&buf, BLOCK_SZ, BLOCK_SZ)) { // 分配 4K 对齐的大小为 4K 的内存块， int posix_memalign(void **memptr, size_t alignment, size_t size);perror("posix_memalign");return 1;}iovecs[current_block].iov_base = buf;iovecs[current_block].iov_len = bytes_to_read; // iov_len: long unsigned int 8bytes, 当前块的大小current_block++;bytes_remaining -= bytes_to_read;}// printf("blocks %d\n", blocks);/** The readv() call will block until all iovec buffers are filled with* file data. Once it returns, we should be able to access the file data* from the iovecs and print them on the console.* */int ret = readv(file_fd, iovecs, blocks);if (ret < 0) {perror("readv");return 1;}for (int i = 0; i < blocks; i++)output_to_console(iovecs[i].iov_base, iovecs[i].iov_len);return 0;
}int main(int argc, char *argv[]) {if (argc < 2) {fprintf(stderr, "Usage: %s <filename1> [<filename2> ...]\n",argv[0]);return 1;}printf("MAX blocks: %ld\n", sysconf(_SC_IOV_MAX)); // 1024，iovecs 最大块数/** For each file that is passed in as the argument, call the* read_and_print_file() function.* */for (int i = 1; i < argc; i++) {if(read_and_print_file(argv[i])) {fprintf(stderr, "Error reading file\n");return 1;}}return 0;
}

io_uring version of cat

#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <sys/syscall.h>
#include <sys/mman.h>
#include <sys/uio.h>
#include <linux/fs.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>/* If your compilation fails because the header file below is missing,* your kernel is probably too old to support io_uring.* */
#include <linux/io_uring.h>#define QUEUE_DEPTH 1
#define BLOCK_SZ    1024/* This is x86 specific */
#define read_barrier()  __asm__ __volatile__("":::"memory")
#define write_barrier() __asm__ __volatile__("":::"memory")struct app_io_sq_ring {unsigned *head;unsigned *tail;unsigned *ring_mask;unsigned *ring_entries;unsigned *flags;unsigned *array;
};struct app_io_cq_ring {unsigned *head;unsigned *tail;unsigned *ring_mask;unsigned *ring_entries;struct io_uring_cqe *cqes;
};struct submitter {int ring_fd;struct app_io_sq_ring sq_ring;struct io_uring_sqe *sqes;struct app_io_cq_ring cq_ring;
};struct file_info {off_t file_sz;struct iovec iovecs[];      /* Referred by readv/writev */
};/** This code is written in the days when io_uring-related system calls are not* part of standard C libraries. So, we roll our own system call wrapper* functions.* */int io_uring_setup(unsigned entries, struct io_uring_params *p)
{return (int) syscall(__NR_io_uring_setup, entries, p);
}int io_uring_enter(int ring_fd, unsigned int to_submit,unsigned int min_complete, unsigned int flags)
{return (int) syscall(__NR_io_uring_enter, ring_fd, to_submit, min_complete,flags, NULL, 0);
}/** Returns the size of the file whose open file descriptor is passed in.* Properly handles regular file and block devices as well. Pretty.* */off_t get_file_size(int fd) {struct stat st;if(fstat(fd, &st) < 0) {perror("fstat");return -1;}if (S_ISBLK(st.st_mode)) {unsigned long long bytes;if (ioctl(fd, BLKGETSIZE64, &bytes) != 0) {perror("ioctl");return -1;}return bytes;} else if (S_ISREG(st.st_mode))return st.st_size;return -1;
}/** io_uring requires a lot of setup which looks pretty hairy, but isn't all* that difficult to understand. Because of all this boilerplate code,* io_uring's author has created liburing, which is relatively easy to use.* However, you should take your time and understand this code. It is always* good to know how it all works underneath. Apart from bragging rights,* it does offer you a certain strange geeky peace.* */
// io_uring 需要很多设置，看起来很麻烦，但并不难理解。由于所有这些样板代码，io_uring 的作者创建了 liburing，它相对易于使用。 但是，您应该花时间理解这段代码。 知道这一切在下面是如何运作的总是很好的。int app_setup_uring(struct submitter *s) {struct app_io_sq_ring *sring = &s->sq_ring;struct app_io_cq_ring *cring = &s->cq_ring;struct io_uring_params p;void *sq_ptr, *cq_ptr;/** We need to pass in the io_uring_params structure to the io_uring_setup()* call zeroed out. We could set any flags if we need to, but for this* example, we don't.* */memset(&p, 0, sizeof(p));s->ring_fd = io_uring_setup(QUEUE_DEPTH, &p); // 调用 io_uring_setup系统调用，返回时将填充 io_uring_param 结构if (s->ring_fd < 0) {perror("io_uring_setup");return 1;}/** io_uring communication happens via 2 shared kernel-user space ring buffers,* which can be jointly mapped with a single mmap() call in recent kernels. * While the completion queue is directly manipulated, the submission queue * has an indirection array in between. We map that in as well.* *//*io_uring 通信通过 2 个共享的内核-用户空间环形缓冲区，这些缓冲区可以与最近内核中的单个 mmap() 调用联合映射。虽然完成队列是直接操作的，但提交队列之间有一个间接数组。*/int sring_sz = p.sq_off.array + p.sq_entries * sizeof(unsigned);int cring_sz = p.cq_off.cqes + p.cq_entries * sizeof(struct io_uring_cqe);/* In kernel version 5.4 and above, it is possible to map the submission and * completion buffers with a single mmap() call. Rather than check for kernel * versions, the recommended way is to just check the features field of the * io_uring_params structure, which is a bit mask. If the * IORING_FEAT_SINGLE_MMAP is set, then we can do away with the second mmap()* call to map the completion ring.* *//*在内核版本 5.4 及更高版本中，可以使用单个 mmap() 调用来映射提交和完成缓冲区。 推荐的方法是检查 io_uring_params 结构的 features 字段，而不是检查内核版本，这是一个位掩码。*/if (p.features & IORING_FEAT_SINGLE_MMAP) {// 新版本内核，CQ 和 SQ 统一，它们的大小也要统一if (cring_sz > sring_sz) {sring_sz = cring_sz;}cring_sz = sring_sz;}/* Map in the submission and completion queue ring buffers.* Older kernels only map in the submission queue, though.* */// 在提交和完成队列环形缓冲区中映射sq_ptr = mmap(0, sring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,s->ring_fd, IORING_OFF_SQ_RING);if (sq_ptr == MAP_FAILED) {perror("mmap");return 1;}if (p.features & IORING_FEAT_SINGLE_MMAP) {// 新版本内核 CQ 和 SQ 统一cq_ptr = sq_ptr;} else {/* Map in the completion queue ring buffer in older kernels separately */// 老版本内核分别映射 CQ 和 SQcq_ptr = mmap(0, cring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,s->ring_fd, IORING_OFF_CQ_RING);if (cq_ptr == MAP_FAILED) {perror("mmap");return 1;}}/* Save useful fields in a global app_io_sq_ring struct for later* easy reference */sring->head = sq_ptr + p.sq_off.head;sring->tail = sq_ptr + p.sq_off.tail;sring->ring_mask = sq_ptr + p.sq_off.ring_mask;sring->ring_entries = sq_ptr + p.sq_off.ring_entries;sring->flags = sq_ptr + p.sq_off.flags;sring->array = sq_ptr + p.sq_off.array;/* Map in the submission queue entries array */s->sqes = mmap(0, p.sq_entries * sizeof(struct io_uring_sqe),PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,s->ring_fd, IORING_OFF_SQES);if (s->sqes == MAP_FAILED) {perror("mmap");return 1;}/* Save useful fields in a global app_io_cq_ring struct for later* easy reference */cring->head = cq_ptr + p.cq_off.head;cring->tail = cq_ptr + p.cq_off.tail;cring->ring_mask = cq_ptr + p.cq_off.ring_mask;cring->ring_entries = cq_ptr + p.cq_off.ring_entries;cring->cqes = cq_ptr + p.cq_off.cqes;return 0;
}/** Output a string of characters of len length to stdout.* We use buffered output here to be efficient,* since we need to output character-by-character.* */
void output_to_console(char *buf, int len) {while (len--) {fputc(*buf++, stdout);}
}/** Read from completion queue.* In this function, we read completion events from the completion queue, get* the data buffer that will have the file data and print it to the console.* */
// 从完成队列中读取。在此函数中，我们从完成队列中读取完成事件，获取将包含文件数据的数据缓冲区并将其打印到控制台。
void read_from_cq(struct submitter *s) {struct file_info *fi;struct app_io_cq_ring *cring = &s->cq_ring;struct io_uring_cqe *cqe;unsigned head, reaped = 0;head = *cring->head;do {read_barrier();/** Remember, this is a ring buffer. If head == tail, it means that the* buffer is empty.* */if (head == *cring->tail)break;/* Get the entry */cqe = &cring->cqes[head & *s->cq_ring.ring_mask];fi = (struct file_info*) cqe->user_data;if (cqe->res < 0)fprintf(stderr, "Error: %s\n", strerror(abs(cqe->res)));int blocks = (int) fi->file_sz / BLOCK_SZ;if (fi->file_sz % BLOCK_SZ) blocks++;for (int i = 0; i < blocks; i++)output_to_console(fi->iovecs[i].iov_base, fi->iovecs[i].iov_len);head++;} while (1);*cring->head = head;write_barrier();
}
/** Submit to submission queue.* In this function, we submit requests to the submission queue. You can submit* many types of requests. Ours is going to be the readv() request, which we* specify via IORING_OP_READV.** */
// 提交到提交队列。在这个函数中，我们将请求提交到提交队列。您可以提交多种类型的请求。我们的将是 readv() 请求，我们通过 IORING_OP_READV 指定。int submit_to_sq(char *file_path, struct submitter *s) {struct file_info *fi;int file_fd = open(file_path, O_RDONLY);if (file_fd < 0 ) {perror("open");return 1;}struct app_io_sq_ring *sring = &s->sq_ring;unsigned index = 0, current_block = 0, tail = 0, next_tail = 0;off_t file_sz = get_file_size(file_fd);if (file_sz < 0)return 1;off_t bytes_remaining = file_sz;int blocks = (int) file_sz / BLOCK_SZ;if (file_sz % BLOCK_SZ) blocks++;fi = malloc(sizeof(*fi) + sizeof(struct iovec) * blocks);if (!fi) {fprintf(stderr, "Unable to allocate memory\n");return 1;}fi->file_sz = file_sz;/** For each block of the file we need to read, we allocate an iovec struct* which is indexed into the iovecs array. This array is passed in as part* of the submission. If you don't understand this, then you need to look* up how the readv() and writev() system calls work.* */// 对于我们需要读取的文件的每个块，我们分配一个 iovec 结构体，该结构体被索引到 iovecs 数组中。该数组作为提交的一部分传入。 如果你不明白这一点，那么你需要查看 readv() 和 writev() 系统调用是如何工作的。while (bytes_remaining) {off_t bytes_to_read = bytes_remaining;if (bytes_to_read > BLOCK_SZ)bytes_to_read = BLOCK_SZ;fi->iovecs[current_block].iov_len = bytes_to_read;void *buf;if( posix_memalign(&buf, BLOCK_SZ, BLOCK_SZ)) {perror("posix_memalign");return 1;}fi->iovecs[current_block].iov_base = buf;current_block++;bytes_remaining -= bytes_to_read;}/* Add our submission queue entry to the tail of the SQE ring buffer */next_tail = tail = *sring->tail;next_tail++;read_barrier();index = tail & *s->sq_ring.ring_mask;struct io_uring_sqe *sqe = &s->sqes[index];sqe->fd = file_fd;sqe->flags = 0;sqe->opcode = IORING_OP_READV;sqe->addr = (unsigned long) fi->iovecs;sqe->len = blocks;sqe->off = 0;sqe->user_data = (unsigned long long) fi;sring->array[index] = index;tail = next_tail;/* Update the tail so the kernel can see it. */if(*sring->tail != tail) {*sring->tail = tail;write_barrier();}/** Tell the kernel we have submitted events with the io_uring_enter() system* call. We also pass in the IOURING_ENTER_GETEVENTS flag which causes the* io_uring_enter() call to wait until min_complete events (the 3rd param)* complete.* */// 使用 io_uring_enter() 系统调用告诉内核我们已经提交了事件。我们还传入了 IOURING_ENTER_GETEVENTS 标志，这会导致 io_uring_enter() 调用等待 min_complete 事件（第三个参数）完成。int ret = io_uring_enter(s->ring_fd, 1,1,IORING_ENTER_GETEVENTS);if(ret < 0) {perror("io_uring_enter");return 1;}return 0;
}int main(int argc, char *argv[]) {struct submitter *s;if (argc < 2) {fprintf(stderr, "Usage: %s <filename>\n", argv[0]);return 1;}s = malloc(sizeof(*s));if (!s) {perror("malloc");return 1;}memset(s, 0, sizeof(*s));if(app_setup_uring(s)) {fprintf(stderr, "Unable to setup uring!\n");return 1;}for (int i = 1; i < argc; i++) {if(submit_to_sq(argv[i], s)) {fprintf(stderr, "Error reading file\n");return 1;}read_from_cq(s);}return 0;
}

liburing version of cat

#include <fcntl.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <liburing.h>
#include <stdlib.h>#define QUEUE_DEPTH 1
#define BLOCK_SZ    1024struct file_info {off_t file_sz;struct iovec iovecs[];      /* Referred by readv/writev */
};/*
* Returns the size of the file whose open file descriptor is passed in.
* Properly handles regular file and block devices as well. Pretty.
* */off_t get_file_size(int fd) {struct stat st;if(fstat(fd, &st) < 0) {perror("fstat");return -1;}if (S_ISBLK(st.st_mode)) {unsigned long long bytes;if (ioctl(fd, BLKGETSIZE64, &bytes) != 0) {perror("ioctl");return -1;}return bytes;} else if (S_ISREG(st.st_mode))return st.st_size;return -1;
}/** Output a string of characters of len length to stdout.* We use buffered output here to be efficient,* since we need to output character-by-character.* */
void output_to_console(char *buf, int len) {while (len--) {fputc(*buf++, stdout);}
}/** Wait for a completion to be available, fetch the data from* the readv operation and print it to the console.* */int get_completion_and_print(struct io_uring *ring) {struct io_uring_cqe *cqe;int ret = io_uring_wait_cqe(ring, &cqe);if (ret < 0) {perror("io_uring_wait_cqe");return 1;}if (cqe->res < 0) {fprintf(stderr, "Async readv failed.\n");return 1;}struct file_info *fi = io_uring_cqe_get_data(cqe);int blocks = (int) fi->file_sz / BLOCK_SZ;if (fi->file_sz % BLOCK_SZ) blocks++;for (int i = 0; i < blocks; i ++)output_to_console(fi->iovecs[i].iov_base, fi->iovecs[i].iov_len);io_uring_cqe_seen(ring, cqe);return 0;
}/** Submit the readv request via liburing* */int submit_read_request(char *file_path, struct io_uring *ring) {int file_fd = open(file_path, O_RDONLY);if (file_fd < 0) {perror("open");return 1;}off_t file_sz = get_file_size(file_fd);off_t bytes_remaining = file_sz;off_t offset = 0;int current_block = 0;int blocks = (int) file_sz / BLOCK_SZ;if (file_sz % BLOCK_SZ) blocks++;struct file_info *fi = malloc(sizeof(*fi) + (sizeof(struct iovec) * blocks));/** For each block of the file we need to read, we allocate an iovec struct* which is indexed into the iovecs array. This array is passed in as part* of the submission. If you don't understand this, then you need to look* up how the readv() and writev() system calls work.* */// 对于我们需要读取的文件的每个块，我们分配一个 iovec 结构体，该结构体被索引到 iovecs 数组中。while (bytes_remaining) {off_t bytes_to_read = bytes_remaining;if (bytes_to_read > BLOCK_SZ)bytes_to_read = BLOCK_SZ;offset += bytes_to_read;fi->iovecs[current_block].iov_len = bytes_to_read;void *buf;if( posix_memalign(&buf, BLOCK_SZ, BLOCK_SZ)) {perror("posix_memalign");return 1;}fi->iovecs[current_block].iov_base = buf;current_block++;bytes_remaining -= bytes_to_read;}fi->file_sz = file_sz;/* Get an SQE */struct io_uring_sqe *sqe = io_uring_get_sqe(ring);/* Setup a readv operation */io_uring_prep_readv(sqe, file_fd, fi->iovecs, blocks, 0);/* Set user data */io_uring_sqe_set_data(sqe, fi);/* Finally, submit the request */io_uring_submit(ring);return 0;
}int main(int argc, char *argv[]) {struct io_uring ring;if (argc < 2) {fprintf(stderr, "Usage: %s [file name] <[file name] ...>\n",argv[0]);return 1;}/* Initialize io_uring */io_uring_queue_init(QUEUE_DEPTH, &ring, 0);for (int i = 1; i < argc; i++) {int ret = submit_read_request(argv[i], &ring);if (ret) {fprintf(stderr, "Error reading file: %s\n", argv[i]);return 1;}get_completion_and_print(&ring);}/* Call the clean-up function. */io_uring_queue_exit(&ring);return 0;
}

readv、io_uring、liburing and command cat相关推荐

linux重命名tar命令,linux常用操作指令4 —— 文件操作相关命令（mkdir、touch、rm、mv、cp、cat 、 find 、tar、chmod）...
文件操作相关命令文件操作相关命令 1.创建文件夹mkdir 2.创建文件touch 3.移动文件夹mv(类似于剪切) 4.删除rm 5.重命名mv 6.复制cp 7.查看文件(cat.head.ta ...
ansible模块command、shell、raw、script
环境: ansible端: ip:192.168.100.129 hostname:node1.lansgg.com client端: ip:192.168.100.131 hostname:v2.l ...
Linux常用命令（echo、date、ls、cd、history、cat）
一.linux常用命令有很多今天我们来总结一下常用的入门命令: 1.linux下关机命令:poweroff.init 0.halt.shutdown -h now 2.linux下重启命令:reboo ...
less加管道tail_linux中cat、more、less、tail、head命令的区别
一.cat 显示文件连接文件内容的工具 cat 是一个文本文件(查看)和(连接)工具,通常与more搭配使用,与more不同的是cat可以合并文件.查看一个文件的内容,用cat比较简单,就是cat后面 ...
linux tac sed,Day08 - 一些常用 Linux 指令 - 3-7 (cat、tac、head、tail、vi、sed)
前言此為 Linux 相關的 3-7 篇,都是很常用的指令,每天學一點,並常使用,便會記下成為反射動作之一 cat 將該檔案內容印在終端機上 command | cat cat -n file ca ...
linux view查看日志命令,linux查看日志文件内容命令tail、cat、tac、head、echo、vi
linux查看日志文件内容命令tail.cat.tac.head.echo tail -f test.log 你会看到屏幕不断有内容被打印出来. 这时候中断第一个进程Ctrl-C, --------- ...
java查看日志命令_[Java教程]【Linux】linux查看日志文件内容命令tail、cat、tac、head、echo...
[Java教程][Linux]linux查看日志文件内容命令tail.cat.tac.head.echo 0 2017-11-14 12:00:29 linux查看日志文件内容命令tail.cat.t ...
linux view查看日志命令,【Linux】linux查看日志文件内容命令tail、cat、tac、head、echo...
linux查看日志文件内容命令tail.cat.tac.head.echo tail -f test.log 你会看到屏幕不断有内容被打印出来. 这时候中断第一个进程Ctrl-C, --------- ...
linux查看日志文件内容命令tail、cat、tac、head、echo、vi
linux查看日志文件内容命令tail.cat.tac.head.echo tail -f test.log 你会看到屏幕不断有内容被打印出来. 这时候中断第一个进程Ctrl-C, --------- ...

readv、io_uring、liburing and command cat

文章目录

readv version of cat

io_uring version of cat

liburing version of cat

readv、io_uring、liburing and command cat相关推荐

最新文章

热门文章