linux V4L2子系统——v4l2架构（3）之video_device

备注：
1. Kernel版本：5.4
2. 使用工具：Source Insight 4.0
3. 参考博客：
（1）Linux V4L2子系统-Video设备框架分析（二）
（2）linux v4l2 学习之-v4l2设备注册过程及各个设备之间的联系
（3）V4L2框架-v4l2 device

文章目录

linux V4L2子系统——v4l2架构（3）之video_device
- 概述
- 主要数据结构体介绍
- 如何注册 video 类型节点
- video_device注册过程分析
- - video_register_device函数
  - __video_register_device函数
- video_device注销过程分析
- videoX设备访问流程梳理
- - file_operations的实现
  - open调用流程
  - ioctl 调用流程

概述

在V4L2子系统中，Video设备是一个字符设备，设备节点为/dev/videoX，主设备号为81，次设备号范围为0-63。在用户空间，应用可以通过 open/close/ioctl/mmap/read/write 系统调用操作Video设备。在内核空间中，Video设备的具体操作方法由驱动中的 struct video_device 提供。驱动使用video_register_device 函数将 struct video_device 注册到V4L2的核心层，然后V4L2的核心层在向上注册一个字符设备，该字符设备实现了虚拟文件系统要求的方法。这样应用就可以使用系统调用访问虚拟文件系统中Video设备提供的方法，然后进一步访问V4L2核心层提供的 v4l2_fops 方法集合，最后通过 struct video_device 结构体中的fops和ioctl_ops方法集合访问Video主设备。Video主设备通过 v4l2_subdev_call 方法访问Video从设备，同时Video从设备可以通过notify回掉方法通知主设备发生了事件。Camera Host控制器为Video主设备，Camear Sensor（摄像头）为Video从设备，一般为I2C设备。

主要数据结构体介绍

详见：linux V4L2子系统——v4l2的结构体（2）之video_device

如何注册 video 类型节点

使用 video_register_device 配合 VFL_TYPE_GRABBER 参数进行注册，此时该函数执行完毕并返回的时候就可以在用户空间看到形如 /dev/videoX 的设备节点了。

// 源码： drivers/media/platform/sunxi/sun6i/sun6i-csi/sun6i-video.c
int sun6i_video_init(struct sun6i_video *video, struct sun6i_csi *csi,const char *name)
{....../* Register video device */strscpy(vdev->name, name, sizeof(vdev->name));vdev->release      = video_device_release_empty; //必须设置此成员，此video_device为静态创建，无动作可执行vdev->fops       = &sun6i_video_fops;vdev->ioctl_ops     = &sun6i_video_ioctl_ops;vdev->vfl_type     = VFL_TYPE_GRABBER; // video设备类型vdev->vfl_dir       = VFL_DIR_RX;            // 接收数据vdev->v4l2_dev      = &csi->v4l2_dev;   // video_device 与 v4l2_device 进行绑定关联vdev->queue      = vidq;                     // video_device 与 vb2_queue 进行绑定关联vdev->lock        = &video->lock;vdev->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_CAPTURE;video_set_drvdata(vdev, video);ret = video_register_device(vdev, VFL_TYPE_GRABBER, -1);if (ret < 0) {v4l2_err(&csi->v4l2_dev,"video_register_device failed: %d\n", ret);goto release_vb2;}......
}

video_device注册过程分析

video_register_device函数

// 源码：include/media/v4l2-dev.h/***  video_register_device - register video4linux devices** @vdev: struct video_device to register* @type: type of device to register, as defined by &enum vfl_devnode_type* @nr:   which device node number is desired:*   (0 == /dev/video0, 1 == /dev/video1, ..., -1 == first free)** Internally, it calls __video_register_device(). Please see its* documentation for more details.** .. note::*    if video_register_device fails, the release() callback of*  &struct video_device structure is *not* called, so the caller*  is responsible for freeing any data. Usually that means that*   you video_device_release() should be called on failure.*/
// 注册video_device结构体
// vdev-video_device结构体指针
// type-注册的设备类型，有效的设备类型如下：
//     VFL_TYPE_GRABBER - A frame grabber
//     VFL_TYPE_VBI - Vertical blank data (undecoded)
//     VFL_TYPE_RADIO - A radio card
//     VFL_TYPE_SUBDEV - A subdevice
//     VFL_TYPE_SDR - Software Defined Radio
// nr-生成的设备节点数量(0 == /dev/video0, 1 == /dev/video1, ...-1 == first free)
// warn_if_nr_in_use-若设备节点编号已被占用则发出警告，同时会选择其他设备节点编号
// owner-video设备节点所属的模块
// 返回值-0成功，小于0失败
static inline int __must_check video_register_device(struct video_device *vdev,enum vfl_devnode_type type,int nr)
{return __video_register_device(vdev, type, nr, 1, vdev->fops->owner);
}

__video_register_device函数

（1）在注册之前必须设置release和v4l2_dev成员，前者用于设备注销时回调释放资源，后者指向了管理video_device的v4l2_device结构体。
（2）检查设备类型并确定设备节点基本名称。
（3）设置设备类型、次设备号（由设备类型和全局video_device数组决定）及设备节点数量。
（4）将要注册的video_device结构体指针保存到全局的video_device数组中。
（5）根据设备类型验证那些ioctl函数可以使用。
（6）分配字符设备结构体。
（7）设置字符设备的操作函数集合为v4l2_fops。
（8）将video设备注册为字符设备。
（9）注册设备。
（10）设置设备引用计数为0时的回调函数，回调函数为 v4l2_device_release。
v4l2_device_release 主要的工作是删除注册的字符设备，回调 v4l2_device 中的 release 函数（通常是video_device_release 函数）释放 video_device 结构体内存，最后减少 v4l2_device 的引用计数（一个v4l2_device 可管理多个 video_device，当 v4l2_device 的引用计数为0时，将进行注销工作）。
（11）增加video_device所属v4l2_device的引用计数。
（12）设置已注册标志 V4L2_FL_REGISTERED。

// 源码：drivers/media/v4l2-core/v4l2-dev.cint __video_register_device(struct video_device *vdev,enum vfl_devnode_type type,int nr, int warn_if_nr_in_use,struct module *owner)
{int i = 0;int ret;int minor_offset = 0;int minor_cnt = VIDEO_NUM_DEVICES;const char *name_base;/* A minor value of -1 marks this video device as neverhaving been registered */vdev->minor = -1;/* the release callback MUST be present */if (WARN_ON(!vdev->release))return -EINVAL;/* the v4l2_dev pointer MUST be present */if (WARN_ON(!vdev->v4l2_dev))return -EINVAL;/* the device_caps field MUST be set for all but subdevs */if (WARN_ON(type != VFL_TYPE_SUBDEV && !vdev->device_caps))return -EINVAL;/* v4l2_fh support */spin_lock_init(&vdev->fh_lock);INIT_LIST_HEAD(&vdev->fh_list);/* Part 1: check device type */// 检查设备类型switch (type) {case VFL_TYPE_GRABBER:name_base = "video";break;case VFL_TYPE_VBI:name_base = "vbi";break;case VFL_TYPE_RADIO:name_base = "radio";break;case VFL_TYPE_SUBDEV:name_base = "v4l-subdev";break;case VFL_TYPE_SDR:/* Use device name 'swradio' because 'sdr' was already taken. */name_base = "swradio";break;case VFL_TYPE_TOUCH:name_base = "v4l-touch";break;default:pr_err("%s called with unknown type: %d\n",__func__, type);return -EINVAL;}vdev->vfl_type = type;vdev->cdev = NULL;if (vdev->dev_parent == NULL)vdev->dev_parent = vdev->v4l2_dev->dev;if (vdev->ctrl_handler == NULL)vdev->ctrl_handler = vdev->v4l2_dev->ctrl_handler;/* If the prio state pointer is NULL, then use the v4l2_deviceprio state. */if (vdev->prio == NULL)vdev->prio = &vdev->v4l2_dev->prio;/* Part 2: find a free minor, device node number and device index. */
#ifdef CONFIG_VIDEO_FIXED_MINOR_RANGES/* Keep the ranges for the first four types for historical* reasons.* Newer devices (not yet in place) should use the range* of 128-191 and just pick the first free minor there* (new style). */switch (type) {case VFL_TYPE_GRABBER:minor_offset = 0;minor_cnt = 64;break;case VFL_TYPE_RADIO:minor_offset = 64;minor_cnt = 64;break;case VFL_TYPE_VBI:minor_offset = 224;minor_cnt = 32;break;default:minor_offset = 128;minor_cnt = 64;break;}
#endif/* Pick a device node number */// 获取一个次设备号mutex_lock(&videodev_lock);nr = devnode_find(vdev, nr == -1 ? 0 : nr, minor_cnt);if (nr == minor_cnt)nr = devnode_find(vdev, 0, minor_cnt);if (nr == minor_cnt) {pr_err("could not get a free device node number\n");mutex_unlock(&videodev_lock);return -ENFILE;}
#ifdef CONFIG_VIDEO_FIXED_MINOR_RANGES/* 1-on-1 mapping of device node number to minor number */i = nr; //这里保存下空闲的次设备号
#else/* The device node number and minor numbers are independent, sowe just find the first free minor number. *//* 查找到第一个空闲的minor号,可以理解成是次设备号*/for (i = 0; i < VIDEO_NUM_DEVICES; i++)if (video_devices[i] == NULL) //所有注册的video_device都会保存到这个数组中break;if (i == VIDEO_NUM_DEVICES) {mutex_unlock(&videodev_lock);pr_err("could not get a free minor\n");return -ENFILE;}
#endif// minor_offset一般是0，i就是查找到的空闲次设备号，// 这里总的次设备支持到256个vdev->minor = i + minor_offset;vdev->num = nr;/* Should not happen since we thought this minor was free */if (WARN_ON(video_devices[vdev->minor])) {mutex_unlock(&videodev_lock);pr_err("video_device not empty!\n");return -ENFILE;}devnode_set(vdev); //将标准位置为已用/* 下面这个方法字面意思看起来是获取一个index，但是查看源代码会发现“这里会去查找不是直系的设备空闲号”，就是说只有video_device不为空，而且v4l2 parent对象相同都会认为是同类，直接跳过相应的index号 */vdev->index = get_index(vdev);video_devices[vdev->minor] = vdev;mutex_unlock(&videodev_lock);if (vdev->ioctl_ops)determine_valid_ioctls(vdev);/* Part 3: Initialize the character device */// 申请一个字符设备vdev->cdev = cdev_alloc();if (vdev->cdev == NULL) {ret = -ENOMEM;goto cleanup;}// 务必留意这个ioctl,后面子设备中的ioctl都是通过这里查找的。vdev->cdev->ops = &v4l2_fops;vdev->cdev->owner = owner;// 添加一个字符设备ret = cdev_add(vdev->cdev, MKDEV(VIDEO_MAJOR, vdev->minor), 1);if (ret < 0) {pr_err("%s: cdev_add failed\n", __func__);kfree(vdev->cdev);vdev->cdev = NULL;goto cleanup;}/* Part 4: register the device with sysfs */// 自动创建设备节点vdev->dev.class = &video_class;vdev->dev.devt = MKDEV(VIDEO_MAJOR, vdev->minor);vdev->dev.parent = vdev->dev_parent;dev_set_name(&vdev->dev, "%s%d", name_base, vdev->num);ret = device_register(&vdev->dev);if (ret < 0) {pr_err("%s: device_register failed\n", __func__);goto cleanup;}/* Register the release callback that will be called when the lastreference to the device goes away. */vdev->dev.release = v4l2_device_release;if (nr != -1 && nr != vdev->num && warn_if_nr_in_use)pr_warn("%s: requested %s%d, got %s\n", __func__,name_base, nr, video_device_node_name(vdev));/* Increase v4l2_device refcount *//* 下面这一步很关键，增加v4l2设备对象引用计数 */v4l2_device_get(vdev->v4l2_dev);/* Part 5: Register the entity. */ret = video_register_media_controller(vdev);/* Part 6: Activate this minor. The char device can now be used. */set_bit(V4L2_FL_REGISTERED, &vdev->flags);return 0;cleanup:mutex_lock(&videodev_lock);if (vdev->cdev)cdev_del(vdev->cdev);video_devices[vdev->minor] = NULL;devnode_clear(vdev);mutex_unlock(&videodev_lock);/* Mark this video device as never having been registered. */vdev->minor = -1;return ret;
}

由这里可以发现，创建video_device时，也创建了一个字符设备。并将该设备的parent节点指定为v4l2_device所依附的那个节点。主要需要注意下面几点。

1 .根据设备类型确定设备名字和次设备数量
由于系统可能包含很多媒体设备，所以v4l2核心将0～255次设备编号划分了区域如下所示：其中VFL_TYPE_GRABBER一般表示提供数据的设备如camera.

类型	此设备号区间	设备基名称
VFL_TYPE_GRABBER	0~63	video
VFL_TYPE_RADIO	64~127	radio
VFL_TYPE_VBI	224~255	vbi
其它(含VFL_TYPE_SUBDEV)	128~191	含“v4l2-subdev”

2.确定设备编号，注册字符设备驱动。

ret = cdev_add(vdev->cdev, MKDEV(VIDEO_MAJOR, vdev->minor), 1);if (ret < 0) {pr_err("%s: cdev_add failed\n", __func__);kfree(vdev->cdev);vdev->cdev = NULL;goto cleanup;}/* Part 4: register the device with sysfs */// 自动创建设备节点vdev->dev.class = &video_class;vdev->dev.devt = MKDEV(VIDEO_MAJOR, vdev->minor);vdev->dev.parent = vdev->dev_parent;dev_set_name(&vdev->dev, "%s%d", name_base, vdev->num);ret = device_register(&vdev->dev);if (ret < 0) {pr_err("%s: device_register failed\n", __func__);goto cleanup;}

3. 确定设备的入口

 /* Part 5: Register the entity. */ret = video_register_media_controller(vdev);

// 源码：drivers/media/v4l2-core/v4l2-dev.c
static int video_register_media_controller(struct video_device *vdev)
{
#if defined(CONFIG_MEDIA_CONTROLLER)u32 intf_type;int ret;/* Memory-to-memory devices are more complex and use* their own function to register its mc entities.*/if (!vdev->v4l2_dev->mdev || vdev->vfl_dir == VFL_DIR_M2M)return 0;vdev->entity.obj_type = MEDIA_ENTITY_TYPE_VIDEO_DEVICE;vdev->entity.function = MEDIA_ENT_F_UNKNOWN;switch (vdev->vfl_type) {case VFL_TYPE_GRABBER:intf_type = MEDIA_INTF_T_V4L_VIDEO;vdev->entity.function = MEDIA_ENT_F_IO_V4L;break;case VFL_TYPE_VBI:intf_type = MEDIA_INTF_T_V4L_VBI;vdev->entity.function = MEDIA_ENT_F_IO_VBI;break;case VFL_TYPE_SDR:intf_type = MEDIA_INTF_T_V4L_SWRADIO;vdev->entity.function = MEDIA_ENT_F_IO_SWRADIO;break;case VFL_TYPE_TOUCH:intf_type = MEDIA_INTF_T_V4L_TOUCH;vdev->entity.function = MEDIA_ENT_F_IO_V4L;break;case VFL_TYPE_RADIO:intf_type = MEDIA_INTF_T_V4L_RADIO;/** Radio doesn't have an entity at the V4L2 side to represent* radio input or output. Instead, the audio input/output goes* via either physical wires or ALSA.*/break;case VFL_TYPE_SUBDEV:intf_type = MEDIA_INTF_T_V4L_SUBDEV;/* Entity will be created via v4l2_device_register_subdev() */break;default:return 0;}// VFL_TYPE_SUBDEV、VFL_TYPE_RADIO将不会进入此处if (vdev->entity.function != MEDIA_ENT_F_UNKNOWN) {vdev->entity.name = vdev->name;/* Needed just for backward compatibility with legacy MC API */vdev->entity.info.dev.major = VIDEO_MAJOR;vdev->entity.info.dev.minor = vdev->minor;ret = media_device_register_entity(vdev->v4l2_dev->mdev,&vdev->entity);if (ret < 0) {pr_warn("%s: media_device_register_entity failed\n",__func__);return ret;}}vdev->intf_devnode = media_devnode_create(vdev->v4l2_dev->mdev,intf_type,0, VIDEO_MAJOR,vdev->minor);if (!vdev->intf_devnode) {media_device_unregister_entity(&vdev->entity);return -ENOMEM;}if (vdev->entity.function != MEDIA_ENT_F_UNKNOWN) {struct media_link *link;link = media_create_intf_link(&vdev->entity,&vdev->intf_devnode->intf,MEDIA_LNK_FL_ENABLED |MEDIA_LNK_FL_IMMUTABLE);if (!link) {media_devnode_remove(vdev->intf_devnode);media_device_unregister_entity(&vdev->entity);return -ENOMEM;}}/* FIXME: how to create the other interface links? */#endifreturn 0;
}

video_device注销过程分析

调用video_unregister_device函数注销注册的video_device，主要执行流程如下：

// 源码：drivers/media/v4l2-core/v4l2-dev.c
/***    video_unregister_device - unregister a video4linux device*  @vdev: the device to unregister**  This unregisters the passed device. Future open calls will* be met with errors.*/
void video_unregister_device(struct video_device *vdev)
{/* Check if vdev was ever registered at all *//* 如果没有注册，则直接返回 */if (!vdev || !video_is_registered(vdev))return;mutex_lock(&videodev_lock);/* This must be in a critical section to prevent a race with v4l2_open.* Once this bit has been cleared video_get may never be called again.*/// 清楚已注册标志clear_bit(V4L2_FL_REGISTERED, &vdev->flags);mutex_unlock(&videodev_lock);// 注销设备device_unregister(&vdev->dev);
}

videoX设备访问流程梳理

Video设备访问流程如下图所示。总结如下：

（1）首先通过系统调用访问/dev/videoX用户空间设备节点。
（2）进入到内核空间，访问字符设备struct file_operations中的方法。对于Vedio设备，该操作集合被V4L2子系统初始化为v4l2_fops集合。
（3）通过V4L2子系统提供的v4l2_fops集合，可直接调用底层驱动实现的Video主设备struct v4l2_file_operations方法，对于ioctl方法，则需要借助中间函数__video_do_ioctl调用底层驱动实现的struct v4l2_ioctl_ops中的ioctl功能。struct v4l2_file_operations方法和struct v4l2_ioctl_ops方法属于主设备方法，需要主设备的驱动实现。
（4）struct v4l2_file_operations和struct v4l2_ioctl_ops中的函数都可以通过v4l2_subdev_call调用Video从设备struct v4l2_subdev_core_ops、struct v4l2_subdev_video_ops、struct v4l2_subdev_pad_ops等方法，这些方法都要在从设备驱动中实现。

file_operations的实现

//源码： drivers/media/v4l2-core/v4l2-dev.cstatic const struct file_operations v4l2_fops = {.owner = THIS_MODULE,.read = v4l2_read,.write = v4l2_write,.open = v4l2_open,.get_unmapped_area = v4l2_get_unmapped_area,.mmap = v4l2_mmap,.unlocked_ioctl = v4l2_ioctl,
#ifdef CONFIG_COMPAT.compat_ioctl = v4l2_compat_ioctl32,
#endif.release = v4l2_release,.poll = v4l2_poll,.llseek = no_llseek,
};

open调用流程

v4l2_open函数的实现：

//源码： drivers/media/v4l2-core/v4l2-dev.c
/* Override for the open function */
static int v4l2_open(struct inode *inode, struct file *filp)
{struct video_device *vdev;int ret = 0;/* Check if the video device is available */mutex_lock(&videodev_lock);vdev = video_devdata(filp); //获取video_device/* return ENODEV if the video device has already been removed. */// 判断video_device是否为空及是否已注册if (vdev == NULL || !video_is_registered(vdev)) {mutex_unlock(&videodev_lock);return -ENODEV;}/* and increase the device refcount */// 引用计数加1video_get(vdev);mutex_unlock(&videodev_lock);//检查 v4l2_file_operations 的 open 函数是否已实现if (vdev->fops->open) {if (video_is_registered(vdev))ret = vdev->fops->open(filp);//回调 v4l2_file_operations 的 open 函数elseret = -ENODEV;}if (vdev->dev_debug & V4L2_DEV_DEBUG_FOP)dprintk("%s: open (%d)\n",video_device_node_name(vdev), ret);/* decrease the refcount in case of an error */// open 函数执行失败，引用计数减1if (ret)video_put(vdev);return ret;
}

ioctl 调用流程

v4l2_ioctl|---> video_ioctl2|---> video_usercopy|---> v4l2_is_known_ioctl // 检查命令是否有效|---> __video_do_ioctl    // 调用 v4l2_ioctl_info 数组中的函数处理命令|---> v4l_s_fmt / v4l_reqbufs / v4l_streamon ... // 回调 v4l2_ioctl_ops 中的函数处理命令|---> vb2_ioctl_streamon|---> vb2_streamon|---> vb2_core_streamon|---> vb2_start_streaming|---> vb2_queue->ops->start_streaming

v4l2_ioctl 函数实现：

//源码： drivers/media/v4l2-core/v4l2-dev.c
static long v4l2_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{struct video_device *vdev = video_devdata(filp);int ret = -ENODEV;//检查 v4l2_file_operations 的 unlocked_ioctl 函数是否已实现if (vdev->fops->unlocked_ioctl) {if (video_is_registered(vdev))// 回调 unlocked_ioctl 函数ret = vdev->fops->unlocked_ioctl(filp, cmd, arg);} elseret = -ENOTTY;return ret;
}

video_ioctl2 函数实现：

// 源码： drivers/media/v4l2-core/v4l2-ioctl.c
long video_ioctl2(struct file *file,unsigned int cmd, unsigned long arg)
{return video_usercopy(file, cmd, arg, __video_do_ioctl);
}
EXPORT_SYMBOL(video_ioctl2);

__video_do_ioctl 函数实现：

// 源码： drivers/media/v4l2-core/v4l2-ioctl.cstatic long __video_do_ioctl(struct file *file,unsigned int cmd, void *arg)
{
......if (v4l2_is_known_ioctl(cmd)) {info = &v4l2_ioctls[_IOC_NR(cmd)];if (!test_bit(_IOC_NR(cmd), vfd->valid_ioctls) &&!((info->flags & INFO_FL_CTRL) && vfh && vfh->ctrl_handler))goto done;if (vfh && (info->flags & INFO_FL_PRIO)) {ret = v4l2_prio_check(vfd->prio, vfh->prio);if (ret)goto done;}} else {default_info.ioctl = cmd;default_info.flags = 0;default_info.debug = v4l_print_default;info = &default_info;}write_only = _IOC_DIR(cmd) == _IOC_WRITE;if (info != &default_info) {ret = info->func(ops, file, fh, arg);} else if (!ops->vidioc_default) {ret = -ENOTTY;} else {ret = ops->vidioc_default(file, fh,vfh ? v4l2_prio_check(vfd->prio, vfh->prio) >= 0 : 0,cmd, arg);}......return ret;
}

v4l_streamon 函数实现：

// 源码： drivers/media/v4l2-core/v4l2-ioctl.cstatic int v4l_streamon(const struct v4l2_ioctl_ops *ops,struct file *file, void *fh, void *arg)
{return ops->vidioc_streamon(file, fh, *(unsigned int *)arg);
}

vb2_ioctl_streamon 函数实现：

// 源码： drivers/media/common/videobuf2/videobuf2-v4l2.cint vb2_ioctl_streamon(struct file *file, void *priv, enum v4l2_buf_type i)
{struct video_device *vdev = video_devdata(file);if (vb2_queue_is_busy(vdev, file))return -EBUSY;return vb2_streamon(vdev->queue, i);
}
EXPORT_SYMBOL_GPL(vb2_ioctl_streamon);