7.camera驱动08-全志-media framework

平台：全志A133 androidQ

启动过程

贯穿全篇的数据流向

sensor0 ==> mipi0 ==> csi0 ==> isp0 ==> scale0(vipp0) ==> vinc0(dma0) ==> video0
sensor1 ==> mipi1 ==> csi1 ==> isp0 ==> scale1(vipp1) ==> vinc1(dma1) ==> video1

sensor、mipi、csi、isp、scale、vinc都是v4l2_subddev,对应/dev/v4l2-subdevX节点,

video是video_device，对应/dev/videoX节点。

全志A133命令查看

ceres-b3:/sys/devices/platform/soc@2900000/2000800.vind # cat vi*****************************************************VIN hardware feature list:mcsi 2, ncsi 1, parser 2, isp 1, vipp 4, dma 4CSI_VERSION: CSI300_100, ISP_VERSION: ISP522_100CSI_TOP: 336000000, CSI_ISP: 300000000*****************************************************vi0:ov13850_mipi => mipi0 => csi0 => isp0 => vipp0input => hoff: 0, voff: 0, w: 2112, h: 1568, fmt: BGGR10output => width: 1920, height: 1080, fmt: NV21interface: MIPI, isp_mode: NORMAL, hflip: 0, vflip: 0prs_in => x: 2112, y: 1568, hb: 3952, hs: 2766buf => cnt: 3 size: 3133440 rest: 3, mode: software_updateframe => cnt: 1046, lost_cnt: 0, error_cnt: 0internal => avg: 32(ms), max: 33(ms), min: 32(ms)*****************************************************

先找到入口

源码位置：longan/kernel/linux-4.9/drivers/media/platform/sunxi-vin/vin.c

static int __init vin_init(void)ret = sunxi_csi_platform_register();    //相应的probe()初始化csi对应的结构体，里面有v4l2_subdev,pads,ctrl等ret = sunxi_isp_platform_register();  //相应的probe()初始化csi对应的结构体，里面有v4l2_subdev,pads,ctrl等ret = sunxi_mipi_platform_register(); //相应的probe()初始化csi对应的结构体，里面有v4l2_subdev,pads,ctrl等ret = sunxi_flash_platform_register();    //相应的probe()初始化csi对应的结构体，里面有v4l2_subdev,pads,ctrl等ret = sunxi_scaler_platform_register();   //相应的probe()初始化csi对应的结构体，里面有v4l2_subdev,pads,ctrl等ret = sunxi_vin_core_register_driver();   //相应的probe()初始化csi对应的结构体，里面有v4l2_subdev,pads,ctrl等ret = platform_driver_register(&vin_driver);//总管理，相应的probe()注册video，v4l2_subdev,media等

是不是还差sensor的v4l2_dev？以gc030a为例：

源码位置：longan/kernel/linux-4.9/drivers/media/platform/sunxi-vin/modules/sensor/gc030a_mipi.c

先看看代表这个sensor的结构体：

struct sensor_info {struct v4l2_subdev sd;struct media_pad sensor_pads[SENSOR_PAD_NUM];struct v4l2_ctrl_handler handler;... ...
};

mipi、csi、isp、scale、vinc对应的结构体也包含这几个结构体，继续看sensor的probe()：

static int sensor_probe(struct i2c_client *client,const struct i2c_device_id *id)struct sensor_info *info;info = kzalloc(sizeof(struct sensor_info), GFP_KERNEL);cci_dev_probe_helper(sd, client, &sensor_ops, &cci_drv);v4l2_i2c_subdev_init(sd, client, sensor_ops);      //初始化v4l2_subdev，设置v4l2_subdev_opssnprintf(sd->name, sizeof(sd->name), "%s", cci_drv->name);  //设置subdev的name，这里就是gc030a_mipiv4l2_set_subdev_hostdata(sd, cci_drv);                        //保存为client的私有数据sd->internal_ops = &sensor_internal_ops;                    //设置v4l2_subdev_internal_ops回掉函数//==============================================================static const struct v4l2_subdev_internal_ops sensor_internal_ops = {.registered = sensor_registered,                    //这里会回调sensor的s_power, init函数};//==============================================================sensor_init_controls(sd, &sensor_ctrl_ops);                        //初始化v4l2_subdev的control/* 其他初始化 */

是不是回调函数挺多的，mipi、csi、isp、scale、vinc也类似，主要在vin回调，下面vin的probe()。

源码位置：longan/kernel/linux-4.9/drivers/media/platform/sunxi-vin/vin.c

static const struct of_device_id sunxi_vin_match[] = {{.compatible = "allwinner,sunxi-vin-media",},  //==> 对应vind{},
};static int vin_probe(struct platform_device *pdev)parse_modules_from_device_tree(vind);   //获取设备树信息，和sensor自动扫描列表parse_sensor_list_info(sensors, sensors->sensor_pos)sprintf(sensor_list_cfg, "/vendor/etc/hawkview/sensor_list_cfg.ini");cfg_section_init(&section);ret = cfg_read_ini(sensor_list_cfg, &section);if (strcmp(pos, "rear") == 0 || strcmp(pos, "REAR") == 0)fetch_sensor_list(sl, "rear_camera_cfg", section);elsefetch_sensor_list(sl, "front_camera_cfg", section);ret = v4l2_device_register(dev, &vind->v4l2_dev);  v4l2的设备注册其实没有注册设备或者设备驱动，只是将v4l2的大结构体与其他设备进行捆绑media_device_init(&vind->media_dev);      //用于运行时数据流的管理ret = media_device_register(&vind->media_dev);     //注册字符设备, 生成/dev/mediaXret = vin_md_register_entities(vind, dev->of_node){__vin_register_module(vind, module, j)modules->sensor[i].sd = __vin_subdev_register(vind, inst, inst->cam_name,inst->cam_addr >> 1,modules->sensor[i].type,module->sensors.sensor_bus_sel);struct i2c_adapter *adapter = i2c_get_adapter(bus_sel);sd = v4l2_i2c_new_subdev(v4l2_dev, adapter, name, addr, NULL);v4l2_i2c_new_subdev_board(v4l2_dev, adapter, &info, probe_addrs);client = i2c_new_device(adapter, info);sd = i2c_get_clientdata(client);                //从client的私有数据中找到subdev实例v4l2_device_register_subdev(v4l2_dev, sd)      //注册sensor subdeverr = sd->internal_ops->registered(sd); //回调到longan/kernel/linux-4.9/drivers/media/platform/sunxi-vin/vin-cci/cci_helper.cstatic int sensor_registered(struct v4l2_subdev *sd)v4l2_subdev_call(sd, core, s_power, PWR_ON);      //回调sensor的sensor_powerret = v4l2_subdev_call(sd, core, init, 0);          //回调sensor的sensor_initlist_add_tail(&sd->list, &v4l2_dev->subdevs);       //只是加入链表//自动对焦subdev注册modules->act[i].sd = __vin_subdev_register(vind, inst, inst->act_name,inst->act_addr >> 1,modules->act[i].type,module->sensors.act_bus_sel);//注册闪光灯的subdevret = v4l2_device_register_subdev(&vind->v4l2_dev,module->modules.flash.sd);//注册vinc subdev和/dev/videoXfor (i = 0; i < VIN_MAX_DEV; i++) {                       //这里VIN_MAX_DEV为4vind->vinc[i] = sunxi_vin_core_get_dev(i);         //从vin_core_gbl数组获取vincvin_md_register_core_entity(vind, vind->vinc[i]); sd = &vinc->vid_cap.subdev;v4l2_set_subdev_hostdata(sd, (void *)&vin_pipe_ops); //pipelineret = v4l2_device_register_subdev(&vind->v4l2_dev, sd);if (sd->internal_ops && sd->internal_ops->registered)err = sd->internal_ops->registered(sd);       //这里回调vinc的vin_capture_subdev_registered，注册video_devicevin_init_video(sd->v4l2_dev, &vinc->vid_cap)ret = video_register_device(&cap->vdev, VFL_TYPE_GRABBER, cap->vinc->id); //会生成/dev/videoX}//注册CSI subdevret = v4l2_device_register_subdev(&vind->v4l2_dev,vind->csi[i].sd);//注册mipi subdevret = v4l2_device_register_subdev(&vind->v4l2_dev,vind->mipi[i].sd);//注册isp subdevret = v4l2_device_register_subdev(&vind->v4l2_dev,vind->isp[i].sd);//注册scaler(vipp) subdevret = v4l2_device_register_subdev(&vind->v4l2_dev,vind->scaler[i].sd);}//上回说到v4l2_device_register_subdev函数只是将subdev加到v4l2_device链表中，现在批量做真正的注册操作，会生成/dev/v4l2-subdevXret = v4l2_device_register_subdev_nodes(&vind->v4l2_dev);}

entitys

entity在v4l2_subdev：

struct v4l2_subdev {#if defined(CONFIG_MEDIA_CONTROLLER)struct media_entity entity;
#endif... ...
}

entity在video_device:

struct video_device
{#if defined(CONFIG_MEDIA_CONTROLLER)struct media_entity entity;... ...
}

由此可见，一个entity代表一个v4l2_subdev/video_device,一个v4l2_subdev/video_device只有一个entity。

pads

sensor的pads

源码路径：drivers/media/platform/sunxi-vin/modules/sensor/gc030a_mipi.c

static int sensor_probe(struct i2c_client *client,const struct i2c_device_id *id)
└-> cci_dev_probe_helper(sd, client, &sensor_ops, &cci_drv);└-> ci_media_entity_init_helper(sd, cci_drv);switch (cci_drv->type) {case CCI_TYPE_SENSOR:     //sensorsi->sensor_pads[SENSOR_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE;     //源,si是sensor_infosd->entity.function = MEDIA_ENT_F_CAM_SENSOR;return media_entity_pads_init(&sd->entity, SENSOR_PAD_NUM, si->sensor_pads);   //这里SENSOR_PAD_NUM为1，只有1个pad, 与entity绑定case CCI_TYPE_ACT:            //对焦sd->entity.function = MEDIA_ENT_F_LENS;return media_entity_pads_init(&sd->entity, 0, NULL);      //没有padcase CCI_TYPE_FLASH:     //闪光灯sd->entity.function = MEDIA_ENT_F_FLASH;return media_entity_pads_init(&sd->entity, 0, NULL);        //没有pad

第6行，sensor_pads是sensor_info的一个数组大小是SENSOR_PAD_NUM, SENSOR_PAD_NUM为1；
第8行，media_entity_pads_init()绑定pads和sensor的entity

其他的mipi,csi,isp,vinc类似

以isp为例

longan/kernel/linux-4.9/drivers/media/platform/sunxi-vin/vin-isp/sunxi_isp.c

int __isp_init_subdev(struct isp_dev *isp)isp->isp_pads[ISP_PAD_SINK].flags = MEDIA_PAD_FL_SINK;         //目的isp->isp_pads[ISP_PAD_SOURCE_ST].flags = MEDIA_PAD_FL_SOURCE;   //源isp->isp_pads[ISP_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE;       //源sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER;ret = media_entity_pads_init(&sd->entity, ISP_PAD_NUM, isp->isp_pads);   //这里ISP_PAD_NUM为3, 表示有3个pad

links

vin_probe () --> vin_create_media_links

源码位置：longan/kernel/linux-4.9/drivers/media/platform/sunxi-vin/vin.c

static int vin_probe(struct platform_device *pdev)media_device_init(&vind->media_dev);            //初始化entities，pads，links等链表ret = media_device_register(&vind->media_dev); //注册字符设备, 生成/dev/mediaXret = vin_md_register_entities(vind, dev->of_node)v4l2_device_register_subdev(... ...)err = media_device_register_entity(v4l2_dev->mdev, entity);        //entity,pad,link会被加到media_device的几个链表中ret = vin_create_media_links(vind);for (i = 0; i < VIN_MAX_DEV; i++) {vinc = vind->vinc[i];                       //找到vincmipi = vind->mipi[vinc->mipi_sel].sd;        //找到mipicsi = vind->csi[vinc->csi_sel].sd;           //找到csimodule = &vind->modules[vinc->rear_sensor];   //找到sensor/******************** 对应的dts是*vinc0:vinc@0 {*    vinc0_csi_sel = <0>;*    vinc0_mipi_sel = <0>;*   vinc0_isp_sel = <0>;*    vinc0_isp_tx_ch = <0>;*  vinc0_tdm_rx_sel = <0xff>;*  vinc0_rear_sensor_sel = <0>;*    vinc0_front_sensor_sel = <1>;*   vinc0_sensor_list = <1>;*    status = "okay";*};******************///创建link: sensor ==> mipisensor_link_to_mipi_csi(module, mipi);ret = media_create_pad_link(source, SENSOR_PAD_SOURCE, sink, 0, 0);//创建link: mipi ==> csisource = &mipi->entity;sink = &csi->entity;ret = media_create_pad_link(source, MIPI_PAD_SOURCE,sink, CSI_PAD_SINK,MEDIA_LNK_FL_ENABLED);//创建link: scaler ==> vincsource = &scaler->entity;sink = &cap_sd->entity;ret = media_create_pad_link(source, SCALER_PAD_SOURCE,sink, VIN_SD_PAD_SINK,MEDIA_LNK_FL_ENABLED);//创建link: vinc ==> videosource = &cap_sd->entity;sink = &vinc->vid_cap.vdev.entity;ret = media_create_pad_link(source, VIN_SD_PAD_SOURCE,sink, 0, MEDIA_LNK_FL_ENABLED);    }//创建link: csi ==> ispfor (i = 0; i < VIN_MAX_CSI; i++) {csi = vind->csi[i].sd;source = &csi->entity;for (j = 0; j < VIN_MAX_ISP; j++) {       //由此可见，每个csi和每个isp相连isp = vind->isp[j].sd;sink = &isp->entity;ret = media_create_pad_link(source, CSI_PAD_SOURCE,sink, ISP_PAD_SINK, 0);}//创建link: isp ==> scalerfor (i = 0; i < VIN_MAX_ISP; i++) {isp = vind->isp[i].sd;source = &isp->entity;stat = vind->stat[i].sd;sink = &stat->entity;//这里多出一个创建link: isp ==> statret = media_create_pad_link(source, ISP_PAD_SOURCE_ST,sink, 0,MEDIA_LNK_FL_IMMUTABLE |MEDIA_LNK_FL_ENABLED);for (j = 0; j < VIN_MAX_SCALER; j++) {scaler = vind->scaler[j].sd;sink = &scaler->entity;ret = media_create_pad_link(source, ISP_PAD_SOURCE,sink, SCALER_PAD_SINK, 0);}}ret = vin_setup_default_links(vind);for (i = 0; i < VIN_MAX_DEV; i++) {isp = vind->isp[vinc->isp_sel].sd;scaler = vind->scaler[vinc->vipp_sel].sd;link = media_entity_find_link(&isp->entity.pads[ISP_PAD_SOURCE],&scaler->entity.pads[SCALER_PAD_SINK]);ret = media_entity_setup_link(link, MEDIA_LNK_FL_ENABLED);p = &vinc->vid_cap.pipe;vin_md_prepare_pipeline(p, &vinc->vid_cap.vdev.entity);}ret = v4l2_device_register_subdev_nodes(&vind->v4l2_dev);struct media_link *link;link = media_create_intf_link(&sd->entity,&vdev->intf_devnode->intf,MEDIA_LNK_FL_ENABLED);

在vin_create_media_links()中调用media_create_pad_link(),你有没发现最后一个参数有些是0，有些是MEDIA_LNK_FL_ENABLED。

所以vin_create_media_links()之后展现的地图应该是这样子的：

灰色的箭头表示还没enable

但是，vin_setup_default_links()又使能了部分isp==>scale，links地图发生了些变化，如下：

选择哪一个isp是由vincX的dts配置来定的，看如下vinc0：

vinc0:vinc@0 {vinc0_csi_sel = <0>;vinc0_mipi_sel = <0>;vinc0_isp_sel = <0>;        <====vinc0_isp_tx_ch = <0>;vinc0_tdm_rx_sel = <0xff>;vinc0_rear_sensor_sel = <0>;vinc0_front_sensor_sel = <1>;vinc0_sensor_list = <1>;status = "okay";
};

由上可见sensor==>mipi和csi==>isp的link是还没enable的，在哪enable呢？

源码位置：longan/kernel/linux-4.9/drivers/media/platform/sunxi-vin/vin-video/vin_video.c

static int vidioc_s_input(struct file *file, void *priv, unsigned int i)__vin_sensor_setup_link(vinc, module, valid_idx, 1)      //module就是sensorstruct v4l2_subdev *sensor = module->modules.sensor[i].sd;subdev = vind->mipi[vinc->mipi_sel].sd;entity = &sensor->entity;list_for_each_entry(link, &entity->links, list)ret = media_entity_setup_link(link, MEDIA_LNK_FL_ENABLED);  //使能sensor==>mipi的link__csi_isp_setup_link(vinc, 1)csi = vind->csi[vinc->csi_sel].sd;isp = vind->isp[vinc->isp_sel].sd;link = media_entity_find_link(&csi->entity.pads[CSI_PAD_SOURCE],&isp->entity.pads[ISP_PAD_SINK]);ret = media_entity_setup_link(link, MEDIA_LNK_FL_ENABLED);

所以最终的links地图是这样子的：

ps:绿色箭头表示已经enable，灰色箭头表示还没enable。

pipeline

pipeline初始化

pipeline相当于一张地图的导航线，导航的起点是sensorX，终点是videoX

先思考个问题，pipeline是保存在哪的呢？

struct vin_core {... ...const struct vin_pipeline_ops *pipeline_ops;struct vin_vid_cap vid_cap;
};

struct vin_vid_cap {struct vin_pipeline pipe;
};

struct vin_pipeline {struct media_pipeline pipe;struct v4l2_subdev *sd[VIN_IND_MAX];
};

由此可见每个vinc都有一个pipeline，vin_pipeline保存着这条“导航线“所经过的v4l2_subdev。

由上文可找到调用流程vin_probe() --> vin_setup_default_links() --> vin_md_prepare_pipeline()

vin_md_prepare_pipeline()是在一个for循环里面执行的

p = &vinc->vid_cap.pipe;
vin_md_prepare_pipeline(p, &vinc->vid_cap.vdev.entity);while (1) {struct media_pad *pad = NULL;/* Find remote source pad */for (i = 0; i < me->num_pads; i++) {struct media_pad *spad = &me->pads[i];if (!(spad->flags & MEDIA_PAD_FL_SINK))continue;pad = media_entity_remote_pad(spad);}if (pad == NULL)   //没找到pad, 退出break;sd = media_entity_to_v4l2_subdev(pad->entity);switch (sd->grp_id) {case VIN_GRP_ID_SENSOR:p->sd[VIN_IND_SENSOR] = sd;break;case VIN_GRP_ID_MIPI:p->sd[VIN_IND_MIPI] = sd;break;case VIN_GRP_ID_CSI:p->sd[VIN_IND_CSI] = sd;break;case VIN_GRP_ID_TDM_RX:p->sd[VIN_IND_TDM_RX] = sd;break;case VIN_GRP_ID_ISP:p->sd[VIN_IND_ISP] = sd;break;case VIN_GRP_ID_SCALER:p->sd[VIN_IND_SCALER] = sd;break;case VIN_GRP_ID_CAPTURE:p->sd[VIN_IND_CAPTURE] = sd;break;default:break;}me = &sd->entity;}

第11行，看看media_entity_remote_pad()的实现

struct media_pad *media_entity_remote_pad(struct media_pad *pad)
{struct media_link *link;list_for_each_entry(link, &pad->entity->links, list) {if (!(link->flags & MEDIA_LNK_FL_ENABLED))continue;if (link->source == pad)return link->sink;if (link->sink == pad)return link->source;}return NULL;}

第5~14行，只有MEDIA_LNK_FL_ENABLED的link才会被返回。

由上文可见，程序运行到这里，部分link并没有enable，那不就是说，这时这个pipeline并不完整？哪什么时候才会完整呢？

这时vinc0的pipeline应该是这样子的：

p
|--sd[VIN_IND_SENSOR]   = NULL
|--sd[VIN_IND_MIPI]     = NULL
|--sd[VIN_IND_CSI]      = NULL
|--sd[VIN_IND_ISP]      = isp0
|--sd[VIN_IND_SCALER]   = scale0
|--sd[VIN_IND_CAPTURE]  = vinc0

源码位置：longan/kernel/linux-4.9/drivers/media/platform/sunxi-vin/vin-video/vin_video.c

static int vidioc_s_input(struct file *file, void *priv, unsigned int i)__vin_sensor_setup_link(vinc, module, valid_idx, 1)      //module就是sensorstruct v4l2_subdev *sensor = module->modules.sensor[i].sd;subdev = vind->mipi[vinc->mipi_sel].sd;entity = &sensor->entity;list_for_each_entry(link, &entity->links, list)ret = media_entity_setup_link(link, MEDIA_LNK_FL_ENABLED);  //使能sensor==>mipi的link__csi_isp_setup_link(vinc, 1)csi = vind->csi[vinc->csi_sel].sd;isp = vind->isp[vinc->isp_sel].sd;link = media_entity_find_link(&csi->entity.pads[CSI_PAD_SOURCE],&isp->entity.pads[ISP_PAD_SINK]);ret = media_entity_setup_link(link, MEDIA_LNK_FL_ENABLED);ret = vin_pipeline_call(vinc, open, &cap->pipe, &cap->vdev.entity, true);//这里的open回调__vin_pipeline_openstatic int __vin_pipeline_open(struct vin_pipeline *p,struct media_entity *me, bool prepare)vin_md_prepare_pipeline(p, me);      //又调了一次

这次vinc0的pipeline应该是这样子的：

p
|--sd[VIN_IND_SENSOR]   = sensor0
|--sd[VIN_IND_MIPI]     = mipi0
|--sd[VIN_IND_CSI]      = csi0
|--sd[VIN_IND_ISP]      = isp0
|--sd[VIN_IND_SCALER]   = scale0
|--sd[VIN_IND_CAPTURE]  = vinc0

pipeline的使用

辛辛苦苦建好一条pipeline，肯定要用起来。

这套SDK貌似并没有使用类似media_entity_pipeline_start()函数。

对pipeline的使用也是简单粗暴，如：

v4l2_subdev_call(cap->pipe.sd[VIN_IND_ISP], core, init, 0);

直接调用subdev的回调函数。

在vidioc_s_input或者vidioc_streamon时，通过vid_cap里的pipe数组就可以清楚地知道操作哪个subdev。

完

以上为笔者的学习手记，如果您发现有不对的地方，请不吝赐教，谢谢。