Gstreamer 硬解码Rtsp流及代码实现

背景

业务需要自己做解码，因为软解码CPU占有率太高，需要硬件加速，也就是硬解码。可以使用ffmpeg或者Gstreamer进行解码，我们选择用Gstreamer做解码。

系统环境：Ubuntu 20.04
代码功能：实现rtsp流的H264硬解码，获取解码后的数据；

一、Gstreamer介绍

Gstreamer是一个用于开发流媒体应用的开源框架，采用了基于插件（plugin）和管道（pipeline）的体系结构，框架中的所有的功能模块都被实现成可以插拔的元素（Element），并且能够很方便地安装到任意一个管道上。由于所有插件都通过管道机制进行统一的数据交换，因此很容易利用已有的各种插件“组装”出一个功能完善的流媒体应用程序。

二、Gstreamer基础概念

2.1 Element

从 GStreamer 自身的观点来看，GstElement 可以描述为一个具有特定属性的黑盒子，它通过连接点（link point）与外界进行交互，向框架中的其余部分表征自己的特性或者功能。

一个Element实现了一个功能（如读取文件，解码，输出等），一个程序需要创建多个element，并按顺序将其串连起来，构成一个完整的pipeline

按照各自功能上的差异，GstElement 可以细分成如下几类：

Source Element 数据源元件，只有输出端。它仅能用来产生供管道消费的数据，而不能对数据做任何处理。一个典型的数据源元件的例子是音频捕获单元，它负责从声卡读取原始的音频数据，然后作为数据源提供给其它模块使用。
Filter Element 过滤器元件，既有输入端又有输出端。它从输入端获得相应的数据，并在经过特殊处理之后传递给输出端。一个典型的过滤器元件的例子是音频编码单元，它首先从外界获得音频数据，然后根据特定的压缩算法对其进行编码，最后再将编码后的结果提供给其它模块使用)
Sink Element 接收器元件，只有输入端。它仅具有消费数据的能力，是整条媒体管道的终端。一个典型的接收器元件的例子是音频回放单元，它负责将接收到的数据写到声卡上，通常这也是音频处理过程中的最后一个环节。

若干elemetns组成一个pipeline

三、Gstreamer编解码一般步骤

创建所需elements
gst_bin_add_many添加创建的elements
链接elements
设置pipeline为Playing状态
设置采样回调

三、完整代码示例

#include <iostream>
#include <string>
#include <gst/gst.h>typedef struct _custom_data
{/* pipeline{source->rtpdepay->rtp_queue->h264parse->hard_decoder->cudaconvert->cudadowload->video_queue->sink} */GstElement *pipeline,*source, *rtpdepay, *rtp_queue, *h264parse, *h264_queue,*hard_decoder, *img_filter, *cudaconvert, *cudadowload, *video_queue, *sink;std::string Rtsp;
} CustomData;/* Create the elements */
GstFlowReturn InitGst(CustomData *data)
{GstFlowReturn retVal;/* Create the elements */data->pipeline = gst_pipeline_new("rtsp-decode-pipeline");/**  param1: element 类型     param2: 名称*  gst_element_factory_make(param1, param2)*/data->source = gst_element_factory_make("rtspsrc", "source");//  rtph264depay: Extracts H264 video from RTP packets (RFC 3984)data->rtpdepay = gst_element_factory_make("rtph264depay", "rtpdepay");data->rtp_queue = gst_element_factory_make("queue", "rtp_queue");// h264parse: Parses H.264 streamsdata->h264parse = gst_element_factory_make("h264parse", "h264parse");data->h264_queue = gst_element_factory_make("queue", "h264_queue");// nvh264dec: NVDEC video decoderdata->hard_decoder = gst_element_factory_make("nvh264dec", "hard_decoder"); // 硬解// cudaconvert: Convert video frames between supported video formats.data->cudaconvert = gst_element_factory_make("cudaconvert", "cudaconvert");// cudadownload: Downloads data from NVIDA GPU via CUDA APIsdata->cudadowload = gst_element_factory_make("cudadownload", "cudadownload");if (!data->hard_decoder || !data->cudaconvert || !data->cudadowload){data->hard_decoder = gst_element_factory_make("avdec_h264", "hard_decoder"); // 软解data->cudaconvert = gst_element_factory_make("videoconvert", "cudaconvert");data->cudadowload = gst_element_factory_make("queue", "cudadownload");/* https://www.nvidia.cn/Download/index.aspx?lang=cn 驱动下载地址*/g_print("Use SoftDecoder, Ensure NVIDIA driver is the lastest version!!!\n");}else{g_print("Use HardDecode!\n");}data->video_queue = gst_element_factory_make("queue", "video_queue");data->sink = gst_element_factory_make("appsink", "sink"); // autovideosink, fakesink, appsink, filesinkif (!data->pipeline || !data->source || !data->rtp_queue || !data->rtpdepay || !data->h264_queue ||!data->h264parse || !data->hard_decoder || !data->cudaconvert || !data->cudadowload || !data->video_queue || !data->sink){g_print("Not all elements could be created!!!\n");return GST_FLOW_ERROR;}g_print("All elements created success\n");/* 设置rtsp的输入地址 */g_object_set(G_OBJECT(data->source), "location", data->Rtsp.c_str(), "latency", 2000, NULL);/* 设置输出格式 */g_object_set(G_OBJECT(data->sink),"sync", FALSE,"emit-signals", TRUE,"caps", gst_caps_new_simple("video/x-raw",// "width", G_TYPE_INT, 1920,// "height", G_TYPE_INT, 1080,// "framerate", GST_TYPE_FRACTION, 10, 1,"format", G_TYPE_STRING, "NV12", NULL),NULL);/* 创建pipeline，注意此时各个组件还没有连接，只是add到管道，也就是说，add要在link之前 */gst_bin_add_many(GST_BIN(data->pipeline), data->source, data->rtpdepay, data->rtp_queue, data->h264parse, data->h264_queue, data->hard_decoder, data->cudaconvert, data->cudadowload, data->video_queue, data->sink, NULL);// 此时rtsp src和 rtph264depay还没有连接，所以必须设置pad-added信号监听，// 在管道开始工作后，确定了数据格式，再把它们连接起来g_signal_connect(data->source, "pad-added", G_CALLBACK(RtspSrcPadAdded_callback), data);// 连接元素，注意顺序不能错，因为还没有确定数据的格式，所以此时rtsp src和rtph264depay还没建立连接，// 先将rtsp src之后的元件连接起来if (!gst_element_link_many(data->rtpdepay, data->rtp_queue, data->h264parse, data->hard_decoder, data->cudaconvert, data->cudadowload, data->video_queue, NULL)){g_printerr("@@@ OpenRtsp: Failed to link rtpdepay -> video_queue\n");return GST_FLOW_ERROR;}if (!gst_element_link_many(data->video_queue, data->sink, NULL)){g_printerr("@@@ OpenRtsp: Failed to link video_queue -> sink\n");return GST_FLOW_ERROR;}g_print("All elements Link success\n");/* 设置 pipeline 状态为 Playing */GstStateChangeReturn ret = gst_element_set_state(data->pipeline, GST_STATE_PLAYING);if (ret == GST_STATE_CHANGE_FAILURE){g_printerr("@@@ OpenRtsp: Unable to set the pipeline to the playing state.\n");return GST_FLOW_ERROR;}// 设置采样回调g_signal_connect(data->sink, "new-sample", G_CALLBACK(ReadvideoFrame_callback), data);return GST_FLOW_OK;
}/* source new src pad create */
static void RtspSrcPadAdded_callback(GstElement *src, GstPad *new_pad, gpointer user_data)
{CustomData *data = (CustomData *)user_data;GstPad *sink_pad = gst_element_get_static_pad(data->rtpdepay, "sink");GstCaps *p_caps;gchar *description;GstPadLinkReturn ret;GstCaps *new_pad_caps = NULL;GstStructure *new_pad_struct = NULL;const gchar *new_pad_type = NULL;g_print("Received new pad '%s' from '%s':\n", GST_PAD_NAME(new_pad), GST_ELEMENT_NAME(src));//使用gst_pad_is_linked(sink_pad)来检查是否已经连接好了，若是则忽略该信号if (gst_pad_is_linked(sink_pad)){g_print("We are already linked. Ignoring.\n");goto exit;}// here, you would setup a new pad link for the newly created pad// so, now find that rtph264depay is needed and link them?p_caps = gst_pad_get_pad_template_caps(new_pad);description = gst_caps_to_string(p_caps);g_print("new pad caps: %s\n", description);g_free(description);if (NULL != p_caps)gst_caps_unref(p_caps);/* Attempt the link *//* Check the new pad's type */new_pad_caps = gst_pad_get_current_caps(new_pad);new_pad_struct = gst_caps_get_structure(new_pad_caps, 0);new_pad_type = gst_structure_get_name(new_pad_struct);//因为rtph264depay要求application/x-rtp格式的输入数据，所以找rtspsrc的pad里面有没有这种格式的数据if (!g_str_has_prefix(new_pad_type, "application/x-rtp")){g_print("It has type '%s' which is not application/x-rtp. Ignoring.\n", new_pad_type);goto exit;}//如果找到了application/x-rtp格式的输入数据，则将rtspsrc的source pad和rtph264depay的sink pad连起来ret = gst_pad_link(new_pad, sink_pad); // linkif (GST_PAD_LINK_FAILED(ret)){g_print("Type is '%s' but link failed.\n", new_pad_type);}else{g_print("Link succeeded (type '%s').\n", new_pad_type);}/* 解引用，释放资源*/if (NULL != new_pad_caps)gst_caps_unref(p_caps);exit:/* 解引用，释放资源*/if (sink_pad != NULL)gst_object_unref(sink_pad);
}/* 每次从视频流中获取一帧数据 */
void ReadVideoFrame_callback(GstElement *sink, gpointer user_data)
{CustomData *data = (CustomData *)user_data;GstSample *sample = nullptr;gsize data_size = 0;gsize stream_size = 0;// 使用pull-sample拉取视频帧，并映射到map变量，通过map拷贝出frame数据g_signal_emit_by_name(sink, "pull-sample", &sample);if (sample){GstBuffer *buffer = gst_sample_get_buffer(sample);if (buffer){GstMapInfo map;if (gst_buffer_map(buffer, &map, GST_MAP_READ)){char buf[4096] = {0};memcpy(buf, map.data, data_size);  // 获取解码后的数据到buf// release buffer mappinggst_buffer_unmap(buffer, &map);}else{printf("fgst_buffer_map error failed!!!\n");}}else{printf("gst_sample_get_buffer failed!!!\n");}gst_sample_unref(sample); // release sample reference}else{printf("sample is null...\n");}return;
}int main()
{GstFlowReturn ret;CustomData *context = (CustomData*)malloc(sizeof(CustomData));if (!context){printf("context malloc failed!\n");return -1;}context->Rtsp = "rtsp://wowzaec2demo.streamlock.net/vod/mp4:BigBuckBunny_115k.mp4";if ((ret = InitGst(context)) == GST_FLOW_OK){printf("InitGst success..\n");}while(1);return 0;
}