FFMPEG+SDL2 实现播放器功能

ffmpeg视频H264压缩，rtsp推流课程教学视频：

https://edu.csdn.net/course/detail/27795

课件中里面提供源码可以直接下载运行！

-------------------------------------------------------------------------------------------------------------------------------------

一、界面

1 打开的pushbutton

2 停止/开始的pushbutton

3 进度条 QSlider

4 播放窗口widget

二、创建videothread线程

该线程用于读取视频数据流具体实现过程可以参考

http://blog.csdn.net/yunge812/article/details/79342089

其中需要添加SDL处理音频的部分进去

2.1 SDL初始化

if (SDL_Init(SDL_INIT_AUDIO)){fprintf(stderr,"Could not initialize SDL - %s. \n", SDL_GetError());exit(1);}

2.2 获取音频流信息

for (i = 0; i < pFormatCtx->nb_streams; i++){    if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO  && audioStream < 0){audioStream = i;}}///如果audioStream为-1 说明没有找到音频流if (audioStream == -1) {printf("Didn't find a audio stream.\n");return;}if (audioStream >= 0){/// 设置SDL音频流信息 音频处理初始化audio_stream_component_open(&mVideoState, audioStream);  //该函数后续会进行完成}

audio_stream_component_open函数在3.3.1中会涉及

2.3 查找音频解码器


///================================ 查找音频解码器 =================================//AVCodecContext *aCodecCtx;aCodecCtx = pFormatCtx->streams[audioStream]->codec;AVCodec *aCodec;aCodec = avcodec_find_decoder(aCodecCtx->codec_id);if (aCodec == NULL){printf("ACodec not found.\n");return;}///================================ 打开音频解码器 ==================================//if (avcodec_open2(aCodecCtx, aCodec, NULL) < 0){printf("Could not open audio codec.\n");return;}    1 SDL初始化vs->audio_st = pFormatCtx->streams[audioStream]; ///设置音频解码packet_queue_init(&vs->videoq); //音频处理：初始化队列

2.4 创建视频解码的线程由SDL进行创建 video_thread函数后续阶段完成

 vs->video_tid = SDL_CreateThread(video_thread, "video_thread", &mVideoState);vs->player = this; //线程指针

2.5 分配AVPacket用于存放读取的视频

 AVPacket *packet = (AVPacket *) malloc(sizeof(AVPacket)); //分配一个packet 用于存放读取的视频

三、开始进行音视频的读取

3.1 音视频统一读取

 //===========================  读取视频信息 ===============================//if (av_read_frame(pFormatCtx, packet) < 0) //读取的是一帧视频  数据存入一个AVPacket的结构中{qDebug()  << "read error." ;return ;}//此时数据存储在packet中if (packet->stream_index == videoStream){packet_queue_put(&vs->videoq, packet); //这里我们将视频数据存入队列}else if( packet->stream_index == audioStream ){packet_queue_put(&vs->audioq, packet);//这里我们将音频数据存入队列}

3.2 视频读取线程将视频转换成QImage显示

3.2.1 视频相关结构体的分配

 ///解码视频相关AVFrame *pFrame, *pFrameRGB;uint8_t *out_buffer_rgb; //解码后的rgb数据struct SwsContext *img_convert_ctx;  //用于解码后的视频格式转换AVCodecContext *pCodecCtx = is->video_st->codec; //视频解码器pFrame = av_frame_alloc();pFrameRGB = av_frame_alloc();///这里我们改成了 将解码后的YUV数据转换成RGB32img_convert_ctx = sws_getContext(pCodecCtx->width, pCodecCtx->height,pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height,PIX_FMT_RGB32, SWS_BICUBIC, NULL, NULL, NULL);numBytes = avpicture_get_size(PIX_FMT_RGB32, pCodecCtx->width,pCodecCtx->height);out_buffer_rgb = (uint8_t *) av_malloc(numBytes * sizeof(uint8_t));avpicture_fill((AVPicture *) pFrameRGB, out_buffer_rgb, PIX_FMT_RGB32, pCodecCtx->width, pCodecCtx->height);// 会将pFrameRGB的数据按RGB格式自动"关联"到out_buffer_rgb  即pFrameRGB中的数据改变了 out_buffer中的数据也会相应的改变

3.2.2 从队列中取出数据

  if (packet_queue_get(&is->videoq, packet, 1) <= 0)break;//队列里面没有数据了  读取完毕了

3.2.3 解码

       ret = avcodec_decode_video2(pCodecCtx, pFrame, &got_picture,packet);if (packet->dts == AV_NOPTS_VALUE && pFrame->opaque&& *(uint64_t*) pFrame->opaque != AV_NOPTS_VALUE){video_pts = *(uint64_t *) pFrame->opaque;}else if (packet->dts != AV_NOPTS_VALUE){video_pts = packet->dts;}else{video_pts = 0;}

3.2.4 音视频同步关键函数synchronize_video( )

        video_pts *= av_q2d(is->video_st->time_base);video_pts = synchronize_video(is, pFrame, video_pts);while(1){audio_pts = is->audio_clock;if (video_pts <= audio_pts) //等待匹配   两者时间匹配 跳出该循环break;int delayTime = (video_pts - audio_pts) * 1000;delayTime = delayTime > 5 ? 5:delayTime;SDL_Delay(delayTime);}

3.2.5 将图像缩放转换格式输出到QImage 并且激活信号

 if (got_picture){sws_scale(img_convert_ctx,(uint8_t const * const *) pFrame->data,pFrame->linesize, 0, pCodecCtx->height, pFrameRGB->data,pFrameRGB->linesize);//把这个RGB数据 用QImage加载QImage tmpImg((uchar *)out_buffer_rgb,pCodecCtx->width,pCodecCtx->height,QImage::Format_RGB32);QImage image = tmpImg.copy(); //把图像复制一份 传递给界面显示is->player->disPlayVideo(image); //调用激发信号的函数}

3.3 音频处理过程

3.3.1 audio_stream_component_open 函数

int audio_stream_component_open(VideoState *is, int stream_index) //audioStream 遍历出来的音频流信息
{AVFormatContext *ic = is->ic;AVCodecContext  *codecCtx;AVCodec         *codec;SDL_AudioSpec   wanted_spec, spec;  // wanted_spec是我们期望设置的属性，spec是系统最终接受的参数int64_t wanted_channel_layout = 0;int wanted_nb_channels; //声道数/*  SDL支持的声道数为 1, 2, 4, 6 *//*  后面我们会使用这个数组来纠正不支持的声道数目 */const int next_nb_channels[] = { 0, 0, 1, 6, 2, 6, 4, 6 };if (stream_index < 0 || stream_index >= ic->nb_streams){return -1;}codecCtx           = ic->streams[stream_index]->codec;wanted_nb_channels = codecCtx->channels;//根据期望的声道数获得期望的声道布局if (!wanted_channel_layout || wanted_nb_channels!= av_get_channel_layout_nb_channels(wanted_channel_layout)) //Return the number of channels in the channel layout.{wanted_channel_layout  = av_get_default_channel_layout(wanted_nb_channels); //Return default channel layout for a given number of channels.wanted_channel_layout &= ~AV_CH_LAYOUT_STEREO_DOWNMIX;}wanted_spec.channels = av_get_channel_layout_nb_channels(wanted_channel_layout);//根据期望的声道布局获得期望的声道数wanted_spec.freq     = codecCtx->sample_rate; //频率  由外部传入流转换得到if (wanted_spec.freq <= 0 || wanted_spec.channels <= 0){return -1;}//设置期望属性wanted_spec.format   = AUDIO_S16SYS;           // 格式wanted_spec.silence  = 0;                      // 0指示静音wanted_spec.samples  = SDL_AUDIO_BUFFER_SIZE;  // 自定义SDL缓冲区大小wanted_spec.callback = audio_callback;         // 音频解码的关键回调函数wanted_spec.userdata = is;                     // 传给上面回调函数的外带数据//打开音频设备，这里使用一个while来循环尝试打开不同的声道数(由上面next_nb_channels数组指定)直到成功打开，或者全部失败//wanted_spec ==> specdo{is->audioID = SDL_OpenAudioDevice(SDL_GetAudioDeviceName(0,0),0,&wanted_spec, &spec,0);//is->audioID = SDL_OpenAudio(&wanted_spec, &spec);fprintf(stderr,"SDL_OpenAudio (%d channels): %s\n",wanted_spec.channels, SDL_GetError());qDebug()<<QString("SDL_OpenAudio (%1 channels): %2").arg(wanted_spec.channels).arg(SDL_GetError());wanted_spec.channels = next_nb_channels[FFMIN(7, wanted_spec.channels)]; // 7 和 wanted_spec.channels的最小值if (!wanted_spec.channels){fprintf(stderr,"No more channel combinations to tyu, audio open failed\n");
//            return -1;break;}wanted_channel_layout = av_get_default_channel_layout(wanted_spec.channels);}while(is->audioID == 0);// 检查实际使用的配置（保存在spec,由SDL_OpenAudio()填充）if (spec.format != AUDIO_S16SYS){fprintf(stderr,"SDL advised audio format %d is not supported!\n",spec.format);return -1;}// 看二者的channels是否相同  得到最终的wanted_channel_layout  该参数需要传给VideoPlayerif (spec.channels != wanted_spec.channels){wanted_channel_layout = av_get_default_channel_layout(spec.channels);if (!wanted_channel_layout){fprintf(stderr,"SDL advised channel count %d is not supported!\n",spec.channels);return -1;}}is->audio_hw_buf_size = spec.size;/* 把设置好的参数保存到大结构VideoState中 */is->audio_src_fmt            = is->audio_tgt_fmt            = AV_SAMPLE_FMT_S16;is->audio_src_freq           = is->audio_tgt_freq           = spec.freq;is->audio_src_channel_layout = is->audio_tgt_channel_layout = wanted_channel_layout;is->audio_src_channels       = is->audio_tgt_channels       = spec.channels;///====================================== 查找解码器 ==========================================//codec = avcodec_find_decoder(codecCtx->codec_id);///====================================== 打开解码器 ==========================================//if (!codec || (avcodec_open2(codecCtx, codec, NULL) < 0)){fprintf(stderr,"Unsupported codec!\n");return -1;}ic->streams[stream_index]->discard = AVDISCARD_DEFAULT; //丢掉大小为0的数据流switch (codecCtx->codec_type){case AVMEDIA_TYPE_AUDIO:is->audio_st = ic->streams[stream_index];is->audio_buf_size = 0;is->audio_buf_index = 0;memset(&is->audio_pkt, 0, sizeof(is->audio_pkt));packet_queue_init(&is->audioq); //初始化PacketQueue队列  全部清零//SDL_PauseAudio(0); // 开始播放静音SDL_PauseAudioDevice(is->audioID,0); //开始播放 调用回调函数audio_callbackbreak;default:break;}return 0;
}

该函数基本上是固定模式需要注意的是回调函数的设置

    wanted_spec.callback = audio_callback;         // 音频解码的关键回调函数

3.3.2 回调函数在开始播放后调用audio_callback

static void audio_callback(void *userdata, Uint8 *stream, int len)
{VideoState *is = (VideoState *) userdata; //传入的数据结构体int len1, audio_data_size;double pts;/*   len是由SDL传入的SDL缓冲区的大小，如果这个缓冲未满，我们就一直往里填充数据 *//*   audio_buf_index 和 audio_buf_size 标示我们自己用来放置解码出来的数据的缓冲区 *//*   这些数据待copy到SDL缓冲区， 当audio_buf_index >= audio_buf_size的时候意味着我们的缓冲为空/*   没有数据可供copy，这时候需要调用audio_decode_frame来解码出更多的桢数据 */while (len > 0){if (is->audio_buf_index >= is->audio_buf_size) //audio_buf为空 需要解码更多帧的数据{///解码之后的数据存储在 is->audio_bufaudio_data_size = audio_decode_frame(is, &pts);   //解码的过程if (audio_data_size < 0)  /* 没能解码出数据，我们默认播放静音 */{/* silence */is->audio_buf_size = 1024;/* 清零，静音 */memset(is->audio_buf, 0, is->audio_buf_size);}else{is->audio_buf_size = audio_data_size;}is->audio_buf_index = 0; //指针归零}/*  查看stream可用空间，决定一次copy多少数据，剩下的下次继续copy */len1 = is->audio_buf_size - is->audio_buf_index;if (len1 > len){len1 = len;}// is->audio_buf  ===>  streammemcpy(stream, (uint8_t *) is->audio_buf + is->audio_buf_index, len1);len    -= len1;stream += len1;is->audio_buf_index += len1;}//qDebug()<<"audio_callback finished";
}

3.3.3 解码的过程audio_decode_frame

 static int audio_decode_frame(VideoState *is, double *pts_ptr) //解码音频{int len1, len2, decoded_data_size;AVPacket *pkt = &is->audio_pkt;int got_frame = 0;int64_t dec_channel_layout;int wanted_nb_samples, resampled_data_size, n;double pts;for (;;){while (is->audio_pkt_size > 0){//判断暂停if (is->isPause == true){SDL_Delay(10);continue;}if (!is->audio_frame){if (!(is->audio_frame = avcodec_alloc_frame())){return AVERROR(ENOMEM);}}elseavcodec_get_frame_defaults(is->audio_frame);//解码 数据存储在is->audio_framelen1 = avcodec_decode_audio4(is->audio_st->codec, is->audio_frame,&got_frame, pkt);if (len1 < 0)  // error, skip the frame{is->audio_pkt_size = 0;break;}is->audio_pkt_data += len1; //指针+len1is->audio_pkt_size -= len1; //缓存-len1if (!got_frame)continue;/* 计算解码出来的桢需要的缓冲大小 */decoded_data_size = av_samples_get_buffer_size(NULL,is->audio_frame->channels, is->audio_frame->nb_samples,(AVSampleFormat)is->audio_frame->format, 1);dec_channel_layout =(is->audio_frame->channel_layout && is->audio_frame->channels== av_get_channel_layout_nb_channels( is->audio_frame->channel_layout))?is->audio_frame->channel_layout :av_get_default_channel_layout(is->audio_frame->channels);wanted_nb_samples = is->audio_frame->nb_samples;if (is->audio_frame->format   != is->audio_src_fmt|| dec_channel_layout != is->audio_src_channel_layout|| is->audio_frame->sample_rate != is->audio_src_freq|| (wanted_nb_samples != is->audio_frame->nb_samples&& !is->swr_ctx)){if (is->swr_ctx)swr_free(&is->swr_ctx);is->swr_ctx = swr_alloc_set_opts(NULL,is->audio_tgt_channel_layout, (AVSampleFormat)is->audio_tgt_fmt,is->audio_tgt_freq, dec_channel_layout,(AVSampleFormat)is->audio_frame->format, is->audio_frame->sample_rate,0, NULL);if (!is->swr_ctx || swr_init(is->swr_ctx) < 0){//fprintf(stderr,"swr_init() failed\n");break;}is->audio_src_channel_layout = dec_channel_layout;is->audio_src_channels = is->audio_st->codec->channels;is->audio_src_freq = is->audio_st->codec->sample_rate;is->audio_src_fmt = is->audio_st->codec->sample_fmt;}/* 这里我们可以对采样数进行调整，增加或者减少，一般可以用来做声画同步 */if (is->swr_ctx){const uint8_t **in =(const uint8_t **) is->audio_frame->extended_data;uint8_t *out[] = { is->audio_buf2 };if (wanted_nb_samples != is->audio_frame->nb_samples){if (swr_set_compensation(is->swr_ctx,(wanted_nb_samples - is->audio_frame->nb_samples)* is->audio_tgt_freq/ is->audio_frame->sample_rate,wanted_nb_samples * is->audio_tgt_freq/ is->audio_frame->sample_rate) < 0){//fprintf(stderr,"swr_set_compensation() failed\n");break;}}//=================================== 转换 ===========================================////转换出来的数据存储在uint8_t *out中。也就是  is->audio_buf2len2 = swr_convert(is->swr_ctx, out,sizeof(is->audio_buf2) / is->audio_tgt_channels/ av_get_bytes_per_sample(is->audio_tgt_fmt),in, is->audio_frame->nb_samples);if (len2 < 0){//fprintf(stderr,"swr_convert() failed\n");break;}if (len2 == sizeof(is->audio_buf2) / is->audio_tgt_channels/ av_get_bytes_per_sample(is->audio_tgt_fmt)){//fprintf(stderr,"warning: audio buffer is probably too small\n");swr_init(is->swr_ctx);}is->audio_buf = is->audio_buf2; //解码之后转换得到的数据resampled_data_size = len2 * is->audio_tgt_channels* av_get_bytes_per_sample(is->audio_tgt_fmt);}else{resampled_data_size = decoded_data_size;is->audio_buf = is->audio_frame->data[0];}//计算时间pts = is->audio_clock;*pts_ptr = pts;n = 2 * is->audio_st->codec->channels;is->audio_clock += (double) resampled_data_size/ (double) (n * is->audio_st->codec->sample_rate);// We have data, return it and come back for more laterreturn resampled_data_size;}//判断暂停if (is->isPause == true){SDL_Delay(10);continue;}if (pkt->data)av_free_packet(pkt);memset(pkt, 0, sizeof(*pkt));if (packet_queue_get(&is->audioq, pkt, 0) <= 0) //从队列is->audioq中取出第一个packet  存放在pkt中return -1;is->audio_pkt_data = pkt->data;is->audio_pkt_size = pkt->size;/* if update, update the audio clock w/pts */if (pkt->pts != AV_NOPTS_VALUE){is->audio_clock = av_q2d(is->audio_st->time_base) * pkt->pts;}}return 0;
}

3.4 synchronize_video( )音视频同步函数

static double synchronize_video(VideoState *is, AVFrame *src_frame, double pts)
{double frame_delay;if (pts != 0) {/* if we have pts, set video clock to it */is->video_clock = pts;} else {/* if we aren't given a pts, set it to the clock */pts = is->video_clock;}/* update the video clock */frame_delay = av_q2d(is->video_st->codec->time_base);/* if we are repeating a frame, adjust clock accordingly */frame_delay += src_frame->repeat_pict * (frame_delay * 0.5);is->video_clock += frame_delay;return pts;
}

总结：

总体来看系统由三个线程

1 音视频处理总线程该线程由继承的QT类得到

2 视频处理线程由SDL创建 vs->video_tid = SDL_CreateThread(video_thread, "video_thread", &mVideoState);

3 音频处理线程由SDL创建 audio_stream_component_open( )中有音频数据的回调函数

同步过程在线程2 视频处理线程中同步

/*===========================================音视频同步===============================================*/video_pts = synchronize_video(is, pFrame, video_pts);while(1){audio_pts = is->audio_clock;if (video_pts <= audio_pts) //等待匹配   两者时间匹配 跳出该循环break;int delayTime = (video_pts - audio_pts) * 1000;delayTime = delayTime > 5 ? 5:delayTime;SDL_Delay(delayTime);}
视频线程会等待音频线程   比

较两者之间的pts 从而继续进行下一帧的播放

目前待解决问题：

1 AVI格式视频无法播放

参考文章：http://blog.yundiantech.com/?log=blog&scat=182

=======================================================================

ffmpeg视频H264压缩，rtsp推流课程教学视频：

https://edu.csdn.net/course/detail/27795

课件中里面提供源码可以直接下载运行！

=======================================================================

最近新开的公众号，文章正在一篇篇的更新，

公众号名称：玩转电子世界

各位朋友有什么问题了可以直接在上面提问，我会一一进行解答的。

跟着阳光非宅男，一步步走进电子的世界。

关注之后回复资料下载关键词可以获得免费海量的视频学习资料下载~~！

已共享的学习视频资料，共享资料正在不断更新中。

共享ffmpeg视频学习资料：

=======================================================================