本文主要分析变速播放框架实现细节，不分析sonic以及soundtouch变速算法。在我的sonic变速变调原理一文中会详细讲解基于基音周期来实现变速变调的原理

1.变速入口分析

从jni层的_setPropertyFloat函数

static void ijkMediaPlayer_setPropertyFloat(JNIEnv *env, jobject thiz, jint id, jfloat value)
{IjkMediaPlayer *mp = jni_get_media_player(env, thiz);JNI_CHECK_GOTO(mp, env, NULL, "mpjni: setPropertyFloat: null mp", LABEL_RETURN);ijkmp_set_property_float(mp, id, value);LABEL_RETURN:ijkmp_dec_ref_p(&mp);return;
}

到ff_ffplay.c中的ffp_set_property_float函数来设置速度

void ffp_set_property_float(FFPlayer *ffp, int id, float value)
{switch (id) {case FFP_PROP_FLOAT_PLAYBACK_RATE:ffp_set_playback_rate(ffp, value);break;case FFP_PROP_FLOAT_PLAYBACK_VOLUME:ffp_set_playback_volume(ffp, value);break;default:return;}
}

跟踪ffp_set_playback_rate函数，我们可以看到这里主要把速度变量设置给了ffp->pf_playback_rate以及把ffp->pf_playback_rate_changed置1.

void ffp_set_playback_rate(FFPlayer *ffp, float rate)
{if (!ffp)return;av_log(ffp, AV_LOG_INFO, "Playback rate: %f\n", rate);ffp->pf_playback_rate = rate;ffp->pf_playback_rate_changed = 1;
}

2.音频变速实现

跟踪这俩个变量，我们可以看到在audio_decode_frame中，我们新增了音频的变速变调算法来处理音频变速，

#if defined(__ANDROID__)if (ffp->soundtouch_enable && ffp->pf_playback_rate != 1.0f && !is->abort_request) {av_fast_malloc(&is->audio_new_buf, &is->audio_new_buf_size, out_size * translate_time);for (int i = 0; i < (resampled_data_size / 2); i++){is->audio_new_buf[i] = (is->audio_buf1[i * 2] | (is->audio_buf1[i * 2 + 1] << 8));}int ret_len = ijk_soundtouch_translate(is->handle, is->audio_new_buf, (float)(ffp->pf_playback_rate), (float)(1.0f/ffp->pf_playback_rate),resampled_data_size / 2, bytes_per_sample, is->audio_tgt.channels, af->frame->sample_rate);if (ret_len > 0) {is->audio_buf = (uint8_t*)is->audio_new_buf;resampled_data_size = ret_len;} else {translate_time++;goto reload;}} else if (ffp->sonic_enabled && ffp->pf_playback_rate != 1.0f && !is->abort_request) {av_fast_malloc(&is->audio_new_buf, &is->audio_new_buf_size, out_size * translate_time * 2);for (int i = 0; i < (resampled_data_size / 2); i++){is->audio_new_buf[i] = (is->audio_buf1[i * 2] | (is->audio_buf1[i * 2 + 1] << 8));}int ret_len = sonicStream_translate(is->sonic_handle,is->audio_new_buf,ffp->pf_playback_rate,(float)(1.0f/ffp->pf_playback_rate),is->audio_tgt.channels, af->frame->sample_rate,resampled_data_size / 2,bytes_per_sample);if (ret_len > 0) {is->audio_buf = (uint8_t*)is->audio_new_buf;resampled_data_size = ret_len;} else {translate_time++;goto reload;}}
#endif

sonic变速是我加的，下面在音频回调sdl_audio_callback函数中,

if (ffp->pf_playback_rate_changed) {ffp->pf_playback_rate_changed = 0;
#if defined(__ANDROID__)if (!ffp->soundtouch_enable && !ffp->sonic_enabled) {SDL_AoutSetPlaybackRate(ffp->aout, ffp->pf_playback_rate);}
#elseSDL_AoutSetPlaybackRate(ffp->aout, ffp->pf_playback_rate);
#endif}if (ffp->pf_playback_volume_changed) {ffp->pf_playback_volume_changed = 0;SDL_AoutSetPlaybackVolume(ffp->aout, ffp->pf_playback_volume);}

上面主要处理对ffp->pf_playback_rate_changed的判断，如果没有开启soundtouch变速或sonic变速，直接走audiotrack的变速来处理。在android6.0系统之下，audiotrack的变速存在变调问题。

从上面我们可以看到这里处理了音频的变速，但是没有处理只有视频流的视频（也就是没有音频流）。

3.视频变速实现

ijk中默认选择音频流作为时间基准的，我们看下视频是如何来做变速的。视频上很容易就可以做到倍速播放，一般的视频格式都是每秒固定的帧数，按比例跳帧就可以了。

static void video_refresh(FFPlayer *opaque, double *remaining_time)
{FFPlayer *ffp = opaque;VideoState *is = ffp->is;double time;Frame *sp, *sp2;if (!is->paused && get_master_sync_type(is) == AV_SYNC_EXTERNAL_CLOCK && is->realtime)check_external_clock_speed(is);if (!ffp->display_disable && is->show_mode != SHOW_MODE_VIDEO && is->audio_st) {time = av_gettime_relative() / 1000000.0;if (is->force_refresh || is->last_vis_time + ffp->rdftspeed < time) {video_display2(ffp);is->last_vis_time = time;}*remaining_time = FFMIN(*remaining_time, is->last_vis_time + ffp->rdftspeed - time);}if (is->video_st) {
retry://add by hxk,support only video change speedif(!is->audio_st && get_master_sync_type(is) == AV_SYNC_EXTERNAL_CLOCK) {if(ffp->pf_playback_rate != 1.0f){change_external_clock_speed(is,ffp->pf_playback_rate);}}//add endif (frame_queue_nb_remaining(&is->pictq) == 0) {// nothing to do, no picture to display in the queue} else {double last_duration, duration, delay;Frame *vp, *lastvp;/* dequeue the picture */lastvp = frame_queue_peek_last(&is->pictq);vp = frame_queue_peek(&is->pictq);if (vp->serial != is->videoq.serial) {frame_queue_next(&is->pictq);goto retry;}if (lastvp->serial != vp->serial)is->frame_timer = av_gettime_relative() / 1000000.0;if (is->paused)goto display;/* compute nominal last_duration */last_duration = vp_duration(is, lastvp, vp);delay = compute_target_delay(ffp, last_duration, is);time= av_gettime_relative()/1000000.0;if (isnan(is->frame_timer) || time < is->frame_timer)is->frame_timer = time;if (time < is->frame_timer + delay) {*remaining_time = FFMIN(is->frame_timer + delay - time, *remaining_time);goto display;}is->frame_timer += delay;if (delay > 0 && time - is->frame_timer > AV_SYNC_THRESHOLD_MAX)is->frame_timer = time;SDL_LockMutex(is->pictq.mutex);if (!isnan(vp->pts))update_video_pts(is, vp->pts, vp->pos, vp->serial);SDL_UnlockMutex(is->pictq.mutex);if (frame_queue_nb_remaining(&is->pictq) > 1) {Frame *nextvp = frame_queue_peek_next(&is->pictq);duration = vp_duration(is, vp, nextvp);if(!is->step && (ffp->framedrop > 0 || (ffp->framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) && time > is->frame_timer + duration) {frame_queue_next(&is->pictq);goto retry;}}if (is->subtitle_st) {while (frame_queue_nb_remaining(&is->subpq) > 0) {sp = frame_queue_peek(&is->subpq);if (frame_queue_nb_remaining(&is->subpq) > 1)sp2 = frame_queue_peek_next(&is->subpq);elsesp2 = NULL;if (sp->serial != is->subtitleq.serial|| (is->vidclk.pts > (sp->pts + ((float) sp->sub.end_display_time / 1000)))|| (sp2 && is->vidclk.pts > (sp2->pts + ((float) sp2->sub.start_display_time / 1000)))){if (sp->uploaded) {ffp_notify_msg4(ffp, FFP_MSG_TIMED_TEXT, 0, 0, "", 1);}frame_queue_next(&is->subpq);} else {break;}}}frame_queue_next(&is->pictq);is->force_refresh = 1;SDL_LockMutex(ffp->is->play_mutex);if (is->step) {is->step = 0;if (!is->paused)stream_update_pause_l(ffp);}SDL_UnlockMutex(ffp->is->play_mutex);}
display:/* display picture */if (!ffp->display_disable && is->force_refresh && is->show_mode == SHOW_MODE_VIDEO && is->pictq.rindex_shown)video_display2(ffp);}is->force_refresh = 0;if (ffp->show_status) {static int64_t last_time;int64_t cur_time;int aqsize, vqsize, sqsize __unused;double av_diff;cur_time = av_gettime_relative();if (!last_time || (cur_time - last_time) >= 30000) {aqsize = 0;vqsize = 0;sqsize = 0;if (is->audio_st)aqsize = is->audioq.size;if (is->video_st)vqsize = is->videoq.size;
#ifdef FFP_MERGEif (is->subtitle_st)sqsize = is->subtitleq.size;
#elsesqsize = 0;
#endifav_diff = 0;if (is->audio_st && is->video_st)av_diff = get_clock(&is->audclk) - get_clock(&is->vidclk);else if (is->video_st)av_diff = get_master_clock(is) - get_clock(&is->vidclk);else if (is->audio_st)av_diff = get_master_clock(is) - get_clock(&is->audclk);av_log(NULL, AV_LOG_INFO,"%7.2f %s:%7.3f fd=%4d aq=%5dKB vq=%5dKB sq=%5dB f=%"PRId64"/%"PRId64"   \r",get_master_clock(is),(is->audio_st && is->video_st) ? "A-V" : (is->video_st ? "M-V" : (is->audio_st ? "M-A" : "   ")),av_diff,is->frame_drops_early + is->frame_drops_late,aqsize / 1024,vqsize / 1024,sqsize,is->video_st ? is->viddec.avctx->pts_correction_num_faulty_dts : 0,is->video_st ? is->viddec.avctx->pts_correction_num_faulty_pts : 0);fflush(stdout);last_time = cur_time;}}
}

其实这里就是视频变速的实现，也是视频同步音频的实现，来实现视频变速。

3.1.没有音频的视频变速

通过前面的分析，我们可以知道ijk没有处理只有视频流的视频变速(没有音频流的视频)。当没有音频流的时候，在read_thread函数中

 /* open the streams */if (st_index[AVMEDIA_TYPE_AUDIO] >= 0) {stream_component_open(ffp, st_index[AVMEDIA_TYPE_AUDIO]);} else {//如果没有音频流ffp->av_sync_type = AV_SYNC_VIDEO_MASTER;is->av_sync_type  = ffp->av_sync_type;}

我们可以看到如果没有音频流，ijk中选择视频流作为时间基准。个人觉得还是外部时钟作为时间基准比较好，比较准。

那么我们可以如何修改呢？

3.1.1.修改没有音频的视频的同步方式为外部时钟

  /* open the streams */if (st_index[AVMEDIA_TYPE_AUDIO] >= 0) {stream_component_open(ffp, st_index[AVMEDIA_TYPE_AUDIO]);} else {ffp->av_sync_type = AV_SYNC_EXTERNAL_CLOCK;is->av_sync_type  = ffp->av_sync_type;}

当没有音频流的时候，选择外部时钟作为时间基准。

3.1.2.修改外部时钟变速

修改video_refresh函数

/* called to display each frame */
static void video_refresh(FFPlayer *opaque, double *remaining_time)
{FFPlayer *ffp = opaque;VideoState *is = ffp->is;double time;Frame *sp, *sp2;if (!is->paused && get_master_sync_type(is) == AV_SYNC_EXTERNAL_CLOCK && is->realtime)check_external_clock_speed(is);if (!ffp->display_disable && is->show_mode != SHOW_MODE_VIDEO && is->audio_st) {time = av_gettime_relative() / 1000000.0;if (is->force_refresh || is->last_vis_time + ffp->rdftspeed < time) {video_display2(ffp);is->last_vis_time = time;}*remaining_time = FFMIN(*remaining_time, is->last_vis_time + ffp->rdftspeed - time);}if (is->video_st) {
retry://当没有音频流的，有视频流时，且时间基准为外部时钟//add by hxk,support only video change speedif(!is->audio_st && get_master_sync_type(is) == AV_SYNC_EXTERNAL_CLOCK) {
//如果速度不等于1，改变外部时钟速度if(ffp->pf_playback_rate != 1.0f){change_external_clock_speed(is,ffp->pf_playback_rate);}}//add endif (frame_queue_nb_remaining(&is->pictq) == 0) {// nothing to do, no picture to display in the queue} else {double last_duration, duration, delay;Frame *vp, *lastvp;/* dequeue the picture */lastvp = frame_queue_peek_last(&is->pictq);vp = frame_queue_peek(&is->pictq);if (vp->serial != is->videoq.serial) {frame_queue_next(&is->pictq);goto retry;}if (lastvp->serial != vp->serial)is->frame_timer = av_gettime_relative() / 1000000.0;if (is->paused)goto display;/* compute nominal last_duration */last_duration = vp_duration(is, lastvp, vp);delay = compute_target_delay(ffp, last_duration, is);time= av_gettime_relative()/1000000.0;if (isnan(is->frame_timer) || time < is->frame_timer)is->frame_timer = time;if (time < is->frame_timer + delay) {*remaining_time = FFMIN(is->frame_timer + delay - time, *remaining_time);goto display;}is->frame_timer += delay;if (delay > 0 && time - is->frame_timer > AV_SYNC_THRESHOLD_MAX)is->frame_timer = time;SDL_LockMutex(is->pictq.mutex);if (!isnan(vp->pts))update_video_pts(is, vp->pts, vp->pos, vp->serial);SDL_UnlockMutex(is->pictq.mutex);if (frame_queue_nb_remaining(&is->pictq) > 1) {Frame *nextvp = frame_queue_peek_next(&is->pictq);duration = vp_duration(is, vp, nextvp);if(!is->step && (ffp->framedrop > 0 || (ffp->framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) && time > is->frame_timer + duration) {frame_queue_next(&is->pictq);goto retry;}}if (is->subtitle_st) {while (frame_queue_nb_remaining(&is->subpq) > 0) {sp = frame_queue_peek(&is->subpq);if (frame_queue_nb_remaining(&is->subpq) > 1)sp2 = frame_queue_peek_next(&is->subpq);elsesp2 = NULL;if (sp->serial != is->subtitleq.serial|| (is->vidclk.pts > (sp->pts + ((float) sp->sub.end_display_time / 1000)))|| (sp2 && is->vidclk.pts > (sp2->pts + ((float) sp2->sub.start_display_time / 1000)))){if (sp->uploaded) {ffp_notify_msg4(ffp, FFP_MSG_TIMED_TEXT, 0, 0, "", 1);}frame_queue_next(&is->subpq);} else {break;}}}frame_queue_next(&is->pictq);is->force_refresh = 1;SDL_LockMutex(ffp->is->play_mutex);if (is->step) {is->step = 0;if (!is->paused)stream_update_pause_l(ffp);}SDL_UnlockMutex(ffp->is->play_mutex);}
display:/* display picture */if (!ffp->display_disable && is->force_refresh && is->show_mode == SHOW_MODE_VIDEO && is->pictq.rindex_shown)video_display2(ffp);}is->force_refresh = 0;if (ffp->show_status) {static int64_t last_time;int64_t cur_time;int aqsize, vqsize, sqsize __unused;double av_diff;cur_time = av_gettime_relative();if (!last_time || (cur_time - last_time) >= 30000) {aqsize = 0;vqsize = 0;sqsize = 0;if (is->audio_st)aqsize = is->audioq.size;if (is->video_st)vqsize = is->videoq.size;
#ifdef FFP_MERGEif (is->subtitle_st)sqsize = is->subtitleq.size;
#elsesqsize = 0;
#endifav_diff = 0;if (is->audio_st && is->video_st)av_diff = get_clock(&is->audclk) - get_clock(&is->vidclk);else if (is->video_st)av_diff = get_master_clock(is) - get_clock(&is->vidclk);else if (is->audio_st)av_diff = get_master_clock(is) - get_clock(&is->audclk);av_log(NULL, AV_LOG_INFO,"%7.2f %s:%7.3f fd=%4d aq=%5dKB vq=%5dKB sq=%5dB f=%"PRId64"/%"PRId64"   \r",get_master_clock(is),(is->audio_st && is->video_st) ? "A-V" : (is->video_st ? "M-V" : (is->audio_st ? "M-A" : "   ")),av_diff,is->frame_drops_early + is->frame_drops_late,aqsize / 1024,vqsize / 1024,sqsize,is->video_st ? is->viddec.avctx->pts_correction_num_faulty_dts : 0,is->video_st ? is->viddec.avctx->pts_correction_num_faulty_pts : 0);fflush(stdout);last_time = cur_time;}}
}

新增外部时钟变速函数。

//add by hxk
static void change_external_clock_speed(VideoState *is,float speed) {if (speed != 1.0f){set_clock_speed(&is->extclk, speed + EXTERNAL_CLOCK_SPEED_STEP * (1.0 - speed) / fabs(1.0 - speed));}
}
//add end

这样我们就可以解决没有音频的视频流变速问题了。当然如果还是选择视频流作为时间基准的话，我们可以修改刷帧速度来实现视频变速。

ijkplayer-音视频变速播放实现相关推荐

Android 音视频变速原理
视频倍速播放假设视频帧率是24fps, 则播放器必须在1000/24 = 41.66ms 内解封装 + 解码 + 渲染完一帧,一般只计算出把YUV数据从渲染队列中取出到渲染结束的时间(Render ...
ffmpeg播放器 android,Android使用FFmpeg(六)--ffmpeg实现音视频同步播放
关于准备工作正文依旧依照流程图来逐步实现同步播放: 从流程图可以看出,实现同步播放需要三个线程,一个开启解码的装置得到packet线程,然后分别是播放音频和视频的线程.这篇简书是以音频播放为基准 ...
关于视频变速播放软件
关于视频变速播放软件通常自学的时候,在BliliBlili上看视频的时候可以开倍速,但是后来我自己从百度云盘上下载了一些视频之后,就需要在本地播放视频,但是在本地播放视频又不能变速播放视频,于是我去 ...
海思Hi3559a音视频同时播放的例子
原文链接:海思Hi3559a音视频同时播放的例子在mpp/sample下新建av目录 (1)编写Makefile,如下: Hisilicon Hi35xx sample Makefile inclu ...
iOS 微信音视频自动播放原生接口WeixinJSBridge API(一些整理小技巧)
原文链接1:https://www.w3ctech.com/topic/1165 原文链接2:https://www.cnblogs.com/jasonduan/p/5635048.html 做一下整 ...
Android WebView加载H5音视频自动播放、关闭Activity停止播放
在Android加载H5,实现H5中的音视频自动播放在Activity中添加代码: if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELL ...
爱奇艺知识的音视频通用播放架构实践
导读随着经济的发展"衣食住行"等基础消费已不再是消费者首要考量,自我认知的提高便成为现阶段消费的必然选择.尤其是在移动互联网崛起的当下,移动支付和不限流业务的普及,人们为他们感兴 ...
如何通过ffmpeg实现视频变速播放
要通过 FFmpeg 实现视频变速播放,您需要使用以下命令: ffmpeg -i input.mp4 -filter:v "setpts=0.5*PTS" output.mp4 其 ...
音视频系列九使用soundTouch实现音视频变速
文章目录前言视频变速音频变速 TSM算法(时域压扩算法) OLA(Overlap-and-Add) WSOLA(Waveform Similarity Overlap-Add) 编译使用参考 ...
iOS ijkplayer 音视频同步
http://www.jianshu.com/p/daf0a61cc1e0 3.3 音视频渲染及同步 3.3.1 音频输出 ijkplayer中Android平台使用OpenSL ES或AudioTr ...

ijkplayer-音视频变速播放实现