前言

音视频开发中，视频编码是另一个重要的部分，基于 FFmpeg 软件解码前面系列文章已经介绍过了。

接下来主要介绍软件编码这一块，包括视频编码、音频编码、为视频添加滤镜等，后续文章安排介绍 Android MediaCodec 硬件编解码。

前文我们对 x264、fdk-aac 及 FFmpeg 进行了整合编译，本文将利用编译好的 FFmpeg 库对 Android Camera2 采集的预览帧先进行渲染，然后利用 OpenGL 添加滤镜，最后读取渲染结果进行编码，生成 mp4 文件。

效果图

FFmpeg 视频编码流程

本文基于 Android Camera 2.0 API 采集的数据源进行编码，编码流程绘制是基于 FFmpeg 4.2.2 版本。

FFmpeg 视频编码流程图

相对于视频解码，编码流程多了一些写文件头尾的操作，需要停止编码时，通过刷入空帧来告诉编码器停止编码。

预览帧添加滤镜、编码流程

写 OpenGL ES 系列文章的时候，很多同学说为啥在 Native 层来写 demo ？

其实就是为了配合 FFmpeg 在视频解码和编码时添加滤镜，那么之前在 native 层写的所有关于滤镜的 demo ,现在可以直接拿过来用了，比如相机基础滤镜，相机抖音滤镜这些。

OpenGLCamera2

https://github.com/githubhaohao/OpenGLCamera2

这个项目有 30 多种滤镜供你参考。

我们首先通过 Android Camera2 预览回调获取预览帧（YUV）：

private ImageReader.OnImageAvailableListener mOnPreviewImageAvailableListener = new ImageReader.OnImageAvailableListener() {@Overridepublic void onImageAvailable(ImageReader reader) {Image image = reader.acquireLatestImage();if (image != null) {if (mCamera2FrameCallback != null) {mCamera2FrameCallback.onPreviewFrame(CameraUtil.YUV_420_888_data(image), image.getWidth(), image.getHeight());}image.close();}}
};

之后利用 GLSurfaceView 来自动创建 OpenGL 环境，创建帧缓冲区对象（FBO），FBO 的主要好处就是保持图像的分辨率不变。

然后在 FBO 离屏渲染时添加滤镜，读取渲染结果作为 FFmpeg 视频编码的输入，最后绑定到 FBO 的纹理再去做屏幕渲染显示出来。

//离屏渲染，添加滤镜
glBindFramebuffer(GL_FRAMEBUFFER, m_DstFboId);
glViewport(0, 0, m_RenderImage.height, m_RenderImage.width); //相机的宽和高反了,
glClear(GL_COLOR_BUFFER_BIT);
glUseProgram (m_ProgramObj);
glBindVertexArray(m_VaoId);UpdateMVPMatrix(0, 0, 1.0, 1.0);
GLUtils::setMat4(m_ProgramObj, "u_MVPMatrix", m_MVPMatrix);glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, m_SrcFboTextureId);
GLUtils::setInt(m_ProgramObj, "s_texture0", 0);
GLUtils::setInt(m_ProgramObj, "u_nImgType", IMAGE_FORMAT_RGBA);
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, (const void *)0);GetRenderFrameFromFBO();
glBindFramebuffer(GL_FRAMEBUFFER, 0);...//GetRenderFrameFromFBO 读取渲染结果，然后通过回调传入 FFmpeg 编码队列
void GLCameraRender::GetRenderFrameFromFBO() {LOGCATE("GLCameraRender::GetRenderFrameFromFBO m_RenderFrameCallback=%p", m_RenderFrameCallback);if(m_RenderFrameCallback != nullptr) {uint8_t *pBuffer = new uint8_t[m_RenderImage.width * m_RenderImage.height * 4];NativeImage nativeImage = m_RenderImage;nativeImage.format = IMAGE_FORMAT_RGBA;nativeImage.width = m_RenderImage.height;nativeImage.height = m_RenderImage.width;nativeImage.pLineSize[0] = nativeImage.width * 4;nativeImage.ppPlane[0] = pBuffer;glReadPixels(0, 0, nativeImage.width, nativeImage.height, GL_RGBA, GL_UNSIGNED_BYTE, pBuffer);m_RenderFrameCallback(m_CallbackContext, &nativeImage);delete []pBuffer;}
}

读取渲染结果的时候除了 glReadPixels , 之前提到高性能的读取方式还有 PBO 、HardwareBuffer等。

FFmpeg 视频编码实现

jni StartRecord 传入视频的宽、高、码率、帧率等参数，OnPreviewFrame 接口传入预览帧。

extern "C"
JNIEXPORT jint JNICALL
Java_com_byteflow_learnffmpeg_media_MediaRecorderContext_native_1StartRecord(JNIEnv *env,jobject thiz,jint recorder_type,jstring out_url,jint frame_width,jint frame_height,jlong video_bit_rate,jint fps) {//MediaRecorderContext 实际上只是对 SingleVideoRecorder 简单封装了一下 const char* url = env->GetStringUTFChars(out_url, nullptr);MediaRecorderContext *pContext = MediaRecorderContext::GetContext(env, thiz);env->ReleaseStringUTFChars(out_url, url);if(pContext) return pContext->StartRecord(recorder_type, url, frame_width, frame_height, video_bit_rate, fps);return 0;
}extern "C"
JNIEXPORT jint JNICALL
Java_com_byteflow_learnffmpeg_media_MediaRecorderContext_native_1StopRecord(JNIEnv *env,jobject thiz) {MediaRecorderContext *pContext = MediaRecorderContext::GetContext(env, thiz);if(pContext) return pContext->StopRecord();return 0;
}extern "C"
JNIEXPORT void JNICALL
Java_com_byteflow_learnffmpeg_media_MediaRecorderContext_native_1OnPreviewFrame(JNIEnv *env,jobject thiz,jint format,jbyteArray data,jint width,jint height) {int len = env->GetArrayLength (data);unsigned char* buf = new unsigned char[len];env->GetByteArrayRegion(data, 0, len, reinterpret_cast<jbyte*>(buf));MediaRecorderContext *pContext = MediaRecorderContext::GetContext(env, thiz);if(pContext) pContext->OnPreviewFrame(format, buf, width, height);delete[] buf;
}

视频编码器主要就是开启一个线程，然后不断地从预览帧队列中读取预览帧进行编码，视频编码器实现：

class SingleVideoRecorder {
public:SingleVideoRecorder(const char* outUrl, int frameWidth, int frameHeight, long bitRate, int fps);~SingleVideoRecorder();int StartRecord();int OnFrame2Encode(NativeImage *inputFrame);int StopRecord();private:static void StartH264EncoderThread(SingleVideoRecorder *context);int EncodeFrame(AVFrame *pFrame);
private:ThreadSafeQueue<NativeImage *> m_frameQueue;char m_outUrl[1024] = {0};int m_frameWidth;int m_frameHeight;int m_frameIndex = 0;long m_bitRate;int m_frameRate;AVPacket m_avPacket;AVFrame  *m_pFrame = nullptr;uint8_t *m_pFrameBuffer = nullptr;AVCodec  *m_pCodec = nullptr;AVStream *m_pStream = nullptr;AVCodecContext *m_pCodecCtx = nullptr;AVFormatContext *m_pFormatCtx = nullptr;thread *m_encodeThread = nullptr;SwsContext *m_SwsContext = nullptr;volatile int m_exit = 0;
};

视频编码循环：

void SingleVideoRecorder::StartH264EncoderThread(SingleVideoRecorder *recorder) {LOGCATE("SingleVideoRecorder::StartH264EncoderThread start");//停止编码且队列为空时退出编码循环while (!recorder->m_exit || !recorder->m_frameQueue.Empty()){if(recorder->m_frameQueue.Empty()) {//队列为空，休眠等待usleep(10 * 1000);continue;}//从队列中取一帧预览帧NativeImage *pImage = recorder->m_frameQueue.Pop();AVFrame *pFrame = recorder->m_pFrame;AVPixelFormat srcPixFmt = AV_PIX_FMT_YUV420P;switch (pImage->format) {case IMAGE_FORMAT_RGBA:srcPixFmt = AV_PIX_FMT_RGBA;break;case IMAGE_FORMAT_NV21:srcPixFmt = AV_PIX_FMT_NV21;break;case IMAGE_FORMAT_NV12:srcPixFmt = AV_PIX_FMT_NV12;break;case IMAGE_FORMAT_I420:srcPixFmt = AV_PIX_FMT_YUV420P;break;default:LOGCATE("SingleVideoRecorder::StartH264EncoderThread unsupport format pImage->format=%d", pImage->format);break;}if(srcPixFmt != AV_PIX_FMT_YUV420P) {if(recorder->m_SwsContext == nullptr) {recorder->m_SwsContext = sws_getContext(pImage->width, pImage->height, srcPixFmt,recorder->m_frameWidth, recorder->m_frameHeight, AV_PIX_FMT_YUV420P,SWS_FAST_BILINEAR, nullptr, nullptr, nullptr);}//格式不同时，需要转换为编码器的目标格式 AV_PIX_FMT_YUV420P，转换之后的图像在 pFrame if(recorder->m_SwsContext != nullptr) {int slice = sws_scale(recorder->m_SwsContext, pImage->ppPlane, pImage->pLineSize, 0,recorder->m_frameHeight, pFrame->data, pFrame->linesize);LOGCATE("SingleVideoRecorder::StartH264EncoderThread sws_scale slice=%d", slice);}}//设置 ptspFrame->pts = recorder->m_frameIndex++;//编码一帧recorder->EncodeFrame(pFrame);//释放预览帧内存NativeImageUtil::FreeNativeImage(pImage);delete pImage;}LOGCATE("SingleVideoRecorder::StartH264EncoderThread end");
}

编码一帧的函数：

int SingleVideoRecorder::EncodeFrame(AVFrame *pFrame) {int result = 0;result = avcodec_send_frame(m_pCodecCtx, pFrame);if(result < 0){LOGCATE("SingleVideoRecorder::EncodeFrame avcodec_send_frame fail. ret=%d", result);return result;}while(!result) {result = avcodec_receive_packet(m_pCodecCtx, &m_avPacket);if (result == AVERROR(EAGAIN) || result == AVERROR_EOF) {return 0;} else if (result < 0) {LOGCATE("SingleVideoRecorder::EncodeFrame avcodec_receive_packet fail. ret=%d", result);return result;}LOGCATE("SingleVideoRecorder::EncodeFrame frame pts=%ld, size=%d", m_avPacket.pts, m_avPacket.size);m_avPacket.stream_index = m_pStream->index;av_packet_rescale_ts(&m_avPacket, m_pCodecCtx->time_base, m_pStream->time_base);m_avPacket.pos = -1;av_interleaved_write_frame(m_pFormatCtx, &m_avPacket);av_packet_unref(&m_avPacket);}return 0;
}

源码地址

https://github.com/githubhaohao/LearnFFmpeg