Android多媒体框架（二）Codec初始化及Omx组件创建

Codec创建流程

Android提供给应用编解码的接口为MediaCodec。我们这里从NuPlayerDecoder开始分析，一是为了衔接之前将的MediaPlayer-NuPlayer流程，二是可以从中参考到MediaCodec是怎么用的，然后去分析内部流程会更好。

之前说到NuPlayer.cpp创建Decoder流程（Video部分）：instantiateDecoder

1965          sp<AMessage> notify = new AMessage(kWhatVideoNotify, this);
1966          ++mVideoDecoderGeneration;
1967          notify->setInt32("generation", mVideoDecoderGeneration);
1968
1969          *decoder = new Decoder(
1970                  notify, mSource, mPID, mUID, mRenderer, mSurface, mCCDecoder);
1971          mVideoDecoderError = false;

NuPlayer::Decoder是继承于DecoderBase的，不过DecoderBase主要就是用于处理一些基本的流程（setParameter、start、stop等）。如果Decoder重写了就更不需要看了。Decoder构造函数中主要是创建了一个looper，每一个Decoder都需要有一个looper，因为MediaCodec是阻塞式的调用，但是NuPlayer需要的是异步操作:
Decoder::Decoder

94      mCodecLooper = new ALooper;
95      mCodecLooper->setName("NPDecoder-CL");
96      mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
97      mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal;

再回来init：NuPlayer.cpp::instantiateDecoder

2032      (*decoder)->init();
2042      (*decoder)->configure(format);

Init()其实主要就是注册handler，主要看下configure，DecoderBase的未重写函数，最终发送kWhatConfigure，调用NuPlayer::Decoder::onConfigure：

293      AString mime;
294      CHECK(format->findString("mime", &mime));
295
296      mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
297      mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
298
299      mComponentName = mime;
300      mComponentName.append(" decoder");
301      ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
302
303      mCodec = MediaCodec::CreateByType(
304              mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid, mUid);
305      int32_t secure = 0;
306      if (format->findInt32("secure", &secure) && secure != 0) {307          if (mCodec != NULL) {308              mCodec->getName(&mComponentName);
309              mComponentName.append(".secure");
310              mCodec->release();
311              ALOGI("[%s] creating", mComponentName.c_str());
312              mCodec = MediaCodec::CreateByComponentName(
313                      mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid, mUid);
314          }
315      }

可以看到MediaCodec创建有两种方式

CreateByType
CreateByComponentName

CreateByType是通过mime创建，而CreateByComponentName是通过组件名创建。当format的secure是1的时候会获取之前mime创建mCodec的组件名，并释放，然后在组件名后拼接secure，然后再通过CreateByComponentName创建codec。Secure类型比较典型的就是youtube视频。（组件名一般为OMX.qcom.video.decoder.avc这种，对应media_codec.xml中）。

再看一下后面几个重要的调用：
NuPlayer::Decoder::onConfigure：

360      err = mCodec->configure(
361              format, mSurface, crypto, 0 /* flags */);
//新建一个AMessage传给mediacodec，当mediacodec做好一些处理（如编码一帧）就把buffer index、flag、size等传入msg，然后post回来，相当于一个callback。
388      sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
389      mCodec->setCallback(reply);
390
391      err = mCodec->start();

然后看下MediaCodec对应的处理吧：
CreateByType

439  sp<MediaCodec> MediaCodec::CreateByType(
440          const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err, pid_t pid,
441          uid_t uid) {442      Vector<AString> matchingCodecs;
443
444      MediaCodecList::findMatchingCodecs(
445              mime.c_str(),
446              encoder,
447              0,
448              &matchingCodecs);
449
450      if (err != NULL) {451          *err = NAME_NOT_FOUND;
452      }
453      for (size_t i = 0; i < matchingCodecs.size(); ++i) {454          sp<MediaCodec> codec = new MediaCodec(looper, pid, uid);
455          AString componentName = matchingCodecs[i];
456          status_t ret = codec->init(componentName);
457          if (err != NULL) {458              *err = ret;
459          }
460          if (ret == OK) {461              return codec;
462          }
463          ALOGD("Allocating component '%s' failed (%d), try next one.",
464                  componentName.c_str(), ret);
465      }
466      return NULL;
467  }

CreateByComponentName

470  sp<MediaCodec> MediaCodec::CreateByComponentName(
471          const sp<ALooper> &looper, const AString &name, status_t *err, pid_t pid, uid_t uid) {472      sp<MediaCodec> codec = new MediaCodec(looper, pid, uid);
473
474      const status_t ret = codec->init(name);
475      if (err != NULL) {476          *err = ret;
477      }
478      return ret == OK ? codec : NULL; // NULL deallocates codec.
479  }

findMatchingCodecs其实就是从所有的mediaCodec数组中选取匹配mime，mediaCodec数组是加载MediaCodecList的时候从media_codecs.xml中读取到的，具体流程放在Extend中跟踪。

CreateByComponentName和CreateByType最终都是new了MediaCodec，调用init传入组件名

887  status_t MediaCodec::init(const AString &name) {906      const sp<IMediaCodecList> mcl = MediaCodecList::getInstance();
...
911      for (const AString &codecName : { name, tmp }) {912          ssize_t codecIdx = mcl->findCodecByName(codecName.c_str());
...
916          mCodecInfo = mcl->getCodecInfo(codecIdx);
917          Vector<AString> mediaTypes;
918          mCodecInfo->getSupportedMediaTypes(&mediaTypes);
...
//根据组件名的开头选择CodecBase，如omx.则为ACodec，C2则为CCodec。
931      mCodec = GetCodecBase(name, mCodecInfo->getOwnerName());
//传给ACodec两个callback，参数都为AMessage，what为kWhatCodecNotify，用于ACodec消息回调。
//CodecCallback主要作用为状态回调，BufferChannel主要作用为buffer状态的回调。
951      mCodec->setCallback(
952              std::unique_ptr<CodecBase::CodecCallback>(
953                      new CodecCallback(new AMessage(kWhatCodecNotify, this))));
954      mBufferChannel = mCodec->getBufferChannel();
955      mBufferChannel->setCallback(
956              std::unique_ptr<CodecBase::BufferCallback>(
957                      new BufferCallback(new AMessage(kWhatCodecNotify,this))));...959      sp<AMessage> msg = new AMessage(kWhatInit, this);986          err = PostAndAwaitResponse(msg, &response);

看一下kWhatInit的消息处理：

2355          case kWhatInit:
2377              mCodec->initiateAllocateComponent(format);

OMX组件创建

上面提到了ACodec/CCodec/MediaFilter都是GetCodecBase通过组件名来创建的。这里拿ACodec当例子，如下为ACodec的构造函数，可以看到ACodec创建了九种状态机，ACodec在不同阶段会通过changeState切换不同的状态，用于管理底层各个Component的各种状态，理解状态机也能帮忙我们理解ACodec工作流程，这里可以看到初始化后状态为mUninitializedState。具体每一个状态机的相互切换留在写数据流的时候再写。

581
582      mUninitializedState = new UninitializedState(this);
583      mLoadedState = new LoadedState(this);
584      mLoadedToIdleState = new LoadedToIdleState(this);
585      mIdleToExecutingState = new IdleToExecutingState(this);
586      mExecutingState = new ExecutingState(this);
587
588      mOutputPortSettingsChangedState =
589          new OutputPortSettingsChangedState(this);
590
591      mExecutingToIdleState = new ExecutingToIdleState(this);
592      mIdleToLoadedState = new IdleToLoadedState(this);
593      mFlushingState = new FlushingState(this);
594
595      mPortEOS[kPortIndexInput] = mPortEOS[kPortIndexOutput] = false;
596      mInputEOSResult = OK;
597
598      mPortMode[kPortIndexInput] = IOMX::kPortModePresetByteBuffer;
599      mPortMode[kPortIndexOutput] = IOMX::kPortModePresetByteBuffer;
600
601      memset(&mLastNativeWindowCrop, 0, sizeof(mLastNativeWindowCrop));
602
603      changeState(mUninitializedState);

之后调用initiateAllocateComponent。看下代码：

634  void ACodec::initiateAllocateComponent(const sp<AMessage> &msg) {635      msg->setWhat(kWhatAllocateComponent);
636      msg->setTarget(this);
637      msg->post();6653          case ACodec::kWhatAllocateComponent:
6654          {6655              onAllocateComponent(msg);
6656              handled = true;
6657              break;6705  bool ACodec::UninitializedState::onAllocateComponent(const sp<AMessage> &msg){6710      sp<AMessage> notify = new AMessage(kWhatOMXMessageList, mCodec);6724      CHECK(msg->findString("componentName", &componentName));6726      sp<CodecObserver> observer = new CodecObserver(notify);6731      OMXClient client;
6732      if (client.connect(owner.c_str()) != OK) {6733          mCodec->signalError(OMX_ErrorUndefined, NO_INIT);
6734          return false;
6735      }
6736      omx = client.interface();
//提高线程调度优先级
6739      int prevPriority = androidGetThreadPriority(tid);
6740      androidSetThreadPriority(tid, ANDROID_PRIORITY_FOREGROUND);
6741      err = omx->allocateNode(componentName.c_str(), observer, &omxNode);
6742      androidSetThreadPriority(tid, prevPriority);
//通过CodecCallback调用MediaCodec的onComponentAllocated来通知MediaCodec当前状态
6769      mCodec->mOMX = omx;
6770      mCodec->mOMXNode = omxNode;
6771      mCodec->mCallback->onComponentAllocated(mCodec->mComponentName.c_str());

接下来主要看下OMX的allocateNode，OMX对象是通过client.interface()获取到的，首先看下OMX是怎么起来的，如下代码可以看到OMX服务是跑在MediaCodec进程中，很标准的HIDL实现。hal接口定义在hardware/hardware/interfaces/media/omx/1.0中。
main_codecservice.cpp

51      using namespace ::android::hardware::media::omx::V1_0;
52      sp<IOmx> omx = new implementation::Omx();55      } else if (omx->registerAsService() != OK) {60      sp<IOmxStore> omxStore = new implementation::OmxStore(omx);63      } else if (omxStore->registerAsService() != OK) {

OmxStore和Omx构造函数除了加载mediacodec xml，主要还new了OMXMaster，OMXMaster比较重要的就是添加了各种Plugin，Plugin如其名，就是加载OMX具体组件的插件。一般android原生都是加载soft plugin，各大平台厂商在实现自己硬编解码的时候也会添加自己的plugin。

32  OMXMaster::OMXMaster() {...
53      addVendorPlugin();
54      addPlatformPlugin();
55  }61  void OMXMaster::addVendorPlugin() {62      addPlugin("libstagefrighthw.so");
63  }
64
65  void OMXMaster::addPlatformPlugin() {66      addPlugin("libstagefright_softomx_plugin.so");
67  }69  void OMXMaster::addPlugin(const char *libname) {70      void *libHandle = android_load_sphal_library(libname, RTLD_NOW);
71
72      if (libHandle == NULL) {73          return;
74      }
75
76      typedef OMXPluginBase *(*CreateOMXPluginFunc)();
77      CreateOMXPluginFunc createOMXPlugin =
78          (CreateOMXPluginFunc)dlsym(
79                  libHandle, "createOMXPlugin");
80      if (!createOMXPlugin)
81          createOMXPlugin = (CreateOMXPluginFunc)dlsym(
82                  libHandle, "_ZN7android15createOMXPluginEv");
83
84      OMXPluginBase *plugin = nullptr;
85      if (createOMXPlugin) {86          plugin = (*createOMXPlugin)();
87      }
88
89      if (plugin) {90          mPlugins.push_back({ plugin, libHandle });
91          addPlugin(plugin);
92      } else {93          android_unload_sphal_library(libHandle);
94      }
95  }97  void OMXMaster::addPlugin(OMXPluginBase *plugin) {98      Mutex::Autolock autoLock(mLock);
104      while ((err = plugin->enumerateComponents(
105                      name, sizeof(name), index++)) == OMX_ErrorNone) {108          if (mPluginByComponentName.indexOfKey(name8) >= 0) {112              continue;
113          }115          mPluginByComponentName.add(name8, plugin);
116      }

回到ACodec，之前调用了omx->allocateNode(componentName.c_str(), observer, &omxNode);

84  Return<void> Omx::allocateNode(
85          const hidl_string& name,
86          const sp<IOmxObserver>& observer,
87          allocateNode_cb _hidl_cb) {//OMXNodeInstance相当于组件的句柄，对象需要传回给ACodec，
//component就是通过Instance回调一些Codec函数，传回一些状态以及buffer信息。
100          instance = new OMXNodeInstance(
101                  this, new LWOmxObserver(observer), name.c_str());
//调用OmxMaster::makeComponentInstance
104          OMX_ERRORTYPE err = mMaster->makeComponentInstance(
105                  name.c_str(), &OMXNodeInstance::kCallbacks,
106                  instance.get(), &handle);116          instance->setHandle(handle);
//传给ACodec instance
144      _hidl_cb(toStatus(OK), new TWOmxNode(instance));
}146  OMX_ERRORTYPE OMXMaster::makeComponentInstance(
147          const char *name,
148          const OMX_CALLBACKTYPE *callbacks,
149          OMX_PTR appData,
150          OMX_COMPONENTTYPE **component) {//mPluginByComponentName是之前加载plugin存入的
156      ssize_t index = mPluginByComponentName.indexOfKey(String8(name));162      OMXPluginBase *plugin = mPluginByComponentName.valueAt(index);
//调用plugin中makeComponentInstance
163      OMX_ERRORTYPE err =
164          plugin->makeComponentInstance(name, callbacks, appData, component);170      mPluginByInstance.add(*component, plugin);

很多平台厂商的plugin文件的实际函数逻辑都在so文件中，无法查看，这里以Soft Plugin作为例子：

87  OMX_ERRORTYPE SoftOMXPlugin::makeComponentInstance(
88          const char *name,
89          const OMX_CALLBACKTYPE *callbacks,
90          OMX_PTR appData,
91          OMX_COMPONENTTYPE **component) {94      for (size_t i = 0; i < kNumComponents; ++i) {95          if (strcmp(name, kComponents[i].mName)) {96              continue;
97          }
//libstagefright_soft_拼接上组件名
99          AString libName = "libstagefright_soft_";
100          libName.append(kComponents[i].mLibNameSuffix);
101          libName.append(".so");
//dlopen打开软编解码器so
117          void *libHandle = dlopen(libName.c_str(), RTLD_NOW|RTLD_NODELETE);129          CreateSoftOMXComponentFunc createSoftOMXComponent =
130              (CreateSoftOMXComponentFunc)dlsym(
131                      libHandle,
132                      "_Z22createSoftOMXComponentPKcPK16OMX_CALLBACKTYPE"
133                      "PvPP17OMX_COMPONENTTYPE");142          sp<SoftOMXComponent> codec =
143              (*createSoftOMXComponent)(name, callbacks, appData, component);160          codec->incStrong(this);
161          codec->setLibHandle(libHandle);

以libstagefright_soft_m4vh263dec.so为例

415  android::SoftOMXComponent *createSoftOMXComponent(
416          const char *name, const OMX_CALLBACKTYPE *callbacks,
417          OMX_PTR appData, OMX_COMPONENTTYPE **component) {418      using namespace android;
419      if (!strcmp(name, "OMX.google.h263.decoder")) {420          return new android::SoftMPEG4(
421                  name, "video_decoder.h263", OMX_VIDEO_CodingH263,
422                  kH263ProfileLevels, ARRAY_SIZE(kH263ProfileLevels),
423                  callbacks, appData, component);
424      } else if (!strcmp(name, "OMX.google.mpeg4.decoder")) {425          return new android::SoftMPEG4(
426                  name, "video_decoder.mpeg4", OMX_VIDEO_CodingMPEG4,
427                  kM4VProfileLevels, ARRAY_SIZE(kM4VProfileLevels),
428                  callbacks, appData, component);
429      } else {430          CHECK(!"Unknown component");
431      }
432      return NULL;
433  }

最终根据传入组件名创建相应解码器SoftMPEG4,看了下SoftMPEG4继承于SoftVideoDecoderOMXComponent，而SoftVideoDecoderOMXComponent又继承于SimpleSoftOMXComponent，所以emptyThisBuffer、fillThisBuffer等omx基本方法，都在SimpleSoftOMXComponent中定义，而SoftMPEG4没有重写。至此omx组件创建完成，后续章节通过OMX各个状态的切换以及ACodec对应状态机的切换来继续分析数据流走向，同时会整理omx框架每个部分的关连。
至于MediaFilter(滤波器)、Codec2，大体流程跟ACodec都差不多，不过多分析，Codec2在Q版本及之前基本都是用于管理软编解码器，R版本应该会兼容硬编解码器，大概率会完全替换OMX框架。补一张整体流程图：

Extend

MediaCodecList加载xml

 void MediaCodecList::findMatchingCodecs(
346          const char *mime, bool encoder, uint32_t flags,
347          Vector<AString> *matches) {348      matches->clear();
349
350      const sp<IMediaCodecList> list = getInstance();
351      if (list == nullptr) {352          return;
353      }
354
355      size_t index = 0;
356      for (;;) {357          ssize_t matchIndex =
358              list->findCodecByType(mime, encoder, index);
359
360          if (matchIndex < 0) {361              break;
362          }
363
364          index = matchIndex + 1;
365
366          const sp<MediaCodecInfo> info = list->getCodecInfo(matchIndex);
367          CHECK(info != nullptr);
368          AString componentName = info->getCodecName();
369
370          if ((flags & kHardwareCodecsOnly) && isSoftwareCodec(componentName)) {371              ALOGV("skipping SW codec '%s'", componentName.c_str());
372          } else {373              matches->push(componentName);
374              ALOGV("matching '%s'", componentName.c_str());
375          }
376      }
377   //所以更改软硬编解码优先可以在这修改
378      if (flags & kPreferSoftwareCodecs ||
379              property_get_bool("debug.stagefright.swcodec", false)) {380          matches->sort(compareSoftwareCodecsFirst);
381      }
382  }
383

getInstance的流程跟起来最后都是到了getLocalInstance:

143  sp<IMediaCodecList> MediaCodecList::getLocalInstance() {MediaCodecList *codecList = new MediaCodecList(GetBuilders());

主要就是new了MediaCodecList，关键是参数GetBuilders（），先看omx的话，MediaCodecList中的builder就是OmxInfoBuilder 。

80  OmxInfoBuilder sOmxInfoBuilder{true /* allowSurfaceEncoders */};
95  std::vector<MediaCodecListBuilderBase *> GetBuilders() {96      std::vector<MediaCodecListBuilderBase *> builders;
97      // if plugin provides the input surface, we cannot use OMX video encoders.
98      // In this case, rely on plugin to provide list of OMX codecs that are usable.
99      sp<PersistentSurface> surfaceTest =
100          StagefrightPluginLoader::GetCCodecInstance()->createInputSurface();
101      if (surfaceTest == nullptr) {102          ALOGD("Allowing all OMX codecs");
103          builders.push_back(&sOmxInfoBuilder);
104      } else {105          ALOGD("Allowing only non-surface-encoder OMX codecs");
106          builders.push_back(&sOmxNoSurfaceEncoderInfoBuilder);
107      }
108      builders.push_back(GetCodec2InfoBuilder());
109      return builders;
110  }

看一下MediaCodecList的构造函数：

203  MediaCodecList::MediaCodecList(std::vector<MediaCodecListBuilderBase*> builders) {204      mGlobalSettings = new AMessage();
205      mCodecInfos.clear();
206      MediaCodecListWriter writer;
207      for (MediaCodecListBuilderBase *builder : builders) {208          if (builder == nullptr) {209              ALOGD("ignored a null builder");
210              continue;
211          }
212          mInitCheck = builder->buildMediaCodecList(&writer);
213          if (mInitCheck != OK) {214              break;
215          }
216      }
217      writer.writeGlobalSettings(mGlobalSettings);
218      writer.writeCodecInfos(&mCodecInfos);
219      std::stable_sort(
220              mCodecInfos.begin(),
221              mCodecInfos.end(),
222              [](const sp<MediaCodecInfo> &info1, const sp<MediaCodecInfo> &info2) {223                  // null is lowest
224                  return info1 == nullptr
225                          || (info2 != nullptr && info1->getRank() < info2->getRank());
226              });
227
228      // remove duplicate entries
229      bool dedupe = property_get_bool("debug.stagefright.dedupe-codecs", true);
230      if (dedupe) {231          std::set<std::string> codecsSeen;
232          for (auto it = mCodecInfos.begin(); it != mCodecInfos.end(); ) {233              std::string codecName = (*it)->getCodecName();
234              if (codecsSeen.count(codecName) == 0) {235                  codecsSeen.emplace(codecName);
236                  it++;
237              } else {238                  it = mCodecInfos.erase(it);
239              }
240          }
241      }
242
================================  buildMediaCodecList  ================================
94  status_t OmxInfoBuilder::buildMediaCodecList(MediaCodecListWriter* writer) {95      // Obtain IOmxStore
96      sp<IOmxStore> omxStore = IOmxStore::getService();
97      if (omxStore == nullptr) {98          ALOGE("Cannot find an IOmxStore service.");
99          return NO_INIT;
100      }
101
102      // List service attributes (global settings)
103      Status status;
104      hidl_vec<IOmxStore::RoleInfo> roles;
105      auto transStatus = omxStore->listRoles(
106              [&roles] (
107              const hidl_vec<IOmxStore::RoleInfo>& inRoleList) {108                  roles = inRoleList;
109              });
================================  OmxStore::listRoles  ================================
136  Return<void> OmxStore::listRoles(listRoles_cb _hidl_cb) {137      _hidl_cb(mRoleList);
138      return Void();
139  }
================================  mRoleList：  ================================
38  OmxStore::OmxStore(
39          const sp<IOmx> &omx,
40          const char* owner,
41          const std::vector<std::string> &searchDirs,
42          const std::vector<std::string> &xmlNames,
43          const char* profilingResultsXmlPath) {44      // retrieve list of omx nodes 57      MediaCodecsXmlParser parser;
58      parser.parseXmlFilesInSearchDirs(xmlNames, searchDirs);
59      if (profilingResultsXmlPath != nullptr) {60          parser.parseXmlPath(profilingResultsXmlPath);
61      }
75      const auto& roleMap = parser.getRoleMap();   for (const auto& rolePair : roleMap) {79          RoleInfo role;
80          role.role = rolePair.first;
81          role.type = rolePair.second.type;
82          role.isEncoder = rolePair.second.isEncoder;
112          mRoleList[i] = std::move(role);OmxStore析构函数参数在头文件里写死的：
================================  OmxStore.h  ================================
46      OmxStore(
47              const sp<IOmx> &omx = nullptr,
48              const char* owner = "default",
49              const std::vector<std::string> &searchDirs =
50                  MediaCodecsXmlParser::getDefaultSearchDirs(),
51              const std::vector<std::string> &xmlFiles =
52                  MediaCodecsXmlParser::getDefaultXmlNames(),
53              const char *xmlProfilingResultsPath =
54                  MediaCodecsXmlParser::defaultProfilingResultsXmlPath);================================  MediaCodecsXmlParser  ================================
37      static std::vector<std::string> getDefaultSearchDirs() {38              return { "/odm/etc", "/vendor/etc", "/etc" };
39      }
40      static std::vector<std::string> getDefaultXmlNames() {41              return { "media_codecs.xml", "media_codecs_performance.xml" };
42      }
43      static constexpr char const* defaultProfilingResultsXmlPath =
44              "/data/misc/media/media_codecs_profiling_results.xml";
45