探索iOS之多摄像头预览架构

在iOS13.0开始支持多摄像头预览AVCaptureMultiCamSession，然后iOS15.0增加支持摄像头画中画预览。在使用之前，我们通过isMultiCamSupported()判断是否支持多Camera同时预览。让我们先看下效果：

一、Camera架构

1、Camera流水线

Camera由AVCaptureDeviceInput、AVCaptureSession、AVCaptureOutput构成。如下图所示：

2、单Camera架构

单个Camera架构意味只有一个AVCaptureDeviceInput，同步输出VideoData和DepthData，支持预览和输出文件。如下图所示：

3、多Camera架构

与单Camera架构相比，多Camera架构包括多个输入源AVCaptureDeviceInput，多个摄像头同时预览，如下图所示：

二、Camera类图结构

Camera类图包括AVCaptureDeviceInput、AVCaptureMultiCamSession、AVCaptureVideoDataOutput、AVCaptureVideoPreviewLayer、AVAssetWriter。如下图所示：

三、Camera输入输出

Camera的输入包括：前置Camera、后置Camera、麦克风，输出包括：预览数据、图片、文件、Metadata，由AVCaptureMultiCamSession进行管理。如下图所示：

四、MultiCamera流同步

多个Camera同时预览，它们共享分辨率和帧率。也需要进行流同步，包括如下：

曝光
对焦
白平衡

五、Camera画中画预览

1、初始化

初始化阶段，主要设置预览图层、配置capture session，示例代码如下：

 override func viewDidLoad() {super.viewDidLoad()// 设置前置、后置camera预览图层backCameraVideoPreviewView.videoPreviewLayer.setSessionWithNoConnection(session)frontCameraVideoPreviewView.videoPreviewLayer.setSessionWithNoConnection(session)// 配置 capture sessionsessionQueue.async {self.configureSession()}}

2、配置session

配置capture session的示例代码如下：

 private func configureSession() {guard setupResult == .success else { return }guard AVCaptureMultiCamSession.isMultiCamSupported else {print("MultiCam not supported on this device")setupResult = .multiCamNotSupportedreturn}session.beginConfiguration()defer {session.commitConfiguration()if setupResult == .success {checkSystemCost()}}guard configureBackCamera() else {setupResult = .configurationFailedreturn}guard configureFrontCamera() else {setupResult = .configurationFailedreturn}}

3、配置后置Camera

配置流程包括：查找后置Camera、添加到session、连接输入设备到输出数据、连接输入设备到预览图层等，示例代码如下：

 private func configureBackCamera() -> Bool {session.beginConfiguration()defer {session.commitConfiguration()}// 查找后置cameraguard let backCamera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back) else {print("Could not find the back camera")return false}// 添加后置camera到sessiondo {backCameraDeviceInput = try AVCaptureDeviceInput(device: backCamera)guard let backCameraDeviceInput = backCameraDeviceInput,session.canAddInput(backCameraDeviceInput) else {return false}session.addInputWithNoConnections(backCameraDeviceInput)} catch {print("Could not create back camera device input: \(error)")return false}// 查找后置camera输入视频端口guard let backCameraDeviceInput = backCameraDeviceInput,let backCameraVideoPort = backCameraDeviceInput.ports(for: .video,sourceDeviceType: backCamera.deviceType,sourceDevicePosition: backCamera.position).first else {return false}// 添加后置camera输出视频数据guard session.canAddOutput(backCameraVideoDataOutput) else {print("Could not add the back camera video data output")return false}session.addOutputWithNoConnections(backCameraVideoDataOutput)backCameraVideoDataOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_32BGRA)]backCameraVideoDataOutput.setSampleBufferDelegate(self, queue: dataOutputQueue)// 连接后置camera输入到数据输出let backCameraVideoDataOutputConnection = AVCaptureConnection(inputPorts: [backCameraVideoPort],output: backCameraVideoDataOutput)guard session.canAddConnection(backCameraVideoDataOutputConnection) else {print("Could not add a connection to the back camera video data output")return false}session.addConnection(backCameraVideoDataOutputConnection)backCameraVideoDataOutputConnection.videoOrientation = .portrait// 连接后置camera输入到预览图层guard let backCameraVideoPreviewLayer = backCameraVideoPreviewLayer else {return false}let backCameraVideoPreviewLayerConnection = AVCaptureConnection(inputPort: backCameraVideoPort, videoPreviewLayer: backCameraVideoPreviewLayer)guard session.canAddConnection(backCameraVideoPreviewLayerConnection) else {print("Could not add a connection to the back camera video preview layer")return false}session.addConnection(backCameraVideoPreviewLayerConnection)return true}

4、配置前置Camera

前置Camera与后置的配置流程类似，只是把back换成front。另外，前置Camera开启镜像。示例代码如下：

    private func configureFrontCamera() -> Bool {session.beginConfiguration()defer {session.commitConfiguration()}// 查找前置cameraguard let frontCamera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .front) else {print("Could not find the front camera")return false}// 添加前置camera到sessiondo {frontCameraDeviceInput = try AVCaptureDeviceInput(device: frontCamera)guard let frontCameraDeviceInput = frontCameraDeviceInput,session.canAddInput(frontCameraDeviceInput) else {return false}session.addInputWithNoConnections(frontCameraDeviceInput)} catch {print("Could not create front camera device input: \(error)")return false}// 查找前置camera输入视频端口guard let frontCameraDeviceInput = frontCameraDeviceInput,let frontCameraVideoPort = frontCameraDeviceInput.ports(for: .video,sourceDeviceType: frontCamera.deviceType,sourceDevicePosition: frontCamera.position).first else {return false}// 添加前置camera输出视频数据guard session.canAddOutput(frontCameraVideoDataOutput) else {print("Could not add the front camera video data output")return false}session.addOutputWithNoConnections(frontCameraVideoDataOutput)frontCameraVideoDataOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_32BGRA)]frontCameraVideoDataOutput.setSampleBufferDelegate(self, queue: dataOutputQueue)// 连接前置camera输入到数据输出let frontCameraVideoDataOutputConnection = AVCaptureConnection(inputPorts: [frontCameraVideoPort],output: frontCameraVideoDataOutput)guard session.canAddConnection(frontCameraVideoDataOutputConnection) else {print("Could not add a connection to the front camera video data output")return false}session.addConnection(frontCameraVideoDataOutputConnection)frontCameraVideoDataOutputConnection.videoOrientation = .portrait// 连接前置camera输入到预览图层guard let frontCameraVideoPreviewLayer = frontCameraVideoPreviewLayer else {return false}let frontCameraVideoPreviewLayerConnection = AVCaptureConnection(inputPort: frontCameraVideoPort, videoPreviewLayer: frontCameraVideoPreviewLayer)guard session.canAddConnection(frontCameraVideoPreviewLayerConnection) else {print("Could not add a connection to the front camera video preview layer")return false}session.addConnection(frontCameraVideoPreviewLayerConnection)// 前置camera开启镜像frontCameraVideoPreviewLayerConnection.isVideoMirrored = truefrontCameraVideoPreviewLayerConnection.automaticallyAdjustsVideoMirroring = falsereturn true}

5、配置双向麦克风

除了提供画中画Camera，还提供前后双向麦克风。示例代码如下：

 private func configureMicrophone() -> Bool {session.beginConfiguration()defer {session.commitConfiguration()}// 查找麦克风guard let microphone = AVCaptureDevice.default(for: .audio) else {print("Could not find the microphone")return false}// 添加麦克风到sessiondo {microphoneDeviceInput = try AVCaptureDeviceInput(device: microphone)guard let microphoneDeviceInput = microphoneDeviceInput,session.canAddInput(microphoneDeviceInput) else {return false}session.addInputWithNoConnections(microphoneDeviceInput)} catch {print("Could not create microphone input: \(error)")return false}// 查找输入设备的后置音频端口guard let microphoneDeviceInput = microphoneDeviceInput,let backMicrophonePort = microphoneDeviceInput.ports(for: .audio,sourceDeviceType: microphone.deviceType,sourceDevicePosition: .back).first else {return false}// 查找输入设备的前置音频端口guard let frontMicrophonePort = microphoneDeviceInput.ports(for: .audio,sourceDeviceType: microphone.deviceType,sourceDevicePosition: .front).first else {return false}// 添加后置麦克风到输出数据guard session.canAddOutput(backMicrophoneAudioDataOutput) else {print("Could not add the back microphone audio data output")return false}session.addOutputWithNoConnections(backMicrophoneAudioDataOutput)backMicrophoneAudioDataOutput.setSampleBufferDelegate(self, queue: dataOutputQueue)// 添加前置麦克风到输出数据guard session.canAddOutput(frontMicrophoneAudioDataOutput) else {print("Could not add the front microphone audio data output")return false}session.addOutputWithNoConnections(frontMicrophoneAudioDataOutput)frontMicrophoneAudioDataOutput.setSampleBufferDelegate(self, queue: dataOutputQueue)// 连接后置麦克风到输出数据let backMicrophoneAudioDataOutputConnection = AVCaptureConnection(inputPorts: [backMicrophonePort],output: backMicrophoneAudioDataOutput)guard session.canAddConnection(backMicrophoneAudioDataOutputConnection) else {print("Could not add a connection to the back microphone audio data output")return false}session.addConnection(backMicrophoneAudioDataOutputConnection)// 连接前置麦克风到输出数据let frontMicrophoneAudioDataOutputConnection = AVCaptureConnection(inputPorts: [frontMicrophonePort],output: frontMicrophoneAudioDataOutput)guard session.canAddConnection(frontMicrophoneAudioDataOutputConnection) else {print("Could not add a connection to the front microphone audio data output")return false}session.addConnection(frontMicrophoneAudioDataOutputConnection)return true}

六、降低功耗

iOS提供API获取硬件功耗：

var hardwareCost: Float { get } // 取值[0.0, 1.0]

同时提供API获取系统压力功耗：

var systemPressureCost: Float { get } // 取值[0.0, 1.0]

关于降低功耗的可行方案如下：

设置最大帧率
降低Camera分辨率
选择低精度像素格式