Mac OS 使用Metal渲染NV12、YUV420、CMSampleBufferRef视频

  • 需求
  • MTKView初始化
  • 摄像头采集CMSampleBufferRef
    • 渲染CMSampleBufferRef
    • yuv420转NV12
    • 渲染NV12
  • END

资料较少，整合后仅作为记录学习使用。

需求

yuv420原始视频数据使用metal渲染。

MTKView初始化

vector_uint2 viewportSize;
MTKView *mMtkview;
id <MTLDevice> mDevice;
id <MTLCommandQueue> mCmdQueue;
id <MTLRenderPipelineState> mPipeline;
id <MTLBuffer> mBuffer;
id <MTLTexture> mTexture;
CVMetalTextureCacheRef mTextureCache;
NSUInteger vertexCount;id<MTLTexture> mTextureY;
id<MTLTexture> mTextureUV;id <MTLBuffer> mConvertMatrix;

setMetal

mMtkview = [[MTKView alloc] initWithFrame:self.view.frame device:MTLCreateSystemDefaultDevice()];
mDevice = mMtkview.device;
self.view = mMtkview;
mMtkview.delegate = self;
mCmdQueue = [mDevice newCommandQueue];CVMetalTextureCacheCreate(NULL, NULL, mDevice, NULL, &mTextureCache);

setPipeline

id <MTLLibrary> library = [mDevice newDefaultLibrary];
id <MTLFunction> vertexfunc = [library newFunctionWithName:@"verfunc"];
id <MTLFunction> fragfunc = [library newFunctionWithName:@"fragfunc"];
MTLRenderPipelineDescriptor *renderdes = [MTLRenderPipelineDescriptor new];
renderdes.vertexFunction = vertexfunc;
renderdes.fragmentFunction = fragfunc;
renderdes.colorAttachments[0].pixelFormat = mMtkview.colorPixelFormat;
mPipeline = [mDevice newRenderPipelineStateWithDescriptor:renderdes error:nil];

setupMatrix

matrix_float3x3 kColorConversion601FullRangeMatrix = (matrix_float3x3){(simd_float3){1.0,    1.0,    1.0},(simd_float3){0.0,    -0.343, 1.765},(simd_float3){1.4,    -0.711, 0.0},
};
vector_float3 kColorConversion601FullRangeOffset = (vector_float3){ -(16.0/255.0), -0.5, -0.5}; // 这个是偏移HQHConvertMatrix converMatrix;
converMatrix.matrix = kColorConversion601FullRangeMatrix;
converMatrix.offset = kColorConversion601FullRangeOffset;
mConvertMatrix = [mDevice newBufferWithBytes:&converMatrix length:sizeof(HQHConvertMatrix) options:MTLResourceStorageModeShared];

MTKViewDelegate

- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size {viewportSize = (vector_uint2){size.width, size.height};
}- (void)drawInMTKView:(MTKView *)view {if (mTextureY && mTextureUV) {id <MTLCommandBuffer> cmdBuffer = [mCmdQueue commandBuffer];MTLRenderPassDescriptor *passdes = view.currentRenderPassDescriptor;if (passdes != nil) {id <MTLRenderCommandEncoder> cmdEncoder = [cmdBuffer renderCommandEncoderWithDescriptor:passdes];[cmdEncoder setViewport:(MTLViewport){0.0,0.0,viewportSize.x,viewportSize.y, -1.0,1.0}];[cmdEncoder setRenderPipelineState:mPipeline];[cmdEncoder setVertexBuffer:mBuffer offset:0 atIndex:0];[cmdEncoder setFragmentTexture:mTextureY atIndex:0];[cmdEncoder setFragmentTexture:mTextureUV atIndex:1];[cmdEncoder setFragmentBuffer:mConvertMatrix offset:0 atIndex:HQHFragmentInputindexMatrix];[cmdEncoder drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:vertexCount];/**rgb 数据[cmdEncoder setFragmentTexture:mTexture atIndex:0];[cmdEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:vertexCount];*/[cmdEncoder endEncoding];[cmdBuffer presentDrawable:view.currentDrawable];[cmdBuffer commit];}}
}

摄像头采集CMSampleBufferRef

AVCaptureSession *mCaptureSession;
AVCaptureDeviceInput *mCaptureInput;
AVCaptureVideoDataOutput *mCaptureOutput;

setupSession

mCaptureSession = [[AVCaptureSession alloc] init];
mCaptureSession.sessionPreset = AVCaptureSessionPreset1280x720;
if (@available(macOS 10.15, *)) {AVCaptureDevice *cameraDevice = [AVCaptureDevice defaultDeviceWithDeviceType:AVCaptureDeviceTypeBuiltInWideAngleCamera mediaType:AVMediaTypeVideo position:AVCaptureDevicePositionBack];mCaptureInput = [[AVCaptureDeviceInput alloc] initWithDevice:cameraDevice error:nil];
} else {AVCaptureDevice *cameraDevice = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo].firstObject;mCaptureInput = [[AVCaptureDeviceInput alloc] initWithDevice:cameraDevice error:nil];
}
if ([mCaptureSession canAddInput:mCaptureInput]) {[mCaptureSession addInput:mCaptureInput];
}mCaptureOutput = [[AVCaptureVideoDataOutput alloc] init];
[mCaptureOutput setAlwaysDiscardsLateVideoFrames:NO];
[mCaptureOutput setSampleBufferDelegate:self queue:dispatch_queue_create("bd", DISPATCH_QUEUE_SERIAL)];if ([mCaptureSession canAddOutput:mCaptureOutput]) {[mCaptureSession addOutput:mCaptureOutput];
}
NSLog(@"out = %@ ary = %@",mCaptureOutput,[mCaptureOutput availableVideoCodecTypes]);
[mCaptureOutput setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange] forKey:(id)kCVPixelBufferPixelFormatTypeKey]];
AVCaptureConnection *connection = [mCaptureOutput connectionWithMediaType:AVMediaTypeVideo];
[connection setVideoOrientation:AVCaptureVideoOrientationPortrait];
[mCaptureSession startRunning];

AVCaptureVideoDataOutputSampleBufferDelegate

- (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection{NSLog(@"data");
//    [self setTexture:sampleBuffer];
}

渲染CMSampleBufferRef

- (void)setTexture:(CMSampleBufferRef)samplebuffer {CVPixelBufferRef pixelbuffer = CMSampleBufferGetImageBuffer(samplebuffer);// textureY{size_t width = CVPixelBufferGetWidthOfPlane(pixelbuffer, 0);size_t height = CVPixelBufferGetHeightOfPlane(pixelbuffer, 0);CVMetalTextureRef texture = NULL;CVReturn status = CVMetalTextureCacheCreateTextureFromImage(NULL, mTextureCache, pixelbuffer, NULL, MTLPixelFormatR8Unorm, width, height, 0, &texture);if (status == kCVReturnSuccess) {mTextureY = CVMetalTextureGetTexture(texture);CFRelease(texture);}}// textureUV{size_t width = CVPixelBufferGetWidthOfPlane(pixelbuffer, 1);size_t height = CVPixelBufferGetHeightOfPlane(pixelbuffer, 1);CVMetalTextureRef texture = NULL;CVReturn status = CVMetalTextureCacheCreateTextureFromImage(NULL, mTextureCache, pixelbuffer, NULL, MTLPixelFormatRG8Unorm, width, height, 1, &texture);if (status == kCVReturnSuccess) {mTextureUV = CVMetalTextureGetTexture(texture);CFRelease(texture);}}size_t width = CVPixelBufferGetWidth(pixelbuffer);size_t heigth = CVPixelBufferGetHeight(pixelbuffer);CVMetalTextureRef temTexture = nil;CVReturn status = CVMetalTextureCacheCreateTextureFromImage(NULL, mTextureCache, pixelbuffer, NULL, MTLPixelFormatBGRA8Unorm, width, heigth, 0, &temTexture);if (status == kCVReturnSuccess) {mMtkview.drawableSize = CGSizeMake(width, heigth);mTexture = CVMetalTextureGetTexture(temTexture);CFRelease(temTexture);}
}

yuv420转NV12

void YUV420PtoNV12(unsigned char *Src, unsigned char* Dst,int Width,int Height){unsigned char* SrcU = Src + Width * Height;unsigned char* SrcV = SrcU + Width * Height / 4 ;memcpy(Dst, Src, Width * Height);unsigned char* DstU = Dst + Width * Height;for(int i = 0 ; i < Width * Height / 4 ; i++ ){( *DstU++) = ( *SrcU++);( *DstU++) = ( *SrcV++);}
}

渲染NV12

- (void)renderYUV420P:(EVFrame *)frame {int w = frame.vsize.width;int h = frame.vsize.height;unsigned char *i420 = frame.data;unsigned char *buffer = malloc(w*h*10);YUV420PtoNV12(i420, buffer, w, h);NSDictionary *pixelAttributes = @{(NSString*)kCVPixelBufferIOSurfacePropertiesKey:@{}};CVPixelBufferRef pixelBuffer = NULL;CVReturn result = CVPixelBufferCreate(kCFAllocatorDefault, w, h, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, (__bridge CFDictionaryRef)(pixelAttributes), &pixelBuffer);CVPixelBufferLockBaseAddress(pixelBuffer,0);void *yDestPlane = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);unsigned char *y_ch0 = buffer;unsigned char *y_ch1 = buffer + w * h;memcpy(yDestPlane, y_ch0, w * h);void *uvDestPlane = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);// NV12memcpy(uvDestPlane, y_ch1, w * h * 0.5);CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);if (result != kCVReturnSuccess) {NSLog(@"Unable to create cvpixelbuffer %d", result);}{size_t width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0);size_t height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0);CVMetalTextureRef texture = NULL;CVReturn status = CVMetalTextureCacheCreateTextureFromImage(NULL, mTextureCache, pixelBuffer, NULL, MTLPixelFormatR8Unorm, width, height, 0, &texture);if (status == kCVReturnSuccess) {mTextureY = CVMetalTextureGetTexture(texture);CFRelease(texture);}}// textureUV{size_t width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 1);size_t height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 1);CVMetalTextureRef texture = NULL;CVReturn status = CVMetalTextureCacheCreateTextureFromImage(NULL, mTextureCache, pixelBuffer, NULL, MTLPixelFormatRG8Unorm, width, height, 1, &texture);if (status == kCVReturnSuccess) {mTextureUV = CVMetalTextureGetTexture(texture);CFRelease(texture);}}size_t width = CVPixelBufferGetWidth(pixelBuffer);size_t heigth = CVPixelBufferGetHeight(pixelBuffer);CVMetalTextureRef temTexture = nil;CVReturn status = CVMetalTextureCacheCreateTextureFromImage(NULL, mTextureCache, pixelBuffer, NULL, MTLPixelFormatBGRA8Unorm, width, heigth, 0, &temTexture);if (status == kCVReturnSuccess) {mMtkview.drawableSize = CGSizeMake(width, heigth);mTexture = CVMetalTextureGetTexture(temTexture);CFRelease(temTexture);}CVPixelBufferRelease(pixelBuffer);free(buffer);buffer = NULL;
}

END

metal确实比OpenGL简单好用很多

Mac OS 使用Metal渲染NV12、YUV420、CMSampleBufferRef视频相关推荐

  1. 让Unraid 也装上黑苹果MAC OS系统

    Hi,大家好,我是热爱开源黑科技的郭大侠. 受到人生观.阿文菌等各位Unraid大神的恩惠,在疫情期间自己在家弄了一套Unraid,并做了直通,但是对于一个极客CTO来讲,虚拟机只弄个群晖和软路由岂不 ...

  2. Mac OS X 背后的故事（九）半导体的丰收（下）

    文 / 王越 随着CPU与GPU合并成技术发展的趋势,苹果开发出了OpenCL框架,能够进行高速并行处理的能力使OpenCL成为了业界标准,被广泛应用. 最近几年,GPU的发展吸引了很多来自科学计算界 ...

  3. Mac OS X 背后的故事（八）半导体的丰收

    原文地址: http://www.programmer.com.cn/10071/ 文/王越 在美国宾夕法尼亚州的东部,有一个风景秀美的城市叫费城.在这个城市诞生了一系列改变世界的奇迹:第一个三权分立 ...

  4. 图像渲染测试软件,检测你的电脑是否支持Metal渲染技术的小工具

    本帖最后由 bizongyi 于 2015-6-20 20:09 编辑 在刚召开的苹果WWDC大会上,与全新操作系统OS X El Capitan一同亮相的还有原来iOS平台上的图形渲染技术Metal ...

  5. Mac os x 系统的发展史

    ·Mac OS 9:发布时间:1999年 于1999年发布的Mac OS 9操作系统(图片来自互联网) 在OS X之前,1999年发布的Mac OS 9看起来就是一个普通的桌面操作系统.并且现在已经被 ...

  6. 什么是mac os?它有哪些好的特点?有哪些应用小技巧

    基本信息 Mac系统是苹果机专用系统,是基于Unix内核的图形化操作系统,一般情况下在普通pc上无法安装的操作系统. 苹果公司不但生产MAC的大部分硬件,连MAC所用的操作系统都是它自行开发的,接下来 ...

  7. mac 10.11 brew php71,MAC OS X 10.11.4下载-OS X 10.11正式版下载 V10.11.6-PC6苹果网

    苹果WWDC 2015全球开发者大会在美国旧金山如期举行,苹果在本次大会上推出了全新的OS X 10.11系统,代号El Capitan,MAC OS X10.11系统并没有带来太多全新的技术和功能, ...

  8. 《MAC OS X 技术内幕》读书笔记第一章：MAC OS X的起源

    <MAC OS X 技术内幕>读书笔记第一章:MAC OS X的起源 前言 1 System x.x系列 1.1System 1.0(1984年1月24日) 1.2System 2.x(1 ...

  9. 通向KDE4之路（三）：完全的Mac OS X撑持

     Troy Unrau 当然KDE的打算在Linux,FreeBSD以及其它UNIX/X11平台上均可乐成移植,但这并不透露表现它在其它平台上就没有冲破.奇趣公司在GPL协议之下颁布了面向Mac,Wi ...

最新文章

  1. SpringDataJpa根据多个id物品清单id查询房源编号
  2. loj #143. 质数判定
  3. oracle－关于时间的sql
  4. linux sh文件case,Shell脚本case语句简明教程
  5. centos ping不通局域网_新手小白初次安装虚拟机，网络不通怎么办，踩过的坑都告诉你...
  6. 北京 || Java 技术、生活、工作交流社区
  7. python判断安全密码_python 字符串实例：检查并判断密码字符串的安全强度
  8. day9 集合基础命令
  9. SmartDrv的前世今生——PrimoCache_2.2.0汉化
  10. 工业通讯 | Profinet协议基础知识(一)
  11. 软文广告的写作技巧分析！
  12. lunix上silk转mp3 和 mp3转silk
  13. SnnGrow快讯：Apple Books上线AI读书功能、有声书市场将迎来颠覆时刻、刚过7岁生日的OpenAI估值达到290亿美元、跻身全球独角兽排行榜前20、中国航天2023全年发射将再破60次
  14. NAND flash基本概念整理
  15. bat 格式化年月日时分秒
  16. win7 计算机不显示收藏夹,win7系统下收藏夹无法使用的原因及解决方法
  17. 关于github上一个关于lytro的代码
  18. cogs184搭建篱笆
  19. streamFile
  20. Android专项测试性能篇整理

热门文章

  1. OCR图像识别技术的自动化复打印服务系统
  2. 从AX2012系统批量生成CSV格式数据
  3. Python深度学习：脑电图癫痫发作检测
  4. MATLAB R2022b Update4安装包下载及安装教程
  5. 可能是一份最适合你的后端面试指南（部分内容前端同样适用）| 掘金技术征文...
  6. 求生之路2服务器指令比较全
  7. 信息安全-网络设备安全(一）
  8. vim 自用/.vimrc配置
  9. ccot 目标跟踪全称_目标跟踪--CamShift
  10. android 电源/充电/库仑