这篇文章处理的是编码后的视频图像数据CVPixelBufferRef，我们使用CAEAGLLayer来作为图像的渲染图层。
该功能使用到的是OpenGL ES的相关技术，如果对该技术不了解的伙伴，请先了解相关技术。
一、着色器
1.1顶点着色器：
const GLchar *shader_vsh = (const GLchar*)"attribute vec4 position;"
"attribute vec2 texCoord;"
"uniform float preferredRotation;"
"varying vec2 texCoordVarying;"
"void main()"
"{"
" mat4 rotationMatrix = mat4(cos(preferredRotation), -sin(preferredRotation), 0.0, 0.0,"
" sin(preferredRotation), cos(preferredRotation), 0.0, 0.0,"
" 0.0, 0.0, 1.0, 0.0,"
" 0.0, 0.0, 0.0, 1.0);"
" gl_Position = position * rotationMatrix;"
" texCoordVarying = texCoord;"
"}";
-
"attribute vec4 position：顶点坐标
attribute vec2 texCoord：纹理坐标
uniform float preferredRotation：旋转角度
attribute vec2 texCoordVarying：用于把texCoord（纹理坐标）传递到片元着器。
mat4 rotationMatrix4x4 的旋转矩阵
gl_Position：内建变量，用来接收顶点坐标。
1.1片元着色器：
const GLchar *shader_fsh = (const GLchar*)"varying highp vec2 texCoordVarying;"
"precision mediump float;"
"uniform sampler2D SamplerY;"
"uniform sampler2D SamplerUV;"
"uniform mat3 colorConversionMatrix;"
"void main()"
"{"
" mediump vec3 yuv;"
" lowp vec3 rgb;"
// Subtract constants to map the video range start at 0
" yuv.x = (texture2D(SamplerY, texCoordVarying).r - (16.0/255.0));"
" yuv.yz = (texture2D(SamplerUV, texCoordVarying).rg - vec2(0.5, 0.5));"
" rgb = colorConversionMatrix * yuv;"
" gl_FragColor = vec4(rgb, 1);"
"}";
-
varying highp vec2 texCoordVarying：从顶点着色器传递过来的纹理坐标。
precision mediump float;：精度设置，中间精度float。
uniform sampler2D SamplerY;：Y亮度纹理采样器。
uniform sampler2D SamplerUV;：UV色度纹理采样器。
uniform mat3 colorConversionMatrix;：颜色转换矩阵。
gl_FragColor：内建函数，输出的颜色用于随后的像素操作。
二、初始化
2.1继承之CAEAGLLayer自定义类CQEAGLLayer。
CGSize size = UIScreen.mainScreen.bounds.size;
_displayLayer = [[CQEAGLLayer alloc] initWithFrame:CGRectMake(size.width*0.5, 100, size.width*0.5, size.height*0.5)];
[self.view.layer addSublayer:_displayLayer];
2.2初始化代码：
- (instancetype)initWithFrame:(CGRect)frame {
self = [super init];
if (self) {
CGFloat scale = [[UIScreen mainScreen] scale];
self.contentsScale = scale;
//一个布尔值，指示层是否包含完全不透明的内容.默认为NO
self.opaque = TRUE;
//kEAGLDrawablePropertyRetainedBacking指定可绘制表面在显示后是否保留其内容的键.默认为NO.
self.drawableProperties = @{ kEAGLDrawablePropertyRetainedBacking :[NSNumber numberWithBool:YES]};
[self setFrame:frame];
// 1、设置绘制框架的上下文.
_context = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES2];
if (!_context || ![EAGLContext setCurrentContext:_context]) {
return nil;
}
// 将默认转换设置为BT.709，这是HDTV的标准
_preferredConversion = kColorConversion709;
//2、设置缓冲区
[self setupBuffers];
//3、加载shaders 着色器
[self loadShaders];
//4、使用着色程序
glUseProgram(self.program);
// 0 和 1 分别代表_lumaTexture 和 _chromaTexture的纹理 ID。
glUniform1i(uniforms[UNIFORM_Y], 0);
glUniform1i(uniforms[UNIFORM_UV], 1);
glUniform1f(uniforms[UNIFORM_ROTATION_ANGLE], 0);
glUniformMatrix3fv(uniforms[UNIFORM_COLOR_CONVERSION_MATRIX], 1, GL_FALSE, _preferredConversion);
}
return self;
}
- 1、创建上下文。
- 2、设置缓冲区。
- 3、使用
program
2.3 设置缓冲区
- (void)setupBuffers {
glDisable(GL_DEPTH_TEST);
//默认顶点属性(position)是关闭的,所以使用前要手动打开
glEnableVertexAttribArray(ATTRIB_VERTEX);
glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(GLfloat), 0);
//打开纹理数据读取通道
glEnableVertexAttribArray(ATTRIB_TEXCOORD);
glVertexAttribPointer(ATTRIB_TEXCOORD, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(GLfloat), 0);
//创建buffer
[self createBuffers];
}
- (void) createBuffers {
//创建帧缓存区 frameBuffer
glGenFramebuffers(1, &_frameBufferHandle);
glBindFramebuffer(GL_FRAMEBUFFER, _frameBufferHandle);
//创建颜色缓存区 RenderBuffer
glGenRenderbuffers(1, &_colorBufferHandle);
glBindRenderbuffer(GL_RENDERBUFFER, _colorBufferHandle);
//绑定渲染缓存区
[_context renderbufferStorage:GL_RENDERBUFFER fromDrawable:self];
//设置渲染缓存区的尺寸
glGetRenderbufferParameteriv(GL_RENDERBUFFER, GL_RENDERBUFFER_WIDTH, &_backingWidth);
glGetRenderbufferParameteriv(GL_RENDERBUFFER, GL_RENDERBUFFER_HEIGHT, &_backingHeight);
//绑定renderBuffer到FrameBuffer
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, _colorBufferHandle);
//检查FrameBuffer状态
if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) {
NSLog(@"Failed to make complete framebuffer object %x", glCheckFramebufferStatus(GL_FRAMEBUFFER));
}
}
-
glVertexAttribPointer顶点数据解析方式:
参数1: 指定从索引0开始取数据，与顶点着色器对应
参数2: 顶点属性大小
参数3: 数据类型
参数4: 归一化
参数5: 步长（Stride)
参数6: 数据在缓冲区起始位置的偏移量
2.4加载着色器
- (BOOL)loadShaders {
GLuint vertShader = 0, fragShader = 0;
// 创建着色program.
self.program = glCreateProgram();
//编译顶点着色器
if(![self compileShaderString:&vertShader type:GL_VERTEX_SHADER shaderString:shader_vsh]) {
NSLog(@"Failed to compile vertex shader");
return NO;
}
//编译片元着色器
if(![self compileShaderString:&fragShader type:GL_FRAGMENT_SHADER shaderString:shader_fsh]) {
NSLog(@"Failed to compile fragment shader");
return NO;
}
// 附着顶点着色器到program.
glAttachShader(self.program, vertShader);
// 附着片元着色器到program.
glAttachShader(self.program, fragShader);
// 绑定属性位置。这需要在链接之前完成.(让ATTRIB_VERTEX/ATTRIB_TEXCOORD 与position/texCoord产生连接)
glBindAttribLocation(self.program, ATTRIB_VERTEX, "position");
glBindAttribLocation(self.program, ATTRIB_TEXCOORD, "texCoord");
// Link the program.
if (![self linkProgram:self.program]) {
NSLog(@"Failed to link program: %d", self.program);
if (vertShader) {
glDeleteShader(vertShader);
vertShader = 0;
}
if (fragShader) {
glDeleteShader(fragShader);
fragShader = 0;
}
if (self.program) {
glDeleteProgram(self.program);
self.program = 0;
}
return NO;
}
//获取uniform的位置
//Y亮度纹理
uniforms[UNIFORM_Y] = glGetUniformLocation(self.program, "SamplerY");
//UV色量纹理
uniforms[UNIFORM_UV] = glGetUniformLocation(self.program, "SamplerUV");
//旋转角度preferredRotation
uniforms[UNIFORM_ROTATION_ANGLE] = glGetUniformLocation(self.program, "preferredRotation");
//YUV->RGB
uniforms[UNIFORM_COLOR_CONVERSION_MATRIX] = glGetUniformLocation(self.program, "colorConversionMatrix");
// Release vertex and fragment shaders.
if (vertShader) {
glDetachShader(self.program, vertShader);
glDeleteShader(vertShader);
}
if (fragShader) {
glDetachShader(self.program, fragShader);
glDeleteShader(fragShader);
}
return YES;
}
2.5编译
- (BOOL)compileShaderString:(GLuint *)shader type:(GLenum)type shaderString:(const GLchar*)shaderString
{
*shader = glCreateShader(type);
glShaderSource(*shader, 1, &shaderString, NULL);
glCompileShader(*shader);
#if defined(DEBUG)
GLint logLength;
glGetShaderiv(*shader, GL_INFO_LOG_LENGTH, &logLength);
if (logLength > 0) {
GLchar *log = (GLchar *)malloc(logLength);
glGetShaderInfoLog(*shader, logLength, &logLength, log);
NSLog(@"Shader compile log:\n%s", log);
free(log);
}
#endif
GLint status = 0;
glGetShaderiv(*shader, GL_COMPILE_STATUS, &status);
if (status == 0) {
glDeleteShader(*shader);
return NO;
}
return YES;
}
2.5链接
- (BOOL)linkProgram:(GLuint)prog {
GLint status;
glLinkProgram(prog);
#if defined(DEBUG)
GLint logLength;
glGetProgramiv(prog, GL_INFO_LOG_LENGTH, &logLength);
if (logLength > 0) {
GLchar *log = (GLchar *)malloc(logLength);
glGetProgramInfoLog(prog, logLength, &logLength, log);
NSLog(@"Program link log:\n%s", log);
free(log);
}
#endif
glGetProgramiv(prog, GL_LINK_STATUS, &status);
if (status == 0) {
return NO;
}
return YES;
}
三、绘制
3.1设置CVPixelBufferRef
- (void)setPixelBuffer:(CVPixelBufferRef)pb {
if(_pixelBuffer) {
CVPixelBufferRelease(_pixelBuffer);
}
_pixelBuffer = CVPixelBufferRetain(pb);
//获取视频帧的宽与高
int frameWidth = (int)CVPixelBufferGetWidth(_pixelBuffer);
int frameHeight = (int)CVPixelBufferGetHeight(_pixelBuffer);
//显示_pixelBuffer
[self display:_pixelBuffer width:frameWidth height:frameHeight];
}
- 在iOS里，我们经常能看到
CVPixelBufferRef这个类型，在Camera 采集返回的数据里得到一个CMSampleBufferRef，而每个CMSampleBufferRef里则包含一个CVPixelBufferRef，在视频硬解码的返回数据里也是一个CVPixelBufferRef（里面包好了所有的压缩的图片信息）。CVPixelBufferRef：是一种像素图片类型，由于CV开头，所以它是属于CoreVideo?？榈?。
3.2显示
- (void)display:(CVPixelBufferRef)pixelBuffer width:(uint32_t)frameWidth height:(uint32_t)frameHeight {
if (!_context || ![EAGLContext setCurrentContext:_context]) {
return;
}
if(pixelBuffer == NULL) {
NSLog(@"PixelBuffer is null");
return;
}
CVReturn errer;
size_t planeCount = CVPixelBufferGetPlaneCount(pixelBuffer);
CFTypeRef colorAttachments = CVBufferGetAttachment(pixelBuffer, kCVImageBufferYCbCrMatrixKey, NULL);
if (CFStringCompare(colorAttachments, kCVImageBufferYCbCrMatrix_ITU_R_601_4, 0) == kCFCompareEqualTo) {
_preferredConversion = kColorConversion601;
}
else {
_preferredConversion = kColorConversion709;
}
CVOpenGLESTextureCacheRef _videoTextureCache;
//创建 CVOpenGLESTextureCacheRef 创建新的纹理缓存
errer = CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, NULL, _context, NULL, &_videoTextureCache);
if (error != noErr) {
NSLog(@"CVOpenGLESTextureCacheCreate error : %d", errer);
return;
}
glActiveTexture(GL_TEXTURE0); //激活纹理
//1.创建Y纹理(亮度纹理)
errer = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_videoTextureCache,
pixelBuffer,
NULL,
GL_TEXTURE_2D,
GL_RED_EXT,
frameWidth,
frameHeight,
GL_RED_EXT,
GL_UNSIGNED_BYTE,
0,
&_lumaTexture);
if (errer) {
NSLog(@"CVOpenGLESTextureCacheCreateTextureFromImage error: %d", errer);
}
//2.配置Y亮度纹理属性
//绑定纹理.
glBindTexture(CVOpenGLESTextureGetTarget(_lumaTexture), CVOpenGLESTextureGetName(_lumaTexture));
//配置纹理放大/缩小过滤方式以及纹理围绕S/T环绕方式
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
//3.UV-plane 纹理
//如果颜色通道个数>1,则除了Y还有UV-Plane.
if(planeCount == 2) {
// UV-plane.
glActiveTexture(GL_TEXTURE1); //激活UV-plane纹理
//4.创建UV-plane纹理
errer = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_videoTextureCache,
pixelBuffer,
NULL,
GL_TEXTURE_2D,
GL_RG_EXT,
frameWidth / 2,
frameHeight / 2,
GL_RG_EXT,
GL_UNSIGNED_BYTE,
1,
&_chromaTexture);
if (error) {
NSLog(@"CVOpenGLESTextureCacheCreateTextureFromImage error: %d", error);
}
//5.绑定纹理
glBindTexture(CVOpenGLESTextureGetTarget(_chromaTexture), CVOpenGLESTextureGetName(_chromaTexture));
//6.配置纹理放大/缩小过滤方式以及纹理围绕S/T环绕方式
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
//绑定帧缓存区
glBindFramebuffer(GL_FRAMEBUFFER, _frameBufferHandle);
//设置视口.
glViewport(0, 0, _backingWidth, _backingHeight);
//清理屏幕
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
//使用shaderProgram
glUseProgram(self.program);
//传递Uniform属性到shader
//UNIFORM_ROTATION_ANGLE 旋转角度
glUniform1f(uniforms[UNIFORM_ROTATION_ANGLE], 0);
//UNIFORM_COLOR_CONVERSION_MATRIX YUV->RGB颜色矩阵
glUniformMatrix3fv(uniforms[UNIFORM_COLOR_CONVERSION_MATRIX], 1, GL_FALSE, _preferredConversion);
// 根据视频的方向和纵横比设置四边形顶点
CGRect viewBounds = self.bounds;
CGSize contentSize = CGSizeMake(frameWidth, frameHeight);
/*
AVMakeRectWithAspectRatioInsideRect
功能: 返回一个按比例缩放的CGRect，该CGRect保持由边界CGRect内的CGSize指定的纵横比
参数1:希望保持的宽高比或纵横比
参数2:填充的rect
*/
CGRect vertexSamplingRect = AVMakeRectWithAspectRatioInsideRect(contentSize, viewBounds);
// 计算标准化的四边形坐标以将帧绘制到其中
//标准化采样大小
CGSize normalizedSamplingSize = CGSizeMake(0.0, 0.0);
//标准化规模
CGSize cropScaleAmount = CGSizeMake(vertexSamplingRect.size.width/viewBounds.size.width,vertexSamplingRect.size.height/viewBounds.size.height);
// 规范化四元顶点
if (cropScaleAmount.width > cropScaleAmount.height) {
normalizedSamplingSize.width = 1.0;
normalizedSamplingSize.height = cropScaleAmount.height/cropScaleAmount.width;
}
else {
normalizedSamplingSize.width = cropScaleAmount.width/cropScaleAmount.height;
normalizedSamplingSize.height = 1.0;;
}
/*
四顶点数据定义了我们绘制像素缓冲区的二维平面区域。
使用（-1，-1）和（1,1）分别作为左下角和右上角坐标形成的顶点数据覆盖整个屏幕。
*/
GLfloat quadVertexData [] = {
-1 * normalizedSamplingSize.width, -1 * normalizedSamplingSize.height,
normalizedSamplingSize.width, -1 * normalizedSamplingSize.height,
-1 * normalizedSamplingSize.width, normalizedSamplingSize.height,
normalizedSamplingSize.width, normalizedSamplingSize.height,
};
// 更新属性值.
//坐标数据
glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, quadVertexData);
glEnableVertexAttribArray(ATTRIB_VERTEX);
/*
纹理顶点的设置使我们垂直翻转纹理。这使得我们的左上角原点缓冲区匹配OpenGL的左下角纹理坐标系
*/
CGRect textureSamplingRect = CGRectMake(0, 0, 1, 1);
GLfloat quadTextureData[] = {
CGRectGetMinX(textureSamplingRect), CGRectGetMaxY(textureSamplingRect),
CGRectGetMaxX(textureSamplingRect), CGRectGetMaxY(textureSamplingRect),
CGRectGetMinX(textureSamplingRect), CGRectGetMinY(textureSamplingRect),
CGRectGetMaxX(textureSamplingRect), CGRectGetMinY(textureSamplingRect)
};
//更新纹理坐标属性值
glVertexAttribPointer(ATTRIB_TEXCOORD, 2, GL_FLOAT, 0, 0, quadTextureData);
glEnableVertexAttribArray(ATTRIB_TEXCOORD);
//绘制图形
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
//绑定渲染缓存区->显示到屏幕
glBindRenderbuffer(GL_RENDERBUFFER, _colorBufferHandle);
[_context presentRenderbuffer:GL_RENDERBUFFER];
//清理纹理,方便下一帧纹理显示
[self cleanUpTextures];
// 定期纹理缓存刷新每帧
CVOpenGLESTextureCacheFlush(_videoTextureCache, 0);
if(_videoTextureCache) {
CFRelease(_videoTextureCache);
}
}
-
CVBufferGetAttachment使用像素缓冲区的颜色附件确定适当的颜色转换矩阵:
参数1: 像素缓存区
参数2: kCVImageBufferYCbCrMatrixKey YCbCr->RGB
参数3: 附件模式,NULL -
CVOpenGLESTextureCacheCreate创建新的纹理缓存：
参数1: kCFAllocatorDefault默认内存分配器.
参数2: NULL
参数3: EAGLContext 图形上下文
参数4: NULL
参数5: 新创建的纹理缓存 -
CVOpenGLESTextureCacheCreateTextureFromImage创建亮度/色度纹理-Y/UV纹理:
参数1: 内存分配器,kCFAllocatorDefault
参数2: 纹理缓存.纹理缓存将管理纹理的纹理缓存对象
参数3: sourceImage.
参数4: 纹理属性.默认给NULL
参数5: 目标纹理,GL_TEXTURE_2D
参数6: 指定纹理中颜色组件的数量(GL_RGBA, GL_LUMINANCE, GL_RGBA8_OES, GL_RG, and GL_RED (NOTE: 在 GLES3 使用 GL_R8 替代 GL_RED).)
参数7: 帧宽度
参数8: 帧高度
参数9: 格式指定像素数据的格式
参数10: 指定像素数据的数据类型,GL_UNSIGNED_BYTE
参数11: planeIndex
参数12: 纹理输出新创建的纹理对象将放置在此处。
