From b446f7dfd3e2fbd981b8cbb075a255cf99a8984b Mon Sep 17 00:00:00 2001 From: Adrien Maglo Date: Fri, 16 Sep 2016 14:00:21 +0200 Subject: [PATCH 19/21] ios2 vout: add subpicture rendering in the zero copy pipeline --- modules/video_output/ios2.m | 338 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 304 insertions(+), 34 deletions(-) diff --git a/modules/video_output/ios2.m b/modules/video_output/ios2.m index 5e56ca5..6d4f1f1 100644 --- a/modules/video_output/ios2.m +++ b/modules/video_output/ios2.m @@ -46,6 +46,7 @@ #import #import #import +#import #import "opengl.h" /** @@ -110,6 +111,20 @@ static NSString *const fragmentShaderString = @" \ } \ "; +static NSString *const fragmentShaderRGBAString = @" \ +varying highp vec2 texCoordVarying; \ +precision mediump float; \ +\ +uniform sampler2D Texture; \ +uniform vec4 FillColor; \ +\ +void main() \ +{ \ + gl_FragColor = texture2D(Texture, texCoordVarying) * FillColor; \ +} \ +"; + + static NSString *const vertexShaderString = @" \ attribute vec4 position; \ attribute vec2 texCoord; \ @@ -139,7 +154,7 @@ static void OpenglESSwap(vlc_gl_t *); static picture_pool_t *ZeroCopyPicturePool(vout_display_t *, unsigned); static void DestroyZeroCopyPoolPicture(picture_t *); -static void ZeroCopyClean(vout_display_t *vd, picture_t *pic, subpicture_t *subpicture); +static void ZeroCopyPrepare(vout_display_t *vd, picture_t *pic, subpicture_t *subpicture); static void ZeroCopyDisplay(vout_display_t *, picture_t *, subpicture_t *); /** @@ -178,6 +193,7 @@ vlc_module_end () @property (readonly) EAGLContext* eaglContext; @property (readonly) BOOL isAppActive; @property GLuint shaderProgram; +@property GLuint shaderProgramSubpictures; - (id)initWithFrame:(CGRect)frame zeroCopy:(bool)zero_copy voutDisplay:(vout_display_t *)vd; @@ -191,6 +207,24 @@ vlc_module_end () - (void)displayPixelBuffer:(CVPixelBufferRef)pixelBuffer; @end +typedef struct { + GLuint texture; + unsigned format; + unsigned type; + unsigned width; + unsigned height; + + float alpha; + + float top; + float left; + float bottom; + float right; + + float tex_width; + float tex_height; +} gl_region_t; + struct vout_display_sys_t { VLCOpenGLES2VideoView *glESView; @@ -205,8 +239,22 @@ struct vout_display_sys_t bool zero_copy; vout_display_place_t place; + + // Subpicture + int region_count; + gl_region_t *region; + + uint8_t *texture_temp_buf; + int texture_temp_buf_size; }; +static inline int GetAlignedSize(unsigned size) +{ + /* Return the smallest larger or equal power of 2 */ + unsigned align = 1 << (8 * sizeof (unsigned) - clz(size)); + return ((align >> 1) == size) ? size : align; +} + static void *OurGetProcAddress(vlc_gl_t *gl, const char *name) { VLC_UNUSED(gl); @@ -281,7 +329,7 @@ static int Open(vlc_object_t *this) if (sys->zero_copy) { vd->pool = ZeroCopyPicturePool; - vd->prepare = ZeroCopyClean; + vd->prepare = ZeroCopyPrepare; vd->display = ZeroCopyDisplay; } else { vd->pool = PicturePool; @@ -325,6 +373,15 @@ void Close (vlc_object_t *this) vout_display_sys_t *sys = vd->sys; @autoreleasepool { + glDeleteProgram(sys->glESView.shaderProgram); + glDeleteProgram(sys->glESView.shaderProgramSubpictures); + + for (int i = 0; i < sys->region_count; i++) { + if (sys->region[i].texture) + glDeleteTextures(1, &sys->region[i].texture); + } + free(sys->region); + if (sys->tapRecognizer) { [sys->tapRecognizer.view removeGestureRecognizer:sys->tapRecognizer]; [sys->tapRecognizer release]; @@ -356,7 +413,8 @@ void Close (vlc_object_t *this) picture_pool_Release(sys->picturePool); sys->picturePool = NULL; } - + + free(sys->texture_temp_buf); free(sys); } } @@ -548,11 +606,154 @@ static void DestroyZeroCopyPoolPicture(picture_t *picture) free(picture); } -static void ZeroCopyClean(vout_display_t *vd, picture_t *pic, subpicture_t *subpicture) +#define ALIGN(x, y) (((x) + ((y) - 1)) & ~((y) - 1)) +static void Upload(vout_display_sys_t *sys, + int in_width, int in_height, + int in_full_width, int in_full_height, + int w_num, int w_den, int h_num, int h_den, + int pitch, int pixel_pitch, + int full_upload, const uint8_t *pixels, + int tex_target, int tex_format, int tex_type) +{ + int width = in_width * w_num / w_den; + int full_width = in_full_width * w_num / w_den; + int height = in_height * h_num / h_den; + int full_height = in_full_height * h_num / h_den; + // This unpack alignment is the default, but setting it just in case. + glPixelStorei(GL_UNPACK_ALIGNMENT, 4); + + int dst_width = full_upload ? full_width : width; + int dst_pitch = ALIGN(dst_width * pixel_pitch, 4); + if ( pitch != dst_pitch ) + { + int buf_size = dst_pitch * full_height * pixel_pitch; + const uint8_t *source = pixels; + uint8_t *destination; + if( !sys->texture_temp_buf || sys->texture_temp_buf_size < buf_size ) + { + free( sys->texture_temp_buf ); + sys->texture_temp_buf = xmalloc( buf_size ); + sys->texture_temp_buf_size = buf_size; + } + destination = sys->texture_temp_buf; + + for( int h = 0; h < height ; h++ ) + { + memcpy( destination, source, width * pixel_pitch ); + source += pitch; + destination += dst_pitch; + } + if (full_upload) + glTexImage2D( tex_target, 0, tex_format, + full_width, full_height, + 0, tex_format, tex_type, sys->texture_temp_buf ); + else + glTexSubImage2D( tex_target, 0, + 0, 0, + width, height, + tex_format, tex_type, sys->texture_temp_buf ); + } else { + if (full_upload) + glTexImage2D(tex_target, 0, tex_format, + full_width, full_height, + 0, tex_format, tex_type, pixels); + else + glTexSubImage2D(tex_target, 0, + 0, 0, + width, height, + tex_format, tex_type, pixels); + } +} + +static void ZeroCopyPrepare(vout_display_t *vd, picture_t *pic, subpicture_t *subpicture) { vout_display_sys_t *sys = vd->sys; if (likely([sys->glESView isAppActive])) [sys->glESView resetBuffers]; + + // Subpictures + int last_count = sys->region_count; + gl_region_t *last = sys->region; + + sys->region_count = 0; + sys->region = NULL; + + if (subpicture) + { + int count = 0; + for (subpicture_region_t *r = subpicture->p_region; r; r = r->p_next) + count++; + + sys->region_count = count; + sys->region = calloc(count, sizeof(*sys->region)); + + glActiveTexture(GL_TEXTURE0 + 0); + + int i = 0; + for (subpicture_region_t *r = subpicture->p_region; r; r = r->p_next, i++) { + gl_region_t *glr = &sys->region[i]; + + glr->format = GL_RGBA; + glr->type = GL_UNSIGNED_BYTE; + glr->width = r->fmt.i_visible_width; + glr->height = r->fmt.i_visible_height; + + glr->width = GetAlignedSize(glr->width); + //glr->height = GetAlignedSize(glr->height); + + glr->tex_width = (float) r->fmt.i_visible_width / glr->width; + glr->tex_height = (float) r->fmt.i_visible_height / glr->height; + + glr->alpha = (float)subpicture->i_alpha * r->i_alpha / 255 / 255; + glr->left = 2.0 * (r->i_x ) / subpicture->i_original_picture_width - 1.0; + glr->top = -2.0 * (r->i_y ) / subpicture->i_original_picture_height + 1.0; + glr->right = 2.0 * (r->i_x + r->fmt.i_visible_width ) / subpicture->i_original_picture_width - 1.0; + glr->bottom = -2.0 * (r->i_y + r->fmt.i_visible_height) / subpicture->i_original_picture_height + 1.0; + + glr->texture = 0; + /* Try to recycle the textures allocated by the previous + call to this function. */ + for (int j = 0; j < last_count; j++) { + if (last[j].texture && + last[j].width == glr->width && + last[j].height == glr->height && + last[j].format == glr->format && + last[j].type == glr->type) { + glr->texture = last[j].texture; + memset(&last[j], 0, sizeof(last[j])); + break; + } + } + + const int pixels_offset = r->fmt.i_y_offset * r->p_picture->p->i_pitch + + r->fmt.i_x_offset * r->p_picture->p->i_pixel_pitch; + if (glr->texture) { + /* A texture was successfully recycled, reuse it. */ + glBindTexture(GL_TEXTURE_2D, glr->texture); + Upload(sys, r->fmt.i_visible_width, r->fmt.i_visible_height, glr->width, glr->height, 1, 1, 1, 1, + r->p_picture->p->i_pitch, r->p_picture->p->i_pixel_pitch, 0, + &r->p_picture->p->p_pixels[pixels_offset], GL_TEXTURE_2D, glr->format, glr->type); + } else { + /* Could not recycle a previous texture, generate a new one. */ + glGenTextures(1, &glr->texture); + glBindTexture(GL_TEXTURE_2D, glr->texture); + + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + Upload(sys, r->fmt.i_visible_width, r->fmt.i_visible_height, glr->width, glr->height, 1, 1, 1, 1, + r->p_picture->p->i_pitch, r->p_picture->p->i_pixel_pitch, 1, + &r->p_picture->p->p_pixels[pixels_offset], GL_TEXTURE_2D, glr->format, glr->type); + } + } + } + + for (int i = 0; i < last_count; i++) { + if (last[i].texture) + glDeleteTextures(1, &last[i].texture); + } + free(last); } static void ZeroCopyDisplay(vout_display_t *vd, picture_t *pic, subpicture_t *subpicture) @@ -961,6 +1162,8 @@ static void ZeroCopyDisplay(vout_display_t *vd, picture_t *pic, subpicture_t *su glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + [self drawSubpictures]; + glBindRenderbuffer(GL_RENDERBUFFER, _renderBuffer); [_eaglContext presentRenderbuffer:GL_RENDERBUFFER]; @@ -971,6 +1174,54 @@ done: [EAGLContext setCurrentContext:previousContext]; } +- (void)drawSubpictures +{ + vout_display_sys_t *sys = _voutDisplay->sys; + + /* Draw the subpictures */ + + glUseProgram(self.shaderProgramSubpictures); + + GLfloat transformMatrix[16]; + orientationTransformMatrix(transformMatrix, _voutDisplay->fmt.orientation); + glUniformMatrix4fv(glGetUniformLocation(self.shaderProgramSubpictures, "transformMatrix"), 1, GL_FALSE, transformMatrix); + + glUniform1i(glGetUniformLocation(self.shaderProgramSubpictures, "Texture"), 0); + + glEnable(GL_BLEND); + glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); + + glActiveTexture(GL_TEXTURE0 + 0); + for (int i = 0; i < sys->region_count; i++) { + gl_region_t *glr = &sys->region[i]; + const GLfloat vertexCoord[] = { + glr->left, glr->top, + glr->left, glr->bottom, + glr->right, glr->top, + glr->right, glr->bottom, + }; + const GLfloat textureCoord[] = { + 0.0, 0.0, + 0.0, glr->tex_height, + glr->tex_width, 0.0, + glr->tex_width, glr->tex_height, + }; + + glBindTexture(GL_TEXTURE_2D, glr->texture); + glUniform4f(glGetUniformLocation(self.shaderProgramSubpictures, "FillColor"), 1.0f, 1.0f, 1.0f, glr->alpha); + + glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, vertexCoord); + glEnableVertexAttribArray(ATTRIB_VERTEX); + + glVertexAttribPointer(ATTRIB_TEXCOORD, 2, GL_FLOAT, 0, 0, textureCoord); + glEnableVertexAttribArray(ATTRIB_TEXCOORD); + + glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + } + glDisable(GL_BLEND); +} + + - (void)setupZeroCopyGL { EAGLContext *previousContext = [EAGLContext currentContext]; @@ -1017,55 +1268,59 @@ done: - (BOOL)loadShaders { - GLuint vertShader, fragShader; - NSURL *vertShaderURL, *fragShaderURL; + GLuint vertShader = 0; + GLuint fragShader = 0; + GLuint fragShaderSub = 0; // Create the shader program. self.shaderProgram = glCreateProgram(); + self.shaderProgramSubpictures = glCreateProgram(); // Create and compile the vertex shader. if (![self compileShader:&vertShader type:GL_VERTEX_SHADER sourceString:vertexShaderString]) { if (_voutDisplay) msg_Err(_voutDisplay, "Failed to compile vertex shader"); - return NO; + goto error; } // Create and compile fragment shader. if (![self compileShader:&fragShader type:GL_FRAGMENT_SHADER sourceString:fragmentShaderString]) { if (_voutDisplay) msg_Err(_voutDisplay, "Failed to compile fragment shader"); - return NO; + goto error; + } + + if (![self compileShader:&fragShaderSub type:GL_FRAGMENT_SHADER sourceString:fragmentShaderRGBAString]) { + if (_voutDisplay) + msg_Err(_voutDisplay, "Failed to compile the RGBA fragment shader"); + goto error; } // Attach vertex shader to program. glAttachShader(self.shaderProgram, vertShader); + glAttachShader(self.shaderProgramSubpictures, vertShader); // Attach fragment shader to program. glAttachShader(self.shaderProgram, fragShader); + glAttachShader(self.shaderProgramSubpictures, fragShaderSub); // Bind attribute locations. This needs to be done prior to linking. glBindAttribLocation(self.shaderProgram, ATTRIB_VERTEX, "position"); glBindAttribLocation(self.shaderProgram, ATTRIB_TEXCOORD, "texCoord"); + glBindAttribLocation(self.shaderProgramSubpictures, ATTRIB_VERTEX, "position"); + glBindAttribLocation(self.shaderProgramSubpictures, ATTRIB_TEXCOORD, "texCoord"); // Link the program. if (![self linkProgram:self.shaderProgram]) { if (_voutDisplay) - msg_Err(_voutDisplay, "Failed to link program: %d", self.shaderProgram); - - if (vertShader) { - glDeleteShader(vertShader); - vertShader = 0; - } - if (fragShader) { - glDeleteShader(fragShader); - fragShader = 0; - } - if (self.shaderProgram) { - glDeleteProgram(self.shaderProgram); - self.shaderProgram = 0; - } + msg_Err(_voutDisplay, "Failed to link program for main pictures: %d", self.shaderProgram); + goto error; + } - return NO; + if (![self linkProgram:self.shaderProgramSubpictures]) { + if (_voutDisplay) + msg_Err(_voutDisplay, "Failed to link program for subpictures: %d", self.shaderProgramSubpictures); + goto error; } // Get uniform locations. @@ -1075,16 +1330,35 @@ done: uniforms[UNIFORM_TRANSFORM_MATRIX] = glGetUniformLocation(self.shaderProgram, "transformMatrix"); // Release vertex and fragment shaders. - if (vertShader) { - glDetachShader(self.shaderProgram, vertShader); + glDetachShader(self.shaderProgram, vertShader); + glDetachShader(self.shaderProgramSubpictures, vertShader); + glDeleteShader(vertShader); + + glDetachShader(self.shaderProgram, fragShader); + glDeleteShader(fragShader); + + glDetachShader(self.shaderProgramSubpictures, fragShaderSub); + glDeleteShader(fragShaderSub); + + return YES; + +error: + if (vertShader) glDeleteShader(vertShader); - } - if (fragShader) { - glDetachShader(self.shaderProgram, fragShader); + + if (fragShader) glDeleteShader(fragShader); - } - return YES; + if (fragShaderSub) + glDeleteShader(fragShaderSub); + + if (self.shaderProgram) + glDeleteProgram(self.shaderProgram); + + if (self.shaderProgramSubpictures) + glDeleteProgram(self.shaderProgramSubpictures); + + return NO; } - (BOOL)compileShader:(GLuint *)shader type:(GLenum)type sourceString:sourceString @@ -1097,7 +1371,6 @@ done: glShaderSource(*shader, 1, &source, NULL); glCompileShader(*shader); -#ifndef NDEBUG GLint logLength; glGetShaderiv(*shader, GL_INFO_LOG_LENGTH, &logLength); if (logLength > 0) { @@ -1107,7 +1380,6 @@ done: msg_Dbg(_voutDisplay, "Shader compile log:\n%s", log); free(log); } -#endif glGetShaderiv(*shader, GL_COMPILE_STATUS, &status); if (status == 0) { @@ -1123,7 +1395,6 @@ done: GLint status; glLinkProgram(prog); -#ifndef NDEBUG GLint logLength; glGetProgramiv(prog, GL_INFO_LOG_LENGTH, &logLength); if (logLength > 0) { @@ -1133,7 +1404,6 @@ done: msg_Dbg(_voutDisplay, "Program link log:\n%s", log); free(log); } -#endif glGetProgramiv(prog, GL_LINK_STATUS, &status); if (status == 0) { -- 2.9.3