From 87cd7de678ddb793468bd64cbdd0a73158bcc524 Mon Sep 17 00:00:00 2001 From: Thomas Guillem Date: Mon, 18 Jun 2018 12:31:13 +0200 Subject: [PATCH 18/24] ffmpeg: backport vtenc patches --- ...lboxenc-fix-mutex-cond-leak-in-error.patch | 35 ++++++ ...videotoolboxenc-split-initialization.patch | 112 ++++++++++++++++++ ...oolboxenc-fix-invalid-session-on-iOS.patch | 48 ++++++++ ...lboxenc-fix-undefined-behavior-with-.patch | 105 ++++++++++++++++ contrib/src/ffmpeg/rules.mak | 4 + 5 files changed, 304 insertions(+) create mode 100644 contrib/src/ffmpeg/0001-avcodec-videotoolboxenc-fix-mutex-cond-leak-in-error.patch create mode 100644 contrib/src/ffmpeg/0002-avcodec-videotoolboxenc-split-initialization.patch create mode 100644 contrib/src/ffmpeg/0003-avcodec-videotoolboxenc-fix-invalid-session-on-iOS.patch create mode 100644 contrib/src/ffmpeg/0004-avcodec-videotoolboxenc-fix-undefined-behavior-with-.patch diff --git a/contrib/src/ffmpeg/0001-avcodec-videotoolboxenc-fix-mutex-cond-leak-in-error.patch b/contrib/src/ffmpeg/0001-avcodec-videotoolboxenc-fix-mutex-cond-leak-in-error.patch new file mode 100644 index 0000000000..679bd70da3 --- /dev/null +++ b/contrib/src/ffmpeg/0001-avcodec-videotoolboxenc-fix-mutex-cond-leak-in-error.patch @@ -0,0 +1,35 @@ +From dea72b148a1250808c625dda0077e43d11689845 Mon Sep 17 00:00:00 2001 +From: Thomas Guillem +Date: Tue, 29 May 2018 18:04:38 +0200 +Subject: [PATCH 1/4] avcodec/videotoolboxenc: fix mutex/cond leak in error + path + +The leak could happen when the vtenc_create_encoder() function failed. +--- + libavcodec/videotoolboxenc.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c +index 086beb41fc..aafef20db0 100644 +--- a/libavcodec/videotoolboxenc.c ++++ b/libavcodec/videotoolboxenc.c +@@ -2473,13 +2473,14 @@ static av_cold int vtenc_close(AVCodecContext *avctx) + { + VTEncContext *vtctx = avctx->priv_data; + ++ pthread_cond_destroy(&vtctx->cv_sample_sent); ++ pthread_mutex_destroy(&vtctx->lock); ++ + if(!vtctx->session) return 0; + + VTCompressionSessionCompleteFrames(vtctx->session, + kCMTimeIndefinite); + clear_frame_queue(vtctx); +- pthread_cond_destroy(&vtctx->cv_sample_sent); +- pthread_mutex_destroy(&vtctx->lock); + CFRelease(vtctx->session); + vtctx->session = NULL; + +-- +2.18.0 + diff --git a/contrib/src/ffmpeg/0002-avcodec-videotoolboxenc-split-initialization.patch b/contrib/src/ffmpeg/0002-avcodec-videotoolboxenc-split-initialization.patch new file mode 100644 index 0000000000..c64aa7521a --- /dev/null +++ b/contrib/src/ffmpeg/0002-avcodec-videotoolboxenc-split-initialization.patch @@ -0,0 +1,112 @@ +From 0a83285f54c5c042104a4d2a03c40e8f69a09a3c Mon Sep 17 00:00:00 2001 +From: Thomas Guillem +Date: Mon, 11 Jun 2018 15:43:56 +0200 +Subject: [PATCH 2/4] avcodec/videotoolboxenc: split initialization + +Split vtenc_init() into vtenc_init() (VTEncContext initialization) and +vtenc_configure_encoder() (creates the vt session). + +This commit will allow to restart the vt session while encoding. +--- + libavcodec/videotoolboxenc.c | 48 +++++++++++++++++++++--------------- + 1 file changed, 28 insertions(+), 20 deletions(-) + +diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c +index aafef20db0..0b47a0abac 100644 +--- a/libavcodec/videotoolboxenc.c ++++ b/libavcodec/videotoolboxenc.c +@@ -1262,19 +1262,16 @@ static int vtenc_create_encoder(AVCodecContext *avctx, + return 0; + } + +-static av_cold int vtenc_init(AVCodecContext *avctx) ++static int vtenc_configure_encoder(AVCodecContext *avctx) + { + CFMutableDictionaryRef enc_info; + CFMutableDictionaryRef pixel_buffer_info; + CMVideoCodecType codec_type; + VTEncContext *vtctx = avctx->priv_data; + CFStringRef profile_level; +- CFBooleanRef has_b_frames_cfbool; + CFNumberRef gamma_level = NULL; + int status; + +- pthread_once(&once_ctrl, loadVTEncSymbols); +- + codec_type = get_cm_codec_type(avctx->codec_id); + if (!codec_type) { + av_log(avctx, AV_LOG_ERROR, "Error: no mapping for AVCodecID %d\n", avctx->codec_id); +@@ -1304,8 +1301,6 @@ static av_cold int vtenc_init(AVCodecContext *avctx) + if (!get_vt_hevc_profile_level(avctx, &profile_level)) return AVERROR(EINVAL); + } + +- vtctx->session = NULL; +- + enc_info = CFDictionaryCreateMutable( + kCFAllocatorDefault, + 20, +@@ -1335,8 +1330,6 @@ static av_cold int vtenc_init(AVCodecContext *avctx) + pixel_buffer_info = NULL; + } + +- pthread_mutex_init(&vtctx->lock, NULL); +- pthread_cond_init(&vtctx->cv_sample_sent, NULL); + vtctx->dts_delta = vtctx->has_b_frames ? -1 : 0; + + get_cv_transfer_function(avctx, &vtctx->transfer_function, &gamma_level); +@@ -1363,8 +1356,32 @@ static av_cold int vtenc_init(AVCodecContext *avctx) + pixel_buffer_info, + &vtctx->session); + +- if (status < 0) +- goto init_cleanup; ++init_cleanup: ++ if (gamma_level) ++ CFRelease(gamma_level); ++ ++ if (pixel_buffer_info) ++ CFRelease(pixel_buffer_info); ++ ++ CFRelease(enc_info); ++ ++ return status; ++} ++ ++static av_cold int vtenc_init(AVCodecContext *avctx) ++{ ++ VTEncContext *vtctx = avctx->priv_data; ++ CFBooleanRef has_b_frames_cfbool; ++ int status; ++ ++ pthread_once(&once_ctrl, loadVTEncSymbols); ++ ++ pthread_mutex_init(&vtctx->lock, NULL); ++ pthread_cond_init(&vtctx->cv_sample_sent, NULL); ++ ++ vtctx->session = NULL; ++ status = vtenc_configure_encoder(avctx); ++ if (status) return status; + + status = VTSessionCopyProperty(vtctx->session, + kVTCompressionPropertyKey_AllowFrameReordering, +@@ -1378,16 +1395,7 @@ static av_cold int vtenc_init(AVCodecContext *avctx) + } + avctx->has_b_frames = vtctx->has_b_frames; + +-init_cleanup: +- if (gamma_level) +- CFRelease(gamma_level); +- +- if (pixel_buffer_info) +- CFRelease(pixel_buffer_info); +- +- CFRelease(enc_info); +- +- return status; ++ return 0; + } + + static void vtenc_get_frame_info(CMSampleBufferRef buffer, bool *is_key_frame) +-- +2.18.0 + diff --git a/contrib/src/ffmpeg/0003-avcodec-videotoolboxenc-fix-invalid-session-on-iOS.patch b/contrib/src/ffmpeg/0003-avcodec-videotoolboxenc-fix-invalid-session-on-iOS.patch new file mode 100644 index 0000000000..9138060be1 --- /dev/null +++ b/contrib/src/ffmpeg/0003-avcodec-videotoolboxenc-fix-invalid-session-on-iOS.patch @@ -0,0 +1,48 @@ +From 72812a3931cda0fcb66d740bbb19c330e97ccbd0 Mon Sep 17 00:00:00 2001 +From: Thomas Guillem +Date: Mon, 11 Jun 2018 16:17:28 +0200 +Subject: [PATCH 3/4] avcodec/videotoolboxenc: fix invalid session on iOS + +Cf. comment. Restart the VT session when the APP goes from foreground to +background and vice versa. +--- + libavcodec/videotoolboxenc.c | 23 +++++++++++++++++++++-- + 1 file changed, 21 insertions(+), 2 deletions(-) + +diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c +index 0b47a0abac..f516ad7d40 100644 +--- a/libavcodec/videotoolboxenc.c ++++ b/libavcodec/videotoolboxenc.c +@@ -2175,8 +2175,27 @@ static int create_cv_pixel_buffer(AVCodecContext *avctx, + #if TARGET_OS_IPHONE + pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session); + if (!pix_buf_pool) { +- av_log(avctx, AV_LOG_ERROR, "Could not get pixel buffer pool.\n"); +- return AVERROR_EXTERNAL; ++ /* On iOS, the VT session is invalidated when the APP switches from ++ * foreground to background and vice versa. Fetch the actual error code ++ * of the VT session to detect that case and restart the VT session ++ * accordingly. */ ++ OSStatus vtstatus; ++ ++ vtstatus = VTCompressionSessionPrepareToEncodeFrames(vtctx->session); ++ if (vtstatus == kVTInvalidSessionErr) { ++ CFRelease(vtctx->session); ++ vtctx->session = NULL; ++ status = vtenc_configure_encoder(avctx); ++ if (status == 0) ++ pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session); ++ } ++ if (!pix_buf_pool) { ++ av_log(avctx, AV_LOG_ERROR, "Could not get pixel buffer pool.\n"); ++ return AVERROR_EXTERNAL; ++ } ++ else ++ av_log(avctx, AV_LOG_WARNING, "VT session restarted because of a " ++ "kVTInvalidSessionErr error.\n"); + } + + status = CVPixelBufferPoolCreatePixelBuffer(NULL, +-- +2.18.0 + diff --git a/contrib/src/ffmpeg/0004-avcodec-videotoolboxenc-fix-undefined-behavior-with-.patch b/contrib/src/ffmpeg/0004-avcodec-videotoolboxenc-fix-undefined-behavior-with-.patch new file mode 100644 index 0000000000..42f4cce1e6 --- /dev/null +++ b/contrib/src/ffmpeg/0004-avcodec-videotoolboxenc-fix-undefined-behavior-with-.patch @@ -0,0 +1,105 @@ +From 029cb11cf13275ec8536d889aae4a307eb987b2f Mon Sep 17 00:00:00 2001 +From: Thomas Guillem +Date: Tue, 3 Jul 2018 16:59:34 +0200 +Subject: [PATCH 4/4] avcodec/videotoolboxenc: fix undefined behavior with + rc_max_rate=0 + +On macOS, a zero rc_max_rate cause an error from +VTSessionSetProperty(kVTCompressionPropertyKey_DataRateLimits). + +on iOS (depending on device/version), a zero rc_max_rate cause invalid +arguments from the vtenc_output_callback after few frames and then a crash +within the VideoToolbox library. +--- + libavcodec/videotoolboxenc.c | 72 ++++++++++++++++++------------------ + 1 file changed, 37 insertions(+), 35 deletions(-) + +diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c +index f516ad7d40..c2f827b9c6 100644 +--- a/libavcodec/videotoolboxenc.c ++++ b/libavcodec/videotoolboxenc.c +@@ -1019,44 +1019,46 @@ static int vtenc_create_encoder(AVCodecContext *avctx, + + if (vtctx->codec_id == AV_CODEC_ID_H264) { + // kVTCompressionPropertyKey_DataRateLimits is not available for HEVC +- bytes_per_second_value = max_rate >> 3; +- bytes_per_second = CFNumberCreate(kCFAllocatorDefault, +- kCFNumberSInt64Type, +- &bytes_per_second_value); +- if (!bytes_per_second) { +- return AVERROR(ENOMEM); +- } +- one_second_value = 1; +- one_second = CFNumberCreate(kCFAllocatorDefault, +- kCFNumberSInt64Type, +- &one_second_value); +- if (!one_second) { +- CFRelease(bytes_per_second); +- return AVERROR(ENOMEM); +- } +- nums[0] = (void *)bytes_per_second; +- nums[1] = (void *)one_second; +- data_rate_limits = CFArrayCreate(kCFAllocatorDefault, +- (const void **)nums, +- 2, +- &kCFTypeArrayCallBacks); +- +- if (!data_rate_limits) { ++ if (max_rate > 0) { ++ bytes_per_second_value = max_rate >> 3; ++ bytes_per_second = CFNumberCreate(kCFAllocatorDefault, ++ kCFNumberSInt64Type, ++ &bytes_per_second_value); ++ if (!bytes_per_second) { ++ return AVERROR(ENOMEM); ++ } ++ one_second_value = 1; ++ one_second = CFNumberCreate(kCFAllocatorDefault, ++ kCFNumberSInt64Type, ++ &one_second_value); ++ if (!one_second) { ++ CFRelease(bytes_per_second); ++ return AVERROR(ENOMEM); ++ } ++ nums[0] = (void *)bytes_per_second; ++ nums[1] = (void *)one_second; ++ data_rate_limits = CFArrayCreate(kCFAllocatorDefault, ++ (const void **)nums, ++ 2, ++ &kCFTypeArrayCallBacks); ++ ++ if (!data_rate_limits) { ++ CFRelease(bytes_per_second); ++ CFRelease(one_second); ++ return AVERROR(ENOMEM); ++ } ++ status = VTSessionSetProperty(vtctx->session, ++ kVTCompressionPropertyKey_DataRateLimits, ++ data_rate_limits); ++ + CFRelease(bytes_per_second); + CFRelease(one_second); +- return AVERROR(ENOMEM); +- } +- status = VTSessionSetProperty(vtctx->session, +- kVTCompressionPropertyKey_DataRateLimits, +- data_rate_limits); ++ CFRelease(data_rate_limits); + +- CFRelease(bytes_per_second); +- CFRelease(one_second); +- CFRelease(data_rate_limits); +- +- if (status) { +- av_log(avctx, AV_LOG_ERROR, "Error setting max bitrate property: %d\n", status); +- return AVERROR_EXTERNAL; ++ if (status) { ++ av_log(avctx, AV_LOG_ERROR, "Error setting max bitrate property: %d\n", status); ++ return AVERROR_EXTERNAL; ++ } + } + + if (profile_level) { +-- +2.18.0 + diff --git a/contrib/src/ffmpeg/rules.mak b/contrib/src/ffmpeg/rules.mak index 7a3b678370..7c3a87068c 100644 --- a/contrib/src/ffmpeg/rules.mak +++ b/contrib/src/ffmpeg/rules.mak @@ -239,6 +239,10 @@ ifdef USE_FFMPEG $(APPLY) $(SRC)/ffmpeg/armv7_fixup.patch $(APPLY) $(SRC)/ffmpeg/dxva_vc1_crash.patch $(APPLY) $(SRC)/ffmpeg/h264_early_SAR.patch + $(APPLY) $(SRC)/ffmpeg/0001-avcodec-videotoolboxenc-fix-mutex-cond-leak-in-error.patch + $(APPLY) $(SRC)/ffmpeg/0002-avcodec-videotoolboxenc-split-initialization.patch + $(APPLY) $(SRC)/ffmpeg/0003-avcodec-videotoolboxenc-fix-invalid-session-on-iOS.patch + $(APPLY) $(SRC)/ffmpeg/0004-avcodec-videotoolboxenc-fix-undefined-behavior-with-.patch endif ifdef USE_LIBAV $(APPLY) $(SRC)/ffmpeg/libav_gsm.patch -- 2.19.1