VLCKit/Resources/MobileVLCKit/patches/0018-ffmpeg-backport-vtenc-patches.patch

From 87cd7de678ddb793468bd64cbdd0a73158bcc524 Mon Sep 17 00:00:00 2001
From: Thomas Guillem <thomas@gllm.fr>
Date: Mon, 18 Jun 2018 12:31:13 +0200
Subject: [PATCH 18/24] ffmpeg: backport vtenc patches

---
 ...lboxenc-fix-mutex-cond-leak-in-error.patch |  35 ++++++
 ...videotoolboxenc-split-initialization.patch | 112 ++++++++++++++++++
 ...oolboxenc-fix-invalid-session-on-iOS.patch |  48 ++++++++
 ...lboxenc-fix-undefined-behavior-with-.patch | 105 ++++++++++++++++
 contrib/src/ffmpeg/rules.mak                  |   4 +
 5 files changed, 304 insertions(+)
 create mode 100644 contrib/src/ffmpeg/0001-avcodec-videotoolboxenc-fix-mutex-cond-leak-in-error.patch
 create mode 100644 contrib/src/ffmpeg/0002-avcodec-videotoolboxenc-split-initialization.patch
 create mode 100644 contrib/src/ffmpeg/0003-avcodec-videotoolboxenc-fix-invalid-session-on-iOS.patch
 create mode 100644 contrib/src/ffmpeg/0004-avcodec-videotoolboxenc-fix-undefined-behavior-with-.patch

diff --git a/contrib/src/ffmpeg/0001-avcodec-videotoolboxenc-fix-mutex-cond-leak-in-error.patch b/contrib/src/ffmpeg/0001-avcodec-videotoolboxenc-fix-mutex-cond-leak-in-error.patch
new file mode 100644
index 0000000000..679bd70da3
--- /dev/null
+++ b/contrib/src/ffmpeg/0001-avcodec-videotoolboxenc-fix-mutex-cond-leak-in-error.patch
@@ -0,0 +1,35 @@
+From dea72b148a1250808c625dda0077e43d11689845 Mon Sep 17 00:00:00 2001
+From: Thomas Guillem <thomas@gllm.fr>
+Date: Tue, 29 May 2018 18:04:38 +0200
+Subject: [PATCH 1/4] avcodec/videotoolboxenc: fix mutex/cond leak in error
+ path
+
+The leak could happen when the vtenc_create_encoder() function failed.
+---
+ libavcodec/videotoolboxenc.c | 5 +++--
+ 1 file changed, 3 insertions(+), 2 deletions(-)
+
+diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c
+index 086beb41fc..aafef20db0 100644
+--- a/libavcodec/videotoolboxenc.c
++++ b/libavcodec/videotoolboxenc.c
+@@ -2473,13 +2473,14 @@ static av_cold int vtenc_close(AVCodecContext *avctx)
+ {
+     VTEncContext *vtctx = avctx->priv_data;
+ 
++    pthread_cond_destroy(&vtctx->cv_sample_sent);
++    pthread_mutex_destroy(&vtctx->lock);
++
+     if(!vtctx->session) return 0;
+ 
+     VTCompressionSessionCompleteFrames(vtctx->session,
+                                        kCMTimeIndefinite);
+     clear_frame_queue(vtctx);
+-    pthread_cond_destroy(&vtctx->cv_sample_sent);
+-    pthread_mutex_destroy(&vtctx->lock);
+     CFRelease(vtctx->session);
+     vtctx->session = NULL;
+ 
+-- 
+2.18.0
+
diff --git a/contrib/src/ffmpeg/0002-avcodec-videotoolboxenc-split-initialization.patch b/contrib/src/ffmpeg/0002-avcodec-videotoolboxenc-split-initialization.patch
new file mode 100644
index 0000000000..c64aa7521a
--- /dev/null
+++ b/contrib/src/ffmpeg/0002-avcodec-videotoolboxenc-split-initialization.patch
@@ -0,0 +1,112 @@
+From 0a83285f54c5c042104a4d2a03c40e8f69a09a3c Mon Sep 17 00:00:00 2001
+From: Thomas Guillem <thomas@gllm.fr>
+Date: Mon, 11 Jun 2018 15:43:56 +0200
+Subject: [PATCH 2/4] avcodec/videotoolboxenc: split initialization
+
+Split vtenc_init() into vtenc_init() (VTEncContext initialization) and
+vtenc_configure_encoder() (creates the vt session).
+
+This commit will allow to restart the vt session while encoding.
+---
+ libavcodec/videotoolboxenc.c | 48 +++++++++++++++++++++---------------
+ 1 file changed, 28 insertions(+), 20 deletions(-)
+
+diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c
+index aafef20db0..0b47a0abac 100644
+--- a/libavcodec/videotoolboxenc.c
++++ b/libavcodec/videotoolboxenc.c
+@@ -1262,19 +1262,16 @@ static int vtenc_create_encoder(AVCodecContext   *avctx,
+     return 0;
+ }
+ 
+-static av_cold int vtenc_init(AVCodecContext *avctx)
++static int vtenc_configure_encoder(AVCodecContext *avctx)
+ {
+     CFMutableDictionaryRef enc_info;
+     CFMutableDictionaryRef pixel_buffer_info;
+     CMVideoCodecType       codec_type;
+     VTEncContext           *vtctx = avctx->priv_data;
+     CFStringRef            profile_level;
+-    CFBooleanRef           has_b_frames_cfbool;
+     CFNumberRef            gamma_level = NULL;
+     int                    status;
+ 
+-    pthread_once(&once_ctrl, loadVTEncSymbols);
+-
+     codec_type = get_cm_codec_type(avctx->codec_id);
+     if (!codec_type) {
+         av_log(avctx, AV_LOG_ERROR, "Error: no mapping for AVCodecID %d\n", avctx->codec_id);
+@@ -1304,8 +1301,6 @@ static av_cold int vtenc_init(AVCodecContext *avctx)
+         if (!get_vt_hevc_profile_level(avctx, &profile_level)) return AVERROR(EINVAL);
+     }
+ 
+-    vtctx->session = NULL;
+-
+     enc_info = CFDictionaryCreateMutable(
+         kCFAllocatorDefault,
+         20,
+@@ -1335,8 +1330,6 @@ static av_cold int vtenc_init(AVCodecContext *avctx)
+         pixel_buffer_info = NULL;
+     }
+ 
+-    pthread_mutex_init(&vtctx->lock, NULL);
+-    pthread_cond_init(&vtctx->cv_sample_sent, NULL);
+     vtctx->dts_delta = vtctx->has_b_frames ? -1 : 0;
+ 
+     get_cv_transfer_function(avctx, &vtctx->transfer_function, &gamma_level);
+@@ -1363,8 +1356,32 @@ static av_cold int vtenc_init(AVCodecContext *avctx)
+                                   pixel_buffer_info,
+                                   &vtctx->session);
+ 
+-    if (status < 0)
+-        goto init_cleanup;
++init_cleanup:
++    if (gamma_level)
++        CFRelease(gamma_level);
++
++    if (pixel_buffer_info)
++        CFRelease(pixel_buffer_info);
++
++    CFRelease(enc_info);
++
++    return status;
++}
++
++static av_cold int vtenc_init(AVCodecContext *avctx)
++{
++    VTEncContext    *vtctx = avctx->priv_data;
++    CFBooleanRef    has_b_frames_cfbool;
++    int             status;
++
++    pthread_once(&once_ctrl, loadVTEncSymbols);
++
++    pthread_mutex_init(&vtctx->lock, NULL);
++    pthread_cond_init(&vtctx->cv_sample_sent, NULL);
++
++    vtctx->session = NULL;
++    status = vtenc_configure_encoder(avctx);
++    if (status) return status;
+ 
+     status = VTSessionCopyProperty(vtctx->session,
+                                    kVTCompressionPropertyKey_AllowFrameReordering,
+@@ -1378,16 +1395,7 @@ static av_cold int vtenc_init(AVCodecContext *avctx)
+     }
+     avctx->has_b_frames = vtctx->has_b_frames;
+ 
+-init_cleanup:
+-    if (gamma_level)
+-        CFRelease(gamma_level);
+-
+-    if (pixel_buffer_info)
+-        CFRelease(pixel_buffer_info);
+-
+-    CFRelease(enc_info);
+-
+-    return status;
++    return 0;
+ }
+ 
+ static void vtenc_get_frame_info(CMSampleBufferRef buffer, bool *is_key_frame)
+-- 
+2.18.0
+
diff --git a/contrib/src/ffmpeg/0003-avcodec-videotoolboxenc-fix-invalid-session-on-iOS.patch b/contrib/src/ffmpeg/0003-avcodec-videotoolboxenc-fix-invalid-session-on-iOS.patch
new file mode 100644
index 0000000000..9138060be1
--- /dev/null
+++ b/contrib/src/ffmpeg/0003-avcodec-videotoolboxenc-fix-invalid-session-on-iOS.patch
@@ -0,0 +1,48 @@
+From 72812a3931cda0fcb66d740bbb19c330e97ccbd0 Mon Sep 17 00:00:00 2001
+From: Thomas Guillem <thomas@gllm.fr>
+Date: Mon, 11 Jun 2018 16:17:28 +0200
+Subject: [PATCH 3/4] avcodec/videotoolboxenc: fix invalid session on iOS
+
+Cf. comment. Restart the VT session when the APP goes from foreground to
+background and vice versa.
+---
+ libavcodec/videotoolboxenc.c | 23 +++++++++++++++++++++--
+ 1 file changed, 21 insertions(+), 2 deletions(-)
+
+diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c
+index 0b47a0abac..f516ad7d40 100644
+--- a/libavcodec/videotoolboxenc.c
++++ b/libavcodec/videotoolboxenc.c
+@@ -2175,8 +2175,27 @@ static int create_cv_pixel_buffer(AVCodecContext   *avctx,
+ #if TARGET_OS_IPHONE
+     pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
+     if (!pix_buf_pool) {
+-        av_log(avctx, AV_LOG_ERROR, "Could not get pixel buffer pool.\n");
+-        return AVERROR_EXTERNAL;
++        /* On iOS, the VT session is invalidated when the APP switches from
++         * foreground to background and vice versa. Fetch the actual error code
++         * of the VT session to detect that case and restart the VT session
++         * accordingly. */
++        OSStatus vtstatus;
++
++        vtstatus = VTCompressionSessionPrepareToEncodeFrames(vtctx->session);
++        if (vtstatus == kVTInvalidSessionErr) {
++            CFRelease(vtctx->session);
++            vtctx->session = NULL;
++            status = vtenc_configure_encoder(avctx);
++            if (status == 0)
++                pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
++        }
++        if (!pix_buf_pool) {
++            av_log(avctx, AV_LOG_ERROR, "Could not get pixel buffer pool.\n");
++            return AVERROR_EXTERNAL;
++        }
++        else
++            av_log(avctx, AV_LOG_WARNING, "VT session restarted because of a "
++                   "kVTInvalidSessionErr error.\n");
+     }
+ 
+     status = CVPixelBufferPoolCreatePixelBuffer(NULL,
+-- 
+2.18.0
+
diff --git a/contrib/src/ffmpeg/0004-avcodec-videotoolboxenc-fix-undefined-behavior-with-.patch b/contrib/src/ffmpeg/0004-avcodec-videotoolboxenc-fix-undefined-behavior-with-.patch
new file mode 100644
index 0000000000..42f4cce1e6
--- /dev/null
+++ b/contrib/src/ffmpeg/0004-avcodec-videotoolboxenc-fix-undefined-behavior-with-.patch
@@ -0,0 +1,105 @@
+From 029cb11cf13275ec8536d889aae4a307eb987b2f Mon Sep 17 00:00:00 2001
+From: Thomas Guillem <thomas@gllm.fr>
+Date: Tue, 3 Jul 2018 16:59:34 +0200
+Subject: [PATCH 4/4] avcodec/videotoolboxenc: fix undefined behavior with
+ rc_max_rate=0
+
+On macOS, a zero rc_max_rate cause an error from
+VTSessionSetProperty(kVTCompressionPropertyKey_DataRateLimits).
+
+on iOS (depending on device/version), a zero rc_max_rate cause invalid
+arguments from the vtenc_output_callback after few frames and then a crash
+within the VideoToolbox library.
+---
+ libavcodec/videotoolboxenc.c | 72 ++++++++++++++++++------------------
+ 1 file changed, 37 insertions(+), 35 deletions(-)
+
+diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c
+index f516ad7d40..c2f827b9c6 100644
+--- a/libavcodec/videotoolboxenc.c
++++ b/libavcodec/videotoolboxenc.c
+@@ -1019,44 +1019,46 @@ static int vtenc_create_encoder(AVCodecContext   *avctx,
+ 
+     if (vtctx->codec_id == AV_CODEC_ID_H264) {
+         // kVTCompressionPropertyKey_DataRateLimits is not available for HEVC
+-        bytes_per_second_value = max_rate >> 3;
+-        bytes_per_second = CFNumberCreate(kCFAllocatorDefault,
+-                                          kCFNumberSInt64Type,
+-                                          &bytes_per_second_value);
+-        if (!bytes_per_second) {
+-            return AVERROR(ENOMEM);
+-        }
+-        one_second_value = 1;
+-        one_second = CFNumberCreate(kCFAllocatorDefault,
+-                                    kCFNumberSInt64Type,
+-                                    &one_second_value);
+-        if (!one_second) {
+-            CFRelease(bytes_per_second);
+-            return AVERROR(ENOMEM);
+-        }
+-        nums[0] = (void *)bytes_per_second;
+-        nums[1] = (void *)one_second;
+-        data_rate_limits = CFArrayCreate(kCFAllocatorDefault,
+-                                         (const void **)nums,
+-                                         2,
+-                                         &kCFTypeArrayCallBacks);
+-
+-        if (!data_rate_limits) {
++        if (max_rate > 0) {
++            bytes_per_second_value = max_rate >> 3;
++            bytes_per_second = CFNumberCreate(kCFAllocatorDefault,
++                                              kCFNumberSInt64Type,
++                                              &bytes_per_second_value);
++            if (!bytes_per_second) {
++                return AVERROR(ENOMEM);
++            }
++            one_second_value = 1;
++            one_second = CFNumberCreate(kCFAllocatorDefault,
++                                        kCFNumberSInt64Type,
++                                        &one_second_value);
++            if (!one_second) {
++                CFRelease(bytes_per_second);
++                return AVERROR(ENOMEM);
++            }
++            nums[0] = (void *)bytes_per_second;
++            nums[1] = (void *)one_second;
++            data_rate_limits = CFArrayCreate(kCFAllocatorDefault,
++                                             (const void **)nums,
++                                             2,
++                                             &kCFTypeArrayCallBacks);
++
++            if (!data_rate_limits) {
++                CFRelease(bytes_per_second);
++                CFRelease(one_second);
++                return AVERROR(ENOMEM);
++            }
++            status = VTSessionSetProperty(vtctx->session,
++                                          kVTCompressionPropertyKey_DataRateLimits,
++                                          data_rate_limits);
++
+             CFRelease(bytes_per_second);
+             CFRelease(one_second);
+-            return AVERROR(ENOMEM);
+-        }
+-        status = VTSessionSetProperty(vtctx->session,
+-                                      kVTCompressionPropertyKey_DataRateLimits,
+-                                      data_rate_limits);
++            CFRelease(data_rate_limits);
+ 
+-        CFRelease(bytes_per_second);
+-        CFRelease(one_second);
+-        CFRelease(data_rate_limits);
+-
+-        if (status) {
+-            av_log(avctx, AV_LOG_ERROR, "Error setting max bitrate property: %d\n", status);
+-            return AVERROR_EXTERNAL;
++            if (status) {
++                av_log(avctx, AV_LOG_ERROR, "Error setting max bitrate property: %d\n", status);
++                return AVERROR_EXTERNAL;
++            }
+         }
+ 
+         if (profile_level) {
+-- 
+2.18.0
+
diff --git a/contrib/src/ffmpeg/rules.mak b/contrib/src/ffmpeg/rules.mak
index 7a3b678370..7c3a87068c 100644
--- a/contrib/src/ffmpeg/rules.mak
+++ b/contrib/src/ffmpeg/rules.mak
@@ -239,6 +239,10 @@ ifdef USE_FFMPEG
 	$(APPLY) $(SRC)/ffmpeg/armv7_fixup.patch
 	$(APPLY) $(SRC)/ffmpeg/dxva_vc1_crash.patch
 	$(APPLY) $(SRC)/ffmpeg/h264_early_SAR.patch
+	$(APPLY) $(SRC)/ffmpeg/0001-avcodec-videotoolboxenc-fix-mutex-cond-leak-in-error.patch
+	$(APPLY) $(SRC)/ffmpeg/0002-avcodec-videotoolboxenc-split-initialization.patch
+	$(APPLY) $(SRC)/ffmpeg/0003-avcodec-videotoolboxenc-fix-invalid-session-on-iOS.patch
+	$(APPLY) $(SRC)/ffmpeg/0004-avcodec-videotoolboxenc-fix-undefined-behavior-with-.patch
 endif
 ifdef USE_LIBAV
 	$(APPLY) $(SRC)/ffmpeg/libav_gsm.patch
-- 
2.19.1