DeviceMotion: make panning and devicemotion interact with each other

0001-DeviceMotion-use-quaternions-and-convert-them-oursel.patch

@@ -0,0 +1,123 @@
+From b124ee92f2b4ef6c44515c19d72f0cd2434ae386 Mon Sep 17 00:00:00 2001
+From: Adrien Maglo <magsoft@videolan.org>
+Date: Wed, 7 Mar 2018 13:45:01 +0100
+Subject: [PATCH] DeviceMotion: use quaternions and convert them ourself
+
+Do not use the Euler angles provided by CoreMotion as they are not defined in the same reference frame as libvlc.
+---
+ Sources/DeviceMotion.swift | 82 +++++++++++++++++++++++++++++++++++++---------
+ 1 file changed, 67 insertions(+), 15 deletions(-)
+
+diff --git a/Sources/DeviceMotion.swift b/Sources/DeviceMotion.swift
+index 4666801f..2ad61639 100644
+--- a/Sources/DeviceMotion.swift
++++ b/Sources/DeviceMotion.swift
+@@ -21,6 +21,12 @@ protocol DeviceMotionDelegate:NSObjectProtocol {
+ 
+ }
+ 
++struct EulerAngles {
++    var yaw: Double = 0
++    var pitch: Double = 0
++    var roll: Double = 0
++}
++
+ @objc(VLCDeviceMotion)
+ class DeviceMotion:NSObject {
+ 
+@@ -28,31 +34,77 @@ class DeviceMotion:NSObject {
+     var referenceAttitude:CMAttitude? = nil
+     @objc weak var delegate: DeviceMotionDelegate? = nil
+ 
++
++    func multQuaternion(q1: CMQuaternion, q2: CMQuaternion) -> CMQuaternion {
++        var ret = CMQuaternion()
++
++        ret.x = q1.x * q2.x - q1.y * q2.y - q1.z * q2.z - q1.w * q2.w
++        ret.y = q1.x * q2.y + q1.y * q2.x + q1.z * q2.w - q1.w * q2.z
++        ret.z = q1.x * q2.z + q1.z * q2.x - q1.y * q2.w + q1.w * q2.y
++        ret.w = q1.w * q2.x + q1.x * q2.w + q1.y * q2.z - q1.z * q2.y
++
++        return ret
++    }
++
++    func quaternionToEuler(qIn: CMQuaternion) -> EulerAngles {
++        // Change the axes
++        var q = CMQuaternion(x:qIn.y, y:qIn.z, z:qIn.x, w:qIn.w)
++
++        // Rotation of 90°
++        let sqrt2 = 0.707106781186548
++        let qRot = CMQuaternion(x: 0, y: 0, z: -sqrt2 / 2, w: sqrt2 / 2)
++
++        // Perform the rotation
++        q = multQuaternion(q1:qRot, q2:q)
++
++        // Now, we can perform the conversion and manage ourself the singularities
++
++        let sqx = q.x * q.x
++        let sqy = q.y * q.y
++        let sqz = q.z * q.z
++        let sqw = q.w * q.w
++
++        let unit = sqx + sqy + sqz + sqw // if normalised is one, otherwise is correction factor
++        let test = q.x * q.y + q.z * q.w
++
++        var vp = EulerAngles()
++
++        if (test > 0.499 * unit) {
++            // singularity at north pole
++            vp.yaw = 2 * atan2(q.x, q.w)
++            vp.pitch = Double.pi / 2
++            vp.roll = 0
++        } else if (test < -0.499 * unit) {
++            // singularity at south pole
++            vp.yaw = -2 * atan2(q.x, q.w)
++            vp.pitch = -Double.pi / 2
++            vp.roll = 0
++        } else {
++            vp.yaw = atan2(2 * q.y * q.w - 2 * q.x * q.z, sqx - sqy - sqz + sqw)
++            vp.pitch = asin(2 * test / unit)
++            vp.roll = atan2(2 * q.x * q.w - 2 * q.y * q.z, -sqx + sqy - sqz + sqw)
++        }
++
++        vp.yaw = -vp.yaw * 180 / Double.pi
++        vp.pitch = vp.pitch * 180 / Double.pi
++        vp.roll = vp.roll * 180 / Double.pi
++
++        return vp
++    }
++
+     @objc func startDeviceMotion() {
+ 
+         if motion.isDeviceMotionAvailable {
+             motion.gyroUpdateInterval = 1.0 / 60.0  // 60 Hz
+-            motion.startDeviceMotionUpdates(using: .xTrueNorthZVertical, to: .main) {
++            motion.startDeviceMotionUpdates(using: .xArbitraryZVertical, to: .main) {
+                 [weak self] (data, error) in
+                 guard let strongSelf = self, let data = data else {
+                     return
+                 }
+ 
+-                //We're using the initial angle of phone as 0.0.0 reference for all axis
+-                //we need to create a copy here, otherwise we just have a reference which is being changed in the next line
+-                if strongSelf.referenceAttitude == nil {
+-                    strongSelf.referenceAttitude = data.attitude.copy() as? CMAttitude
++                if let euler = self?.quaternionToEuler(qIn: data.attitude.quaternion) {
++                    strongSelf.delegate?.deviceMotionHasAttitude(deviceMotion:strongSelf, pitch:euler.pitch, yaw:euler.yaw, roll:euler.roll)
+                 }
+-                // this line basically substracts the reference attitude so that we have yaw, pitch and roll changes in
+-                // relation to the very first angle
+-                data.attitude.multiply(byInverseOf: strongSelf.referenceAttitude!)
+-
+-                let pitch = -(180/Double.pi)*data.attitude.pitch // -90; 90
+-                let yaw = -(180/Double.pi)*data.attitude.yaw // -180; 180
+-                let roll = -(180/Double.pi)*data.attitude.roll// -180; 180
+-
+-                //print(pitch,yaw,roll)
+-                strongSelf.delegate?.deviceMotionHasAttitude(deviceMotion:strongSelf, pitch:pitch, yaw:yaw, roll:roll)
+             }
+         }
+     }
+-- 
+2.13.0
+

Sources/DeviceMotion.swift

@@ -32,6 +32,8 @@ class DeviceMotion:NSObject {
 
     let motion = CMMotionManager()
     let sqrt2 = 0.5.squareRoot()
+    var lastEulerAngle = EulerAngles()
+    var lastQuaternion: CMQuaternion? = nil
 
     @objc weak var delegate: DeviceMotionDelegate? = nil
 
@@ -100,14 +102,31 @@ class DeviceMotion:NSObject {
                     return
                 }
 
-                let euler = strongSelf.quaternionToEuler(qIn: data.attitude.quaternion)
+                var euler = strongSelf.quaternionToEuler(qIn: data.attitude.quaternion)
+                if let lastQuaternion = strongSelf.lastQuaternion {
+                    let lastEuler = strongSelf.quaternionToEuler(qIn: lastQuaternion)
+                    let diffYaw = euler.yaw - lastEuler.yaw
+                    let diffPitch = euler.pitch - lastEuler.pitch
+                    let diffRoll = euler.roll - lastEuler.roll
+
+                    euler.yaw = strongSelf.lastEulerAngle.yaw + diffYaw
+                    euler.pitch = strongSelf.lastEulerAngle.pitch + diffPitch
+                    euler.pitch = strongSelf.lastEulerAngle.roll + diffRoll
+                }
                 strongSelf.delegate?.deviceMotionHasAttitude(deviceMotion:strongSelf, pitch:euler.pitch, yaw:euler.yaw, roll:euler.roll)
             }
         }
     }
 
+    @objc func lastAngle(yaw:Double, pitch:Double, roll:Double) {
+        lastEulerAngle.yaw = yaw
+        lastEulerAngle.pitch = pitch
+        lastEulerAngle.roll = roll
+    }
+
     @objc func stopDeviceMotion() {
         if motion.isDeviceMotionActive {
+            lastQuaternion = motion.deviceMotion?.attitude.quaternion
             motion.stopDeviceMotionUpdates()
         }
     }

Sources/VLCMovieViewController.m

@@ -113,7 +113,7 @@ typedef NS_ENUM(NSInteger, VLCPanType) {
     VLCDeviceMotion *_deviceMotion;
     CGFloat _fov;
     CGPoint _saveLocation;
-    CGSize _screenSizePixel;
+    CGSize _screenPixelSize;
 }
 @property (nonatomic, strong) VLCMovieViewControlPanelView *controllerPanel;
 @property (nonatomic, strong) UIPopoverController *masterPopoverController;
@@ -233,8 +233,7 @@ typedef NS_ENUM(NSInteger, VLCPanType) {
 
     CGRect screenBounds = [[UIScreen mainScreen] bounds];
     CGFloat screenScale = [[UIScreen mainScreen] scale];
-    _screenSizePixel = CGSizeMake(screenBounds.size.width * screenScale, screenBounds.size.height * screenScale);
-    _saveLocation = CGPointMake(-1.f, -1.f);
+    _screenPixelSize = CGSizeMake(screenBounds.size.width * screenScale, screenBounds.size.height * screenScale);
 
     [self setupConstraints];
 
@@ -639,7 +638,7 @@ typedef NS_ENUM(NSInteger, VLCPanType) {
 
 - (void)handlePinchGesture:(UIPinchGestureRecognizer *)recognizer
 {
-    CGFloat diff = DEFAULT_FOV * -(ZOOM_SENSITIVITY * recognizer.velocity / _screenSizePixel.width);
+    CGFloat diff = DEFAULT_FOV * -(ZOOM_SENSITIVITY * recognizer.velocity / _screenPixelSize.width);
 
     if ([_vpc currentMediaIs360Video]) {
         [self zoom360Video:diff];
@@ -651,7 +650,7 @@ typedef NS_ENUM(NSInteger, VLCPanType) {
 - (void)zoom360Video:(CGFloat)zoom
 {
     if ([_vpc updateViewpoint:0 pitch:0 roll:0 fov:zoom absolute:NO]) {
-        //Checking for fov value in case of
+        //clamp Fov between min and max fov
         _fov = MAX(MIN(_fov + zoom, MAX_FOV), MIN_FOV);
     }
 }
@@ -861,7 +860,9 @@ typedef NS_ENUM(NSInteger, VLCPanType) {
 
 - (void)deviceMotionHasAttitudeWithDeviceMotion:(VLCDeviceMotion *)deviceMotion pitch:(double)pitch yaw:(double)yaw roll:(double)roll
 {
-    [_vpc updateViewpoint:yaw pitch:pitch roll:roll fov:_fov absolute:YES];
+    if (_panRecognizer.state != UIGestureRecognizerStateChanged || UIGestureRecognizerStateBegan) {
+        [_vpc updateViewpoint:yaw pitch:pitch roll:roll fov:_fov absolute:YES];
+    }
 }
 #pragma mark - controls
 
@@ -1349,80 +1350,98 @@ currentMediaHasTrackToChooseFrom:(BOOL)currentMediaHasTrackToChooseFrom
 {
     CGFloat panDirectionX = [panRecognizer velocityInView:self.view].x;
     CGFloat panDirectionY = [panRecognizer velocityInView:self.view].y;
+    if (panRecognizer.state == UIGestureRecognizerStateBegan) {
+        _currentPanType = [self detectPanTypeForPan:panRecognizer];
+        if ([_vpc currentMediaIs360Video]) {
+            _saveLocation =  [panRecognizer locationInView:self.view];
+            [_deviceMotion stopDeviceMotion];
+        }
+    }
+    
+    switch (_currentPanType) {
+        case VLCPanTypeSeek: {
+            if (!_seekGestureEnabled)
+                return;
+            double timeRemainingDouble = (-[_vpc remainingTime].intValue*0.001);
+            int timeRemaining = timeRemainingDouble;
+
+            if (panDirectionX > 0) {
+                if (timeRemaining > 2 ) // to not go outside duration , video will stop
+                    [_vpc jumpForward:1];
+            } else
+                [_vpc jumpBackward:1];
 
-    if (_currentPanType == VLCPanTypeSeek) {
-        if (!_seekGestureEnabled)
-            return;
-        double timeRemainingDouble = (-[_vpc remainingTime].intValue*0.001);
-        int timeRemaining = timeRemainingDouble;
+            break;
+        case VLCPanTypeVolume:
 
-        if (panDirectionX > 0) {
-            if (timeRemaining > 2 ) // to not go outside duration , video will stop
-                [_vpc jumpForward:1];
-        } else
-            [_vpc jumpBackward:1];
-    } else if (_currentPanType == VLCPanTypeVolume) {
-        if (!_volumeGestureEnabled)
-            return;
-        MPMusicPlayerController *musicPlayer = [MPMusicPlayerController applicationMusicPlayer];
+            if (!_volumeGestureEnabled)
+                return;
+            MPMusicPlayerController *musicPlayer = [MPMusicPlayerController applicationMusicPlayer];
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wdeprecated"
-        // there is no replacement for .volume which we want to use since Apple's susggestion is to not use their overlays
-        // but switch to the volume slider exclusively. meh.
-        if (panDirectionY > 0)
-            musicPlayer.volume -= 0.01;
-        else
-            musicPlayer.volume += 0.01;
+            // there is no replacement for .volume which we want to use since Apple's susggestion is to not use their overlays
+            // but switch to the volume slider exclusively. meh.
+            if (panDirectionY > 0)
+                musicPlayer.volume -= 0.01;
+            else
+                musicPlayer.volume += 0.01;
 #pragma clang diagnostic pop
-    } else if (_currentPanType == VLCPanTypeBrightness) {
-        if (!_brightnessGestureEnabled)
-            return;
-        CGFloat brightness = [UIScreen mainScreen].brightness;
-
-        if (panDirectionY > 0)
-            brightness = brightness - 0.01;
-        else
-            brightness = brightness + 0.01;
-
-        // Sanity check since -[UIScreen brightness] does not go by 0.01 steps
-        if (brightness > 1.0)
-            brightness = 1.0;
-        else if (brightness < 0.0)
-            brightness = 0.0;
-
-        NSAssert(brightness >= 0 && brightness <= 1, @"Brightness must be within 0 and 1 (it is %f)", brightness);
-
-        [[UIScreen mainScreen] setBrightness:brightness];
-
-        NSString *brightnessHUD = [NSString stringWithFormat:@"%@: %@ %%", NSLocalizedString(@"VFILTER_BRIGHTNESS", nil), [[[NSString stringWithFormat:@"%f",(brightness*100)] componentsSeparatedByString:@"."] objectAtIndex:0]];
-        [self.statusLabel showStatusMessage:brightnessHUD];
-    } else if (_currentPanType == VLCPanTypeProjection) {
-
-        CGPoint tmp = [panRecognizer locationInView:self.view];
-        CGFloat changeX = 0.f;
-        CGFloat changeY = 0.f;
+        } break;
+        case VLCPanTypeBrightness: {
+            if (!_brightnessGestureEnabled)
+                return;
+            CGFloat brightness = [UIScreen mainScreen].brightness;
+
+            if (panDirectionY > 0)
+                brightness = brightness - 0.01;
+            else
+                brightness = brightness + 0.01;
+
+            // Sanity check since -[UIScreen brightness] does not go by 0.01 steps
+            if (brightness > 1.0)
+                brightness = 1.0;
+            else if (brightness < 0.0)
+                brightness = 0.0;
+
+            NSAssert(brightness >= 0 && brightness <= 1, @"Brightness must be within 0 and 1 (it is %f)", brightness);
+
+            [[UIScreen mainScreen] setBrightness:brightness];
+
+            NSString *brightnessHUD = [NSString stringWithFormat:@"%@: %@ %%", NSLocalizedString(@"VFILTER_BRIGHTNESS", nil), [[[NSString stringWithFormat:@"%f",(brightness*100)] componentsSeparatedByString:@"."] objectAtIndex:0]];
+            [self.statusLabel showStatusMessage:brightnessHUD];
+        } break;
+        case VLCPanTypeProjection: {
+            [self updateProjectionWithPanRecognizer:panRecognizer];
+        } break;
+        case VLCPanTypeNone: {
+        } break;
+    }
 
-        if (_saveLocation.x != -1.f && _saveLocation.y != -1.f) {
-            changeX = tmp.x - _saveLocation.x;
-            changeY = tmp.y - _saveLocation.y;
+    if (panRecognizer.state == UIGestureRecognizerStateEnded) {
+        _currentPanType = VLCPanTypeNone;
+        if ([_vpc currentMediaIs360Video]) {
+            [_deviceMotion lastAngleWithYaw:_vpc.yaw pitch:_vpc.pitch roll:_vpc.roll];
+            [_deviceMotion startDeviceMotion];
         }
+    }
+}
 
-        _saveLocation = [panRecognizer locationInView:self.view];
-
-        //screenSizePixel width is used twice to get a constant speed on the movement.
-        CGFloat yaw = _fov * -changeX / _screenSizePixel.width;
-        CGFloat pitch = _fov * -changeY / _screenSizePixel.width;
+- (void)updateProjectionWithPanRecognizer:(UIPanGestureRecognizer *)panGestureRecognizer
+{
+    CGPoint newLocationInView = [panGestureRecognizer locationInView:self.view];
+    
+    CGFloat diffX = newLocationInView.x - _saveLocation.x;
+    CGFloat diffY = newLocationInView.y - _saveLocation.y;
+    _saveLocation = newLocationInView;
 
-        [_vpc updateViewpoint:yaw pitch:pitch roll:0 fov:0 absolute:NO];
-    }
+    //screenSizePixel width is used twice to get a constant speed on the movement.
+    CGFloat yaw = _fov * -diffX / _screenPixelSize.width;
+    CGFloat pitch = _fov * -diffY / _screenPixelSize.width;
 
-    if (panRecognizer.state == UIGestureRecognizerStateEnded) {
-        _currentPanType = VLCPanTypeNone;
+    CGFloat newYaw = _vpc.yaw + yaw;
+    CGFloat newPitch = _vpc.pitch + pitch;
 
-        //Invalidate saved location when the gesture is ended
-        if ([_vpc currentMediaIs360Video])
-            _saveLocation = CGPointMake(-1.f, -1.f);
-    }
+    [_vpc updateViewpoint:newYaw pitch:newPitch roll:_vpc.roll fov:_vpc.fov absolute:YES];
 }
 
 - (void)swipeRecognized:(UISwipeGestureRecognizer*)swipeRecognizer

Sources/VLCPlaybackController.h

@@ -71,6 +71,11 @@ currentMediaHasTrackToChooseFrom:(BOOL)currentMediaHasTrackToChooseFrom
 @property (nonatomic, readwrite) float saturation; // default = 1.0
 @property (nonatomic, readwrite) float gamma; // default = 1.0
 
+@property (nonatomic, readonly) CGFloat yaw; //  between ]-180;180]
+@property (nonatomic, readonly) CGFloat pitch; // ]-90;90]
+@property (nonatomic, readonly) CGFloat roll; // ]-180;180]
+@property (nonatomic, readonly) CGFloat fov; // ]0;180[ (default 80.)
+
 @property (readonly) NSInteger indexOfCurrentAudioTrack;
 @property (readonly) NSInteger indexOfCurrentSubtitleTrack;
 @property (readonly) NSInteger indexOfCurrentTitle;

Sources/VLCPlaybackController.m

@@ -967,9 +967,30 @@ typedef NS_ENUM(NSUInteger, VLCAspectRatio) {
 #if !TARGET_OS_TV
 - (BOOL)updateViewpoint:(CGFloat)yaw pitch:(CGFloat)pitch roll:(CGFloat)roll fov:(CGFloat)fov absolute:(BOOL)absolute
 {
+    NSLog(@"update with yaw:%f pitch:%f roll:%f, fov:%f", yaw, pitch, roll, fov);
     return [_mediaPlayer updateViewpoint:yaw pitch:pitch roll:roll fov:fov absolute:absolute];
 }
 
+- (CGFloat)yaw
+{
+    return _mediaPlayer.yaw;
+}
+
+- (CGFloat)pitch
+{
+    return _mediaPlayer.pitch;
+}
+
+- (CGFloat)roll
+{
+    return _mediaPlayer.roll;
+}
+
+- (CGFloat)fov
+{
+    return _mediaPlayer.fov;
+}
+
 - (BOOL)currentMediaIs360Video
 {
     return [self currentMediaProjection] == VLCMediaProjectionEquiRectangular;