From 8bcc38c24e25d27697235a93d046e78320075276 Mon Sep 17 00:00:00 2001 From: Ilya Laktyushin Date: Mon, 15 Jan 2024 02:26:12 +0400 Subject: [PATCH] Video message recording improvements --- submodules/Camera/Sources/Camera.swift | 15 ++-- submodules/Camera/Sources/CameraInput.swift | 5 +- submodules/Camera/Sources/CameraOutput.swift | 61 ++++++++++++-- .../Camera/Sources/CameraPreviewView.swift | 40 +++++++--- submodules/Camera/Sources/VideoRecorder.swift | 4 +- .../Sources/ThemeGridControllerNode.swift | 20 +++-- .../Sources/VideoMessageCameraScreen.swift | 80 +++++++++++++++---- .../Sources/ChatControllerNode.swift | 3 + 8 files changed, 182 insertions(+), 46 deletions(-) diff --git a/submodules/Camera/Sources/Camera.swift b/submodules/Camera/Sources/Camera.swift index 5f831246e17..55c6816e6a1 100644 --- a/submodules/Camera/Sources/Camera.swift +++ b/submodules/Camera/Sources/Camera.swift @@ -74,12 +74,12 @@ final class CameraDeviceContext { self.device.resetZoom(neutral: self.exclusive || !self.additional) } - func invalidate() { + func invalidate(switchAudio: Bool = true) { guard let session = self.session else { return } - self.output.invalidate(for: session) - self.input.invalidate(for: session) + self.output.invalidate(for: session, switchAudio: switchAudio) + self.input.invalidate(for: session, switchAudio: switchAudio) } private func maxDimensions(additional: Bool, preferWide: Bool) -> CMVideoDimensions { @@ -248,7 +248,8 @@ private final class CameraContext { mainDeviceContext.output.markPositionChange(position: targetPosition) } else { self.configure { - self.mainDeviceContext?.invalidate() + let isRoundVideo = self.initialConfiguration.isRoundVideo + self.mainDeviceContext?.invalidate(switchAudio: !isRoundVideo) let targetPosition: Camera.Position if case .back = mainDeviceContext.device.position { @@ -260,8 +261,8 @@ private final class CameraContext { self._positionPromise.set(targetPosition) self.modeChange = .position - let isRoundVideo = self.initialConfiguration.isRoundVideo - let preferWide = self.initialConfiguration.preferWide || (self.positionValue == .front && isRoundVideo) + + let preferWide = self.initialConfiguration.preferWide || isRoundVideo let preferLowerFramerate = self.initialConfiguration.preferLowerFramerate || isRoundVideo mainDeviceContext.configure(position: targetPosition, previewView: self.simplePreviewView, audio: self.initialConfiguration.audio, photo: self.initialConfiguration.photo, metadata: self.initialConfiguration.metadata, preferWide: preferWide, preferLowerFramerate: preferLowerFramerate, switchAudio: !isRoundVideo) @@ -352,7 +353,7 @@ private final class CameraContext { self.additionalDeviceContext?.invalidate() self.additionalDeviceContext = nil - let preferWide = self.initialConfiguration.preferWide || (self.positionValue == .front && self.initialConfiguration.isRoundVideo) + let preferWide = self.initialConfiguration.preferWide || self.initialConfiguration.isRoundVideo let preferLowerFramerate = self.initialConfiguration.preferLowerFramerate || self.initialConfiguration.isRoundVideo self.mainDeviceContext = CameraDeviceContext(session: self.session, exclusive: true, additional: false, ciContext: self.ciContext, use32BGRA: self.initialConfiguration.isRoundVideo) diff --git a/submodules/Camera/Sources/CameraInput.swift b/submodules/Camera/Sources/CameraInput.swift index c058c2127bd..29adeca209d 100644 --- a/submodules/Camera/Sources/CameraInput.swift +++ b/submodules/Camera/Sources/CameraInput.swift @@ -14,8 +14,11 @@ class CameraInput { } } - func invalidate(for session: CameraSession) { + func invalidate(for session: CameraSession, switchAudio: Bool = true) { for input in session.session.inputs { + if !switchAudio && input === self.audioInput { + continue + } session.session.removeInput(input) } } diff --git a/submodules/Camera/Sources/CameraOutput.swift b/submodules/Camera/Sources/CameraOutput.swift index 483a7fead16..ff09574e624 100644 --- a/submodules/Camera/Sources/CameraOutput.swift +++ b/submodules/Camera/Sources/CameraOutput.swift @@ -190,7 +190,7 @@ final class CameraOutput: NSObject { } } - func invalidate(for session: CameraSession) { + func invalidate(for session: CameraSession, switchAudio: Bool = true) { if #available(iOS 13.0, *) { if let previewConnection = self.previewConnection { if session.session.connections.contains(where: { $0 === previewConnection }) { @@ -214,7 +214,7 @@ final class CameraOutput: NSObject { if session.session.outputs.contains(where: { $0 === self.videoOutput }) { session.session.removeOutput(self.videoOutput) } - if session.session.outputs.contains(where: { $0 === self.audioOutput }) { + if switchAudio, session.session.outputs.contains(where: { $0 === self.audioOutput }) { session.session.removeOutput(self.audioOutput) } if session.session.outputs.contains(where: { $0 === self.photoOutput }) { @@ -409,6 +409,14 @@ final class CameraOutput: NSObject { private weak var masterOutput: CameraOutput? private var lastSampleTimestamp: CMTime? + + private var needsCrossfadeTransition = false + private var crossfadeTransitionStart: Double = 0.0 + + private var needsSwitchSampleOffset = false + private var lastAudioSampleTime: CMTime? + private var videoSwitchSampleTimeOffset: CMTime? + func processVideoRecording(_ sampleBuffer: CMSampleBuffer, fromAdditionalOutput: Bool) { guard let formatDescriptor = CMSampleBufferGetFormatDescription(sampleBuffer) else { return @@ -417,10 +425,10 @@ final class CameraOutput: NSObject { if let videoRecorder = self.videoRecorder, videoRecorder.isRecording { if case .roundVideo = self.currentMode, type == kCMMediaType_Video { + let currentTimestamp = CACurrentMediaTime() + let duration: Double = 0.2 if !self.exclusive { var transitionFactor: CGFloat = 0.0 - let currentTimestamp = CACurrentMediaTime() - let duration: Double = 0.2 if case .front = self.currentPosition { transitionFactor = 1.0 if self.lastSwitchTimestamp > 0.0, currentTimestamp - self.lastSwitchTimestamp < duration { @@ -446,13 +454,51 @@ final class CameraOutput: NSObject { videoRecorder.appendSampleBuffer(sampleBuffer) } } else { - if let processedSampleBuffer = self.processRoundVideoSampleBuffer(sampleBuffer, additional: self.currentPosition == .front, transitionFactor: self.currentPosition == .front ? 1.0 : 0.0) { + var additional = self.currentPosition == .front + var transitionFactor = self.currentPosition == .front ? 1.0 : 0.0 + if self.lastSwitchTimestamp > 0.0 { + if self.needsCrossfadeTransition { + self.needsCrossfadeTransition = false + self.crossfadeTransitionStart = currentTimestamp + 0.03 + self.needsSwitchSampleOffset = true + } + if self.crossfadeTransitionStart > 0.0, currentTimestamp - self.crossfadeTransitionStart < duration { + if case .front = self.currentPosition { + transitionFactor = max(0.0, (currentTimestamp - self.crossfadeTransitionStart) / duration) + } else { + transitionFactor = 1.0 - max(0.0, (currentTimestamp - self.crossfadeTransitionStart) / duration) + } + } else if currentTimestamp - self.lastSwitchTimestamp < 0.05 { + additional = !additional + transitionFactor = 1.0 - transitionFactor + self.needsCrossfadeTransition = true + } + } + if let processedSampleBuffer = self.processRoundVideoSampleBuffer(sampleBuffer, additional: additional, transitionFactor: transitionFactor) { videoRecorder.appendSampleBuffer(processedSampleBuffer) } else { videoRecorder.appendSampleBuffer(sampleBuffer) } } } else { + if type == kCMMediaType_Audio { + if self.needsSwitchSampleOffset { + self.needsSwitchSampleOffset = false + + if let lastAudioSampleTime = self.lastAudioSampleTime { + let videoSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + let offset = videoSampleTime - lastAudioSampleTime + if let current = self.videoSwitchSampleTimeOffset { + self.videoSwitchSampleTimeOffset = current + offset + } else { + self.videoSwitchSampleTimeOffset = offset + } + self.lastAudioSampleTime = nil + } + } + + self.lastAudioSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + CMSampleBufferGetDuration(sampleBuffer) + } videoRecorder.appendSampleBuffer(sampleBuffer) } } @@ -494,6 +540,11 @@ final class CameraOutput: NSObject { var sampleTimingInfo: CMSampleTimingInfo = .invalid CMSampleBufferGetSampleTimingInfo(sampleBuffer, at: 0, timingInfoOut: &sampleTimingInfo) + if let videoSwitchSampleTimeOffset = self.videoSwitchSampleTimeOffset { + sampleTimingInfo.decodeTimeStamp = sampleTimingInfo.decodeTimeStamp - videoSwitchSampleTimeOffset + sampleTimingInfo.presentationTimeStamp = sampleTimingInfo.presentationTimeStamp - videoSwitchSampleTimeOffset + } + var newSampleBuffer: CMSampleBuffer? status = CMSampleBufferCreateForImageBuffer( allocator: kCFAllocatorDefault, diff --git a/submodules/Camera/Sources/CameraPreviewView.swift b/submodules/Camera/Sources/CameraPreviewView.swift index 73046cbe8dd..54a3666eb97 100644 --- a/submodules/Camera/Sources/CameraPreviewView.swift +++ b/submodules/Camera/Sources/CameraPreviewView.swift @@ -21,6 +21,29 @@ private extension UIInterfaceOrientation { } } +private class SimpleCapturePreviewLayer: AVCaptureVideoPreviewLayer { + public var didEnterHierarchy: (() -> Void)? + public var didExitHierarchy: (() -> Void)? + + override open func action(forKey event: String) -> CAAction? { + if event == kCAOnOrderIn { + self.didEnterHierarchy?() + } else if event == kCAOnOrderOut { + self.didExitHierarchy?() + } + return nullAction + } + + override public init(layer: Any) { + super.init(layer: layer) + } + + required public init?(coder: NSCoder) { + fatalError("init(coder:) has not been implemented") + } +} + + public class CameraSimplePreviewView: UIView { func updateOrientation() { guard self.videoPreviewLayer.connection?.isVideoOrientationSupported == true else { @@ -72,11 +95,16 @@ public class CameraSimplePreviewView: UIView { private var previewingDisposable: Disposable? private let placeholderView = UIImageView() - public init(frame: CGRect, main: Bool) { + public init(frame: CGRect, main: Bool, roundVideo: Bool = false) { super.init(frame: frame) - self.videoPreviewLayer.videoGravity = main ? .resizeAspectFill : .resizeAspect - self.placeholderView.contentMode = main ? .scaleAspectFill : .scaleAspectFit + if roundVideo { + self.videoPreviewLayer.videoGravity = .resizeAspectFill + self.placeholderView.contentMode = .scaleAspectFill + } else { + self.videoPreviewLayer.videoGravity = main ? .resizeAspectFill : .resizeAspect + self.placeholderView.contentMode = main ? .scaleAspectFill : .scaleAspectFit + } self.addSubview(self.placeholderView) } @@ -567,35 +595,29 @@ public class CameraPreviewView: MTKView { var scaleX: CGFloat var scaleY: CGFloat - // Rotate the layer into screen orientation. switch UIDevice.current.orientation { case .portraitUpsideDown: rotation = 180 scaleX = videoPreviewRect.width / captureDeviceResolution.width scaleY = videoPreviewRect.height / captureDeviceResolution.height - case .landscapeLeft: rotation = 90 scaleX = videoPreviewRect.height / captureDeviceResolution.width scaleY = scaleX - case .landscapeRight: rotation = -90 scaleX = videoPreviewRect.height / captureDeviceResolution.width scaleY = scaleX - default: rotation = 0 scaleX = videoPreviewRect.width / captureDeviceResolution.width scaleY = videoPreviewRect.height / captureDeviceResolution.height } - // Scale and mirror the image to ensure upright presentation. let affineTransform = CGAffineTransform(rotationAngle: radiansForDegrees(rotation)) .scaledBy(x: scaleX, y: -scaleY) overlayLayer.setAffineTransform(affineTransform) - // Cover entire screen UI. let rootLayerBounds = self.bounds overlayLayer.position = CGPoint(x: rootLayerBounds.midX, y: rootLayerBounds.midY) } diff --git a/submodules/Camera/Sources/VideoRecorder.swift b/submodules/Camera/Sources/VideoRecorder.swift index 7f9bcb05f34..1b1d11900fd 100644 --- a/submodules/Camera/Sources/VideoRecorder.swift +++ b/submodules/Camera/Sources/VideoRecorder.swift @@ -146,7 +146,7 @@ private final class VideoRecorderImpl { } if failed { - print("error") + print("append video error") return } @@ -256,7 +256,7 @@ private final class VideoRecorderImpl { } if failed { - print("error") + print("append audio error") return } diff --git a/submodules/TelegramUI/Components/Settings/WallpaperGridScreen/Sources/ThemeGridControllerNode.swift b/submodules/TelegramUI/Components/Settings/WallpaperGridScreen/Sources/ThemeGridControllerNode.swift index 50ba9af0cdd..5fe89baff5f 100644 --- a/submodules/TelegramUI/Components/Settings/WallpaperGridScreen/Sources/ThemeGridControllerNode.swift +++ b/submodules/TelegramUI/Components/Settings/WallpaperGridScreen/Sources/ThemeGridControllerNode.swift @@ -329,7 +329,10 @@ final class ThemeGridControllerNode: ASDisplayNode { if let strongSelf = self, !strongSelf.currentState.editing { let entries = previousEntries.with { $0 } if let entries = entries, !entries.isEmpty { - let wallpapers = entries.map { $0.wallpaper }.filter { !$0.isColorOrGradient } + var wallpapers = entries.map { $0.wallpaper } + if case .peer = mode { + wallpapers = wallpapers.filter { !$0.isColorOrGradient } + } var options = WallpaperPresentationOptions() if wallpaper == strongSelf.presentationData.chatWallpaper, let settings = wallpaper.settings { @@ -575,7 +578,14 @@ final class ThemeGridControllerNode: ASDisplayNode { transition.updateFrame(node: strongSelf.bottomBackgroundNode, frame: CGRect(origin: CGPoint(x: 0.0, y: gridLayout.contentSize.height), size: CGSize(width: layout.size.width, height: 500.0))) transition.updateFrame(node: strongSelf.bottomSeparatorNode, frame: CGRect(origin: CGPoint(x: 0.0, y: gridLayout.contentSize.height), size: CGSize(width: layout.size.width, height: UIScreenPixel))) - let params = ListViewItemLayoutParams(width: layout.size.width, leftInset: layout.safeInsets.left, rightInset: layout.safeInsets.right, availableHeight: layout.size.height) + let sideInset = max(16.0, floor((layout.size.width - 674.0) / 2.0)) + var listInsets = layout.safeInsets + if layout.size.width >= 375.0 { + listInsets.left = sideInset + listInsets.right = sideInset + } + + let params = ListViewItemLayoutParams(width: layout.size.width, leftInset: listInsets.left, rightInset: listInsets.right, availableHeight: layout.size.height) let makeResetLayout = strongSelf.resetItemNode.asyncLayout() let makeResetDescriptionLayout = strongSelf.resetDescriptionItemNode.asyncLayout() @@ -588,8 +598,8 @@ final class ThemeGridControllerNode: ASDisplayNode { transition.updateFrame(node: strongSelf.resetItemNode, frame: CGRect(origin: CGPoint(x: 0.0, y: gridLayout.contentSize.height + 35.0), size: resetLayout.contentSize)) transition.updateFrame(node: strongSelf.resetDescriptionItemNode, frame: CGRect(origin: CGPoint(x: 0.0, y: gridLayout.contentSize.height + 35.0 + resetLayout.contentSize.height), size: resetDescriptionLayout.contentSize)) - let sideInset = strongSelf.leftOverlayNode.frame.maxX - strongSelf.maskNode.frame = CGRect(origin: CGPoint(x: sideInset, y: strongSelf.separatorNode.frame.minY + UIScreenPixel + 4.0), size: CGSize(width: layout.size.width - sideInset * 2.0, height: gridLayout.contentSize.height + 6.0)) + let maskSideInset = strongSelf.leftOverlayNode.frame.maxX + strongSelf.maskNode.frame = CGRect(origin: CGPoint(x: maskSideInset, y: strongSelf.separatorNode.frame.minY + UIScreenPixel + 4.0), size: CGSize(width: layout.size.width - sideInset * 2.0, height: gridLayout.contentSize.height + 6.0)) } } } @@ -934,7 +944,7 @@ final class ThemeGridControllerNode: ASDisplayNode { let (resetDescriptionLayout, _) = makeResetDescriptionLayout(self.resetDescriptionItem, params, ItemListNeighbors(top: .none, bottom: .none)) if !isChannel { - insets.bottom += buttonHeight + 35.0 + resetDescriptionLayout.contentSize.height + 32.0 + listInsets.bottom += buttonHeight + 35.0 + resetDescriptionLayout.contentSize.height + 32.0 } self.gridNode.frame = CGRect(x: 0.0, y: 0.0, width: layout.size.width, height: layout.size.height) diff --git a/submodules/TelegramUI/Components/VideoMessageCameraScreen/Sources/VideoMessageCameraScreen.swift b/submodules/TelegramUI/Components/VideoMessageCameraScreen/Sources/VideoMessageCameraScreen.swift index 81cb98617b6..afc3a21091f 100644 --- a/submodules/TelegramUI/Components/VideoMessageCameraScreen/Sources/VideoMessageCameraScreen.swift +++ b/submodules/TelegramUI/Components/VideoMessageCameraScreen/Sources/VideoMessageCameraScreen.swift @@ -609,11 +609,11 @@ public class VideoMessageCameraScreen: ViewController { self.previewContainerView = UIView() self.previewContainerView.clipsToBounds = true - let isDualCameraEnabled = Camera.isDualCameraSupported //!"".isEmpty // + let isDualCameraEnabled = Camera.isDualCameraSupported let isFrontPosition = "".isEmpty - self.mainPreviewView = CameraSimplePreviewView(frame: .zero, main: true) - self.additionalPreviewView = CameraSimplePreviewView(frame: .zero, main: false) + self.mainPreviewView = CameraSimplePreviewView(frame: .zero, main: true, roundVideo: true) + self.additionalPreviewView = CameraSimplePreviewView(frame: .zero, main: false, roundVideo: true) self.progressView = RecordingProgressView(frame: .zero) @@ -746,6 +746,11 @@ public class VideoMessageCameraScreen: ViewController { return } self.cameraState = self.cameraState.updatedPosition(position) + + if !self.cameraState.isDualCameraEnabled { + self.animatePositionChange() + } + self.requestUpdateLayout(transition: .easeInOut(duration: 0.2)) }) @@ -807,6 +812,31 @@ public class VideoMessageCameraScreen: ViewController { self.previewContainerView.layer.animateAlpha(from: 1.0, to: 0.0, duration: 0.25, removeOnCompletion: false) } + private func animatePositionChange() { + if let snapshotView = self.mainPreviewView.snapshotView(afterScreenUpdates: false) { + self.previewContainerView.insertSubview(snapshotView, belowSubview: self.progressView) + self.previewSnapshotView = snapshotView + + let action = { [weak self] in + guard let self else { + return + } + UIView.animate(withDuration: 0.2, animations: { + self.previewSnapshotView?.alpha = 0.0 + }, completion: { _ in + self.previewSnapshotView?.removeFromSuperview() + self.previewSnapshotView = nil + }) + } + + Queue.mainQueue().after(1.0) { + action() + } + + self.requestUpdateLayout(transition: .immediate) + } + } + func pauseCameraCapture() { self.mainPreviewView.isEnabled = false self.additionalPreviewView.isEnabled = false @@ -850,10 +880,10 @@ public class VideoMessageCameraScreen: ViewController { action() }) } else { - Queue.mainQueue().after(1.0) { - action() - } - } + Queue.mainQueue().after(1.0) { + action() + } + } self.cameraIsActive = true self.requestUpdateLayout(transition: .immediate) @@ -1084,21 +1114,37 @@ public class VideoMessageCameraScreen: ViewController { } transition.setCornerRadius(layer: self.previewContainerView.layer, cornerRadius: previewSide / 2.0) - let previewInnerFrame = CGRect(origin: .zero, size: previewFrame.size) + let previewBounds = CGRect(origin: .zero, size: previewFrame.size) + + let previewInnerSize: CGSize + let additionalPreviewInnerSize: CGSize + + if self.cameraState.isDualCameraEnabled { + previewInnerSize = CGSize(width: previewFrame.size.width, height: previewFrame.size.width / 9.0 * 16.0) + additionalPreviewInnerSize = CGSize(width: previewFrame.size.width, height: previewFrame.size.width / 3.0 * 4.0) + } else { + previewInnerSize = CGSize(width: previewFrame.size.width, height: previewFrame.size.width / 3.0 * 4.0) + additionalPreviewInnerSize = CGSize(width: previewFrame.size.width, height: previewFrame.size.width / 3.0 * 4.0) + } - let additionalPreviewSize = CGSize(width: previewFrame.size.width, height: previewFrame.size.width / 3.0 * 4.0) - let additionalPreviewInnerFrame = CGRect(origin: CGPoint(x: 0.0, y: floorToScreenPixels((previewFrame.height - additionalPreviewSize.height) / 2.0)), size: additionalPreviewSize) - self.mainPreviewView.frame = previewInnerFrame - self.additionalPreviewView.frame = additionalPreviewInnerFrame + let previewInnerFrame = CGRect(origin: CGPoint(x: 0.0, y: floorToScreenPixels((previewFrame.height - previewInnerSize.height) / 2.0)), size: previewInnerSize) + + let additionalPreviewInnerFrame = CGRect(origin: CGPoint(x: 0.0, y: floorToScreenPixels((previewFrame.height - additionalPreviewInnerSize.height) / 2.0)), size: additionalPreviewInnerSize) + if self.cameraState.isDualCameraEnabled { + self.mainPreviewView.frame = previewInnerFrame + self.additionalPreviewView.frame = additionalPreviewInnerFrame + } else { + self.mainPreviewView.frame = self.cameraState.position == .front ? additionalPreviewInnerFrame : previewInnerFrame + } - self.progressView.frame = previewInnerFrame + self.progressView.frame = previewBounds self.progressView.value = CGFloat(self.cameraState.duration / 60.0) transition.setAlpha(view: self.additionalPreviewView, alpha: self.cameraState.position == .front ? 1.0 : 0.0) - self.previewBlurView.frame = previewInnerFrame - self.previewSnapshotView?.frame = previewInnerFrame - self.loadingView.update(size: previewInnerFrame.size, transition: .immediate) + self.previewBlurView.frame = previewBounds + self.previewSnapshotView?.center = previewBounds.center + self.loadingView.update(size: previewBounds.size, transition: .immediate) let componentSize = self.componentHost.update( transition: transition, @@ -1168,7 +1214,7 @@ public class VideoMessageCameraScreen: ViewController { resultPreviewView.addGestureRecognizer(UITapGestureRecognizer(target: self, action: #selector(self.resultTapped))) } - resultPreviewView.frame = previewInnerFrame + resultPreviewView.frame = previewBounds } else if let resultPreviewView = self.resultPreviewView { self.resultPreviewView = nil resultPreviewView.removeFromSuperview() diff --git a/submodules/TelegramUI/Sources/ChatControllerNode.swift b/submodules/TelegramUI/Sources/ChatControllerNode.swift index de5355c1e19..c031ddad322 100644 --- a/submodules/TelegramUI/Sources/ChatControllerNode.swift +++ b/submodules/TelegramUI/Sources/ChatControllerNode.swift @@ -3729,6 +3729,9 @@ class ChatControllerNode: ASDisplayNode, UIScrollViewDelegate { if self.chatPresentationInterfaceState.inputTextPanelState.mediaRecordingState != nil { return false } + if self.chatPresentationInterfaceState.recordedMediaPreview != nil { + return false + } if let inputPanelNode = self.inputPanelNode as? ChatTextInputPanelNode { if inputPanelNode.isFocused { return false