From bb4d8c80251013e057c1e24f830480b851a2ff1d Mon Sep 17 00:00:00 2001 From: Christopher Poile Date: Thu, 7 Dec 2023 13:45:25 -0500 Subject: [PATCH 01/11] captions on videos from posts and searches --- app/actions/remote/search.ts | 24 +++++- app/components/files/files.tsx | 5 +- app/components/files/index.ts | 1 + app/products/calls/context.ts | 6 ++ app/products/calls/types/calls.ts | 12 +++ app/products/calls/utils.ts | 30 +++++++- app/screens/gallery/footer/actions/index.tsx | 17 ++++- .../footer/actions/inverted_action.tsx | 73 +++++++++++++++++++ app/screens/gallery/footer/footer.tsx | 7 ++ app/screens/gallery/index.tsx | 28 ++++++- app/screens/gallery/video_renderer/index.tsx | 18 ++++- app/utils/gallery/index.ts | 3 +- types/api/files.d.ts | 1 + types/screens/gallery.ts | 1 + 14 files changed, 211 insertions(+), 15 deletions(-) create mode 100644 app/products/calls/context.ts create mode 100644 app/screens/gallery/footer/actions/inverted_action.tsx diff --git a/app/actions/remote/search.ts b/app/actions/remote/search.ts index af41b7e09c3..86ab63a8cb4 100644 --- a/app/actions/remote/search.ts +++ b/app/actions/remote/search.ts @@ -1,6 +1,7 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. +import {getPosts} from '@actions/local/post'; import {SYSTEM_IDENTIFIERS} from '@constants/database'; import DatabaseManager from '@database/manager'; import NetworkManager from '@managers/network_manager'; @@ -17,6 +18,7 @@ import {fetchPostAuthors, fetchMissingChannelsFromPosts} from './post'; import {forceLogoutIfNecessary} from './session'; import type Model from '@nozbe/watermelondb/Model'; +import type PostModel from '@typings/database/models/servers/post'; export async function fetchRecentMentions(serverUrl: string): Promise { try { @@ -132,13 +134,27 @@ export const searchFiles = async (serverUrl: string, teamId: string, params: Fil const client = NetworkManager.getClient(serverUrl); const result = await client.searchFiles(teamId, params.terms); const files = result?.file_infos ? Object.values(result.file_infos) : []; - const allChannelIds = files.reduce((acc, f) => { + const [allChannelIds, allPostIds] = files.reduce<[Set, Set]>((acc, f) => { if (f.channel_id) { - acc.push(f.channel_id); + acc[0].add(f.channel_id); } + if (f.post_id) { + acc[1].add(f.post_id); + } + return acc; + }, [new Set(), new Set()]); + const channels = Array.from(allChannelIds.values()); + + // Attach the file's post's props to the FileInfo (needed for captioning videos) + const postIds = Array.from(allPostIds); + const posts = await getPosts(serverUrl, postIds); + const idToPost = posts.reduce>((acc, p) => { + acc[p.id] = p; return acc; - }, []); - const channels = [...new Set(allChannelIds)]; + }, {}); + files.forEach((f) => { + f.postProps = idToPost[f.post_id]?.props; + }); return {files, channels}; } catch (error) { logDebug('error on searchFiles', getFullErrorMessage(error)); diff --git a/app/components/files/files.tsx b/app/components/files/files.tsx index 3747fea43bc..da11d676f2d 100644 --- a/app/components/files/files.tsx +++ b/app/components/files/files.tsx @@ -24,6 +24,7 @@ type FilesProps = { location: string; isReplyPost: boolean; postId: string; + postProps: Record; publicLinkEnabled: boolean; } @@ -48,7 +49,7 @@ const styles = StyleSheet.create({ }, }); -const Files = ({canDownloadFiles, failed, filesInfo, isReplyPost, layoutWidth, location, postId, publicLinkEnabled}: FilesProps) => { +const Files = ({canDownloadFiles, failed, filesInfo, isReplyPost, layoutWidth, location, postId, postProps, publicLinkEnabled}: FilesProps) => { const galleryIdentifier = `${postId}-fileAttachments-${location}`; const [inViewPort, setInViewPort] = useState(false); const isTablet = useIsTablet(); @@ -63,7 +64,7 @@ const Files = ({canDownloadFiles, failed, filesInfo, isReplyPost, layoutWidth, l }; const handlePreviewPress = preventDoubleTap((idx: number) => { - const items = filesForGallery.value.map((f) => fileToGalleryItem(f, f.user_id)); + const items = filesForGallery.value.map((f) => fileToGalleryItem(f, f.user_id, postProps)); openGalleryAtIndex(galleryIdentifier, idx, items); }); diff --git a/app/components/files/index.ts b/app/components/files/index.ts index 339fa7ad12c..f2b7b15b7db 100644 --- a/app/components/files/index.ts +++ b/app/components/files/index.ts @@ -45,6 +45,7 @@ const enhance = withObservables(['post'], ({database, post}: EnhanceProps) => { return { canDownloadFiles: observeCanDownloadFiles(database), postId: of$(post.id), + postProps: of$(post.props), publicLinkEnabled, filesInfo, }; diff --git a/app/products/calls/context.ts b/app/products/calls/context.ts new file mode 100644 index 00000000000..3453e1ed51e --- /dev/null +++ b/app/products/calls/context.ts @@ -0,0 +1,6 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +import {createContext} from 'react'; + +export const CaptionsEnabledContext = createContext([]); diff --git a/app/products/calls/types/calls.ts b/app/products/calls/types/calls.ts index c2bd051d0e6..3d0caa06efd 100644 --- a/app/products/calls/types/calls.ts +++ b/app/products/calls/types/calls.ts @@ -193,3 +193,15 @@ export type CallsVersion = { version?: string; build?: string; }; + +export type SubtitleTrack = { + title?: string | undefined; + language?: string | undefined; + type: 'application/x-subrip' | 'application/ttml+xml' | 'text/vtt'; + uri: string; +}; + +export type SelectedSubtitleTrack = { + type: 'system' | 'disabled' | 'title' | 'language' | 'index'; + value?: string | number | undefined; +}; diff --git a/app/products/calls/utils.ts b/app/products/calls/utils.ts index bcfd80a7733..3e7373fe126 100644 --- a/app/products/calls/utils.ts +++ b/app/products/calls/utils.ts @@ -3,13 +3,15 @@ import {makeCallsBaseAndBadgeRGB, rgbToCSS} from '@mattermost/calls'; import {Alert} from 'react-native'; +import {TextTrackType} from 'react-native-video'; +import {buildFileUrl} from '@actions/remote/file'; import {Calls, Post} from '@constants'; import {NOTIFICATION_SUB_TYPE} from '@constants/push_notification'; import {isMinimumServerVersion} from '@utils/helpers'; import {displayUsername} from '@utils/user'; -import type {CallSession, CallsTheme, CallsVersion} from '@calls/types/calls'; +import type {CallSession, CallsTheme, CallsVersion, SelectedSubtitleTrack, SubtitleTrack} from '@calls/types/calls'; import type {CallsConfig} from '@mattermost/calls/lib/types'; import type PostModel from '@typings/database/models/servers/post'; import type UserModel from '@typings/database/models/servers/user'; @@ -201,3 +203,29 @@ export function isCallsStartedMessage(payload?: NotificationData) { // calls will be >= 0.21.0, and push proxy will be >= 5.27.0 return (payload?.message === 'You\'ve been invited to a call' || callsMessageRegex.test(payload?.message || '')); } + +export const getHasTranscript = (postProps?: Record): boolean => { + return !(!postProps || !postProps.captions_file_id); +}; + +export const getTranscriptionUri = (serverUrl: string, postProps?: Record): { + track?: SubtitleTrack; + selected?: SelectedSubtitleTrack; +} => { + if (!postProps || !postProps.captions_file_id) { + return {}; + } + + const language = 'en'; // Note: will be changed when we support multiple languages. + const uri = buildFileUrl(serverUrl, postProps.captions_file_id); + + return { + track: { + title: 'subtitles', + language, + type: TextTrackType.VTT, + uri, + }, + selected: {type: 'index', value: 0}, + }; +}; diff --git a/app/screens/gallery/footer/actions/index.tsx b/app/screens/gallery/footer/actions/index.tsx index 7c0bf3a5cf0..4b9978cb5f1 100644 --- a/app/screens/gallery/footer/actions/index.tsx +++ b/app/screens/gallery/footer/actions/index.tsx @@ -5,6 +5,8 @@ import {useManagedConfig} from '@mattermost/react-native-emm'; import React from 'react'; import {StyleSheet, View} from 'react-native'; +import InvertedAction from '@screens/gallery/footer/actions/inverted_action'; + import Action from './action'; type Props = { @@ -15,11 +17,15 @@ type Props = { onCopyPublicLink: () => void; onDownload: () => void; onShare: () => void; + hasCaptions: boolean; + captionEnabled: boolean; + onCaptionsPress: () => void; } const styles = StyleSheet.create({ container: { flexDirection: 'row', + alignItems: 'center', }, action: { marginLeft: 24, @@ -27,8 +33,8 @@ const styles = StyleSheet.create({ }); const Actions = ({ - canDownloadFiles, disabled, enablePublicLinks, fileId, - onCopyPublicLink, onDownload, onShare, + canDownloadFiles, disabled, enablePublicLinks, fileId, onCopyPublicLink, + onDownload, onShare, hasCaptions, captionEnabled, onCaptionsPress, }: Props) => { const managedConfig = useManagedConfig(); const canCopyPublicLink = !fileId.startsWith('uid') && enablePublicLinks && managedConfig.copyAndPasteProtection !== 'true'; @@ -41,6 +47,13 @@ const Actions = ({ iconName='link-variant' onPress={onCopyPublicLink} />} + {hasCaptions && + + } {canDownloadFiles && <> void; + style?: StyleProp; +} + +const pressedStyle = ({pressed}: PressableStateCallbackType) => { + let opacity = 1; + if (Platform.OS === 'ios' && pressed) { + opacity = 0.5; + } + + return [{opacity}]; +}; + +const baseStyle = StyleSheet.create({ + container: { + width: 40, + height: 40, + borderRadius: 4, + alignItems: 'center', + justifyContent: 'center', + backgroundColor: '#000', + }, + containerActivated: { + backgroundColor: '#fff', + }, +}); + +const androidRippleConfig: PressableAndroidRippleConfig = {borderless: true, radius: 24, color: '#FFF'}; + +const InvertedAction = ({activated, iconName, onPress, style}: Props) => { + const pressableStyle = useCallback((pressed: PressableStateCallbackType) => ([ + pressedStyle(pressed), + baseStyle.container, + activated && baseStyle.containerActivated, + style, + ]), [style, activated]); + + return ( + + + + ); +}; + +export default InvertedAction; diff --git a/app/screens/gallery/footer/footer.tsx b/app/screens/gallery/footer/footer.tsx index 310805fb587..09ce90dbc00 100644 --- a/app/screens/gallery/footer/footer.tsx +++ b/app/screens/gallery/footer/footer.tsx @@ -35,6 +35,9 @@ type Props = { post?: PostModel; style: StyleProp; teammateNameDisplay: string; + hasCaptions: boolean; + captionEnabled: boolean; + onCaptionsPress: () => void; } const AnimatedSafeAreaView = Animated.createAnimatedComponent(SafeAreaView); @@ -58,6 +61,7 @@ const Footer = ({ author, canDownloadFiles, channelName, currentUserId, enablePostIconOverride, enablePostUsernameOverride, enablePublicLink, hideActions, isDirectChannel, item, post, style, teammateNameDisplay, + hasCaptions, captionEnabled, onCaptionsPress, }: Props) => { const showActions = !hideActions && Boolean(item.id) && !item.id?.startsWith('uid'); const [action, setAction] = useState('none'); @@ -142,6 +146,9 @@ const Footer = ({ onCopyPublicLink={handleCopyLink} onDownload={handleDownload} onShare={handleShare} + hasCaptions={hasCaptions} + captionEnabled={captionEnabled} + onCaptionsPress={onCaptionsPress} /> } diff --git a/app/screens/gallery/index.tsx b/app/screens/gallery/index.tsx index 40d33290fab..b555808b4e7 100644 --- a/app/screens/gallery/index.tsx +++ b/app/screens/gallery/index.tsx @@ -1,9 +1,11 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -import React, {useCallback, useMemo, useRef, useState} from 'react'; +import React, {useCallback, useEffect, useMemo, useRef, useState} from 'react'; import {NativeModules, useWindowDimensions, Platform} from 'react-native'; +import {CaptionsEnabledContext} from '@calls/context'; +import {getHasTranscript} from '@calls/utils'; import useAndroidHardwareBackHandler from '@hooks/android_back_handler'; import {useIsTablet} from '@hooks/device'; import {useGalleryControls} from '@hooks/gallery'; @@ -29,10 +31,27 @@ const GalleryScreen = ({componentId, galleryIdentifier, hideActions, initialInde const dim = useWindowDimensions(); const isTablet = useIsTablet(); const [localIndex, setLocalIndex] = useState(initialIndex); + const [captionsEnabled, setCaptionsEnabled] = useState(new Array(items.length).fill(true)); + const [hasCaptions, setHasCaptions] = useState([]); const {setControlsHidden, headerStyles, footerStyles} = useGalleryControls(); const dimensions = useMemo(() => ({width: dim.width, height: dim.height}), [dim.width]); const galleryRef = useRef(null); + useEffect(() => { + const captions = items.reduce((acc, item) => { + acc.push(getHasTranscript(item.postProps)); + return acc; + }, [] as boolean[]); + setHasCaptions(captions); + }, [items]); + + const onCaptionsPressIdx = useCallback((idx: number) => { + const enabled = [...captionsEnabled]; + enabled[idx] = !enabled[idx]; + setCaptionsEnabled(enabled); + }, [captionsEnabled, setCaptionsEnabled]); + const onCaptionsPress = useCallback(() => onCaptionsPressIdx(localIndex), [localIndex, onCaptionsPressIdx]); + const onClose = useCallback(() => { // We keep the un freeze here as we want // the screen to be visible when the gallery @@ -63,7 +82,7 @@ const GalleryScreen = ({componentId, galleryIdentifier, hideActions, initialInde useAndroidHardwareBackHandler(componentId, close); return ( - <> +
- + ); }; diff --git a/app/screens/gallery/video_renderer/index.tsx b/app/screens/gallery/video_renderer/index.tsx index f61ed59668b..8491b41250a 100644 --- a/app/screens/gallery/video_renderer/index.tsx +++ b/app/screens/gallery/video_renderer/index.tsx @@ -1,13 +1,22 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -import React, {useCallback, useEffect, useMemo, useRef, useState} from 'react'; +import React, {useCallback, useContext, useEffect, useMemo, useRef, useState} from 'react'; import {DeviceEventEmitter, Platform, StyleSheet, useWindowDimensions} from 'react-native'; -import Animated, {Easing, useAnimatedRef, useAnimatedStyle, useSharedValue, withTiming, type WithTimingConfig} from 'react-native-reanimated'; +import Animated, { + Easing, + useAnimatedRef, + useAnimatedStyle, + useSharedValue, + withTiming, + type WithTimingConfig, +} from 'react-native-reanimated'; import {useSafeAreaInsets} from 'react-native-safe-area-context'; import Video, {type OnPlaybackRateData} from 'react-native-video'; import {updateLocalFilePath} from '@actions/local/file'; +import {CaptionsEnabledContext} from '@calls/context'; +import {getTranscriptionUri} from '@calls/utils'; import CompassIcon from '@components/compass_icon'; import {Events} from '@constants'; import {GALLERY_FOOTER_HEIGHT, VIDEO_INSET} from '@constants/gallery'; @@ -59,6 +68,7 @@ const VideoRenderer = ({height, index, initialIndex, item, isPageActive, onShoul const serverUrl = useServerUrl(); const videoRef = useAnimatedRef