MM-54866 - Calls: Transcription support

app/actions/remote/search.ts

@@ -1,6 +1,7 @@
 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
 // See LICENSE.txt for license information.
 
+import {getPosts} from '@actions/local/post';
 import {SYSTEM_IDENTIFIERS} from '@constants/database';
 import DatabaseManager from '@database/manager';
 import NetworkManager from '@managers/network_manager';
@@ -17,6 +18,7 @@ import {fetchPostAuthors, fetchMissingChannelsFromPosts} from './post';
 import {forceLogoutIfNecessary} from './session';
 
 import type Model from '@nozbe/watermelondb/Model';
+import type PostModel from '@typings/database/models/servers/post';
 
 export async function fetchRecentMentions(serverUrl: string): Promise<PostSearchRequest> {
     try {
@@ -132,13 +134,27 @@ export const searchFiles = async (serverUrl: string, teamId: string, params: Fil
         const client = NetworkManager.getClient(serverUrl);
         const result = await client.searchFiles(teamId, params.terms);
         const files = result?.file_infos ? Object.values(result.file_infos) : [];
-        const allChannelIds = files.reduce<string[]>((acc, f) => {
+        const [allChannelIds, allPostIds] = files.reduce<[Set<string>, Set<string>]>((acc, f) => {
             if (f.channel_id) {
-                acc.push(f.channel_id);
+                acc[0].add(f.channel_id);
             }
+            if (f.post_id) {
+                acc[1].add(f.post_id);
+            }
+            return acc;
+        }, [new Set<string>(), new Set<string>()]);
+        const channels = Array.from(allChannelIds.values());
+
+        // Attach the file's post's props to the FileInfo (needed for captioning videos)
+        const postIds = Array.from(allPostIds);
+        const posts = await getPosts(serverUrl, postIds);
+        const idToPost = posts.reduce<Dictionary<PostModel>>((acc, p) => {
+            acc[p.id] = p;
             return acc;
-        }, []);
-        const channels = [...new Set(allChannelIds)];
+        }, {});
+        files.forEach((f) => {
+            f.postProps = idToPost[f.post_id]?.props;
+        });
         return {files, channels};
     } catch (error) {
         logDebug('error on searchFiles', getFullErrorMessage(error));

app/components/files/files.tsx

@@ -24,6 +24,7 @@ type FilesProps = {
     location: string;
     isReplyPost: boolean;
     postId: string;
+    postProps: Record<string, any>;
     publicLinkEnabled: boolean;
 }
 
@@ -48,7 +49,7 @@ const styles = StyleSheet.create({
     },
 });
 
-const Files = ({canDownloadFiles, failed, filesInfo, isReplyPost, layoutWidth, location, postId, publicLinkEnabled}: FilesProps) => {
+const Files = ({canDownloadFiles, failed, filesInfo, isReplyPost, layoutWidth, location, postId, postProps, publicLinkEnabled}: FilesProps) => {
     const galleryIdentifier = `${postId}-fileAttachments-${location}`;
     const [inViewPort, setInViewPort] = useState(false);
     const isTablet = useIsTablet();
@@ -63,7 +64,7 @@ const Files = ({canDownloadFiles, failed, filesInfo, isReplyPost, layoutWidth, l
     };
 
     const handlePreviewPress = preventDoubleTap((idx: number) => {
-        const items = filesForGallery.value.map((f) => fileToGalleryItem(f, f.user_id));
+        const items = filesForGallery.value.map((f) => fileToGalleryItem(f, f.user_id, postProps));
         openGalleryAtIndex(galleryIdentifier, idx, items);
     });
 

app/components/files/index.ts

@@ -45,6 +45,7 @@ const enhance = withObservables(['post'], ({database, post}: EnhanceProps) => {
     return {
         canDownloadFiles: observeCanDownloadFiles(database),
         postId: of$(post.id),
+        postProps: of$(post.props),
         publicLinkEnabled,
         filesInfo,
     };

app/products/calls/context.ts

@@ -0,0 +1,6 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+import {createContext} from 'react';
+
+export const CaptionsEnabledContext = createContext<boolean[]>([]);

app/products/calls/types/calls.ts

@@ -193,3 +193,15 @@ export type CallsVersion = {
     version?: string;
     build?: string;
 };
+
+export type SubtitleTrack = {
+    title?: string | undefined;
+    language?: string | undefined;
+    type: 'application/x-subrip' | 'application/ttml+xml' | 'text/vtt';
+    uri: string;
+};
+
+export type SelectedSubtitleTrack = {
+    type: 'system' | 'disabled' | 'title' | 'language' | 'index';
+    value?: string | number | undefined;
+};

app/products/calls/utils.ts

@@ -3,14 +3,16 @@
 
 import {makeCallsBaseAndBadgeRGB, rgbToCSS} from '@mattermost/calls';
 import {Alert} from 'react-native';
+import {TextTrackType} from 'react-native-video';
 
+import {buildFileUrl} from '@actions/remote/file';
 import {Calls, Post} from '@constants';
 import {NOTIFICATION_SUB_TYPE} from '@constants/push_notification';
 import {isMinimumServerVersion} from '@utils/helpers';
 import {displayUsername} from '@utils/user';
 
-import type {CallSession, CallsTheme, CallsVersion} from '@calls/types/calls';
-import type {CallsConfig} from '@mattermost/calls/lib/types';
+import type {CallSession, CallsTheme, CallsVersion, SelectedSubtitleTrack, SubtitleTrack} from '@calls/types/calls';
+import type {CallsConfig, Caption} from '@mattermost/calls/lib/types';
 import type PostModel from '@typings/database/models/servers/post';
 import type UserModel from '@typings/database/models/servers/user';
 import type {IntlShape} from 'react-intl';
@@ -201,3 +203,33 @@ export function isCallsStartedMessage(payload?: NotificationData) {
     // calls will be >= 0.21.0, and push proxy will be >= 5.27.0
     return (payload?.message === 'You\'ve been invited to a call' || callsMessageRegex.test(payload?.message || ''));
 }
+
+export const hasCaptions = (postProps?: Record<string, any> & { captions?: Caption[] }): boolean => {
+    return !(!postProps || !postProps.captions?.[0]);
+};
+
+export const getTranscriptionUri = (serverUrl: string, postProps?: Record<string, any> & { captions?: Caption[] }): {
+    tracks?: SubtitleTrack[];
+    selected: SelectedSubtitleTrack;
+} => {
+    // Note: We're not using hasCaptions above because this tells typescript that the caption exists later.
+    // We could use some fancy typescript to do the same, but it's not worth the complexity.
+    if (!postProps || !postProps.captions?.[0]) {
+        return {
+            tracks: undefined,
+            selected: {type: 'disabled'},
+        };
+    }
+
+    const tracks: SubtitleTrack[] = postProps.captions.map((t) => ({
+        title: t.title,
+        language: t.language,
+        type: TextTrackType.VTT,
+        uri: buildFileUrl(serverUrl, t.file_id),
+    }));
+
+    return {
+        tracks,
+        selected: {type: 'index', value: 0},
+    };
+};

app/screens/gallery/footer/actions/action.tsx

@@ -5,7 +5,15 @@
 // See LICENSE.txt for license information.
 
 import React, {useCallback} from 'react';
-import {Platform, Pressable, type PressableAndroidRippleConfig, type PressableStateCallbackType, type StyleProp, type ViewStyle} from 'react-native';
+import {
+    Platform,
+    Pressable,
+    type PressableAndroidRippleConfig,
+    type PressableStateCallbackType,
+    type StyleProp,
+    StyleSheet,
+    type ViewStyle,
+} from 'react-native';
 
 import CompassIcon from '@components/compass_icon';
 import {changeOpacity} from '@utils/theme';
@@ -26,19 +34,29 @@ const pressedStyle = ({pressed}: PressableStateCallbackType) => {
     return [{opacity}];
 };
 
+const baseStyle = StyleSheet.create({
+    container: {
+        width: 40,
+        height: 40,
+        alignItems: 'center',
+        justifyContent: 'center',
+    },
+});
+
 const androidRippleConfig: PressableAndroidRippleConfig = {borderless: true, radius: 24, color: '#FFF'};
 
 const Action = ({disabled, iconName, onPress, style}: Props) => {
     const pressableStyle = useCallback((pressed: PressableStateCallbackType) => ([
         pressedStyle(pressed),
+        baseStyle.container,
         style,
     ]), [style]);
 
     return (
         <Pressable
             android_ripple={androidRippleConfig}
             disabled={disabled}
-            hitSlop={24}
+            hitSlop={4}
             onPress={onPress}
             style={pressableStyle}
         >

app/screens/gallery/footer/actions/index.tsx

@@ -5,6 +5,8 @@ import {useManagedConfig} from '@mattermost/react-native-emm';
 import React from 'react';
 import {StyleSheet, View} from 'react-native';
 
+import InvertedAction from '@screens/gallery/footer/actions/inverted_action';
+
 import Action from './action';
 
 type Props = {
@@ -15,20 +17,22 @@ type Props = {
     onCopyPublicLink: () => void;
     onDownload: () => void;
     onShare: () => void;
+    hasCaptions: boolean;
+    captionEnabled: boolean;
+    onCaptionsPress: () => void;
 }
 
 const styles = StyleSheet.create({
     container: {
         flexDirection: 'row',
-    },
-    action: {
-        marginLeft: 24,
+        alignItems: 'center',
+        gap: 8,
     },
 });
 
 const Actions = ({
-    canDownloadFiles, disabled, enablePublicLinks, fileId,
-    onCopyPublicLink, onDownload, onShare,
+    canDownloadFiles, disabled, enablePublicLinks, fileId, onCopyPublicLink,
+    onDownload, onShare, hasCaptions, captionEnabled, onCaptionsPress,
 }: Props) => {
     const managedConfig = useManagedConfig<ManagedConfig>();
     const canCopyPublicLink = !fileId.startsWith('uid') && enablePublicLinks && managedConfig.copyAndPasteProtection !== 'true';
@@ -41,19 +45,24 @@ const Actions = ({
                 iconName='link-variant'
                 onPress={onCopyPublicLink}
             />}
+            {hasCaptions &&
+            <InvertedAction
+                activated={captionEnabled}
+                iconName='closed-caption-outline'
+                onPress={onCaptionsPress}
+            />
+            }
             {canDownloadFiles &&
             <>
                 <Action
                     disabled={disabled}
                     iconName='download-outline'
                     onPress={onDownload}
-                    style={styles.action}
                 />
                 <Action
                     disabled={disabled}
                     iconName='export-variant'
                     onPress={onShare}
-                    style={styles.action}
                 />
             </>
             }

app/screens/gallery/footer/actions/inverted_action.tsx

@@ -0,0 +1,73 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+import React, {useCallback} from 'react';
+import {
+    Platform,
+    Pressable,
+    type PressableAndroidRippleConfig,
+    type PressableStateCallbackType,
+    type StyleProp,
+    StyleSheet,
+    type ViewStyle,
+} from 'react-native';
+
+import CompassIcon from '@components/compass_icon';
+
+type Props = {
+    activated: boolean;
+    iconName: string;
+    onPress: () => void;
+    style?: StyleProp<ViewStyle>;
+}
+
+const pressedStyle = ({pressed}: PressableStateCallbackType) => {
+    let opacity = 1;
+    if (Platform.OS === 'ios' && pressed) {
+        opacity = 0.5;
+    }
+
+    return [{opacity}];
+};
+
+const baseStyle = StyleSheet.create({
+    container: {
+        width: 40,
+        height: 40,
+        borderRadius: 4,
+        alignItems: 'center',
+        justifyContent: 'center',
+        backgroundColor: '#000',
+    },
+    containerActivated: {
+        backgroundColor: '#fff',
+    },
+});
+
+const androidRippleConfig: PressableAndroidRippleConfig = {borderless: true, radius: 24, color: '#FFF'};
+
+const InvertedAction = ({activated, iconName, onPress, style}: Props) => {
+    const pressableStyle = useCallback((pressed: PressableStateCallbackType) => ([
+        pressedStyle(pressed),
+        baseStyle.container,
+        activated && baseStyle.containerActivated,
+        style,
+    ]), [style, activated]);
+
+    return (
+        <Pressable
+            android_ripple={androidRippleConfig}
+            hitSlop={4}
+            onPress={onPress}
+            style={pressableStyle}
+        >
+            <CompassIcon
+                color={activated ? '#000' : '#fff'}
+                name={iconName}
+                size={24}
+            />
+        </Pressable>
+    );
+};
+
+export default InvertedAction;

app/screens/gallery/footer/footer.tsx

@@ -35,6 +35,9 @@ type Props = {
     post?: PostModel;
     style: StyleProp<ViewStyle>;
     teammateNameDisplay: string;
+    hasCaptions: boolean;
+    captionEnabled: boolean;
+    onCaptionsPress: () => void;
 }
 
 const AnimatedSafeAreaView = Animated.createAnimatedComponent(SafeAreaView);
@@ -58,6 +61,7 @@ const Footer = ({
     author, canDownloadFiles, channelName, currentUserId,
     enablePostIconOverride, enablePostUsernameOverride, enablePublicLink,
     hideActions, isDirectChannel, item, post, style, teammateNameDisplay,
+    hasCaptions, captionEnabled, onCaptionsPress,
 }: Props) => {
     const showActions = !hideActions && Boolean(item.id) && !item.id?.startsWith('uid');
     const [action, setAction] = useState<GalleryAction>('none');
@@ -142,6 +146,9 @@ const Footer = ({
                     onCopyPublicLink={handleCopyLink}
                     onDownload={handleDownload}
                     onShare={handleShare}
+                    hasCaptions={hasCaptions}
+                    captionEnabled={captionEnabled}
+                    onCaptionsPress={onCaptionsPress}
                 />
                 }
             </View>