From ed5a88d9d807c471a548bd9f23e0dcbf60c6cf6e Mon Sep 17 00:00:00 2001 From: Paul Frazee Date: Fri, 30 Jun 2023 11:35:29 -0500 Subject: [APP-718] Improvements and fixes to language handling (#931) * Add locale helpers for narrowing languages * Add a translate link to posts in a different language * Update language filtering to use narrowing when multiple declared * Fix a few more RTL layout cases * Fix types --- src/lib/api/feed-manip.ts | 58 ++------------------- src/locale/helpers.ts | 81 +++++++++++++++++++++++++++++ src/locale/languages.ts | 4 ++ src/view/com/lists/ListCard.tsx | 2 +- src/view/com/lists/ListItems.tsx | 1 + src/view/com/post-thread/PostThreadItem.tsx | 70 +++++++++++++++++++------ src/view/com/post/Post.tsx | 13 ++--- src/view/com/posts/FeedItem.tsx | 32 ++++++++---- src/view/com/profile/ProfileHeader.tsx | 1 + 9 files changed, 174 insertions(+), 88 deletions(-) create mode 100644 src/locale/helpers.ts (limited to 'src') diff --git a/src/lib/api/feed-manip.ts b/src/lib/api/feed-manip.ts index da89ca88f..97665429d 100644 --- a/src/lib/api/feed-manip.ts +++ b/src/lib/api/feed-manip.ts @@ -4,10 +4,7 @@ import { AppBskyEmbedRecordWithMedia, AppBskyEmbedRecord, } from '@atproto/api' -import * as bcp47Match from 'bcp-47-match' -import lande from 'lande' -import {hasProp} from 'lib/type-guards' -import {LANGUAGES_MAP_CODE2} from '../../locale/languages' +import {isPostInLanguage} from '../../locale/helpers' type FeedViewPost = AppBskyFeedDefs.FeedViewPost export type FeedTunerFn = ( @@ -245,76 +242,29 @@ export class FeedTuner { * returns an array of `FeedViewPostsSlice` objects. */ static preferredLangOnly(preferredLangsCode2: string[]) { - const langsCode3 = preferredLangsCode2.map( - l => LANGUAGES_MAP_CODE2[l]?.code3 || l, - ) return ( tuner: FeedTuner, slices: FeedViewPostsSlice[], ): FeedViewPostsSlice[] => { - // 1. Early return if no languages have been specified + // early return if no languages have been specified if (!preferredLangsCode2.length || preferredLangsCode2.length === 0) { return slices } for (let i = slices.length - 1; i >= 0; i--) { - // 2. Set a flag to indicate whether the item has text in a preferred language let hasPreferredLang = false for (const item of slices[i].items) { - // 3. check if the post has a `langs` property and if it is in the list of preferred languages - // if it is, set the flag to true - // if language is declared, regardless of a match, break out of the loop - if ( - hasProp(item.post.record, 'langs') && - Array.isArray(item.post.record.langs) - ) { - if ( - bcp47Match.basicFilter( - item.post.record.langs, - preferredLangsCode2, - ).length > 0 - ) { - hasPreferredLang = true - } - break - } - // 4. FALLBACK if no language declared : - // Get the most likely language of the text in the post from the `lande` library and - // check if it is in the list of preferred languages - // if it is, set the flag to true and break out of the loop - else if ( - hasProp(item.post.record, 'text') && - typeof item.post.record.text === 'string' - ) { - // Treat empty text the same as no text - if (item.post.record.text.length === 0) { - hasPreferredLang = true - break - } - const langsProbabilityMap = lande(item.post.record.text) - const mostLikelyLang = langsProbabilityMap[0][0] - // const secondMostLikelyLang = langsProbabilityMap[1][0] - // const thirdMostLikelyLang = langsProbabilityMap[2][0] - - // we check for code3 here because that is what the `lande` library returns - if (langsCode3.includes(mostLikelyLang)) { - hasPreferredLang = true - break - } - } - // 5. no text? roll with it (eg: image-only posts, reposts, etc.) - else { + if (isPostInLanguage(item.post, preferredLangsCode2)) { hasPreferredLang = true break } } - // 6. if item does not fit preferred language, remove it + // if item does not fit preferred language, remove it if (!hasPreferredLang) { slices.splice(i, 1) } } - // 7. return the filtered list of items return slices } } diff --git a/src/locale/helpers.ts b/src/locale/helpers.ts new file mode 100644 index 000000000..4b9002586 --- /dev/null +++ b/src/locale/helpers.ts @@ -0,0 +1,81 @@ +import {AppBskyFeedDefs, AppBskyFeedPost} from '@atproto/api' +import lande from 'lande' +import {hasProp} from 'lib/type-guards' +import * as bcp47Match from 'bcp-47-match' +import {LANGUAGES_MAP_CODE2, LANGUAGES_MAP_CODE3} from './languages' + +export function code2ToCode3(lang: string): string { + if (lang.length === 2) { + return LANGUAGES_MAP_CODE2[lang]?.code3 || lang + } + return lang +} + +export function code3ToCode2(lang: string): string { + if (lang.length === 3) { + return LANGUAGES_MAP_CODE3[lang]?.code2 || lang + } + return lang +} + +export function getPostLanguage( + post: AppBskyFeedDefs.PostView, +): string | undefined { + let candidates: string[] = [] + let postText: string = '' + if (hasProp(post.record, 'text') && typeof post.record.text === 'string') { + postText = post.record.text + } + + if ( + AppBskyFeedPost.isRecord(post.record) && + hasProp(post.record, 'langs') && + Array.isArray(post.record.langs) + ) { + candidates = post.record.langs + } + + // if there's only one declared language, use that + if (candidates?.length === 1) { + return candidates[0] + } + + // no text? can't determine + if (postText.trim().length === 0) { + return undefined + } + + // run the language model + let langsProbabilityMap = lande(postText) + + // filter down using declared languages + if (candidates?.length) { + langsProbabilityMap = langsProbabilityMap.filter( + ([lang, _probability]: [string, number]) => { + return candidates.includes(code3ToCode2(lang)) + }, + ) + } + + if (langsProbabilityMap[0]) { + return code3ToCode2(langsProbabilityMap[0][0]) + } +} + +export function isPostInLanguage( + post: AppBskyFeedDefs.PostView, + targetLangs: string[], +): boolean { + const lang = getPostLanguage(post) + if (!lang) { + // the post has no text, so we just say "yes" for now + return true + } + return bcp47Match.basicFilter(lang, targetLangs).length > 0 +} + +export function getTranslatorLink(lang: string, text: string): string { + return encodeURI( + `https://translate.google.com/?sl=auto&tl=${lang}&text=${text}`, + ) +} diff --git a/src/locale/languages.ts b/src/locale/languages.ts index 269e2fa9a..3983c213f 100644 --- a/src/locale/languages.ts +++ b/src/locale/languages.ts @@ -555,3 +555,7 @@ export const LANGUAGES_MAP_CODE2 = Object.fromEntries( export const LANGUAGES_MAP_CODE3 = Object.fromEntries( LANGUAGES.map(lang => [lang.code3, lang]), ) +// some additional manual mappings (not clear if these should be in the "official" mappings) +if (LANGUAGES_MAP_CODE2.fa) { + LANGUAGES_MAP_CODE3.pes = LANGUAGES_MAP_CODE2.fa +} diff --git a/src/view/com/lists/ListCard.tsx b/src/view/com/lists/ListCard.tsx index 2293dbeca..b70fa3773 100644 --- a/src/view/com/lists/ListCard.tsx +++ b/src/view/com/lists/ListCard.tsx @@ -96,7 +96,7 @@ export const ListCard = ({ {descriptionRichText ? ( diff --git a/src/view/com/lists/ListItems.tsx b/src/view/com/lists/ListItems.tsx index 47fa4a943..289ba000b 100644 --- a/src/view/com/lists/ListItems.tsx +++ b/src/view/com/lists/ListItems.tsx @@ -347,6 +347,7 @@ const styles = StyleSheet.create({ borderTopWidth: 1, }, headerDescription: { + flex: 1, marginTop: 8, }, headerBtns: { diff --git a/src/view/com/post-thread/PostThreadItem.tsx b/src/view/com/post-thread/PostThreadItem.tsx index 002795d77..692fac9e9 100644 --- a/src/view/com/post-thread/PostThreadItem.tsx +++ b/src/view/com/post-thread/PostThreadItem.tsx @@ -2,7 +2,7 @@ import React, {useCallback, useMemo} from 'react' import {observer} from 'mobx-react-lite' import {AccessibilityActionEvent, Linking, StyleSheet, View} from 'react-native' import Clipboard from '@react-native-clipboard/clipboard' -import {AtUri} from '@atproto/api' +import {AtUri, AppBskyFeedDefs} from '@atproto/api' import { FontAwesomeIcon, FontAwesomeIconStyle, @@ -18,6 +18,7 @@ import {s} from 'lib/styles' import {ago, niceDate} from 'lib/strings/time' import {sanitizeDisplayName} from 'lib/strings/display-names' import {pluralize} from 'lib/strings/helpers' +import {getTranslatorLink, isPostInLanguage} from '../../../locale/helpers' import {useStores} from 'state/index' import {PostMeta} from '../util/PostMeta' import {PostEmbeds} from '../util/post-embeds' @@ -65,6 +66,13 @@ export const PostThreadItem = observer(function PostThreadItem({ }, [item.post.uri, item.post.author.handle]) const repostsTitle = 'Reposts of this post' + const primaryLanguage = store.preferences.contentLanguages[0] || 'en' + const translatorUrl = getTranslatorLink(primaryLanguage, record?.text || '') + const needsTranslation = useMemo( + () => !isPostInLanguage(item.post, store.preferences.contentLanguages), + [item.post, store.preferences.contentLanguages], + ) + const onPressReply = React.useCallback(() => { store.shell.openComposer({ replyTo: { @@ -98,17 +106,9 @@ export const PostThreadItem = observer(function PostThreadItem({ Toast.show('Copied to clipboard') }, [record]) - const primaryLanguage = store.preferences.contentLanguages[0] || 'en' - const onOpenTranslate = React.useCallback(() => { - Linking.openURL( - encodeURI( - `https://translate.google.com/?sl=auto&tl=${primaryLanguage}&text=${ - record?.text || '' - }`, - ), - ) - }, [record, primaryLanguage]) + Linking.openURL(translatorUrl) + }, [translatorUrl]) const onToggleThreadMute = React.useCallback(async () => { try { @@ -276,6 +276,7 @@ export const PostThreadItem = observer(function PostThreadItem({ type="post-text-lg" richText={item.richText} lineHeight={1.3} + style={s.flex1} /> ) : undefined} @@ -283,9 +284,11 @@ export const PostThreadItem = observer(function PostThreadItem({ - - {niceDate(item.post.indexedAt)} - + {hasEngagement ? ( {item.post.repostCount ? ( @@ -411,7 +414,7 @@ export const PostThreadItem = observer(function PostThreadItem({ @@ -419,6 +422,15 @@ export const PostThreadItem = observer(function PostThreadItem({ + {needsTranslation && ( + + + + Translate this post + + + + )} + {niceDate(post.indexedAt)} + {needsTranslation && ( + <> + • + + Translate + + + )} + + ) +} + const styles = StyleSheet.create({ outer: { borderTopWidth: 1, @@ -540,6 +577,9 @@ const styles = StyleSheet.create({ paddingHorizontal: 0, paddingBottom: 10, }, + translateLink: { + marginBottom: 6, + }, contentHider: { marginTop: 4, }, diff --git a/src/view/com/post/Post.tsx b/src/view/com/post/Post.tsx index 3eac7ee7b..fac27b842 100644 --- a/src/view/com/post/Post.tsx +++ b/src/view/com/post/Post.tsx @@ -30,6 +30,7 @@ import {UserAvatar} from '../util/UserAvatar' import {useStores} from 'state/index' import {s, colors} from 'lib/styles' import {usePalette} from 'lib/hooks/usePalette' +import {getTranslatorLink} from '../../../locale/helpers' export const Post = observer(function Post({ uri, @@ -167,16 +168,11 @@ const PostLoaded = observer( }, [record]) const primaryLanguage = store.preferences.contentLanguages[0] || 'en' + const translatorUrl = getTranslatorLink(primaryLanguage, record?.text || '') const onOpenTranslate = React.useCallback(() => { - Linking.openURL( - encodeURI( - `https://translate.google.com/?sl=auto&tl=${primaryLanguage}&text=${ - record?.text || '' - }`, - ), - ) - }, [record, primaryLanguage]) + Linking.openURL(translatorUrl) + }, [translatorUrl]) const onToggleThreadMute = React.useCallback(async () => { try { @@ -299,6 +295,7 @@ const PostLoaded = observer( type="post-text" richText={item.richText} lineHeight={1.3} + style={s.flex1} /> ) : undefined} diff --git a/src/view/com/posts/FeedItem.tsx b/src/view/com/posts/FeedItem.tsx index d83e64073..7354c8a67 100644 --- a/src/view/com/posts/FeedItem.tsx +++ b/src/view/com/posts/FeedItem.tsx @@ -27,6 +27,7 @@ import {useStores} from 'state/index' import {usePalette} from 'lib/hooks/usePalette' import {useAnalytics} from 'lib/analytics/analytics' import {sanitizeDisplayName} from 'lib/strings/display-names' +import {getTranslatorLink, isPostInLanguage} from '../../../locale/helpers' export const FeedItem = observer(function ({ item, @@ -62,6 +63,12 @@ export const FeedItem = observer(function ({ const urip = new AtUri(record.reply.parent?.uri || record.reply.root.uri) return urip.hostname }, [record?.reply]) + const primaryLanguage = store.preferences.contentLanguages[0] || 'en' + const translatorUrl = getTranslatorLink(primaryLanguage, record?.text || '') + const needsTranslation = useMemo( + () => !isPostInLanguage(item.post, store.preferences.contentLanguages), + [item.post, store.preferences.contentLanguages], + ) const onPressReply = React.useCallback(() => { track('FeedItem:PostReply') @@ -98,17 +105,9 @@ export const FeedItem = observer(function ({ Toast.show('Copied to clipboard') }, [record]) - const primaryLanguage = store.preferences.contentLanguages[0] || 'en' - const onOpenTranslate = React.useCallback(() => { - Linking.openURL( - encodeURI( - `https://translate.google.com/?sl=auto&tl=${primaryLanguage}&text=${ - record?.text || '' - }`, - ), - ) - }, [record, primaryLanguage]) + Linking.openURL(translatorUrl) + }, [translatorUrl]) const onToggleThreadMute = React.useCallback(async () => { track('FeedItem:ThreadMute') @@ -301,12 +300,22 @@ export const FeedItem = observer(function ({ type="post-text" richText={item.richText} lineHeight={1.3} + style={s.flex1} /> ) : undefined} + {needsTranslation && ( + + + + Translate this post + + + + )}