fix: repeatedly fetching posts, improve federation logic

.github/workflows/cloudflare_pages.yml

@@ -0,0 +1,54 @@
+name: Deploy to Cloudflare Pages
+
+on:
+  # Uncomment *ONLY* this to enable CD for every new release
+  # push:
+  #   tags:
+  #     - 'v*'
+  # Uncomment *ONLY* this to enable CD for every push to the `main` branch
+  # push:
+  #   branch: release
+  workflow_dispatch: {}
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      deployments: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - run: corepack enable
+
+      - name: Set node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: pnpm
+
+      - name: Install build dependencies
+        run: npm install -g pnpm && pnpm i --frozen-lockfile
+
+      - name: Build Elk
+        run: pnpm run build
+        env:
+          HOST: 0.0.0.0
+          NODE_ENV: production
+          NUXT_DEPLOY_URL: ${{ vars.NUXT_DEPLOY_URL }}
+          NUXT_PUBLIC_DEFAULT_SERVER: ${{ vars.NUXT_PUBLIC_DEFAULT_SERVER }}
+          NUXT_STORAGE_DRIVER: kv-binding
+          VITE_DEV_PWA: true
+          NITRO_PRESET: ${{ vars.NITRO_PRESET }}
+
+      - name: Publish
+        uses: cloudflare/pages-action@1
+        with:
+          directory: .output/public
+          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          projectName: ${{ secrets.CLOUDFLARE_PAGES_PROJECT_NAME }}
+          apiToken: ${{ secrets.CLOUDFLARE_PAGES_API_TOKEN }}
+          gitHubToken: ${{ secrets.GITHUB_TOKEN }}

composables/cache.ts

@@ -1,8 +1,16 @@
 import { LRUCache } from 'lru-cache'
 import type { mastodon } from 'masto'
 
+// expire in an hour
 const cache = new LRUCache<string, any>({
   max: 1000,
+  ttl: 3600000,
+  ttlAutopurge: true,
+  allowStaleOnFetchAbort: true,
+  allowStaleOnFetchRejection: true,
+  allowStale: true,
+  noUpdateTTL: true,
+  ttlResolution: 60000,
 })
 
 if (process.dev && process.client)
@@ -17,19 +25,138 @@ function removeCached(key: string) {
   cache.delete(key)
 }
 
-export function fetchStatus(id: string, force = false): Promise<mastodon.v1.Status> {
-  const server = currentServer.value
-  const userId = currentUser.value?.account.id
-  const key = `${server}:${userId}:status:${id}`
-  const cached = cache.get(key)
-  if (cached && !force)
-    return cached
-  const promise = useMastoClient().v1.statuses.fetch(id)
-    .then((status) => {
-      cacheStatus(status)
-      return status
+function generateStatusIdCacheKeyAccessibleToCurrentUser(statusId: string) {
+  return `${currentServer.value}:${currentUser.value?.account.id}:status:${statusId}`
+}
+
+async function federateRemoteStatus(statusUri: string, force = false): Promise<mastodon.v1.Status | null> {
+  if (cache.has(`stop:${statusUri}`)) {
+    if (process.dev)
+      // eslint-disable-next-line no-console
+      console.debug(`Skipping further processing for invalid status URI: ${statusUri}`)
+    return Promise.resolve(null)
+  }
+
+  if (statusUri.startsWith(`https://${currentServer.value}`)) {
+    if (process.dev)
+      // eslint-disable-next-line no-console
+      console.info(`Local domain is authoritative, so redirecting resolution request for status: ${statusUri}`)
+
+    return fetchStatus(statusUri.split('/').pop() ?? statusUri.replace(`https://${currentServer.value}/`, ''))
+  }
+
+  if (statusUri.search(/^\d+$/) !== -1) {
+    if (process.dev)
+      // eslint-disable-next-line no-console
+      console.info(`statusUri parameter was passed an ID, so redirecting resolution request: ${statusUri}`)
+
+    return fetchStatus(statusUri, force)
+  }
+
+  const localStatusIdCacheKey = generateStatusIdCacheKeyAccessibleToCurrentUser(statusUri)
+
+  const cached: mastodon.v1.Status | Promise<mastodon.v1.Status> | undefined | null | number = cache.get(localStatusIdCacheKey, { allowStale: false, updateAgeOnGet: false })
+  if (cached) {
+    if (
+      !!cached
+      && !(typeof cached === 'number')
+      && !(cached instanceof Promise)
+      && (cached.uri === statusUri)
+      && !force
+    ) {
+      return cached
+    }
+    else if (cached instanceof Promise) {
+      return cached
+    }
+    else if (typeof cached === 'number') {
+      if ([401, 403, 418].includes(cached))
+        console.error(`Current user is forbidden or lacks authorization to fetch status: ${statusUri}`)
+      if ([404].includes(cached))
+        console.error(`The requested status URI cannot be found: ${statusUri}`)
+      if ([429].includes(cached))
+        console.error('The request was rate-limited by the Mastodon server')
+      if ([500, 501, 503].includes(cached))
+        console.error('The Mastodon server is unresponsive')
+      return Promise.resolve(null)
+    }
+  }
+
+  const promise = useMastoClient().v2.search({ q: statusUri, type: 'statuses', resolve: (!!currentUser.value), limit: 1 })
+    .then(async (results) => {
+      const post = results.statuses.pop()
+      if (!post) {
+        console.error(`Status could not be federated, perhaps it no longer exists: '${statusUri}'`)
+        cache.set(localStatusIdCacheKey, 404)
+        return Promise.resolve(null)
+      }
+
+      const splitUri = post.account.url.replace('https://', '').split('/@')
+      const accountWebfinger = `${splitUri[1]}@${splitUri[0]}`
+      post.account.acct = accountWebfinger
+
+      cache.set(localStatusIdCacheKey, post)
+      return post
     })
-  cache.set(key, promise)
+    .catch((e) => {
+      console.error(`Encountered error while federating status using URI '${statusUri}' | ${(e as Error)}`)
+      cache.set(localStatusIdCacheKey, null)
+      return Promise.resolve(null)
+    })
+  cache.set(localStatusIdCacheKey, promise)
+  return promise
+}
+
+export async function fetchStatus(statusId: string, force = false): Promise<mastodon.v1.Status | null> {
+  if (cache.has(`stop:${statusId}`)) {
+    if (process.dev)
+      // eslint-disable-next-line no-console
+      console.debug(`Skipping further processing for invalid status Id: ${statusId}`)
+    return Promise.resolve(null)
+  }
+
+  // Handle scenario where the value of statusId is actually an URI
+  if (statusId.startsWith('h')) {
+    if (process.dev)
+      // eslint-disable-next-line no-console
+      console.info(`statusId parameter was passed an URI, so redirecting resolution request: ${statusId}`)
+    return federateRemoteStatus(statusId, force)
+  }
+
+  // handle invalid statusId
+  if ((statusId.search(/^\d+$/) === -1)) {
+    console.error(`Malformed or unrecognized Status ID: ${statusId}`)
+    cache.set(`stop:${statusId}`, 418)
+    return Promise.resolve(null)
+  }
+
+  const localStatusIdCacheKey = generateStatusIdCacheKeyAccessibleToCurrentUser(statusId)
+  const cached: mastodon.v1.Status | Promise<mastodon.v1.Status> | undefined | null = cache.get(localStatusIdCacheKey, { allowStale: false, updateAgeOnGet: false })
+  if (cached) {
+    // avoid race condition by returning the existing promise instead of restarting the chain of events all over again
+    if (cached instanceof Promise)
+      return cached
+    if (typeof cached === 'number') {
+      // wait for the cached value to expire before trying again
+      if ([401, 403, 404, 418, 429, 500, 501, 503].includes(cached))
+        return null
+    }
+    else if (cached.id === statusId) {
+      // if we don't care about authoritative values then return cached value
+      if (!force)
+        return cached
+    }
+  }
+
+  const promise = useMastoClient().v1.statuses.fetch(statusId)
+    .then(async (post) => {
+      const splitUri = post.account.url.replace('https://', '').split('/@')
+      const accountWebfinger = `${splitUri[1]}@${splitUri[0]}`
+      post.account.acct = accountWebfinger
+      cache.set(localStatusIdCacheKey, post)
+      return post
+    })
+  cache.set(localStatusIdCacheKey, promise)
   return promise
 }
 

docs/content/2.deployment/10.getting_started.md

@@ -0,0 +1,82 @@
+# Getting Started
+## Introduction
+
+Want to host Elk for your Mastodon instance? You came to the right place! This section covers topics that are common to several kinds of deployments, including:
+- [Preparing for deployment](#prerequisites)
+- [Obtaining the Elk source code](#creating-a-fork)
+- [Configuring persistent storage](#setting-up-cloudflare-workers-kv)
+
+
+## Prerequisites
+Before getting started, please ensure that you have access to the following:
+- If you don't have a GitHub account, [create one](https://github.com/signup?ref_cta=Sign+up&ref_loc=header+logged+out&ref_page=%2F&source=header-home) before proceeding. The _Free_ tier includes everything you need to deploy Elk.
+- If you plan to deploy Elk using serverless infrastructure (e.g. Netlify, Cloudflare Pages) _**or**_ intend to use continuous integration/continuous deployment tools (e.g. GitHub Actions), then you'll need a Cloudflare account in order to setup a [key-value store](https://www.cloudflare.com/products/workers-kv/) that will be used to save Elk application and user credentials. If you don't have a Cloudflare account, [create one](https://dash.cloudflare.com/sign-up/workers) before proceeding. The _Workers Free_ plan includes everything you need to deploy Elk and run a personal/low-traffic instance, **however**, you may incur charges if you [exceed plan limits](https://www.cloudflare.com/plans/developer-platform/) 
+
+![Cloudflare Workers Free plan limits as of February 19, 2023.](/docs/images/selfhosting-guide/cf-workers-limits.png)
+
+- If you plan to deploy Elk to a containerized environment, you'll need to provision a file system-based persistent storage and mount it as a volume. Please refer to [Docker's official documentation](https://docs.docker.com/engine/reference/commandline/volume_create/) and/or your hosting provider's product guide for detailed instructions on how to do this.
+
+<br>[back to top](#getting-started)
+
+## Creating a fork
+
+[Click here to fork the Elk repository](https://github.com/elk-zone/elk/fork). Make sure to _deselect_ "Copy the main branch only" so that you have access to the [`release`](https://github.com/elk-zone/elk/tree/release) branch, which is more stable than the [`main`](https://github.com/elk-zone/elk) branch. 
+
+
+![The settings to use for forking the Elk repository](/docs/images/selfhosting-guide/github-fork.png)
+
+For a more detailed guide to forking a repository, please refer to [GitHub's official documentation](https://docs.github.com/en/get-started/quickstart/fork-a-repo).
+
+<br>[back to top](#getting-started)
+
+### Which branch(es) to deploy?
+
+> **_A note about alpha software:_** Elk is under active development. Many core features are ready for daily use, but many others remain a work-in-progress. Please be mindful of this fact as you consider which branch to deploy (and to which audiences). 
+
+<br>
+
+If you followed [these instructions](#creating-a-fork), you have access to 2 branches for your deployment: `release` and `main`. You can deploy one, or the other, or both. 
+
+If you're looking to Elk as a replacement for daily use, we strongly recommend sticking to the `release` branch. If you are a developer that likes to live on the cutting edge, even if that means dealing with feature regressions and new bugs, then the `main` branch will be more to your liking.
+
+Unless you're an experienced developer that's actively contributing to the Elk project, deploying any branch *other than* `release` and `main` is **_strongly discouraged_**; these other branches contain features in various stages of testing and development－ all of them have the potential to permanently corrupt or destroy data without warning. 
+
+If you're a developer that's interested in contributing to the Elk project, please review the [Contributing](https://github.com/elk-zone/elk/tree/release#-contributing) section on the repository's main landing page.
+
+<br>[back to top](#getting-started)
+
+## Setting up Cloudflare Workers KV
+### Create a KV Namespace
+As [mentioned earlier](#prerequisites), you'll need to create a Cloudflare KV Namespace if you are deploying Elk to a serverless environment, such as [Netlify Functions](https://www.netlify.com/products/#netlify-functions) or [Cloudflare Pages](https://pages.cloudflare.com/). The following instructions will guide you through the process by using the Cloudflare Dashboard:[^1]
+
+1. Log into your CloudFlare account.
+2. From the left navigation bar on the *main* account Dashboard (this is one navigation level _above_ the website-level dashboards):
+   - Click "Workers",
+     > _Note:_ if this is your first time opening this tab, CloudFlare will ask you to set up a free custom Cloudflare Workers subdomain. Follow the instructions to set up the domain. For the purposes of this guide, it doesn't matter what you choose as your subdomain (besides, you can change it later)
+
+   - Then, click "KV" in the navigation bar:
+
+   ![Where to find the Workers and KV Namespace links](/docs/images/selfhosting-guide/cf-nav-bar.png)
+
+3. Click the blue "Create namespace" button that will appear in the main body of the page. Follow the instructions to name the KV Namespace. For Elk's purposes, you can label the KV Namespace whatever you want, but Cloudflare will not allow you to change the label. Click "Add" then make a note of the newly-created KV Namespace **ID**
+
+![How to create a KV Namespace using the Cloudflare Dashboard](/docs/images/selfhosting-guide/cf-create-kv-namespace.png)
+
+<br>[back to top](#getting-started)
+
+### Find your account ID and create an API token
+If your hosting provider is _**not**_ Cloudflare, then you'll need to create an API token that will allow Elk to read and write to the newly-created KV Namespace.
+
+1. Again, from the left-side navigation bar on the *main* account Dashboard, click on "Workers"
+2. Halfway down the **right-side** navigation bar, you will see a gray box labeled, `Account ID`－ make a note of that value and save it for later. 
+3. Beneath `Account ID` field you will see a link to "API Tokens", click on that link and then click on the blue button labeled "Create Token" (or "Get started" if this is the first Custom Token you create in your account):
+4. Follow the directions to label and configure your API Token. For Elk's purposes, the only required permission is **Account | Workers KV Storage | Edit** :
+
+![The settings to use for the CloudFlare API token](/docs/images/selfhosting-guide/cf-api-token-settings.png)
+
+5. Save the newly made token in a safe spot and **DO NOT** add it to any file that you may (intentionally or unintentionally) commit to any GitHub repository. You may want to keep the tab open while you configure the rest of your deployment.
+
+<br>[back to top](#getting-started)
+
+
+[^1]: If you prefer using the command line, then please refer to [Cloudflare's official documentation](https://developers.cloudflare.com/workers/wrangler/workers-kv/)