Merge branch 'main' into zod-feed-cleanup

db/init/100_views.sql

@@ -87,7 +87,7 @@ SELECT
     boards.slug as board_slug,
     realms.id as realm_id,
     realms.slug as realm_slug,
-    realms.string_id as realm_string_id,
+    realms.string_id as realm_external_id,
     first_post.id as first_post_id,
     first_post.string_id AS first_post_string_id,
     first_post.content AS content,

db/test_db_init/050_memes_insert.sql

@@ -7,8 +7,9 @@ new_thread_id AS (
 root_post_id AS (
   INSERT INTO posts (string_id,parent_thread,parent_post,author,created,"content","type",whisper_tags,is_deleted,anonymity_type) OVERRIDING SYSTEM VALUE VALUES 
     ('b2c57275-512e-4821-8cf8-b3ac76e1e044',(SELECT id FROM new_thread_id), NULL, 1,'2020-08-22 03:34:39.806','[{"insert":"Somnium Files Funny Memes"},{"attributes":{"header":2},"insert":""}]','text','{}',false,'everyone')
-  RETURNING id)
-INSERT INTO posts (string_id,parent_thread,parent_post,author,created,"content","type",whisper_tags,is_deleted,anonymity_type) VALUES 
+  RETURNING id),
+post_inserts AS (
+  INSERT INTO posts (string_id,parent_thread,parent_post,author,created,"content","type",whisper_tags,is_deleted,anonymity_type) VALUES 
  ('d14e1001a-a22a-4673-8b8a-cd0423e38a3',(SELECT id FROM new_thread_id),(SELECT id FROM root_post_id),1,'2020-08-22 03:35:03.191','[{"insert":{"block-image":{"src":"https://firebasestorage.googleapis.com/v0/b/bobaboard-fb.appspot.com/o/images%2Fmemes%2F2765f36a-b4f9-4efe-96f2-cb34f055d032%2F45c75c3e-1086-4e72-8742-549335cdfbff?alt=media&token=97efd46b-f464-4620-a1d1-a97f61e96d91","spoilers":false,"width":1280,"height":720}}},{"insert":""}]','text','{}',false,'strangers')
 ,('a16d4b39-4e63-434f-9502-3c1e67cb253b',(SELECT id FROM new_thread_id),(SELECT id FROM root_post_id),1,'2020-08-22 03:35:24.049','[{"insert":{"block-image":{"src":"https://firebasestorage.googleapis.com/v0/b/bobaboard-fb.appspot.com/o/images%2Fmemes%2F2765f36a-b4f9-4efe-96f2-cb34f055d032%2F768fced9-5f46-42f0-8aeb-ad135b412ae9?alt=media&token=0840c22f-0781-4a83-8f17-f41338772c99","spoilers":false,"width":690,"height":388}}},{"insert":""}]','text','{}',false,'strangers')
 ,('41b1ed49-2235-4100d-bbf5-d7bd304a2d6',(SELECT id FROM new_thread_id),(SELECT id FROM root_post_id),1,'2020-08-22 03:35:44.507','[{"insert":{"block-image":{"src":"https://firebasestorage.googleapis.com/v0/b/bobaboard-fb.appspot.com/o/images%2Fmemes%2F2765f36a-b4f9-4efe-96f2-cb34f055d032%2F0c9c7e0e-2bc0-4a71-9c6e-201890f8bbf8?alt=media&token=caa232a5-9cb6-46c7-8968-c979364d255d","spoilers":false,"width":3840,"height":2160}}},{"insert":""}]','text','{}',false,'strangers')
@@ -144,8 +145,14 @@ INSERT INTO posts (string_id,parent_thread,parent_post,author,created,"content",
 ,('d169fd3a-e819-4401-bbf4-950e82598901',(SELECT id FROM new_thread_id),(SELECT id FROM root_post_id),1,'2020-08-22 03:35:44.507','[{"insert":{"block-image":{"src":"https://firebasestorage.googleapis.com/v0/b/bobaboard-fb.appspot.com/o/images%2Fmemes%2F2765f36a-b4f9-4efe-96f2-cb34f055d032%2F0c9c7e0e-2bc0-4a71-9c6e-201890f8bbf8?alt=media&token=caa232a5-9cb6-46c7-8968-c979364d255d","spoilers":false,"width":3840,"height":2160}}},{"insert":""}]','text','{}',false,'strangers')
 ,('cf781193-4cff-466c-8b34-a38b94ec8014',(SELECT id FROM new_thread_id),(SELECT id FROM root_post_id),1,'2020-08-22 03:36:18.729','[{"insert":{"block-image":{"src":"https://firebasestorage.googleapis.com/v0/b/bobaboard-fb.appspot.com/o/images%2Fmemes%2F2765f36a-b4f9-4efe-96f2-cb34f055d032%2F7707f104-044c-4111-b422-74e11ccef4a2?alt=media&token=7cdf3edb-0d63-467e-ade6-05447cc602c3","spoilers":false,"width":1920,"height":1080}}},{"insert":""}]','text','{}',false,'strangers')
 ,('b8321446-3ebe-4177-80ce-7a5adcb38e36',(SELECT id FROM new_thread_id),(SELECT id FROM root_post_id),1,'2020-08-22 03:36:55.850','[{"insert":{"block-image":{"src":"https://firebasestorage.googleapis.com/v0/b/bobaboard-fb.appspot.com/o/images%2Fmemes%2F2765f36a-b4f9-4efe-96f2-cb34f055d032%2F894374bd-fed2-42af-a21e-18ed9c5040fc?alt=media&token=f5afda1f-2a14-43a3-9717-7cdc0aeddc2d","spoilers":false,"width":3840,"height":2160}}},{"insert":""}]','text','{}',false,'strangers')
-;
+)
 
+INSERT INTO user_thread_identities(thread_id, user_id, identity_id)
+  VALUES
+    ((SELECT id FROM new_thread_id),
+    (SELECT id FROM Users WHERE username = 'bobatan'),
+    (SELECT id FROM secret_identities WHERE display_name = 'Old Time-y Anon'));
+
 INSERT INTO categories(category) VALUES 
   ('aiba'), 
   ('release');

server/feeds/queries.ts

@@ -9,6 +9,59 @@ const info = debug("bobaserver:feeds:queries-info");
 const log = debug("bobaserver:feeds:queries-log");
 
 const DEFAULT_PAGE_SIZE = 10;
+
+export const getRealmActivityByExternalId = async({
+  realmExternalId,
+  firebaseId,
+  cursor,
+  pageSize,
+}: {
+  realmExternalId: string;
+  firebaseId: string | null;
+  cursor: string | null;
+  pageSize?: number;
+}): Promise<ZodDbFeedType | null | false> => {
+  const decodedCursor = cursor ? decodeCursor(cursor) : null;
+
+  const finalPageSize =
+    decodedCursor?.page_size || pageSize || DEFAULT_PAGE_SIZE;
+  const rows = await pool.manyOrNone(sql.getRealmActivity, {
+    realm_id: realmExternalId,
+    firebase_id: firebaseId,
+    last_activity_cursor: decodedCursor?.last_activity_cursor || null,
+    page_size: finalPageSize,
+  });
+
+  if (!rows) {
+    log(`Realm not found: ${realmExternalId}`);
+    return null;
+  };
+
+  if (rows.length == 1 && rows[0].thread_id == null) {
+    // Only one row with just the null thread)
+    log(`Realm empty: ${realmExternalId}`);
+    return { cursor: null, activity: [] };
+  };
+
+  let result = rows;
+  let nextCursor = null;
+  info(`Got getRealmActivityByExternalId query result`, result);
+  if (result.length > finalPageSize) {
+    nextCursor = encodeCursor({
+      last_activity_cursor:
+        result[result.length - 1].thread_last_activity_at_micro,
+      page_size: finalPageSize,
+    });
+    // remove last element from array
+    result.pop();
+  };
+
+  log(
+    `Fetched realm ${realmExternalId} activity data for user ${firebaseId}`
+  );
+  return { cursor: nextCursor, activity: rows };
+};
+
 export const getBoardActivityByExternalId = async ({
   boardExternalId,
   firebaseId,

server/feeds/routes.ts

@@ -5,6 +5,7 @@ import {
 } from "utils/response-utils";
 import {
   getBoardActivityByExternalId,
+  getRealmActivityByExternalId,
   getUserActivity,
   getUserStarFeed,
 } from "./queries";
@@ -21,6 +22,77 @@ const log = debug("bobaserver:feeds:routes");
 
 const router = express.Router();
 
+/**
+ * @openapi
+ * /feeds/realms/{realm_id}:
+ *   get:
+ *     summary: Get latest activity on entire realm
+ *     operationId: getRealmActivity
+ *     tags:
+ *       - /feeds/
+ *     parameters:
+ *       - name: realm_id
+ *         in: path
+ *         description: The external id of the realm to fetch the activity of.
+ *         required: true
+ *         schema:
+ *           type: string
+ *       - name: cursor
+ *         in: query
+ *         description: The cursor to start feeding the activity of the board from.
+ *         schema:
+ *           type: string
+ *         allowEmptyValue: true
+ *     responses:
+ *        404:
+ *          description: The realm was not found.
+ *        200:
+ *         description: The realm's activity.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: "#/components/schemas/FeedActivity"
+ */
+router.get("/realms/:realm_id", ensureLoggedIn, async (req, res) => {
+  const { realm_id: realmExternalId } = req.params;
+  const { cursor } = req.query;
+  log(
+    `Fetching activity data for realm with slug ${realmExternalId} with cursor ${cursor}`
+  );
+
+  log(cursor);
+  const result = await getRealmActivityByExternalId({
+    realmExternalId,
+    firebaseId: req.currentUser?.uid || null,
+    cursor: (cursor as string) || null,
+  });
+  info(`Found activity for realm ${realmExternalId}:`, result);
+
+  if (!result) {
+    throw new NotFound404Error(
+      `Realm with id ${realmExternalId} was not found`
+    );
+  }
+  if (!result.activity.length) {
+    res.sendStatus(204);
+    return;
+  }
+
+  const threadsWithIdentity = result.activity.map(makeServerThreadSummary);
+  const response: ZodFeed = {
+    cursor: {
+      next: result.cursor,
+    },
+    activity: threadsWithIdentity,
+  };
+
+  response.activity.map((post) => ensureNoIdentityLeakage(post));
+  log(
+    `Returning board activity data for board ${realmExternalId} for user ${req.currentUser?.uid}.`
+  );
+  res.status(200).json(FeedActivitySchema.parse(response));
+});
+
 /**
  * @openapi
  * /feeds/boards/{board_id}:

server/feeds/sql/board-activity-by-external-id.sql

@@ -4,7 +4,7 @@ SELECT
     board_slug,
     board_external_id as board_id,
     realm_slug,
-    realm_string_id as realm_id,
+    realm_external_id as realm_id,
     TO_CHAR(last_update_timestamp, 'YYYY-MM-DD"T"HH24:MI:SS.00"Z') as thread_last_activity_at,
     thread_details.default_view,
     -- Amount details

server/feeds/sql/index.ts

@@ -2,6 +2,9 @@ import { QueryFile } from "pg-promise";
 import path from "path";
 
 export default {
+  getRealmActivity: new QueryFile(
+    path.join(__dirname, "realm-activity.sql")
+  ),
   getUserFeedActivity: new QueryFile(
     path.join(__dirname, "user-feed-activity.sql")
   ),

server/feeds/sql/realm-activity.sql

@@ -0,0 +1,65 @@
+SELECT
+    -- Thread details (DbThreadType)
+    thread_external_id as thread_id,
+    board_slug,
+    board_external_id as board_id,
+    realm_slug,
+    realm_external_id as realm_id,
+    TO_CHAR(last_update_timestamp, 'YYYY-MM-DD"T"HH24:MI:SS.00"Z') as thread_last_activity_at,
+    thread_details.default_view,
+    -- Amount details
+    COALESCE(posts_amount, 0) as thread_total_posts_amount,
+    COALESCE(threads_amount, 0) as thread_direct_threads_amount,
+    COALESCE(comments_amount, 0) as thread_total_comments_amount,
+    COALESCE(new_posts_board_amount, 0) as thread_new_posts_amount,
+    COALESCE(new_comments_board_amount, 0) as thread_new_comments_amount,
+    COALESCE(muted, FALSE) as muted,
+    COALESCE(hidden, FALSE) as hidden,
+    COALESCE(starred, FALSE) as starred,
+    -- Contribution details (DbContributionType)
+    first_post_string_id as post_id,
+    thread_external_id as parent_thread_id,
+    NULL as parent_post_id,
+    board_slug as parent_board_slug,
+    board_external_id as parent_board_id,
+    -- Author details
+    author,
+    username,
+    user_avatar,
+    secret_identity_name,
+    secret_identity_avatar,
+    secret_identity_color,
+    accessory_avatar,
+    COALESCE(friend_thread, FALSE) as friend,
+    COALESCE(own_thread, FALSE) as self,
+    TO_CHAR(first_post_timestamp, 'YYYY-MM-DD"T"HH24:MI:SS.00"Z"') as created_at,
+    -- Generic details
+    content,
+    -- Contribution tags
+    index_tags,
+    category_tags,
+    content_warnings,
+    whisper_tags,
+    COALESCE(own_thread, FALSE) as is_own,
+    COALESCE(is_new_board, FALSE) as is_new,
+    -- This last activity must have the .US at the end or it will trigger a bug
+    -- where some posts are skipped by the last activity cursor.
+    -- See documentation on the queries JS file.
+    TO_CHAR(last_update_timestamp, 'YYYY-MM-DD"T"HH24:MI:SS.US') as thread_last_activity_at_micro
+FROM threads
+INNER JOIN thread_details
+  ON threads.id = thread_details.thread_id AND thread_details.realm_external_id = ${realm_id}
+LEFT JOIN thread_identities
+    ON thread_identities.user_id = thread_details.author AND thread_identities.thread_id = thread_details.thread_id
+LEFT JOIN thread_user_details
+   ON ${firebase_id} IS NOT NULL AND thread_user_details.user_id = (SELECT id FROM users WHERE users.firebase_id = ${firebase_id} LIMIT 1)
+         AND thread_details.thread_id = thread_user_details.thread_id
+WHERE
+   -- Activity cursor conditions
+   thread_details.realm_external_id = ${realm_id}
+   AND last_update_timestamp <= COALESCE(${last_activity_cursor}, NOW())
+   AND is_new IS TRUE
+   AND muted IS FALSE
+   AND hidden IS FALSE
+ORDER BY thread_last_activity_at DESC
+LIMIT ${page_size} + 1

server/feeds/sql/star-feed-activity.sql

@@ -19,7 +19,7 @@ SELECT
     NULL as parent_post_id,
     board_slug as parent_board_slug,
     realm_slug,
-    realm_string_id as realm_id,
+    realm_external_id as realm_id,
     -- Author details
     author,
     author_identity.username,

server/feeds/sql/user-feed-activity.sql

@@ -5,7 +5,7 @@ SELECT
     board_slug,
     board_external_id as board_id,
     realm_slug,
-    realm_string_id as realm_id,
+    realm_external_id as realm_id,
     TO_CHAR(last_update_timestamp, 'YYYY-MM-DD"T"HH24:MI:SS.00"Z"') as thread_last_activity_at,
     thread_details.default_view,
     -- Amount details
@@ -69,6 +69,6 @@ WHERE
    AND (${own_only} IS FALSE OR own_thread IS TRUE)
    AND muted IS FALSE
    AND hidden IS FALSE
-   AND realm_string_id = ${realm_id}
+   AND realm_external_id = ${realm_id}
 ORDER BY thread_last_activity_at DESC
 LIMIT ${page_size} + 1

server/feeds/tests/REST/REST-realm-feed.test.ts

@@ -0,0 +1,32 @@
+import { BOBATAN_USER_ID, ONCEST_USER_ID } from "test/data/auth";
+import { setLoggedInUser, startTestServer } from "utils/test-utils";
+
+import { BOBATAN_REALM_FEED } from "test/data/realmfeed";
+import { TWISTED_MINDS_REALM_EXTERNAL_ID } from "test/data/realms";
+import debug from "debug";
+import request from "supertest";
+import router from "../../routes";
+
+jest.mock("handlers/auth");
+
+const log = debug("bobaserver:board:routes");
+
+  describe("Tests Realm Activity Feed Endpoint", () => {
+    const server = startTestServer(router);
+
+    test("should return realm activity data" , async () => {
+        setLoggedInUser(BOBATAN_USER_ID)
+        const res = await request(server.app).get(`/realms/${TWISTED_MINDS_REALM_EXTERNAL_ID}`);
+
+        expect(res.status).toBe(200);
+        expect(res.body).toEqual(BOBATAN_REALM_FEED);
+    });
+
+    test("should return 401 when user isn't logged in", async () => { 
+        const res = await request(server.app).get(`/realms/${TWISTED_MINDS_REALM_EXTERNAL_ID}`);
+
+        expect(res.status).toBe(401);
+        expect(res.body).toEqual({message: "No authenticated user found."});
+    });
+  });
+

server/feeds/tests/queries.test.ts

@@ -2,6 +2,7 @@ import { BOBATAN_USER_ID, ONCEST_USER_ID } from "test/data/auth";
 import {
   FAVORITE_CHARACTER_THREAD_ID,
   FAVORITE_MURDER_THREAD_ID,
+  FUNNY_MEMES_THREAD_ID,
 } from "test/data/threads";
 
 import { TWISTED_MINDS_REALM_EXTERNAL_ID } from "test/data/realms";
@@ -75,6 +76,19 @@ describe("feed activity queries", () => {
         thread_last_activity_at: "2020-10-04T05:44:00.00Z",
         threads_amount: 0,
       },
+      {
+        comments_amount: 0,
+        created_at: "2020-08-22T03:34:39.00Z",
+        is_new: false,
+        new_comments_amount: 0,
+        new_posts_amount: 0,
+        post_id: "b2c57275-512e-4821-8cf8-b3ac76e1e044",
+        posts_amount: 136,
+        thread_id: FUNNY_MEMES_THREAD_ID,
+        thread_last_activity_at: "2020-08-22T03:36:55.00Z",
+        thread_last_activity_at_micro: "2020-08-22T03:36:55.850000",
+        threads_amount: 135,
+      },
       {
         comments_amount: 0,
         created_at: "2020-04-24T05:42:00.00Z",
@@ -140,6 +154,19 @@ describe("feed activity queries", () => {
         thread_last_activity_at: "2020-10-04T05:44:00.00Z",
         threads_amount: 0,
       },
+      {
+        comments_amount: 0,
+        created_at: "2020-08-22T03:34:39.00Z",
+        is_new: false,
+        new_comments_amount: 0,
+        new_posts_amount: 0,
+        post_id: "b2c57275-512e-4821-8cf8-b3ac76e1e044",
+        posts_amount: 136,
+        thread_id: FUNNY_MEMES_THREAD_ID,
+        thread_last_activity_at: "2020-08-22T03:36:55.00Z",
+        thread_last_activity_at_micro: "2020-08-22T03:36:55.850000",
+        threads_amount: 135,
+      },
       {
         comments_amount: 2,
         created_at: "2020-04-30T03:23:00.00Z",
@@ -180,7 +207,7 @@ describe("feed activity queries", () => {
     })) as ZodDbFeedType;
 
     expect(feed.cursor).toBe(
-      "eyJsYXN0X2FjdGl2aXR5X2N1cnNvciI6IjIwMjAtMDUtMjNUMDU6NTI6MDAuMDAwMDAwIiwicGFnZV9zaXplIjoxfQ=="
+      "eyJsYXN0X2FjdGl2aXR5X2N1cnNvciI6IjIwMjAtMDgtMjJUMDM6MzY6NTUuODUwMDAwIiwicGFnZV9zaXplIjoxfQ=="
     );
     expect(feed.activity.map(extractActivity)).toEqual([
       {