Better get_feed_es error logging. Fix fetch related saves + reposts

discovery-provider/src/queries/get_feed.py

@@ -1,4 +1,5 @@
 import datetime
+import logging
 
 from flask import request
 from sqlalchemy import and_, desc, func, or_
@@ -24,6 +25,8 @@
 
 trackDedupeMaxMinutes = 10
 
+logger = logging.getLogger(__name__)
+
 
 def get_feed(args):
     skip_es = request.args.get("es") == "0"
@@ -32,7 +35,8 @@ def get_feed(args):
         try:
             (limit, _) = get_pagination_vars()
             return get_feed_es(args, limit)
-        except:
+        except Exception as e:
+            logger.error(f"elasticsearch get_feed_es failed: {e}")
             return get_feed_sql(args)
     else:
         return get_feed_sql(args)

discovery-provider/src/queries/get_feed_es.py

@@ -218,7 +218,7 @@ def get_feed_es(args, limit=10):
     item_keys = [i["item_key"] for i in sorted_feed]
 
     (follow_saves, follow_reposts) = fetch_followed_saves_and_reposts(
-        current_user_id, item_keys, limit * 20
+        current_user_id, item_keys, limit
     )
 
     for item in sorted_feed:
@@ -266,7 +266,7 @@ def fetch_followed_saves_and_reposts(current_user_id, item_keys, limit):
                 ]
             }
         },
-        "size": limit * 20,  # how much to overfetch?
+        "size": limit * 10,  # how much to overfetch?
         "sort": {"created_at": "desc"},
     }
     mdsl = [

discovery-provider/src/utils/elasticdsl.py

@@ -26,6 +26,9 @@ def listify(things):
 
 
 def pluck_hits(found):
+    if "error" in found:
+        raise Exception(found["error"])
+
     res = [h["_source"] for h in found["hits"]["hits"]]
 
     # add score for search_quality.py script
@@ -34,21 +37,6 @@ def pluck_hits(found):
     return res
 
 
-def docs_and_ids(found, id_set=False):
-    docs = []
-    ids = []
-    for hit in found["hits"]["hits"]:
-        docs.append(hit["_source"])
-        ids.append(hit["_id"])
-    if id_set:
-        ids = set(ids)
-    return docs, ids
-
-
-def hits_by_id(found):
-    return {h["_id"]: h["_source"] for h in found["hits"]["hits"]}
-
-
 def populate_user_metadata_es(user, current_user):
     user["total_balance"] = str(
         int(user.get("balance", "0") or "0")