fewer writes (#30)

ccbrown · Nov 26, 2018 · bd2def4 · bd2def4
1 parent 8aa1cff
commit bd2def4
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 8 deletions.
diff --git a/server/forum_indexer.go b/server/forum_indexer.go
@@ -198,7 +198,8 @@ func (indexer *ForumIndexer) index(locale *Locale, poster string, timezone *time
 		"poster": poster,
 	})
 
-	cutoff := time.Now().Add(time.Hour * -12)
+	pageCutoff := time.Now().Add(-12 * time.Hour)
+	cutoff := time.Now().Add(-14 * 24 * time.Hour)
 	activity := []Activity(nil)
 
 	for page := 1; ; page++ {
@@ -209,13 +210,16 @@ func (indexer *ForumIndexer) index(locale *Locale, poster string, timezone *time
 
 		done := len(posts) == 0
 		for _, post := range posts {
-			if post.Time.Before(cutoff) {
+			if post.Time.Before(pageCutoff) {
 				done = true
 			}
+			if post.Time.Before(cutoff) {
+				break
+			}
 			activity = append(activity, post)
 		}
 
-		logger.WithField("count", len(posts)).Info("received forum posts")
+		logger.WithField("count", len(activity)).Info("received forum posts")
 
 		if done {
 			break

diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go
@@ -57,7 +57,7 @@ func (indexer *RedditIndexer) run() {
 			if next >= len(users) {
 				next = 0
 			}
-			time.Sleep(time.Second * 2)
+			time.Sleep(time.Second * 3)
 		}
 	}
 }
@@ -151,7 +151,8 @@ func (indexer *RedditIndexer) index(user string) error {
 		"user": user,
 	})
 
-	cutoff := time.Now().Add(time.Hour * -12)
+	pageCutoff := time.Now().Add(-12 * time.Hour)
+	cutoff := time.Now().Add(-14 * 24 * time.Hour)
 	activity := []Activity(nil)
 
 	for page := ""; ; {
@@ -163,20 +164,23 @@ func (indexer *RedditIndexer) index(user string) error {
 
 		done := len(things) == 0
 		for _, thing := range things {
-			if thing.ActivityTime().Before(cutoff) {
+			if thing.ActivityTime().Before(pageCutoff) {
 				done = true
 			}
+			if thing.ActivityTime().Before(cutoff) {
+				break
+			}
 			activity = append(activity, thing)
 		}
 
 		logger.WithFields(log.Fields{
-			"count": len(things),
+			"count": len(activity),
 		}).Info("received reddit activity")
 
 		if done {
 			break
 		}
-		time.Sleep(time.Second * 2)
+		time.Sleep(time.Second * 3)
 	}
 
 	if len(activity) == 0 {