Only fetch 10k articles at most from categories

- On refresh for a feed or folder in FreshRSS, cap
the fetch count to 10,000
- Once collected, only fetch the articles that are missing
from the database.

Author: jocmp

capy/src/main/java/com/jocmp/capy/accounts/reader/ReaderAccountDelegate.kt

@@ -53,6 +53,8 @@ internal class ReaderAccountDelegate(
             } else {
                 refreshCategoryArticles(filter)
             }
+
+            fetchMissingArticles()
         }
     }
 
@@ -206,7 +208,6 @@ internal class ReaderAccountDelegate(
         refreshFeeds()
         refreshArticleState()
         fetchPaginatedArticles(since = since, stream = Stream.Read())
-        fetchMissingArticles()
     }
 
     private fun upsertTaggings(subscription: Subscription) {
@@ -270,17 +271,21 @@ internal class ReaderAccountDelegate(
     }
 
     /**
-     * This is a slightly different algorithm that `refreshTopLevelArticles`.
+     * This is a slightly different algorithm than [refreshTopLevelArticles].
      *
-     *   - Assume the category (folder or feed) exists so it skips refreshing
-     *   the subscription list
-     *   - Refresh all IDs in a given feed and save its content
+     *   - Assume the category (folder or feed) exists so it skips refreshing the subscription list
+     *   - Fetches up to 10k IDs
+     *   - On return, the [fetchMissingArticles] will only fetch articles that are not already
+     *     saved
      */
     private suspend fun refreshCategoryArticles(filter: ArticleFilter) {
         val stream = filter.toStream(source)
 
         refreshArticleState()
-        fetchPaginatedArticles(stream = stream)
+
+        withResult(googleReader.streamItemsIDs(stream = stream)) { result ->
+            articleRecords.createStatuses(articleIDs = result.itemRefs.map { it.hexID })
+        }
     }
 
     private suspend fun fetchMissingArticles() {
@@ -317,7 +322,6 @@ internal class ReaderAccountDelegate(
             stream = stream,
             since = since,
             continuation = continuation,
-            count = MAX_PAGINATED_ITEM_LIMIT,
         )
 
         val result = response.body()

capy/src/main/java/com/jocmp/capy/persistence/ArticleRecords.kt

@@ -46,6 +46,16 @@ internal class ArticleRecords internal constructor(
         )
     }
 
+    /**
+     * Create placeholder statuses to be updated
+     * by the [findMissingArticles] query
+     */
+    fun createStatuses(articleIDs: List<String>, updatedAt: ZonedDateTime = nowUTC()) {
+        database.transactionWithErrorHandling {
+            articleIDs.forEach { createStatus(articleID = it, updatedAt = updatedAt, read = false) }
+        }
+    }
+
     /**
      * Upserts a record status. On conflict it overwrites "read" metadata.
      */

capy/src/main/sqldelight/com/jocmp/capy/db/articles.sq

@@ -154,8 +154,7 @@ findMissingArticles:
 SELECT article_id
 FROM article_statuses
 LEFT OUTER JOIN articles ON article_statuses.article_id = articles.id
-WHERE articles.id IS NULL
-AND article_statuses.read = 0 OR article_statuses.starred = 1;
+WHERE articles.id IS NULL;
 
 create:
 INSERT INTO articles(

capy/src/test/java/com/jocmp/capy/accounts/feedbin/FeedbinAccountDelegateTest.kt

@@ -191,7 +191,7 @@ class FeedbinAccountDelegateTest {
             ),
         )
 
-        val starredEntries = listOf(readEntry, unreadEntry)
+        val starredEntries = listOf(unreadEntry, readEntry)
 
         coEvery { feedbin.subscriptions() }.returns(Response.success(subscriptions))
         coEvery { feedbin.unreadEntries() }.returns(Response.success(listOf(unreadEntry.id)))

capy/src/test/java/com/jocmp/capy/accounts/reader/ReaderAccountDelegateTest.kt

@@ -623,7 +623,7 @@ class ReaderAccountDelegateTest {
             googleReader.streamItemsIDs(
                 streamID = stream.id,
                 since = any(),
-                count = 100,
+                count = 10_000,
             )
         }.returns(Response.success(StreamItemIDsResult(itemRefs = itemRefs, continuation = null)))
 

readerclient/src/main/java/com/jocmp/readerclient/ItemIdentifiers.kt

@@ -0,0 +1,5 @@
+package com.jocmp.readerclient
+
+object ItemIdentifiers {
+    fun parseToHexID(numericID: String) = String.format("%016x", numericID.toLong())
+}

readerclient/src/main/java/com/jocmp/readerclient/ItemRef.kt

@@ -1,12 +1,11 @@
 package com.jocmp.readerclient
 
+import com.jocmp.readerclient.ItemIdentifiers.parseToHexID
 import com.squareup.moshi.JsonClass
 
 @JsonClass(generateAdapter = true)
 data class ItemRef(
     val id: String,
 ) {
-    val hexID = buildHexID(id)
+    val hexID = parseToHexID(id)
 }
-
-fun buildHexID(id: String) = String.format("%016x", id.toLong())