fix(wordpress): ensure all file links are rewritten (#31652)

- in the case of either having a small `schema.perPage` setting
  or many many referenced resources on a single page only the first
  page of links were properly rewritten
- now we ensure that all pages are fully processed until we resolve the
  promise

fixes #31646

Co-authored-by: Ward Peeters <[email protected]>

Author: rburgst

packages/gatsby-source-wordpress/__tests__/fetch-referenced-media-items.test.js

@@ -0,0 +1,96 @@
+jest.mock(`../dist/utils/fetch-graphql`, () => jest.fn())
+
+import { getHelpers } from "../dist/utils/get-gatsby-api"
+import fetchGraphql from "../dist/utils/fetch-graphql"
+import { fetchMediaItemsBySourceUrl } from "../dist/steps/source-nodes/fetch-nodes/fetch-referenced-media-items"
+import { createContentDigest } from "gatsby-core-utils"
+import store from "../dist/store"
+
+const fakeReporter = {
+  panic: msg => {
+    console.error(msg)
+  },
+  info: msg => {
+    console.log(msg)
+  },
+}
+
+const createApi = () => {
+  return {
+    actions: {
+      createTypes: jest.fn(),
+      createNode: jest.fn(),
+      deleteNode: jest.fn(),
+    },
+    reporter: fakeReporter,
+    createNodeId: jest.fn(),
+    async getNode(id) {
+      return {
+        localFile: {
+          id: id,
+        },
+      }
+    },
+  }
+}
+
+describe(`fetchMediaItemsBySourceUrl`, () => {
+  beforeAll(() => {
+    store.dispatch.gatsbyApi.setState({
+      pluginOptions: {
+        schema: {
+          perPage: 2,
+        },
+      },
+    })
+  })
+
+  afterEach(() => {
+    jest.resetAllMocks()
+  })
+
+  it(`should properly download multiple pages`, async () => {
+    fetchGraphql
+      .mockResolvedValueOnce({
+        data: {
+          mediaItem__index_0: null,
+          mediaItem__index_1: null,
+        },
+      })
+      .mockResolvedValueOnce({
+        data: {
+          mediaItem__index_2: {
+            id: 2,
+            mediaItemUrl: `https://wordpress.host/wp-content/uploads/2018/05/file1.mp3`,
+          },
+          mediaItem__index_3: {
+            id: 3,
+            mediaItemUrl: `https://wordpress.host/wp-content/uploads/2018/05/file1.mp3`,
+          },
+        },
+      })
+      .mockResolvedValueOnce({
+        data: {
+          mediaItem__index_4: null,
+          mediaItem__index_5: null,
+        },
+      })
+      .mockResolvedValueOnce({
+        data: {
+          mediaItem__index_6: null,
+          mediaItem__index_7: null,
+        },
+      })
+    const result = await fetchMediaItemsBySourceUrl({
+      mediaItemUrls: [
+        `https://wordpress.host/wp-content/uploads/2018/05/file1.mp3?_=7`,
+        `https://wordpress.host/wp-content/uploads/2018/05/file2.mp3?_=7`,
+        `https://wordpress.host/wp-content/uploads/2018/05/file1.mp3`,
+        `https://wordpress.host/wp-content/uploads/2018/05/file2.mp3`,
+      ],
+      createContentDigest,
+      helpers: createApi(),
+    })
+    expect(result).toHaveLength(2)
+  })
+})

packages/gatsby-source-wordpress/src/steps/source-nodes/fetch-nodes/fetch-referenced-media-items.js

@@ -271,7 +271,7 @@ const processAndDedupeImageUrls = urls =>
     }, urls)
   )
 
-const fetchMediaItemsBySourceUrl = async ({
+export const fetchMediaItemsBySourceUrl = async ({
   mediaItemUrls,
   selectionSet,
   builtFragments,
@@ -319,10 +319,16 @@ const fetchMediaItemsBySourceUrl = async ({
   // we pass this resolve function into the queue function so it can let us
   // know when it's finished
   let resolveFutureNodes
+  const allResolvedNodes = [...previouslyCachedMediaItemNodes]
+  let resolveCountTogo = mediaItemUrlsPages.length
   const futureNodes = new Promise(resolve => {
-    resolveFutureNodes = (nodes = []) =>
-      // combine our resolved nodes we fetched with our cached nodes
-      resolve([...nodes, ...previouslyCachedMediaItemNodes])
+    // combine our resolved nodes we fetched with our cached nodes
+    resolveFutureNodes = (nodes = []) => {
+      allResolvedNodes.push(...nodes)
+      if (--resolveCountTogo === 0) {
+        resolve(allResolvedNodes)
+      }
+    }
   })
 
   // we have no media items to fetch,