fix(gatsby-source-drupal): cache backlink records (#33444) (#33701)

GatsbyJS Bot · KyleAMathews · web-flow · commit 26c51c0c4ecb · 2021-10-28T02:29:34.000+07:00
* fix(gatsby-source-drupal): check relationships type exists on node before filtering (#33181) (#33228) * fix(gatsby-source-drupal): check relationships type exists on node before filtering * Update packages/gatsby-source-drupal/src/utils.js Co-authored-by: Dustin Schau <DSchau@users.noreply.github.com> * format Co-authored-by: Dustin Schau <DSchau@users.noreply.github.com> (cherry picked from commit d4f8355) Co-authored-by: Kyle Mathews <mathews.kyle@gmail.com> * chore(release): Publish - babel-plugin-remove-graphql-queries@3.14.0 - babel-preset-gatsby-package@1.14.0 - babel-preset-gatsby@1.14.0 - create-gatsby@1.14.0 - gatsby-admin@0.24.0 - gatsby-cli@3.14.0 - gatsby-codemods@2.14.0 - gatsby-core-utils@2.14.0 - gatsby-cypress@1.14.0 - gatsby-design-tokens@3.14.0 - gatsby-dev-cli@3.14.0 - gatsby-graphiql-explorer@1.14.0 - gatsby-legacy-polyfills@1.14.0 - gatsby-link@3.14.0 - gatsby-page-utils@1.14.0 - gatsby-plugin-benchmark-reporting@1.14.0 - gatsby-plugin-canonical-urls@3.14.0 - gatsby-plugin-catch-links@3.14.0 - gatsby-plugin-coffeescript@3.14.0 - gatsby-plugin-create-client-paths@3.14.0 - gatsby-plugin-cxs@3.14.0 - gatsby-plugin-emotion@6.14.0 - gatsby-plugin-facebook-analytics@3.14.0 - gatsby-plugin-feed@3.14.0 - gatsby-plugin-flow@2.14.0 - gatsby-plugin-fullstory@3.14.0 - gatsby-plugin-gatsby-cloud@3.2.0 - gatsby-plugin-google-analytics@3.14.0 - gatsby-plugin-google-gtag@3.14.0 - gatsby-plugin-google-tagmanager@3.14.0 - gatsby-plugin-graphql-config@0.23.0 - gatsby-plugin-image@1.14.0 - gatsby-plugin-jss@3.14.0 - gatsby-plugin-layout@2.14.0 - gatsby-plugin-less@5.14.0 - gatsby-plugin-lodash@4.14.0 - gatsby-plugin-manifest@3.14.0 - gatsby-plugin-mdx@2.14.0 - gatsby-plugin-netlify-cms@5.14.0 - gatsby-plugin-no-sourcemaps@3.14.0 - gatsby-plugin-nprogress@3.14.0 - gatsby-plugin-offline@4.14.0 - gatsby-plugin-page-creator@3.14.0 - gatsby-plugin-postcss@4.14.0 - gatsby-plugin-preact@5.14.0 - gatsby-plugin-preload-fonts@2.14.0 - gatsby-plugin-react-css-modules@3.14.0 - gatsby-plugin-react-helmet@4.14.0 - gatsby-plugin-remove-trailing-slashes@3.14.0 - gatsby-plugin-sass@4.14.0 - gatsby-plugin-schema-snapshot@2.14.0 - gatsby-plugin-sharp@3.14.0 - gatsby-plugin-sitemap@4.10.0 - gatsby-plugin-styled-components@4.14.0 - gatsby-plugin-styled-jsx@4.14.0 - gatsby-plugin-styletron@6.14.0 - gatsby-plugin-stylus@3.14.0 - gatsby-plugin-subfont@3.14.0 - gatsby-plugin-twitter@3.14.0 - gatsby-plugin-typescript@3.14.0 - gatsby-plugin-typography@3.14.0 - gatsby-plugin-utils@1.14.0 - gatsby-react-router-scroll@4.14.0 - gatsby-recipes@0.25.0 - gatsby-remark-autolink-headers@4.11.0 - gatsby-remark-code-repls@5.11.0 - gatsby-remark-copy-linked-files@4.11.0 - gatsby-remark-custom-blocks@3.14.0 - gatsby-remark-embed-snippet@6.11.0 - gatsby-remark-graphviz@3.11.0 - gatsby-remark-images-contentful@4.11.0 - gatsby-remark-images@5.11.0 - gatsby-remark-katex@5.11.0 - gatsby-remark-prismjs@5.11.0 - gatsby-remark-responsive-iframe@4.11.0 - gatsby-remark-smartypants@4.11.0 - gatsby-source-contentful@5.14.0 - gatsby-source-drupal@4.14.0 - gatsby-source-faker@3.14.0 - gatsby-source-filesystem@3.14.0 - gatsby-source-graphql@3.14.0 - gatsby-source-hacker-news@3.14.0 - gatsby-source-lever@3.14.0 - gatsby-source-medium@3.14.0 - gatsby-source-mongodb@3.14.0 - gatsby-source-npm-package-search@3.14.0 - gatsby-source-shopify@5.6.0 - gatsby-source-wikipedia@3.14.0 - gatsby-source-wordpress@5.14.0 - gatsby-telemetry@2.14.0 - gatsby-transformer-asciidoc@2.14.0 - gatsby-transformer-csv@3.14.0 - gatsby-transformer-documentationjs@5.14.0 - gatsby-transformer-excel@3.14.0 - gatsby-transformer-hjson@3.14.0 - gatsby-transformer-javascript-frontmatter@3.14.0 - gatsby-transformer-javascript-static-exports@3.14.0 - gatsby-transformer-json@3.14.0 - gatsby-transformer-pdf@2.14.0 - gatsby-transformer-react-docgen@6.14.0 - gatsby-transformer-remark@4.11.0 - gatsby-transformer-screenshot@3.14.0 - gatsby-transformer-sharp@3.14.0 - gatsby-transformer-sqip@3.14.0 - gatsby-transformer-toml@3.14.0 - gatsby-transformer-xml@3.14.0 - gatsby-transformer-yaml@3.14.0 - gatsby-worker@0.5.0 - gatsby@3.14.0 * Preserve backlink/reference lookup tables in cache * test * fix bad merge * try try again * Improve log * Get working with tests Co-authored-by: GatsbyJS Bot <mathews.kyle+gatsbybot@gmail.com> Co-authored-by: Vladimir Razuvaev <vladimir.razuvaev@gmail.com> Co-authored-by: Krzysztof Szot <kszot@reference.be> (cherry picked from commit 0def3ac) Co-authored-by: Kyle Mathews <mathews.kyle@gmail.com>
diff --git a/packages/gatsby-source-drupal/src/__tests__/index.js b/packages/gatsby-source-drupal/src/__tests__/index.js
@@ -20,6 +20,15 @@ jest.mock(`gatsby-source-filesystem`, () => {
   }
 })
 
+function makeCache() {
+  const store = new Map()
+  return {
+    get: async id => store.get(id),
+    set: async (key, value) => store.set(key, value),
+    store,
+  }
+}
+
 const normalize = require(`../normalize`)
 const downloadFileSpy = jest.spyOn(normalize, `downloadFile`)
 
@@ -75,6 +84,7 @@ describe(`gatsby-source-drupal`, () => {
     store,
     getNode: id => nodes[id],
     getNodes,
+    cache: makeCache(),
   }
 
   beforeAll(async () => {
diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js
@@ -12,6 +12,8 @@ const { setOptions, getOptions } = require(`./plugin-options`)
 
 const { nodeFromData, downloadFile, isFileNode } = require(`./normalize`)
 const {
+  initRefsLookups,
+  storeRefsLookups,
   handleReferences,
   handleWebhookUpdate,
   handleDeletedNode,
@@ -150,6 +152,8 @@ exports.sourceNodes = async (
   } = pluginOptions
   const { createNode, setPluginStatus, touchNode } = actions
 
+  await initRefsLookups({ cache, getNode })
+
   // Update the concurrency limit from the plugin options
   requestQueue.concurrency = concurrentAPIRequests
 
@@ -202,6 +206,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
         }
 
         changesActivity.end()
+        await storeRefsLookups({ cache })
         return
       }
 
@@ -232,6 +237,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
       return
     }
     changesActivity.end()
+    await storeRefsLookups({ cache })
     return
   }
 
@@ -362,6 +368,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
 
         drupalFetchIncrementalActivity.end()
         fastBuildsSpan.finish()
+        await storeRefsLookups({ cache })
         return
       }
 
@@ -372,6 +379,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
       initialSourcing = false
 
       if (!requireFullRebuild) {
+        await storeRefsLookups({ cache })
         return
       }
     }
@@ -635,6 +643,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
   initialSourcing = false
 
   createNodesSpan.finish()
+  await storeRefsLookups({ cache, getNodes })
   return
 }
 
diff --git a/packages/gatsby-source-drupal/src/utils.js b/packages/gatsby-source-drupal/src/utils.js
@@ -9,8 +9,38 @@ const {
 
 const { getOptions } = require(`./plugin-options`)
 
-const backRefsNamesLookup = new Map()
-const referencedNodesLookup = new Map()
+let backRefsNamesLookup = new Map()
+let referencedNodesLookup = new Map()
+
+const initRefsLookups = async ({ cache }) => {
+  const backRefsNamesLookupStr = await cache.get(`backRefsNamesLookup`)
+  const referencedNodesLookupStr = await cache.get(`referencedNodesLookup`)
+
+  if (backRefsNamesLookupStr) {
+    backRefsNamesLookup = new Map(JSON.parse(backRefsNamesLookupStr))
+  }
+
+  if (referencedNodesLookupStr) {
+    referencedNodesLookup = new Map(JSON.parse(referencedNodesLookupStr))
+  }
+}
+
+exports.initRefsLookups = initRefsLookups
+
+const storeRefsLookups = async ({ cache }) => {
+  await Promise.all([
+    cache.set(
+      `backRefsNamesLookup`,
+      JSON.stringify(Array.from(backRefsNamesLookup.entries()))
+    ),
+    cache.set(
+      `referencedNodesLookup`,
+      JSON.stringify(Array.from(referencedNodesLookup.entries()))
+    ),
+  ])
+}
+
+exports.storeRefsLookups = storeRefsLookups
 
 const handleReferences = (
   node,
@@ -333,7 +363,9 @@ ${JSON.stringify(nodeToUpdate, null, 4)}
     }
     node.internal.contentDigest = createContentDigest(node)
     createNode(node)
-    reporter.log(`Updated Gatsby node: ${node.id}`)
+    reporter.log(
+      `Updated Gatsby node: id: ${node.id} — type: ${node.internal.type}`
+    )
   }
 }
 

Original file line number	Diff line number	Diff line change
@@ -12,6 +12,8 @@ const { setOptions, getOptions } = require(`./plugin-options`)
`12`	`12`
`13`	`13`	const { nodeFromData, downloadFile, isFileNode } = require(`./normalize`)
`14`	`14`	`const {`
	`15`	`+ initRefsLookups,`
	`16`	`+ storeRefsLookups,`
`15`	`17`	`handleReferences,`
`16`	`18`	`handleWebhookUpdate,`
`17`	`19`	`handleDeletedNode,`
`@@ -150,6 +152,8 @@ exports.sourceNodes = async (`
`150`	`152`	`} = pluginOptions`
`151`	`153`	`const { createNode, setPluginStatus, touchNode } = actions`
`152`	`154`
	`155`	`+ await initRefsLookups({ cache, getNode })`
	`156`	`+`
`153`	`157`	`// Update the concurrency limit from the plugin options`
`154`	`158`	`requestQueue.concurrency = concurrentAPIRequests`
`155`	`159`
@@ -202,6 +206,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
`202`	`206`	`}`
`203`	`207`
`204`	`208`	`changesActivity.end()`
	`209`	`+ await storeRefsLookups({ cache })`
`205`	`210`	`return`
`206`	`211`	`}`
`207`	`212`
@@ -232,6 +237,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
`232`	`237`	`return`
`233`	`238`	`}`
`234`	`239`	`changesActivity.end()`
	`240`	`+ await storeRefsLookups({ cache })`
`235`	`241`	`return`
`236`	`242`	`}`
`237`	`243`
@@ -362,6 +368,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
`362`	`368`
`363`	`369`	`drupalFetchIncrementalActivity.end()`
`364`	`370`	`fastBuildsSpan.finish()`
	`371`	`+ await storeRefsLookups({ cache })`
`365`	`372`	`return`
`366`	`373`	`}`
`367`	`374`
@@ -372,6 +379,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
`372`	`379`	`initialSourcing = false`
`373`	`380`
`374`	`381`	`if (!requireFullRebuild) {`
	`382`	`+ await storeRefsLookups({ cache })`
`375`	`383`	`return`
`376`	`384`	`}`
`377`	`385`	`}`
@@ -635,6 +643,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
`635`	`643`	`initialSourcing = false`
`636`	`644`
`637`	`645`	`createNodesSpan.finish()`
	`646`	`+ await storeRefsLookups({ cache, getNodes })`
`638`	`647`	`return`
`639`	`648`	`}`
`640`	`649`