Skip to content

Commit ad2938c

Browse files
KyleAMathewsGatsbyJS Botvladarkszot-ref
authored and
axe312ger
committed
fix(gatsby-source-drupal): cache backlink records (#33444)
* fix(gatsby-source-drupal): check relationships type exists on node before filtering (#33181) (#33228) * fix(gatsby-source-drupal): check relationships type exists on node before filtering * Update packages/gatsby-source-drupal/src/utils.js Co-authored-by: Dustin Schau <[email protected]> * format Co-authored-by: Dustin Schau <[email protected]> (cherry picked from commit d4f8355) Co-authored-by: Kyle Mathews <[email protected]> * chore(release): Publish - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] - [email protected] * Preserve backlink/reference lookup tables in cache * test * fix bad merge * try try again * Improve log * Get working with tests Co-authored-by: GatsbyJS Bot <[email protected]> Co-authored-by: Vladimir Razuvaev <[email protected]> Co-authored-by: Krzysztof Szot <[email protected]>
1 parent 6bc4acd commit ad2938c

File tree

3 files changed

+54
-3
lines changed

3 files changed

+54
-3
lines changed

‎packages/gatsby-source-drupal/src/__tests__/index.js

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,15 @@ jest.mock(`gatsby-source-filesystem`, () => {
2020
}
2121
})
2222

23+
function makeCache() {
24+
const store = new Map()
25+
return {
26+
get: async id => store.get(id),
27+
set: async (key, value) => store.set(key, value),
28+
store,
29+
}
30+
}
31+
2332
const normalize = require(`../normalize`)
2433
const downloadFileSpy = jest.spyOn(normalize, `downloadFile`)
2534

@@ -75,6 +84,7 @@ describe(`gatsby-source-drupal`, () => {
7584
store,
7685
getNode: id => nodes[id],
7786
getNodes,
87+
cache: makeCache(),
7888
}
7989

8090
beforeAll(async () => {

‎packages/gatsby-source-drupal/src/gatsby-node.js

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ const { setOptions, getOptions } = require(`./plugin-options`)
1212

1313
const { nodeFromData, downloadFile, isFileNode } = require(`./normalize`)
1414
const {
15+
initRefsLookups,
16+
storeRefsLookups,
1517
handleReferences,
1618
handleWebhookUpdate,
1719
handleDeletedNode,
@@ -150,6 +152,8 @@ exports.sourceNodes = async (
150152
} = pluginOptions
151153
const { createNode, setPluginStatus, touchNode } = actions
152154

155+
await initRefsLookups({ cache, getNode })
156+
153157
// Update the concurrency limit from the plugin options
154158
requestQueue.concurrency = concurrentAPIRequests
155159

@@ -202,6 +206,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
202206
}
203207

204208
changesActivity.end()
209+
await storeRefsLookups({ cache })
205210
return
206211
}
207212

@@ -232,6 +237,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
232237
return
233238
}
234239
changesActivity.end()
240+
await storeRefsLookups({ cache })
235241
return
236242
}
237243

@@ -362,6 +368,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
362368

363369
drupalFetchIncrementalActivity.end()
364370
fastBuildsSpan.finish()
371+
await storeRefsLookups({ cache })
365372
return
366373
}
367374

@@ -372,6 +379,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
372379
initialSourcing = false
373380

374381
if (!requireFullRebuild) {
382+
await storeRefsLookups({ cache })
375383
return
376384
}
377385
}
@@ -635,6 +643,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
635643
initialSourcing = false
636644

637645
createNodesSpan.finish()
646+
await storeRefsLookups({ cache, getNodes })
638647
return
639648
}
640649

‎packages/gatsby-source-drupal/src/utils.js

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,38 @@ const {
99

1010
const { getOptions } = require(`./plugin-options`)
1111

12-
const backRefsNamesLookup = new Map()
13-
const referencedNodesLookup = new Map()
12+
let backRefsNamesLookup = new Map()
13+
let referencedNodesLookup = new Map()
14+
15+
const initRefsLookups = async ({ cache }) => {
16+
const backRefsNamesLookupStr = await cache.get(`backRefsNamesLookup`)
17+
const referencedNodesLookupStr = await cache.get(`referencedNodesLookup`)
18+
19+
if (backRefsNamesLookupStr) {
20+
backRefsNamesLookup = new Map(JSON.parse(backRefsNamesLookupStr))
21+
}
22+
23+
if (referencedNodesLookupStr) {
24+
referencedNodesLookup = new Map(JSON.parse(referencedNodesLookupStr))
25+
}
26+
}
27+
28+
exports.initRefsLookups = initRefsLookups
29+
30+
const storeRefsLookups = async ({ cache }) => {
31+
await Promise.all([
32+
cache.set(
33+
`backRefsNamesLookup`,
34+
JSON.stringify(Array.from(backRefsNamesLookup.entries()))
35+
),
36+
cache.set(
37+
`referencedNodesLookup`,
38+
JSON.stringify(Array.from(referencedNodesLookup.entries()))
39+
),
40+
])
41+
}
42+
43+
exports.storeRefsLookups = storeRefsLookups
1444

1545
const handleReferences = (
1646
node,
@@ -333,7 +363,9 @@ ${JSON.stringify(nodeToUpdate, null, 4)}
333363
}
334364
node.internal.contentDigest = createContentDigest(node)
335365
createNode(node)
336-
reporter.log(`Updated Gatsby node: ${node.id}`)
366+
reporter.log(
367+
`Updated Gatsby node: id: ${node.id} — type: ${node.internal.type}`
368+
)
337369
}
338370
}
339371

0 commit comments

Comments
 (0)