Skip to content

Commit c40a2c9

Browse files
cstamascstamas
authored
[4.0.x] Consolidate caches (#11379)
Problem: Our caches (all of them) are over 10GB allowed cache space. Due this, GH is dropping caches and our jobs end up exposed to HTTP 500 that since migration Central happens quite often. We used the plain `actions/cache` to store Mimir caches for each node (200-400 MB depending on which node we talk about), and we have 3 active branches (3.9, 4.0 and master), that simply totals out the 10 GB limit. This PR makes we have one "cache blob" per OS, so each OS has one cache blob (times three, for 3.9, 4.0 and master). Changes: * implement "always save" pattern (see cache doco) * we keep cache "per lane" (per OS) * 3 kind of builds (initial, full and integration-tests) all use same (per OS) cache at start and at end uploads cache as artifact (1 day retention) * at end there is a matrix job "consolidate caches" (runs on all 3 OSes) that downloads caches and consolidate them and save cache * hence, we will have 3 OS specific caches Backport of 304791e
1 parent d77220f commit c40a2c9

File tree

1 file changed

+90
-28
lines changed

1 file changed

+90
-28
lines changed

‎.github/workflows/maven.yml‎

Lines changed: 90 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ env:
3535
MIMIR_VERSION: 0.10.4
3636
MIMIR_BASEDIR: ~/.mimir
3737
MIMIR_LOCAL: ~/.mimir/local
38+
MAVEN_OPTS: -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./target/java_heapdump.hprof
3839

3940
jobs:
4041
initial-build:
@@ -61,10 +62,12 @@ jobs:
6162
6263
- name: Restore Mimir caches
6364
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
64-
id: restore-cache
6565
with:
6666
path: ${{ env.MIMIR_LOCAL }}
67-
key: mvn40-${{ runner.os }}-initial
67+
key: mvn40-${{ runner.os }}-${{ github.run_id }}
68+
restore-keys: |
69+
mvn40-${{ runner.os }}-
70+
mvn40-
6871
6972
- name: Set up Maven
7073
shell: bash
@@ -85,12 +88,13 @@ jobs:
8588
shell: bash
8689
run: ls -la apache-maven/target
8790

88-
- name: Save Mimir caches
89-
uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
90-
if: ${{ github.event_name != 'pull_request' && !cancelled() && !failure() }}
91+
- name: Upload Mimir caches
92+
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4
93+
if: ${{ !cancelled() && !failure() }}
9194
with:
95+
name: cache-${{ runner.os }}-initial
96+
retention-days: 1
9297
path: ${{ env.MIMIR_LOCAL }}
93-
key: ${{ steps.restore-cache.outputs.cache-primary-key }}
9498

9599
- name: Upload Maven distributions
96100
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4
@@ -102,13 +106,24 @@ jobs:
102106
103107
- name: Upload test artifacts
104108
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4
105-
if: failure() || cancelled()
109+
if: ${{ failure() || cancelled() }}
106110
with:
107-
name: ${{ github.run_number }}-initial
111+
name: initial-logs
112+
retention-days: 1
108113
path: |
109114
**/target/surefire-reports/*
110115
**/target/java_heapdump.hprof
111116
117+
- name: Upload Mimir logs
118+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
119+
if: always()
120+
with:
121+
name: initial-mimir-logs
122+
include-hidden-files: true
123+
retention-days: 1
124+
path: |
125+
~/.mimir/*.log
126+
112127
full-build:
113128
needs: initial-build
114129
runs-on: ${{ matrix.os }}
@@ -152,13 +167,12 @@ jobs:
152167
153168
- name: Restore Mimir caches
154169
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
155-
id: restore-cache
156170
with:
157171
path: ${{ env.MIMIR_LOCAL }}
158-
key: mvn40-full-${{ matrix.os }}-${{ matrix.java }}
172+
key: mvn40-${{ runner.os }}-${{ github.run_id }}
159173
restore-keys: |
160-
mvn40-full-${{ matrix.os }}-
161-
mvn40-full-
174+
mvn40-${{ runner.os }}-
175+
mvn40-
162176
163177
- name: Download Maven distribution
164178
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v4
@@ -195,19 +209,33 @@ jobs:
195209
shell: bash
196210
run: mvn site -e -B -V -Preporting
197211

198-
- name: Save Mimir caches
199-
uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
200-
if: ${{ github.event_name != 'pull_request' && !cancelled() && !failure() }}
212+
- name: Upload Mimir caches
213+
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4
214+
if: ${{ !cancelled() && !failure() }}
201215
with:
216+
name: cache-${{ runner.os }}-full-build-${{ matrix.java }}
217+
retention-days: 1
202218
path: ${{ env.MIMIR_LOCAL }}
203-
key: ${{ steps.restore-cache.outputs.cache-primary-key }}
204219

205220
- name: Upload test artifacts
206221
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4
207222
if: failure() || cancelled()
208223
with:
209-
name: ${{ github.run_number }}-full-build-artifact-${{ runner.os }}-${{ matrix.java }}
210-
path: '**/target/surefire-reports/*'
224+
name: full-build-logs-${{ runner.os }}-${{ matrix.java }}
225+
retention-days: 1
226+
path: |
227+
**/target/surefire-reports/*
228+
**/target/java_heapdump.hprof
229+
230+
- name: Upload Mimir logs
231+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
232+
if: always()
233+
with:
234+
name: full-build-mimir-logs-${{ runner.os }}-${{ matrix.java }}
235+
include-hidden-files: true
236+
retention-days: 1
237+
path: |
238+
~/.mimir/*.log
211239
212240
integration-tests:
213241
needs: initial-build
@@ -240,13 +268,12 @@ jobs:
240268
241269
- name: Restore Mimir caches
242270
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
243-
id: restore-cache
244271
with:
245272
path: ${{ env.MIMIR_LOCAL }}
246-
key: mvn40-its-${{ matrix.os }}-${{ matrix.java }}
273+
key: mvn40-${{ runner.os }}-${{ github.run_id }}
247274
restore-keys: |
248-
mvn40-its-${{ matrix.os }}-
249-
mvn40-its-
275+
mvn40-${{ runner.os }}-
276+
mvn40-
250277
251278
- name: Download Maven distribution
252279
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v4
@@ -279,20 +306,55 @@ jobs:
279306
shell: bash
280307
run: mvn install -e -B -V -Prun-its,mimir
281308

282-
- name: Save Mimir caches
283-
uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
284-
if: ${{ github.event_name != 'pull_request' && !cancelled() && !failure() }}
309+
- name: Upload Mimir caches
310+
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4
311+
if: ${{ !cancelled() && !failure() }}
285312
with:
313+
name: cache-${{ runner.os }}-integration-tests-${{ matrix.java }}
314+
retention-days: 1
286315
path: ${{ env.MIMIR_LOCAL }}
287-
key: ${{ steps.restore-cache.outputs.cache-primary-key }}
288316

289317
- name: Upload test artifacts
290318
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4
291-
if: failure() || cancelled()
319+
if: ${{ failure() || cancelled() }}
292320
with:
293-
name: ${{ github.run_number }}-integration-test-artifact-${{ runner.os }}-${{ matrix.java }}
321+
name: integration-test-logs-${{ runner.os }}-${{ matrix.java }}
322+
retention-days: 1
294323
path: |
295324
**/target/surefire-reports/*
296325
**/target/failsafe-reports/*
297326
./its/core-it-suite/target/test-classes/**
298327
**/target/java_heapdump.hprof
328+
329+
- name: Upload Mimir logs
330+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
331+
if: always()
332+
with:
333+
name: integration-test-mimir-logs-${{ runner.os }}-${{ matrix.java }}
334+
include-hidden-files: true
335+
retention-days: 1
336+
path: |
337+
~/.mimir/*.log
338+
339+
consolidate-caches:
340+
runs-on: ${{ matrix.os }}
341+
strategy:
342+
fail-fast: false
343+
matrix:
344+
os: [ubuntu-latest, macos-latest, windows-latest]
345+
needs:
346+
- full-build
347+
- integration-tests
348+
steps:
349+
- name: Download Caches
350+
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v4
351+
with:
352+
merge-multiple: true
353+
pattern: 'cache-${{ runner.os }}*'
354+
path: ${{ env.MIMIR_LOCAL }}
355+
- name: Publish cache
356+
uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
357+
if: ${{ github.event_name != 'pull_request' && !cancelled() && !failure() }}
358+
with:
359+
path: ${{ env.MIMIR_LOCAL }}
360+
key: mvn40-${{ runner.os }}-${{ github.run_id }}

0 commit comments

Comments
 (0)