Skip to content
This repository was archived by the owner on Aug 18, 2020. It is now read-only.

Commit 75aa12f

Browse files
committed
[DEVOPS-1113] New buildkite configuration
1 parent fc4f14e commit 75aa12f

File tree

3 files changed

+66
-182
lines changed

3 files changed

+66
-182
lines changed

.buildkite/pipeline.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ steps:
22
- label: 'Windows'
33
command: 'PowerShell.exe -executionpolicy remotesigned -File .\.buildkite\windows.ps1'
44
agents:
5-
system: x86_64-win2016-legacy
5+
system: x86_64-win2016
66

77
- label: 'stack2nix'
88
command: 'nix-shell -Q -j 4 --run scripts/check-stack2nix.sh'

.buildkite/windows.ps1

Lines changed: 64 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,76 @@
11
############################################################################
22
# Windows build script for Buildkite
33
#
4-
# Requires the Windows 10 VM to be set up with the compiler,
5-
# and other tools.
6-
#
4+
# Requires the devops buildkite-windows AMI
75
############################################################################
86

9-
Set-PSDebug -Trace 1
7+
#Set-PSDebug -Trace 1
108

119
Write-Output "--- Prepare work directory"
1210

1311
# Avoid long paths on Windows
14-
$env:STACK_ROOT="D:\s"
12+
$env:STACK_ROOT="Z:\s"
1513
$env:STACK_WORK=".w"
16-
$env:WORK_DIR="D:\w"
14+
$env:WORK_DIR="Z:\w"
1715
# Override the temp directory to avoid sed escaping issues
1816
# See https://github.com/haskell/cabal/issues/5386
19-
$env:TMP="D:\\tmp"
17+
$env:TMP="Z:\\tmp"
2018

2119
# Store the original checkout directory
2220
$env:CHECKOUT_PATH=(Get-Item -Path ".\").FullName
2321

2422
# Temp directory needs to exist
2523
New-Item -ItemType Directory -Force -Path $env:TMP
2624

27-
$env:PATH="$env:PATH;D:\ghc\ghc-8.2.2\bin;D:\stack;$Env:Programfiles\7-Zip;D:\w"
25+
New-Item -ItemType Directory -Force -Path z:\ghc
26+
27+
New-Item -ItemType Directory -Force -Path z:\Downloads
28+
New-Item -ItemType Directory -Force -Path $env:STACK_ROOT
29+
30+
$StackConfig = @"
31+
templates:
32+
params: null
33+
system-ghc: true
34+
local-programs-path: "z:\\s\\programs"
35+
local-bin-path: "z:\\s\\bin"
36+
"@
37+
38+
$StackConfig | Out-File -FilePath $env:STACK_ROOT\config.yaml -Encoding ASCII
39+
40+
#
41+
if (!([System.IO.File]::Exists("Z:\ghc\ghc-8.2.2.tar.xz"))) {
42+
echo "Downloading and extracting GHC"
43+
curl.exe https://s3.eu-central-1.amazonaws.com/ci-static/ghc-8.2.2-x86_64-unknown-mingw32.tar.xz -o Z:\ghc\ghc-8.2.2.tar.xz -L
44+
7z x Z:\ghc\ghc-8.2.2.tar.xz -oZ:\ghc
45+
7z x Z:\ghc\ghc-8.2.2.tar -oZ:\ghc
46+
}
47+
48+
# OpenSSL
49+
#
50+
#$env:USERPROFILE
51+
if (!([System.IO.File]::Exists("Z:\Downloads\Win64OpenSSL-1_0_2q.exe"))) {
52+
echo "Downloading and installing OpenSSL"
53+
rd -r -fo z:\OpenSSL-Win64-v102
54+
curl.exe https://s3.eu-central-1.amazonaws.com/ci-static/Win64OpenSSL-1_0_2q.exe -o Z:\Downloads\Win64OpenSSL-1_0_2q.exe -L
55+
cmd /c start /wait "Z:\Downloads\Win64OpenSSL-1_0_2q.exe" /silent /verysilent /sp- /suppressmsgboxes /DIR=Z:\OpenSSL-Win64-v102
56+
}
57+
## Install stack
58+
if (!([System.IO.File]::Exists("Z:\Downloads\stack-1.9.1-windows-x86_64.zip"))) {
59+
echo "Downloading and extracting stack"
60+
rd -r -fo z:\stack
61+
curl.exe https://s3.eu-central-1.amazonaws.com/ci-static/stack-1.9.1-windows-x86_64.zip -o z:\Downloads\stack-1.9.1.zip -L
62+
7z -oZ:\stack x Z:\Downloads\stack-1.9.1-windows-x86_64.zip
63+
}
64+
65+
$env:PATH="$env:PATH;Z:\ghc\ghc-8.2.2\bin;Z:\stack;Z:\w"
66+
67+
# Install liblzma/xz
68+
if (!([System.IO.File]::Exists("Z:\Downloads\xz-5.2.3-windows.zip"))) {
69+
echo "Downloading and extracting xz"
70+
rd -r -fo z:\xz_extracted
71+
curl.exe -L https://s3.eu-central-1.amazonaws.com/ci-static/xz-5.2.3-windows.zip -o Z:\Downloads\xz-5.2.3-windows.zip
72+
7z -oZ:\xz_extracted x Z:\Downloads\xz-5.2.3-windows.zip
73+
}
2874

2975
# Clear out work directory, copy source tree there, then cd into it.
3076
rd -r -fo $env:WORK_DIR
@@ -36,15 +82,21 @@ cd $env:WORK_DIR
3682
# headers and the zip provides rocksdb.dll. This is needed to build
3783
# rocksdb-haskell.
3884
git.exe clone https://github.com/facebook/rocksdb.git --branch v4.13.5
39-
curl.exe -L 'https://s3.eu-central-1.amazonaws.com/ci-static/serokell-rocksdb-haskell-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip' -o D:\Downloads\rocksdb.zip
40-
7z x D:\Downloads\rocksdb.zip
85+
if (!([System.IO.File]::Exists("Z:\Downloads\rocksdb-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip"))) {
86+
echo "Downloading rocksdb"
87+
curl.exe -L 'https://s3.eu-central-1.amazonaws.com/ci-static/serokell-rocksdb-haskell-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip' -o Z:\Downloads\rocksdb-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip
88+
}
89+
7z x Z:\Downloads\rocksdb-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip
4190

4291
# CSL-1509: After moving the 'cardano-sl' project itself into a separate folder ('lib/'), the 'cardano-text.exe' executable fails on AppVeyor CI.
4392
# After some investigation, it was discovered that this was because 'rocksdb.dll' has to be located in this folder as well, or else the test executable doesn't work.
4493
copy rocksdb.dll node
4594
copy rocksdb.dll lib
4695
copy rocksdb.dll wallet-new
4796

97+
# Terminate on all errors
98+
$ErrorActionPreference = "Stop"
99+
48100
############################################################################
49101
# Prepare stack and build tools
50102
#
@@ -64,7 +116,7 @@ stack.exe exec -- ghc-pkg recache
64116
stack.exe --verbosity warn setup --no-reinstall
65117

66118
# Install happy separately: https://github.com/commercialhaskell/stack/issues/3151#issuecomment-310642487. Also install cpphs because it's a build-tool and Stack can't figure out by itself that it should be installed
67-
stack.exe --verbosity warn install happy cpphs -j 2 --no-terminal --local-bin-path $env:SYSTEMROOT\system32 --extra-include-dirs="D:\OpenSSL-Win64-v102\include" --extra-lib-dirs="D:\OpenSSL-Win64-v102" --extra-include-dirs="D:\xz_extracted\include" --extra-lib-dirs="D:\xz_extracted\bin_x86-64" --extra-include-dirs="$env:WORK_DIR\rocksdb\include" --extra-lib-dirs="$env:WORK_DIR"
119+
stack.exe --verbosity warn install happy cpphs -j 2 --no-terminal --local-bin-path $env:SYSTEMROOT\system32 --extra-include-dirs="Z:\OpenSSL-Win64-v102\include" --extra-lib-dirs="Z:\OpenSSL-Win64-v102" --extra-include-dirs="Z:\xz_extracted\include" --extra-lib-dirs="Z:\xz_extracted\bin_x86-64" --extra-include-dirs="$env:WORK_DIR\rocksdb\include" --extra-lib-dirs="$env:WORK_DIR"
68120

69121
############################################################################
70122
# Build and test.
@@ -74,18 +126,14 @@ stack.exe --verbosity warn install happy cpphs -j 2 --no-terminal --local-bin-pa
74126

75127
Write-Output "+++ Stack build and test"
76128

77-
stack.exe --dump-logs install cardano-sl cardano-sl-tools cardano-sl-wallet-new -j 3 --no-terminal --local-bin-path $env:WORK_DIR --no-haddock-deps --flag cardano-sl-core:-asserts --flag cardano-sl-tools:for-installer --extra-include-dirs="D:\OpenSSL-Win64-v102\include" --extra-lib-dirs="D:\OpenSSL-Win64-v102" --extra-include-dirs="D:\xz_extracted\include" --extra-lib-dirs="D:\xz_extracted\bin_x86-64" --extra-include-dirs="$env:WORK_DIR\rocksdb\include" --extra-lib-dirs="$env:WORK_DIR"
78-
129+
stack.exe --dump-logs install cardano-sl cardano-sl-tools cardano-sl-wallet-new -j 3 --no-terminal --local-bin-path $env:WORK_DIR --no-haddock-deps --flag cardano-sl-core:-asserts --flag cardano-sl-tools:for-installer --extra-include-dirs="Z:\OpenSSL-Win64-v102\include" --extra-lib-dirs="Z:\OpenSSL-Win64-v102" --extra-include-dirs="Z:\xz_extracted\include" --extra-lib-dirs="Z:\xz_extracted\bin_x86-64" --extra-include-dirs="$env:WORK_DIR\rocksdb\include" --extra-lib-dirs="$env:WORK_DIR"
79130

80131
############################################################################
81132
# Assemble artifact zip file for use by the Daedalus installers build
82133
#
83134

84135
Write-Output "--- Create zip file"
85136

86-
# from here onwards, errors terminate the script
87-
$ErrorActionPreference = "Stop"
88-
89137
# Cardano pieces, modulo the frontend
90138
mkdir daedalus
91139
## log config is called `log-config-prod.yaml` just in case, it's the old name

appveyor.yml

Lines changed: 1 addition & 165 deletions
Original file line numberDiff line numberDiff line change
@@ -3,169 +3,5 @@ image: Visual Studio 2015
33

44
build: off
55

6-
environment:
7-
global:
8-
# Avoid long paths on Windows
9-
STACK_ROOT: "c:\\s"
10-
STACK_WORK: ".w"
11-
WORK_DIR: "c:\\w"
12-
CACHE_S3_VERSION: v0.1.4
13-
CACHE_S3_MAX_SIZE: 1600MB # AppVeyor limits the amount uploaded to approx 2GB
14-
AWS_REGION: us-west-1
15-
S3_BUCKET: appveyor-ci-cache
16-
AWS_ACCESS_KEY_ID:
17-
secure: sQWt5CpaN0H+jwUVoTsrET46pADUDEcrJ5D9MHmKX0M=
18-
AWS_SECRET_ACCESS_KEY:
19-
secure: m5sQYd16K8HA0zoZaD0gOl4EEWUso1D51L5rp+kT3hLaIE3tt4iT+b+iW8F4F0FU
20-
21-
init:
22-
- ps: $env:CACHE_S3_READY = (("$env:CACHE_S3_VERSION" -ne "") -and ("$env:S3_BUCKET" -ne "") -and ("$env:AWS_ACCESS_KEY_ID" -ne "") -and ("$env:AWS_SECRET_ACCESS_KEY" -ne ""))
23-
24-
before_test:
25-
# Avoid long paths not to each MAX_PATH of 260 chars
26-
- xcopy /q /s /e /r /k /i /v /h /y "%APPVEYOR_BUILD_FOLDER%" "%WORK_DIR%"
27-
- cd "%WORK_DIR%"
28-
# Restore cache
29-
- Echo %APPVEYOR_BUILD_VERSION% > build-id
30-
- ps: >-
31-
Write-Host "in pagefile script" ;
32-
$c = Get-WmiObject Win32_computersystem -EnableAllPrivileges ;
33-
if($c.AutomaticManagedPagefile){
34-
Write-Host "disabling managed page file settings"
35-
$c.AutomaticManagedPagefile = $false
36-
$c.Put() | Out-Null
37-
} ;
38-
$new_page_size=25000 ;
39-
$CurrentPageFile = Get-WmiObject -Class Win32_PageFileSetting ;
40-
if ($CurrentPageFile.InitialSize -ne $new_page_size) {
41-
Write-Host "setting new page file size to $new_page_size"
42-
$CurrentPageFile.InitialSize=$new_page_size
43-
$CurrentPageFile.MaximumSize=$new_page_size
44-
$CurrentPageFile.Put() | Out-Null
45-
} ;
46-
if ( $env:CACHE_S3_READY -eq $true ) {
47-
Start-FileDownload https://github.com/fpco/cache-s3/releases/download/$env:CACHE_S3_VERSION/cache-s3-$env:CACHE_S3_VERSION-windows-x86_64.zip -FileName cache-s3.zip
48-
7z x cache-s3.zip cache-s3.exe
49-
.\cache-s3 --max-size=$env:CACHE_S3_MAX_SIZE --prefix=$env:APPVEYOR_PROJECT_NAME --git-branch=$env:APPVEYOR_REPO_BRANCH --suffix=windows -v info -c restore stack --base-branch=develop
50-
.\cache-s3 --max-size=$env:CACHE_S3_MAX_SIZE --prefix=$env:APPVEYOR_PROJECT_NAME --git-branch=$env:APPVEYOR_REPO_BRANCH --suffix=windows -v info -c restore stack work --base-branch=develop
51-
}
52-
53-
# Get custom GHC
54-
- ps: >-
55-
mkdir C:\ghc
56-
57-
Invoke-WebRequest "https://s3.eu-central-1.amazonaws.com/ci-static/ghc-8.2.2-x86_64-unknown-mingw32.tar.xz" -OutFile "C:\ghc\ghc.tar.xz" -UserAgent "Curl"
58-
59-
7z x C:\ghc\ghc.tar.xz -oC:\ghc
60-
61-
7z x C:\ghc\ghc.tar -oC:\ghc
62-
63-
$env:PATH="$env:PATH;C:\ghc\ghc-8.2.2\bin"
64-
65-
# Install OpenSSL 1.0.2 (see https://github.com/appveyor/ci/issues/1665)
66-
- ps: (New-Object Net.WebClient).DownloadFile('https://slproweb.com/download/Win64OpenSSL-1_0_2q.exe', "$($env:USERPROFILE)\Win64OpenSSL.exe")
67-
- ps: cmd /c start /wait "$($env:USERPROFILE)\Win64OpenSSL.exe" /silent /verysilent /sp- /suppressmsgboxes /DIR=C:\OpenSSL-Win64-v102
68-
- ps: Install-Product node 6
69-
# Install stack
70-
- ps: Start-FileDownload http://www.stackage.org/stack/windows-x86_64 -FileName stack.zip
71-
- 7z x stack.zip stack.exe
72-
73-
74-
# Install rocksdb
75-
- git clone https://github.com/facebook/rocksdb.git --branch v4.13.5
76-
- ps: Start-FileDownload 'https://s3.eu-central-1.amazonaws.com/ci-static/serokell-rocksdb-haskell-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip' -FileName rocksdb.zip
77-
- 7z x rocksdb.zip
78-
79-
# CSL-1509: After moving the 'cardano-sl' project itself into a separate folder ('lib/'), the 'cardano-text.exe' executable fails on AppVeyor CI.
80-
# After some investigation, it was discovered that this was because 'rocksdb.dll' has to be located in this folder as well, or else the test executable doesn't work.
81-
- copy rocksdb.dll node
82-
- copy rocksdb.dll lib
83-
- copy rocksdb.dll wallet
84-
- copy rocksdb.dll wallet-new
85-
86-
# Install liblzma/xz
87-
- ps: Start-FileDownload https://tukaani.org/xz/xz-5.2.3-windows.zip -Filename xz-5.2.3-windows.zip
88-
- 7z -oC:\xz_extracted x xz-5.2.3-windows.zip
89-
906
test_script:
91-
- cd "%WORK_DIR%"
92-
- stack config --system-ghc set system-ghc --global true
93-
- stack exec -- ghc-pkg recache
94-
- stack --verbosity warn setup --no-reinstall > nul
95-
# Install happy separately: https://github.com/commercialhaskell/stack/issues/3151#issuecomment-310642487. Also install cpphs because it's a build-tool and Stack can't figure out by itself that it should be installed
96-
- scripts\ci\appveyor-retry call stack --verbosity warn install happy cpphs
97-
-j 2
98-
--no-terminal
99-
--local-bin-path %SYSTEMROOT%\system32
100-
--extra-include-dirs="C:\OpenSSL-Win64-v102\include"
101-
--extra-lib-dirs="C:\OpenSSL-Win64-v102"
102-
--extra-include-dirs="C:\xz_extracted\include"
103-
--extra-lib-dirs="C:\xz_extracted\bin_x86-64"
104-
--extra-include-dirs="%WORK_DIR%\rocksdb\include"
105-
--extra-lib-dirs="%WORK_DIR%"
106-
# TODO: CSL-1133. To be reenabled.
107-
# - stack test --coverage
108-
# - stack hpc report cardano-sl cardano-sl-txp cardano-sl-core cardano-sl-db cardano-sl-update cardano-sl-infra cardano-sl-lrc cardano-sl-ssc
109-
# Retry transient failures due to https://github.com/haskell/cabal/issues/4005
110-
# We intentionally don't build auxx here, because this build is for installer.
111-
- scripts\ci\appveyor-retry call stack --dump-logs install cardano-sl cardano-sl-tools cardano-sl-wallet cardano-sl-wallet-new
112-
-j 3
113-
--no-terminal
114-
--local-bin-path %WORK_DIR%
115-
--no-haddock-deps
116-
--flag cardano-sl-core:-asserts
117-
--flag cardano-sl-tools:for-installer
118-
--flag cardano-sl-wallet:for-installer
119-
--extra-include-dirs="C:\OpenSSL-Win64-v102\include"
120-
--extra-lib-dirs="C:\OpenSSL-Win64-v102"
121-
--extra-include-dirs="C:\xz_extracted\include"
122-
--extra-lib-dirs="C:\xz_extracted\bin_x86-64"
123-
--extra-include-dirs="%WORK_DIR%\rocksdb\include"
124-
--extra-lib-dirs="%WORK_DIR%"
125-
# Cardano pieces, modulo the frontend
126-
- mkdir daedalus
127-
# log config is called `log-config-prod.yaml` just in case, it's the old name
128-
- copy log-configs\daedalus.yaml daedalus\log-config-prod.yaml
129-
- copy lib\configuration.yaml daedalus\
130-
- copy lib\*genesis*.json daedalus\
131-
- copy cardano-launcher.exe daedalus\
132-
- copy cardano-node.exe daedalus\
133-
- copy cardano-x509-certificates.exe daedalus\
134-
- cd daedalus
135-
- Echo %APPVEYOR_BUILD_VERSION% > build-id
136-
- Echo %APPVEYOR_REPO_COMMIT% > commit-id
137-
- Echo https://ci.appveyor.com/project/%APPVEYOR_ACCOUNT_NAME%/%APPVEYOR_PROJECT_SLUG%/build/%APPVEYOR_BUILD_VERSION% > ci-url
138-
139-
after_test:
140-
- xcopy /q /s /e /r /k /i /v /h /y "%WORK_DIR%\daedalus" "%APPVEYOR_BUILD_FOLDER%\daedalus"
141-
- cd "%WORK_DIR%/daedalus"
142-
- 7z a "%APPVEYOR_REPO_COMMIT%.zip" *
143-
- appveyor PushArtifact "%APPVEYOR_REPO_COMMIT%.zip"
144-
- cd "%WORK_DIR%" # Get back to where cache-s3.exe is located
145-
- ps: >-
146-
if ( ($env:CACHE_S3_READY -eq $true) -and (-not $env:APPVEYOR_PULL_REQUEST_NUMBER) ) {
147-
if ($env:APPVEYOR_REPO_BRANCH -eq "master" -Or $env:APPVEYOR_REPO_BRANCH -eq "develop" -Or $env:APPVEYOR_REPO_BRANCH -like "release*") {
148-
Write-Host "saving stack"
149-
.\cache-s3 --max-size=$env:CACHE_S3_MAX_SIZE --prefix=$env:APPVEYOR_PROJECT_NAME --git-branch=$env:APPVEYOR_REPO_BRANCH --suffix=windows -c -v info save stack
150-
Write-Host "done stack"
151-
}
152-
Write-Host "saving stack work"
153-
.\cache-s3 --max-size=$env:CACHE_S3_MAX_SIZE --prefix=$env:APPVEYOR_PROJECT_NAME --git-branch=$env:APPVEYOR_REPO_BRANCH --suffix=windows -c -v info save stack work
154-
Write-Host "done stack work"
155-
}
156-
artifacts:
157-
- path: daedalus/
158-
name: CardanoSL
159-
type: zip
160-
161-
deploy:
162-
provider: S3
163-
access_key_id:
164-
secure: IEky6PsMzHaKHNBMxR8tQaQI8X7qWRB9+HuEroTVRBk=
165-
secret_access_key:
166-
secure: cqjzG96hWB1x3JDbVSbF9E+aJ5jKvIGacJRUDWATHaTOYfSt6Rvive/NrF4lKBIm
167-
bucket: appveyor-ci-deploy
168-
region: ap-northeast-1
169-
set_public: true
170-
folder: cardano-sl
171-
artifact: $(APPVEYOR_REPO_COMMIT).zip
7+
- ps: echo "No Longer Used, check buildkite windows job"

0 commit comments

Comments
 (0)