input-output-hk · iohk-bors · Nov 29, 2018 · Nov 28, 2018
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -2,7 +2,7 @@ steps:
   - label: 'Windows'
     command: 'PowerShell.exe -executionpolicy remotesigned -File .\.buildkite\windows.ps1'
     agents:
-      system: x86_64-win2016-legacy
+      system: x86_64-win2016
 
   - label: 'stack2nix'
     command: 'nix-shell -Q -j 4 --run scripts/check-stack2nix.sh'

diff --git a/.buildkite/windows.ps1 b/.buildkite/windows.ps1
@@ -1,30 +1,76 @@
 ############################################################################
 # Windows build script for Buildkite
 #
-# Requires the Windows 10 VM to be set up with the compiler,
-# and other tools.
-#
+# Requires the devops buildkite-windows AMI
 ############################################################################
 
-Set-PSDebug -Trace 1
+#Set-PSDebug -Trace 1
 
 Write-Output "--- Prepare work directory"
 
 # Avoid long paths on Windows
-$env:STACK_ROOT="D:\s"
+$env:STACK_ROOT="Z:\s"
 $env:STACK_WORK=".w"
-$env:WORK_DIR="D:\w"
+$env:WORK_DIR="Z:\w"
 # Override the temp directory to avoid sed escaping issues
 # See https://github.com/haskell/cabal/issues/5386
-$env:TMP="D:\\tmp"
+$env:TMP="Z:\\tmp"
 
 # Store the original checkout directory
 $env:CHECKOUT_PATH=(Get-Item -Path ".\").FullName
 
 # Temp directory needs to exist
 New-Item -ItemType Directory -Force -Path $env:TMP
 
-$env:PATH="$env:PATH;D:\ghc\ghc-8.2.2\bin;D:\stack;$Env:Programfiles\7-Zip;D:\w"
+New-Item -ItemType Directory -Force -Path z:\ghc
+
+New-Item -ItemType Directory -Force -Path z:\Downloads
+New-Item -ItemType Directory -Force -Path $env:STACK_ROOT
+
+$StackConfig = @"
+templates:
+  params: null
+system-ghc: true
+local-programs-path: "z:\\s\\programs"
+local-bin-path: "z:\\s\\bin"
+"@
+
+$StackConfig | Out-File -FilePath $env:STACK_ROOT\config.yaml -Encoding ASCII
+
+#
+if (!([System.IO.File]::Exists("Z:\ghc\ghc-8.2.2.tar.xz"))) {
+  echo "Downloading and extracting GHC"
+  curl.exe https://s3.eu-central-1.amazonaws.com/ci-static/ghc-8.2.2-x86_64-unknown-mingw32.tar.xz -o Z:\ghc\ghc-8.2.2.tar.xz -L
+  7z x Z:\ghc\ghc-8.2.2.tar.xz -oZ:\ghc
+  7z x Z:\ghc\ghc-8.2.2.tar -oZ:\ghc
+}
+
+# OpenSSL
+#
+#$env:USERPROFILE
+if (!([System.IO.File]::Exists("Z:\Downloads\Win64OpenSSL-1_0_2q.exe"))) {
+  echo "Downloading and installing OpenSSL"
+  rd -r -fo z:\OpenSSL-Win64-v102
+  curl.exe https://s3.eu-central-1.amazonaws.com/ci-static/Win64OpenSSL-1_0_2q.exe -o Z:\Downloads\Win64OpenSSL-1_0_2q.exe -L
+  cmd /c start /wait "Z:\Downloads\Win64OpenSSL-1_0_2q.exe" /silent /verysilent /sp- /suppressmsgboxes /DIR=Z:\OpenSSL-Win64-v102
+}
+## Install stack
+if (!([System.IO.File]::Exists("Z:\Downloads\stack-1.9.1-windows-x86_64.zip"))) {
+  echo "Downloading and extracting stack"
+  rd -r -fo z:\stack
+  curl.exe https://s3.eu-central-1.amazonaws.com/ci-static/stack-1.9.1-windows-x86_64.zip -o z:\Downloads\stack-1.9.1-windows-x86_64.zip -L
+  7z -oZ:\stack x Z:\Downloads\stack-1.9.1-windows-x86_64.zip
+}
+
+$env:PATH="$env:PATH;Z:\ghc\ghc-8.2.2\bin;Z:\stack;Z:\w"
+
+# Install liblzma/xz
+if (!([System.IO.File]::Exists("Z:\Downloads\xz-5.2.3-windows.zip"))) {
+  echo "Downloading and extracting xz"
+  rd -r -fo z:\xz_extracted
+  curl.exe -L https://s3.eu-central-1.amazonaws.com/ci-static/xz-5.2.3-windows.zip -o Z:\Downloads\xz-5.2.3-windows.zip
+  7z -oZ:\xz_extracted x Z:\Downloads\xz-5.2.3-windows.zip
+}
 
 # Clear out work directory, copy source tree there, then cd into it.
 rd -r -fo $env:WORK_DIR
@@ -36,15 +82,21 @@ cd $env:WORK_DIR
 # headers and the zip provides rocksdb.dll. This is needed to build
 # rocksdb-haskell.
 git.exe clone https://github.com/facebook/rocksdb.git --branch v4.13.5
-curl.exe -L 'https://s3.eu-central-1.amazonaws.com/ci-static/serokell-rocksdb-haskell-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip' -o D:\Downloads\rocksdb.zip
-7z x D:\Downloads\rocksdb.zip
+if (!([System.IO.File]::Exists("Z:\Downloads\rocksdb-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip"))) {
+  echo "Downloading rocksdb"
+  curl.exe -L 'https://s3.eu-central-1.amazonaws.com/ci-static/serokell-rocksdb-haskell-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip' -o Z:\Downloads\rocksdb-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip
+}
+7z x Z:\Downloads\rocksdb-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip
 
 # CSL-1509: After moving the 'cardano-sl' project itself into a separate folder ('lib/'), the 'cardano-text.exe' executable fails on AppVeyor CI.
 # After some investigation, it was discovered that this was because 'rocksdb.dll' has to be located in this folder as well, or else the test executable doesn't work.
 copy rocksdb.dll node
 copy rocksdb.dll lib
 copy rocksdb.dll wallet-new
 
+# Terminate on all errors
+$ErrorActionPreference = "Stop"
+
 ############################################################################
 # Prepare stack and build tools
 #
@@ -64,7 +116,7 @@ stack.exe exec -- ghc-pkg recache
 stack.exe --verbosity warn setup --no-reinstall
 
 # Install happy separately: https://github.com/commercialhaskell/stack/issues/3151#issuecomment-310642487. Also install cpphs because it's a build-tool and Stack can't figure out by itself that it should be installed
-stack.exe --verbosity warn install happy cpphs -j 2 --no-terminal --local-bin-path $env:SYSTEMROOT\system32 --extra-include-dirs="D:\OpenSSL-Win64-v102\include" --extra-lib-dirs="D:\OpenSSL-Win64-v102" --extra-include-dirs="D:\xz_extracted\include" --extra-lib-dirs="D:\xz_extracted\bin_x86-64" --extra-include-dirs="$env:WORK_DIR\rocksdb\include" --extra-lib-dirs="$env:WORK_DIR"
+stack.exe --verbosity warn install happy cpphs -j 2 --no-terminal --local-bin-path $env:SYSTEMROOT\system32 --extra-include-dirs="Z:\OpenSSL-Win64-v102\include" --extra-lib-dirs="Z:\OpenSSL-Win64-v102" --extra-include-dirs="Z:\xz_extracted\include" --extra-lib-dirs="Z:\xz_extracted\bin_x86-64" --extra-include-dirs="$env:WORK_DIR\rocksdb\include" --extra-lib-dirs="$env:WORK_DIR"
 
 ############################################################################
 # Build and test.
@@ -74,18 +126,14 @@ stack.exe --verbosity warn install happy cpphs -j 2 --no-terminal --local-bin-pa
 
 Write-Output "+++ Stack build and test"
 
-stack.exe --dump-logs install cardano-sl cardano-sl-tools cardano-sl-wallet-new -j 3 --no-terminal --local-bin-path $env:WORK_DIR --no-haddock-deps --flag cardano-sl-core:-asserts --flag cardano-sl-tools:for-installer --extra-include-dirs="D:\OpenSSL-Win64-v102\include" --extra-lib-dirs="D:\OpenSSL-Win64-v102" --extra-include-dirs="D:\xz_extracted\include" --extra-lib-dirs="D:\xz_extracted\bin_x86-64" --extra-include-dirs="$env:WORK_DIR\rocksdb\include" --extra-lib-dirs="$env:WORK_DIR"
-
+stack.exe --dump-logs install cardano-sl cardano-sl-tools cardano-sl-wallet-new -j 3 --no-terminal --local-bin-path $env:WORK_DIR --no-haddock-deps --flag cardano-sl-core:-asserts --flag cardano-sl-tools:for-installer --extra-include-dirs="Z:\OpenSSL-Win64-v102\include" --extra-lib-dirs="Z:\OpenSSL-Win64-v102" --extra-include-dirs="Z:\xz_extracted\include" --extra-lib-dirs="Z:\xz_extracted\bin_x86-64" --extra-include-dirs="$env:WORK_DIR\rocksdb\include" --extra-lib-dirs="$env:WORK_DIR"
 
 ############################################################################
 # Assemble artifact zip file for use by the Daedalus installers build
 #
 
 Write-Output "--- Create zip file"
 
-# from here onwards, errors terminate the script
-$ErrorActionPreference = "Stop"
-
 # Cardano pieces, modulo the frontend
 mkdir daedalus
 ## log config is called `log-config-prod.yaml` just in case, it's the old name

diff --git a/appveyor.yml b/appveyor.yml
@@ -3,169 +3,5 @@ image: Visual Studio 2015
 
 build: off
 
-environment:
-  global:
-    # Avoid long paths on Windows
-    STACK_ROOT: "c:\\s"
-    STACK_WORK: ".w"
-    WORK_DIR: "c:\\w"
-    CACHE_S3_VERSION: v0.1.4
-    CACHE_S3_MAX_SIZE: 1600MB  # AppVeyor limits the amount uploaded to approx 2GB
-    AWS_REGION: us-west-1
-    S3_BUCKET: appveyor-ci-cache
-    AWS_ACCESS_KEY_ID:
-      secure: sQWt5CpaN0H+jwUVoTsrET46pADUDEcrJ5D9MHmKX0M=
-    AWS_SECRET_ACCESS_KEY:
-      secure: m5sQYd16K8HA0zoZaD0gOl4EEWUso1D51L5rp+kT3hLaIE3tt4iT+b+iW8F4F0FU
-
-init:
-- ps: $env:CACHE_S3_READY = (("$env:CACHE_S3_VERSION" -ne "") -and ("$env:S3_BUCKET" -ne "") -and ("$env:AWS_ACCESS_KEY_ID" -ne "") -and ("$env:AWS_SECRET_ACCESS_KEY" -ne ""))
-
-before_test:
-# Avoid long paths not to each MAX_PATH of 260 chars
-- xcopy /q /s /e /r /k /i /v /h /y "%APPVEYOR_BUILD_FOLDER%" "%WORK_DIR%"
-- cd "%WORK_DIR%"
-# Restore cache
-- Echo %APPVEYOR_BUILD_VERSION% > build-id
-- ps: >-
-    Write-Host "in pagefile script" ;
-    $c = Get-WmiObject Win32_computersystem -EnableAllPrivileges ;
-    if($c.AutomaticManagedPagefile){
-      Write-Host "disabling managed page file settings"
-      $c.AutomaticManagedPagefile = $false
-      $c.Put() | Out-Null
-    } ;
-    $new_page_size=25000 ;
-    $CurrentPageFile = Get-WmiObject -Class Win32_PageFileSetting ;
-    if ($CurrentPageFile.InitialSize -ne $new_page_size) {
-      Write-Host "setting new page file size to $new_page_size"
-      $CurrentPageFile.InitialSize=$new_page_size
-      $CurrentPageFile.MaximumSize=$new_page_size
-      $CurrentPageFile.Put() | Out-Null
-    } ;
-    if ( $env:CACHE_S3_READY -eq $true ) {
-      Start-FileDownload https://github.com/fpco/cache-s3/releases/download/$env:CACHE_S3_VERSION/cache-s3-$env:CACHE_S3_VERSION-windows-x86_64.zip -FileName cache-s3.zip
-      7z x cache-s3.zip cache-s3.exe
-      .\cache-s3 --max-size=$env:CACHE_S3_MAX_SIZE --prefix=$env:APPVEYOR_PROJECT_NAME --git-branch=$env:APPVEYOR_REPO_BRANCH --suffix=windows -v info -c restore stack --base-branch=develop
-      .\cache-s3 --max-size=$env:CACHE_S3_MAX_SIZE --prefix=$env:APPVEYOR_PROJECT_NAME --git-branch=$env:APPVEYOR_REPO_BRANCH --suffix=windows -v info -c restore stack work --base-branch=develop
-    }
-
-# Get custom GHC
-- ps: >-
-    mkdir C:\ghc
-
-    Invoke-WebRequest "https://s3.eu-central-1.amazonaws.com/ci-static/ghc-8.2.2-x86_64-unknown-mingw32.tar.xz" -OutFile "C:\ghc\ghc.tar.xz" -UserAgent "Curl"
-
-    7z x C:\ghc\ghc.tar.xz -oC:\ghc
-
-    7z x C:\ghc\ghc.tar -oC:\ghc
-
-    $env:PATH="$env:PATH;C:\ghc\ghc-8.2.2\bin"
-
-# Install OpenSSL 1.0.2 (see https://github.com/appveyor/ci/issues/1665)
-- ps: (New-Object Net.WebClient).DownloadFile('https://slproweb.com/download/Win64OpenSSL-1_0_2q.exe', "$($env:USERPROFILE)\Win64OpenSSL.exe")
-- ps: cmd /c start /wait "$($env:USERPROFILE)\Win64OpenSSL.exe" /silent /verysilent /sp- /suppressmsgboxes /DIR=C:\OpenSSL-Win64-v102
-- ps: Install-Product node 6
-# Install stack
-- ps: Start-FileDownload http://www.stackage.org/stack/windows-x86_64 -FileName stack.zip
-- 7z x stack.zip stack.exe
-
-
-# Install rocksdb
-- git clone https://github.com/facebook/rocksdb.git --branch v4.13.5
-- ps: Start-FileDownload 'https://s3.eu-central-1.amazonaws.com/ci-static/serokell-rocksdb-haskell-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip' -FileName rocksdb.zip
-- 7z x rocksdb.zip
-
-# CSL-1509: After moving the 'cardano-sl' project itself into a separate folder ('lib/'), the 'cardano-text.exe' executable fails on AppVeyor CI.
-# After some investigation, it was discovered that this was because 'rocksdb.dll' has to be located in this folder as well, or else the test executable doesn't work.
-- copy rocksdb.dll node
-- copy rocksdb.dll lib
-- copy rocksdb.dll wallet
-- copy rocksdb.dll wallet-new
-
-# Install liblzma/xz
-- ps: Start-FileDownload https://tukaani.org/xz/xz-5.2.3-windows.zip -Filename xz-5.2.3-windows.zip
-- 7z -oC:\xz_extracted x xz-5.2.3-windows.zip
-
 test_script:
-  - cd "%WORK_DIR%"
-  - stack config --system-ghc set system-ghc --global true
-  - stack exec -- ghc-pkg recache
-  - stack --verbosity warn setup --no-reinstall > nul
-  # Install happy separately: https://github.com/commercialhaskell/stack/issues/3151#issuecomment-310642487. Also install cpphs because it's a build-tool and Stack can't figure out by itself that it should be installed
-  - scripts\ci\appveyor-retry call stack --verbosity warn install happy cpphs
-      -j 2
-      --no-terminal
-      --local-bin-path %SYSTEMROOT%\system32
-      --extra-include-dirs="C:\OpenSSL-Win64-v102\include"
-      --extra-lib-dirs="C:\OpenSSL-Win64-v102"
-      --extra-include-dirs="C:\xz_extracted\include"
-      --extra-lib-dirs="C:\xz_extracted\bin_x86-64"
-      --extra-include-dirs="%WORK_DIR%\rocksdb\include"
-      --extra-lib-dirs="%WORK_DIR%"
-#   TODO: CSL-1133. To be reenabled.
-#   - stack test --coverage
-#   - stack hpc report cardano-sl cardano-sl-txp cardano-sl-core cardano-sl-db cardano-sl-update cardano-sl-infra cardano-sl-lrc cardano-sl-ssc
-# Retry transient failures due to https://github.com/haskell/cabal/issues/4005
-  # We intentionally don't build auxx here, because this build is for installer.
-  - scripts\ci\appveyor-retry call stack --dump-logs install cardano-sl cardano-sl-tools cardano-sl-wallet cardano-sl-wallet-new
-      -j 3
-      --no-terminal
-      --local-bin-path %WORK_DIR%
-      --no-haddock-deps
-      --flag cardano-sl-core:-asserts
-      --flag cardano-sl-tools:for-installer
-      --flag cardano-sl-wallet:for-installer
-      --extra-include-dirs="C:\OpenSSL-Win64-v102\include"
-      --extra-lib-dirs="C:\OpenSSL-Win64-v102"
-      --extra-include-dirs="C:\xz_extracted\include"
-      --extra-lib-dirs="C:\xz_extracted\bin_x86-64"
-      --extra-include-dirs="%WORK_DIR%\rocksdb\include"
-      --extra-lib-dirs="%WORK_DIR%"
-  # Cardano pieces, modulo the frontend
-  - mkdir daedalus
-    # log config is called `log-config-prod.yaml` just in case, it's the old name
-  - copy log-configs\daedalus.yaml daedalus\log-config-prod.yaml
-  - copy lib\configuration.yaml daedalus\
-  - copy lib\*genesis*.json daedalus\
-  - copy cardano-launcher.exe daedalus\
-  - copy cardano-node.exe daedalus\
-  - copy cardano-x509-certificates.exe daedalus\
-  - cd daedalus
-  - Echo %APPVEYOR_BUILD_VERSION% > build-id
-  - Echo %APPVEYOR_REPO_COMMIT% > commit-id
-  - Echo https://ci.appveyor.com/project/%APPVEYOR_ACCOUNT_NAME%/%APPVEYOR_PROJECT_SLUG%/build/%APPVEYOR_BUILD_VERSION% > ci-url
-
-after_test:
- - xcopy /q /s /e /r /k /i /v /h /y "%WORK_DIR%\daedalus" "%APPVEYOR_BUILD_FOLDER%\daedalus"
- - cd "%WORK_DIR%/daedalus"
- - 7z a "%APPVEYOR_REPO_COMMIT%.zip" *
- - appveyor PushArtifact "%APPVEYOR_REPO_COMMIT%.zip"
- - cd "%WORK_DIR%" # Get back to where cache-s3.exe is located
- - ps: >-
-    if ( ($env:CACHE_S3_READY -eq $true) -and (-not $env:APPVEYOR_PULL_REQUEST_NUMBER) ) {
-      if ($env:APPVEYOR_REPO_BRANCH -eq "master" -Or $env:APPVEYOR_REPO_BRANCH -eq "develop" -Or $env:APPVEYOR_REPO_BRANCH -like "release*") {
-        Write-Host "saving stack"
-        .\cache-s3 --max-size=$env:CACHE_S3_MAX_SIZE --prefix=$env:APPVEYOR_PROJECT_NAME --git-branch=$env:APPVEYOR_REPO_BRANCH --suffix=windows -c -v info save stack
-        Write-Host "done stack"
-      }
-      Write-Host "saving stack work"
-      .\cache-s3 --max-size=$env:CACHE_S3_MAX_SIZE --prefix=$env:APPVEYOR_PROJECT_NAME --git-branch=$env:APPVEYOR_REPO_BRANCH --suffix=windows -c -v info save stack work
-      Write-Host "done stack work"
-    }
-artifacts:
-  - path: daedalus/
-    name: CardanoSL
-    type: zip
-
-deploy:
-  provider: S3
-  access_key_id:
-    secure: IEky6PsMzHaKHNBMxR8tQaQI8X7qWRB9+HuEroTVRBk=
-  secret_access_key:
-    secure: cqjzG96hWB1x3JDbVSbF9E+aJ5jKvIGacJRUDWATHaTOYfSt6Rvive/NrF4lKBIm
-  bucket: appveyor-ci-deploy
-  region: ap-northeast-1
-  set_public: true
-  folder: cardano-sl
-  artifact: $(APPVEYOR_REPO_COMMIT).zip
+- ps: echo "No Longer Used, check buildkite windows job"