pipeline(depls): trigger update-pipeline on each secrets-repo commit

 We also trigger `init-concourse-boshrelease-and-stemcell` on each
 `<root-depls>-versions.yml` update, to reduce manual job triggering. We
 use `tee -a` to replace `set -x`.

 We also rename `paas-template` to `paas-templates`, to avoid typo errors.

Author: o-orand

concourse/pipelines/template/depls-pipeline.yml.erb

@@ -89,7 +89,7 @@ resources:
     branch: ((secrets-branch))
     skip_ssl_verification: true
 
-- name: paas-template-<%=depls %>
+- name: paas-templates-<%=depls %>
   type: git
   source:
     uri: ((paas-templates-uri))
@@ -136,7 +136,7 @@ resources:
     branch: ((secrets-branch))
     skip_ssl_verification: true
 
-- name: paas-template-<%= name %>
+- name: paas-templates-<%= name %>
   type: git
   source:
     uri: ((paas-templates-uri))
@@ -208,6 +208,16 @@ resources:
     skip_ssl_verification: true
 <% end %>
 
+<% if enabled_deployments.any? %>
+- name: paas-templates-<%=depls %>-versions
+  type: git
+  source:
+    uri: ((paas-templates-uri))
+    paths: ["<%= depls %>/<%= depls %>-versions.yml"]
+    branch: ((paas-templates-branch))
+    skip_ssl_verification: true
+<% end %>
+
 <% if all_ci_deployments.any? || disabled_deployments.any? %>
 - name: paas-templates-full
   type: git
@@ -264,6 +274,7 @@ jobs:
         params: { submodules: none}
         <%= "passed: [update-pipeline-#{depls}-generated]" if all_ci_deployments.any? %>
       - get: secrets-full
+        version: every
         trigger: true
         <%= "passed: [update-pipeline-#{depls}-generated]" if all_ci_deployments.any? %>
         params: { submodules: none}
@@ -317,15 +328,15 @@ jobs:
       - get: secrets-<%= depls %>
         params: { submodules: none}
 #        <!--<%= "passed: [update-pipeline-#{depls}-generated]" if all_ci_deployments.any? %>-->
-      - get: paas-template-<%= depls %>
+      - get: paas-templates-<%= depls %>
         params: { submodules: none}
 #        <!--<%= "passed: [update-pipeline-#{depls}-generated]" if all_ci_deployments.any? %>-->
       - get: cf-ops-automation
         params: { submodules: none}
         <%= "passed: [update-pipeline-#{depls}-generated]" if all_ci_deployments.any? %>
 
     - task: run-deploy.sh
-      input_mapping: {scripts-resource: cf-ops-automation, templates: paas-template-<%= depls %>, secrets: secrets-<%= depls %>}
+      input_mapping: {scripts-resource: cf-ops-automation, templates: paas-templates-<%= depls %>, secrets: secrets-<%= depls %>}
       file: cf-ops-automation/concourse/tasks/execute_deploy_script.yml
       params:
         CURRENT_DEPLS: <%= depls %>/template
@@ -353,7 +364,7 @@ jobs:
       params: { submodules: none}
       trigger: true
 #      <!--<%= "passed: [update-pipeline-#{depls}-generated]" if all_ci_deployments.any? %>-->
-    - get: paas-template-<%=depls %>
+    - get: paas-templates-<%=depls %>
       params: { submodules: none}
       trigger: true
 #      <!--<%= "passed: [update-pipeline-#{depls}-generated]" if all_ci_deployments.any? %>-->
@@ -362,7 +373,7 @@ jobs:
       trigger: true
       <%= "passed: [update-pipeline-#{depls}-generated]" if all_ci_deployments.any? %>
     - task: generate-<%= depls %>-all-config
-      input_mapping: {scripts-resource: cf-ops-automation, credentials-resource: secrets-<%=depls %>, additional-resource: paas-template-<%=depls %>}
+      input_mapping: {scripts-resource: cf-ops-automation, credentials-resource: secrets-<%=depls %>, additional-resource: paas-templates-<%=depls %>}
       output_mapping: {generated-files: config-manifest}
       file: cf-ops-automation/concourse/tasks/generate-manifest.yml
       params:
@@ -487,13 +498,13 @@ jobs:
       params: { submodules: none}
       trigger: true
 #     <!--<%= "passed: [update-pipeline-#{depls}-generated]" if all_ci_deployments.any? %>-->
-    - get: paas-template-<%= name %>
+    - get: paas-templates-<%= name %>
       trigger: true
   <% current_git_submodule = git_submodules[depls][name] if git_submodules[depls] %>
       params: { submodules: <%= current_git_submodule || 'none' %>}
 #     <!--<%= "passed: [update-pipeline-#{depls}-generated]" if all_ci_deployments.any? %>-->
   - task: generate-<%= name %>-manifest
-    input_mapping: {scripts-resource: cf-ops-automation, credentials-resource: secrets-<%= name %>, additional-resource: paas-template-<%= name %>}
+    input_mapping: {scripts-resource: cf-ops-automation, credentials-resource: secrets-<%= name %>, additional-resource: paas-templates-<%= name %>}
     output_mapping: {generated-files: release-manifest}
     file: cf-ops-automation/concourse/tasks/generate-manifest.yml
     params:
@@ -506,14 +517,14 @@ jobs:
       CUSTOM_SCRIPT_DIR: additional-resource/<%= depls %>/<%= name %>/template
       IAAS_TYPE: ((iaas-type))
   - task: execute-<%= name %>-spiff-pre-bosh-deploy
-    input_mapping: {scripts-resource: cf-ops-automation, template-resource: paas-template-<%= name %>, credentials-resource: secrets-<%= name %>, additional-resource: release-manifest}
+    input_mapping: {scripts-resource: cf-ops-automation, template-resource: paas-templates-<%= name %>, credentials-resource: secrets-<%= name %>, additional-resource: release-manifest}
     output_mapping: {generated-files: pre-bosh-deploy-resource}
     file: cf-ops-automation/concourse/tasks/spiff_pre_bosh_deploy.yml
     params:
       CUSTOM_SCRIPT_DIR: template-resource/<%= depls %>/<%= name %>/template
       SECRETS_DIR: credentials-resource/<%= depls %>/<%= name %>
   - task: copy-<%= name %>-required-files
-    input_mapping: {scripts-resource: cf-ops-automation, template-resource: paas-template-<%= name %>, credentials-resource: secrets-<%= name %>, additional-resource: pre-bosh-deploy-resource}
+    input_mapping: {scripts-resource: cf-ops-automation, template-resource: paas-templates-<%= name %>, credentials-resource: secrets-<%= name %>, additional-resource: pre-bosh-deploy-resource}
     output_mapping: {generated-files: final-release-manifest}
     file: cf-ops-automation/concourse/tasks/copy_deployment_required_files.yml
     params:
@@ -631,7 +642,7 @@ jobs:
   <% end %>
       cleanup: true
   - task: execute-<%= name %>-post-bosh-deploy
-    input_mapping: {scripts-resource: cf-ops-automation, template-resource: paas-template-<%= name %>, credentials-resource: secrets-<%= name %>, additional-resource: final-release-manifest}
+    input_mapping: {scripts-resource: cf-ops-automation, template-resource: paas-templates-<%= name %>, credentials-resource: secrets-<%= name %>, additional-resource: final-release-manifest}
     output_mapping: {generated-files: post-deploy-result}
     file: cf-ops-automation/concourse/tasks/post_bosh_deploy.yml
     params:
@@ -668,7 +679,7 @@ jobs:
       params: { submodules: none}
       trigger: true
       passed: [deploy-<%= name %>]
-    - get: paas-template-<%= name %>
+    - get: paas-templates-<%= name %>
       params: { submodules: none}
       trigger: true
       passed: [deploy-<%= name %>]
@@ -741,7 +752,7 @@ jobs:
      - get: secrets-<%= depls %>
        params: { submodules: none}
        <%= "passed: [update-pipeline-#{depls}-generated]" if all_ci_deployments.any? %>
-     - get: paas-template-<%= depls %>
+     - get: paas-templates-<%= depls %>
        params: { submodules: none}
        <%= "passed: [update-pipeline-#{depls}-generated]" if all_ci_deployments.any? %>
      - get: cf-ops-automation
@@ -766,7 +777,7 @@ jobs:
        params: { submodules: none}
        passed: [recreate-all]
        trigger: true
-     - get: paas-template-<%= depls %>
+     - get: paas-templates-<%= depls %>
        params: { submodules: none}
        passed: [recreate-all]
        trigger: true
@@ -823,7 +834,10 @@ jobs:
     - get: secrets-full
       params: { submodules: none}
       attempts: 2
-#      trigger: true
+    - get: paas-templates-<%= depls %>-versions
+      params: { submodules: none}
+      attempts: 2
+      trigger: true
   - task: generate-<%= depls %>-flight-plan
     output_mapping: {result-dir: init-<%= depls %>-plan}
     config:
@@ -836,19 +850,19 @@ jobs:
       run:
         path: sh
         args:
-        - -exc
+        - -ec
         - |
   <% uniq_releases.sort.each do |name, info| %>
     <% if config['offline-mode'] && config['offline-mode']['boshreleases'] %>
-          echo "check-resource -r $BUILD_PIPELINE_NAME/<%= name %> --from path:<%= info['repository']&.split('/')&.first %>/<%= name %>-((<%= name %>-version)).tgz" >> result-dir/flight-plan
+          echo "check-resource -r $BUILD_PIPELINE_NAME/<%= name %> --from path:<%= info['repository']&.split('/')&.first %>/<%= name %>-((<%= name %>-version)).tgz" | tee -a result-dir/flight-plan
     <% else %>
-          echo "check-resource -r $BUILD_PIPELINE_NAME/<%= name %> --from version:((<%= name %>-version))" >> result-dir/flight-plan
+          echo "check-resource -r $BUILD_PIPELINE_NAME/<%= name %> --from version:((<%= name %>-version))" | tee -a result-dir/flight-plan
     <% end %>
   <% end %>
     <% if offline_stemcells_enabled %>
-          echo "check-resource -r $BUILD_PIPELINE_NAME/((stemcell-main-name)) --from path:((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz" >> result-dir/flight-plan
+          echo "check-resource -r $BUILD_PIPELINE_NAME/((stemcell-main-name)) --from path:((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz" | tee -a result-dir/flight-plan
     <% else %>
-          echo "check-resource -r $BUILD_PIPELINE_NAME/((stemcell-main-name)) --from version:((stemcell-version))" >> result-dir/flight-plan
+          echo "check-resource -r $BUILD_PIPELINE_NAME/((stemcell-main-name)) --from version:((stemcell-version))" | tee -a result-dir/flight-plan
     <% end %>
       params:
         BUILD_PIPELINE_NAME: <%= depls %>-generated
@@ -881,7 +895,7 @@ jobs:
   - aggregate:
     - get: secrets-full-writer
       params: { submodules: none}
-    - get: paas-template-<%= depls %>
+    - get: paas-templates-<%= depls %>
       params: { submodules: none}
       attempts: 2
     - get: paas-templates-wip
@@ -900,6 +914,7 @@ jobs:
       params: { submodules: none}
       attempts: 2
     - get: secrets-full
+      version: every
       params: { submodules: none}
       attempts: 2
 
@@ -992,15 +1007,15 @@ jobs:
       params: { submodules: none}
       passed: [update-pipeline-<%= depls %>-generated]
       trigger: true
-    - get: paas-template-<%=depls %>
+    - get: paas-templates-<%=depls %>
       params: { submodules: none }
       trigger: true
     - get: secrets-<%= depls %>
       passed: [update-pipeline-<%= depls %>-generated]
       params: { submodules: none }
       trigger: true
   - task: generate-terraform-tfvars
-    input_mapping: {scripts-resource: cf-ops-automation, credentials-resource: secrets-<%= depls %>, additional-resource: paas-template-<%=depls %>}
+    input_mapping: {scripts-resource: cf-ops-automation, credentials-resource: secrets-<%= depls %>, additional-resource: paas-templates-<%=depls %>}
     output_mapping: {generated-files: terraform-tfvars}
     file: cf-ops-automation/concourse/tasks/generate-manifest.yml
     params:
@@ -1013,7 +1028,7 @@ jobs:
       SUFFIX: -tpl.tfvars.yml
       IAAS_TYPE: ((iaas-type))
   - task: terraform-plan
-    input_mapping: {secret-state-resource: secrets-<%= depls %>,spec-resource: paas-template-<%=depls %>}
+    input_mapping: {secret-state-resource: secrets-<%= depls %>,spec-resource: paas-templates-<%=depls %>}
     file: cf-ops-automation/concourse/tasks/terraform_plan_cloudfoundry.yml
     params:
       SPEC_PATH: "<%= terraform_config_path %>/spec"
@@ -1037,15 +1052,15 @@ jobs:
     - get: cf-ops-automation
       params: { submodules: none}
       passed: [check-terraform-consistency]
-    - get: paas-template-<%=depls %>
+    - get: paas-templates-<%=depls %>
       params: { submodules: none}
       passed: [check-terraform-consistency]
       trigger: false
     - get: secrets-<%= depls %>
       params: { submodules: none}
       passed: [check-terraform-consistency]
   - task: generate-terraform-tfvars
-    input_mapping: {scripts-resource: cf-ops-automation, credentials-resource: secrets-<%= depls %>, additional-resource: paas-template-<%=depls %>}
+    input_mapping: {scripts-resource: cf-ops-automation, credentials-resource: secrets-<%= depls %>, additional-resource: paas-templates-<%=depls %>}
     output_mapping: {generated-files: terraform-tfvars}
     file: cf-ops-automation/concourse/tasks/generate-manifest.yml
     params:
@@ -1058,7 +1073,7 @@ jobs:
       SUFFIX: -tpl.tfvars.yml
       IAAS_TYPE: ((iaas-type))
   - task: terraform-apply
-    input_mapping: {secret-state-resource: secrets-<%= depls %>,spec-resource: paas-template-<%=depls %>}
+    input_mapping: {secret-state-resource: secrets-<%= depls %>,spec-resource: paas-templates-<%=depls %>}
     output_mapping: {generated-files: terraform-cf}
     file: cf-ops-automation/concourse/tasks/terraform_apply_cloudfoundry.yml
     params:

concourse/tasks/fly_execute_commands.yml

@@ -26,23 +26,24 @@ outputs:
 run:
   path: sh
   args:
-  - -exc
+  - -ec
   - |
     mkdir -p /usr/local/bin
     FLY=/usr/local/bin/fly
     echo "Fetching fly...";
     curl -SsL -u "$FLY_USERNAME:$FLY_PASSWORD" "$ATC_EXTERNAL_URL/api/v1/cli?arch=amd64&platform=linux" -k > $FLY;
     chmod +x $FLY;
 
+    echo "Login into Concourse@${ATC_EXTERNAL_URL}"
     fly login -t concourse-target --team-name="$FLY_TEAM" -c "$ATC_EXTERNAL_URL" --username="$FLY_USERNAME" --password="$FLY_PASSWORD" -k 2>&1
     PREFIX="fly -t concourse-target"
 
     old_IFS=$IFS     # sauvegarde du séparateur de champ
     IFS=$'\n'     # nouveau séparateur de champ, le caractère fin de ligne
     for command in $(cat fly-cmd/flight-plan)
     do
-      echo "Executing '$PREFIX $command': " >> flight-report/flight-report.txt 2>&1
-      eval "$PREFIX $command" >> flight-report/flight-report.txt 2>&1
+      echo "Executing '$PREFIX $command': " 2>&1 | tee -a flight-report/flight-report.txt
+      eval "$PREFIX $command" 2>&1 | tee -a flight-report/flight-report.txt
     done
     IFS=$old_IFS     # rétablissement du séparateur de champ par défaut
     cat flight-report/flight-report.txt

spec/lib/template_processor/template_processor_for_depls_pipeline_spec.rb

@@ -188,7 +188,7 @@
               params: { submodules: none}
               trigger: true
               passed: [deploy-shield-expe]
-            - get: paas-template-shield-expe
+            - get: paas-templates-shield-expe
               params: { submodules: none}
               trigger: true
               passed: [deploy-shield-expe]
@@ -344,7 +344,7 @@
       end
       let(:expected_stemcell_deploy_get) { ['((stemcell-version))'] * 2 }
       let(:expected_stemcell_deploy_put) { ['((stemcell-main-name))/stemcell.tgz'] * 2 }
-      let(:expected_stemcell_init) { 'echo "check-resource -r $BUILD_PIPELINE_NAME/((stemcell-main-name)) --from version:((stemcell-version))" >> result-dir/flight-plan' }
+      let(:expected_stemcell_init) { 'echo "check-resource -r $BUILD_PIPELINE_NAME/((stemcell-main-name)) --from version:((stemcell-version))" | tee -a result-dir/flight-plan' }
 
       it 'generates bosh-io stemcell' do
         bosh_io_stemcell = generated_pipeline['resources'].select { |resource| resource['type'] == 'bosh-io-stemcell' }
@@ -451,7 +451,7 @@
           [
             { "task" => 'generate-terraform-tfvars',
               "input_mapping" =>
-                { "scripts-resource" => "cf-ops-automation", "credentials-resource" => "secrets-my-root-depls", "additional-resource" => "paas-template-my-root-depls" },
+                { "scripts-resource" => "cf-ops-automation", "credentials-resource" => "secrets-my-root-depls", "additional-resource" => "paas-templates-my-root-depls" },
               "output_mapping" => { "generated-files" => "terraform-tfvars" },
               "file" => "cf-ops-automation/concourse/tasks/generate-manifest.yml",
               "params" =>
@@ -464,7 +464,7 @@
             { "task" => "terraform-plan",
               "input_mapping" =>
                    { "secret-state-resource" => "secrets-my-root-depls",
-                     "spec-resource" => "paas-template-my-root-depls" },
+                     "spec-resource" => "paas-templates-my-root-depls" },
               "file" => "cf-ops-automation/concourse/tasks/terraform_plan_cloudfoundry.yml",
               "params" =>
                    { "SPEC_PATH" => "my-tfstate-location/spec",
@@ -486,12 +486,12 @@
       let(:check_terraform_plans) { check_terraform_jobs.flat_map { |job| job['plan'] } }
       let(:check_terraform_aggregate) { check_terraform_plans.flat_map { |tasks| tasks['aggregate'] }.compact }
       let(:check_terraform_secrets_triggering) { check_terraform_aggregate.select { |task| task['get'].start_with?('secrets-') }.flat_map { |task| task['trigger'] } }
-      let(:check_terraform_paas_templates_triggering) { check_terraform_aggregate.select { |task| task['get'].start_with?('paas-template-') }.flat_map { |task| task['trigger'] } }
+      let(:check_terraform_paas_templates_triggering) { check_terraform_aggregate.select { |task| task['get'].start_with?('paas-templates-') }.flat_map { |task| task['trigger'] } }
       let(:enforce_terraform_jobs) { generated_pipeline['jobs'].select { |resource| resource['name'].start_with?('approve-and-enforce-terraform-consistency') } }
       let(:enforce_terraform_plans) { enforce_terraform_jobs.flat_map { |job| job['plan'] } }
       let(:enforce_terraform_aggregate) { enforce_terraform_plans.flat_map { |tasks| tasks['aggregate'] }.compact }
       let(:enforce_terraform_all_secrets_triggering) { enforce_terraform_aggregate.select { |task| task['get'].start_with?('secrets-') }.flat_map { |task| task['trigger'] } }
-      let(:enforce_terraform_paas_templates_triggering) { enforce_terraform_aggregate.select { |task| task['get'].start_with?('paas-template-') }.flat_map { |task| task['trigger'] } }
+      let(:enforce_terraform_paas_templates_triggering) { enforce_terraform_aggregate.select { |task| task['get'].start_with?('paas-templates-') }.flat_map { |task| task['trigger'] } }
       let(:all_ci_deployments) { enable_root_deployment_terraform }
 
       it 'triggers check-consistency automatically on each commit on secrets' do

spec/scripts/generate-depls/fixtures/references/delete-depls-ref.yml

@@ -60,7 +60,7 @@ resources:
       branch: ((secrets-branch))
       skip_ssl_verification: true
 
-  - name: paas-template-delete-depls
+  - name: paas-templates-delete-depls
     type: git
     source:
       uri: ((paas-templates-uri))
@@ -109,6 +109,7 @@ jobs:
         params:
           submodules: none
       - get: secrets-full
+        version: every
         trigger: true
         params:
           submodules: none
@@ -165,7 +166,7 @@ jobs:
       - get: secrets-delete-depls
         params:
           submodules: none
-      - get: paas-template-delete-depls
+      - get: paas-templates-delete-depls
         params:
           submodules: none
       - get: cf-ops-automation
@@ -174,7 +175,7 @@ jobs:
     - task: run-deploy.sh
       input_mapping:
         scripts-resource: cf-ops-automation
-        templates: paas-template-delete-depls
+        templates: paas-templates-delete-depls
         secrets: secrets-delete-depls
       file: cf-ops-automation/concourse/tasks/execute_deploy_script.yml
       params:
@@ -202,7 +203,7 @@ jobs:
       params:
         submodules: none
       trigger: true
-    - get: paas-template-delete-depls
+    - get: paas-templates-delete-depls
       params:
         submodules: none
       trigger: true
@@ -214,7 +215,7 @@ jobs:
       input_mapping:
         scripts-resource: cf-ops-automation
         credentials-resource: secrets-delete-depls
-        additional-resource: paas-template-delete-depls
+        additional-resource: paas-templates-delete-depls
       output_mapping:
         generated-files: config-manifest
       file: cf-ops-automation/concourse/tasks/generate-manifest.yml
@@ -318,7 +319,7 @@ jobs:
       - get: secrets-delete-depls
         params:
           submodules: none
-      - get: paas-template-delete-depls
+      - get: paas-templates-delete-depls
         params:
           submodules: none
       - get: cf-ops-automation