Merge branch 'main' into fix-hb-proc-io

Author: vigneshshanmugam

.buildkite/hooks/pre-command

@@ -2,6 +2,27 @@
 
 set -euo pipefail
 
+AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth"
+
+retry() {
+  local retries=$1
+  shift
+  local count=0
+  until "$@"; do
+    exit=$?
+    wait=$((2 ** count))
+    count=$((count + 1))
+    if [ $count -lt "$retries" ]; then
+      >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+      sleep $wait
+    else
+      >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+      return $exit
+    fi
+  done
+  return 0
+}
+
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then
   source .buildkite/env-scripts/env.sh
   source .buildkite/env-scripts/util.sh
@@ -12,10 +33,17 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
   fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
   source .buildkite/scripts/setenv.sh
   if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then
     echo "Skipped pre-command when running the Upload pipeline"
     exit 0
   fi
 fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "$BUILDKITE_STEP_KEY" == "extended-cloud-test" ]]; then
+  BEATS_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+  export BEATS_AWS_SECRET_KEY
+  BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+  export BEATS_AWS_ACCESS_KEY
+fi

.buildkite/scripts/cloud_tests.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# What Terraform Module will run
+export MODULE_DIR="x-pack/metricbeat/module/aws"
+
+source .buildkite/scripts/install_tools.sh
+
+set -euo pipefail
+
+trap 'teardown || true; unset_secrets' EXIT
+
+# Set the MODULE env variable if possible
+defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}"
+
+# Prepare the cloud resources using Terraform
+startCloudTestEnv "${MODULE_DIR}"
+
+# Run tests
+echo "--- Run Cloud Tests for $BEATS_PROJECT_NAME"
+pushd "${BEATS_PROJECT_NAME}" > /dev/null
+
+mage build test
+
+popd > /dev/null

.buildkite/scripts/common.sh

@@ -9,6 +9,8 @@ arch_type="$(uname -m)"
 GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""}
 GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""}
 ONLY_DOCS=${ONLY_DOCS:-"true"}
+OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
+XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
 [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")"
 [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")"
 [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")"
@@ -17,6 +19,9 @@ ONLY_DOCS=${ONLY_DOCS:-"true"}
 [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")"
 [ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")"
 [ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")"
+[ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")"
+[ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")"
+[ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")"
 
 metricbeat_changeset=(
   "^metricbeat/.*"
@@ -81,6 +86,30 @@ packaging_changeset=(
   ".go-version"
   )
 
+case "${BUILDKITE_PIPELINE_SLUG}" in
+  "beats-metricbeat")
+    BEAT_CHANGESET_REFERENCE=${metricbeat_changeset[@]}
+    ;;
+  "beats-libbeat")
+    BEAT_CHANGESET_REFERENCE=${libbeat_changeset[@]}
+    ;;
+  "beats-packetbeat")
+    BEAT_CHANGESET_REFERENCE=${packetbeat_changeset[@]}
+    ;;
+  "beats-winlogbeat")
+    BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]}
+    ;;
+  "beats-xpack-libbeat")
+    BEAT_CHANGESET_REFERENCE=${xpack_libbeat_changeset[@]}
+    ;;
+  "beats-xpack-metricbeat")
+    BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]}
+    ;;
+  *)
+  echo "The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet."
+  ;;
+esac
+
 check_and_set_beat_vars() {
   if [[ -n "$BEATS_PROJECT_NAME" && "$BEATS_PROJECT_NAME" == *"x-pack/"* ]]; then
     BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/}              #remove -
@@ -89,8 +118,8 @@ check_and_set_beat_vars() {
     BEATS_GH_LABEL=${BEATS_XPACK_LABEL_PROJECT_NAME}
     TRIGGER_SPECIFIC_BEAT="run_${BEATS_XPACK_PROJECT_NAME}"
     TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_arm_tests"
+    TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_aws_tests"
     TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests"
-    declare -n BEAT_CHANGESET_REFERENCE="${BEATS_XPACK_PROJECT_NAME}_changeset"
     echo "Beats project name is $BEATS_XPACK_PROJECT_NAME"
     mandatory_changeset=(
       "${BEAT_CHANGESET_REFERENCE[@]}"
@@ -101,8 +130,8 @@ check_and_set_beat_vars() {
     BEATS_GH_LABEL=${BEATS_PROJECT_NAME}
     TRIGGER_SPECIFIC_BEAT="run_${BEATS_PROJECT_NAME}"
     TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_PROJECT_NAME}_arm_tests"
+    TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_PROJECT_NAME}_aws_tests"
     TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests"
-    declare -n BEAT_CHANGESET_REFERENCE="${BEATS_PROJECT_NAME}_changeset"
     echo "Beats project name is $BEATS_PROJECT_NAME"
     mandatory_changeset=(
       "${BEAT_CHANGESET_REFERENCE[@]}"
@@ -113,8 +142,10 @@ check_and_set_beat_vars() {
   BEATS_GH_COMMENT="/test ${BEATS_PROJECT_NAME}"
   BEATS_GH_MACOS_COMMENT="${BEATS_GH_COMMENT} for macos"
   BEATS_GH_ARM_COMMENT="${BEATS_GH_COMMENT} for arm"
-  BAETS_GH_MACOS_LABEL="macOS"
-  BAETS_GH_ARM_LABEL="arm"
+  BEATS_GH_AWS_COMMENT="${BEATS_GH_COMMENT} for aws cloud"
+  BEATS_GH_MACOS_LABEL="macOS"
+  BEATS_GH_ARM_LABEL="arm"
+  BEATS_GH_AWS_LABEL="aws"
 }
 
 with_docker_compose() {
@@ -127,6 +158,19 @@ with_docker_compose() {
   docker-compose version
 }
 
+with_Terraform() {
+    echo "Setting up the Terraform environment..."
+    local path_to_file="${WORKSPACE}/terraform.zip"
+    create_workspace
+    check_platform_architeture
+    retry 5 curl -sSL -o ${path_to_file} "https://releases.hashicorp.com/terraform/${ASDF_TERRAFORM_VERSION}/terraform_${ASDF_TERRAFORM_VERSION}_${platform_type_lowercase}_${go_arch_type}.zip"
+    unzip -q ${path_to_file} -d ${BIN}/
+    rm ${path_to_file}
+    chmod +x ${BIN}/terraform
+    export PATH="${BIN}:${PATH}"
+    terraform version
+}
+
 create_workspace() {
   if [[ ! -d "${BIN}" ]]; then
     mkdir -p "${BIN}"
@@ -168,6 +212,8 @@ with_mage() {
   for pkg in "${install_packages[@]}"; do
     go install "${pkg}@latest"
   done
+  echo "Download modules to local cache"
+  retry 3 go mod download
 }
 
 with_go() {
@@ -283,9 +329,15 @@ are_paths_changed() {
 
 are_changed_only_paths() {
   local patterns=("${@}")
-  local changelist=()
-  local changed_files=$(git diff --name-only HEAD@{1} HEAD)
-  if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then
+  local changed_files=($(git diff --name-only HEAD@{1} HEAD))
+  local matched_files=()
+  for pattern in "${patterns[@]}"; do
+    local matched=($(grep -E "${pattern}" <<< "${changed_files[@]}"))
+    if [ "${#matched[@]}" -gt 0 ]; then
+      matched_files+=("${matched[@]}")
+    fi
+  done
+  if [ "${#matched_files[@]}" -eq "${#changed_files[@]}" ] || [ "${#changed_files[@]}" -eq 0 ]; then
     return 0
   fi
   return 1
@@ -301,7 +353,7 @@ are_conditions_met_mandatory_tests() {
 are_conditions_met_arm_tests() {
   if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
     if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
-      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_ARM_LABEL}" || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then
+      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then
         return 0
       fi
     fi
@@ -311,8 +363,19 @@ are_conditions_met_arm_tests() {
 
 are_conditions_met_macos_tests() {
   if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
-    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
-      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_MACOS_LABEL}" || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
+      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
+        return 0
+      fi
+    fi
+  fi
+  return 1
+}
+
+are_conditions_met_aws_tests() {
+  if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
+      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_AWS_LABEL} || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
         return 0
       fi
     fi
@@ -336,19 +399,101 @@ config_git() {
   fi
 }
 
+defineModuleFromTheChangeSet() {
+  # This method gathers the module name, if required, in order to run the ITs only if the changeset affects a specific module.
+  # For such, it's required to look for changes under the module folder and exclude anything else such as asciidoc and png files.
+  # This method defines and exports the MODULE variable with a particular module name or '' if changeset doesn't affect a specific module
+  local project_path=$1
+  local project_path_transformed=$(echo "$project_path" | sed 's/\//\\\//g')
+  local project_path_exclussion="((?!^${project_path_transformed}\\/).)*\$"
+  local exclude=("^(${project_path_exclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)")
+
+  if [[ "$project_path" == *"x-pack/"* ]]; then
+    local pattern=("$XPACK_MODULE_PATTERN")
+  else
+    local pattern=("$OSS_MODULE_PATTERN")
+  fi
+  local changed_modules=""
+  local module_dirs=$(find "$project_path/module" -mindepth 1 -maxdepth 1 -type d)
+  for module_dir in $module_dirs; do
+    if are_paths_changed $module_dir && ! are_changed_only_paths "${exclude[@]}"; then
+      if [[ -z "$changed_modules" ]]; then
+        changed_modules=$(basename "$module_dir")
+      else
+        changed_modules+=",$(basename "$module_dir")"
+      fi
+    fi
+  done
+  if [[ -z "$changed_modules" ]]; then # TODO: remove this condition and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved
+    export MODULE="aws"
+  else
+    export MODULE="${changed_modules}"  # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved
+  # export MODULE="${changed_modules}"     # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved
+  fi
+}
+
+terraformInit() {
+  local dir=$1
+  echo "Terraform Init on $dir"
+  pushd "${dir}" > /dev/null
+  terraform init
+  popd > /dev/null
+}
+
+withAWS() {
+  # This method gathers the masked AWS credentials from pre-command hook and sets the right AWS variable names.
+  export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY
+  export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY
+  export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws"
+}
+
+startCloudTestEnv() {
+  local dir=$1
+  withAWS
+  echo "--- Run docker-compose services for emulated cloud env"
+  docker-compose -f .ci/jobs/docker-compose.yml up -d                     #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
+  with_Terraform
+  terraformInit "$dir"
+  export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g')
+  export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}"
+  export TF_VAR_CREATED_DATE=$(date +%s)
+  export TF_VAR_ENVIRONMENT="ci"
+  export TF_VAR_REPO="${REPO}"
+  pushd "${dir}" > /dev/null
+  terraform apply -auto-approve
+  popd > /dev/null
+}
+
+teardown() {
+  # Teardown resources after using them
+  echo "---Terraform Cleanup"
+  .ci/scripts/terraform-cleanup.sh "${MODULE_DIR}"              #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
+
+  echo "---Docker Compose Cleanup"
+  docker-compose -f .ci/jobs/docker-compose.yml down -v         #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
+}
+
+unset_secrets () {
+  for var in $(printenv | sed 's;=.*;;' | sort); do
+    if [[ "$var" == AWS_* || "$var" == BEATS_AWS_* ]]; then
+      unset "$var"
+    fi
+  done
+}
+
 if ! are_changed_only_paths "${docs_changeset[@]}" ; then
-  ONLY_DOCS="false"
+  export ONLY_DOCS="false"
   echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS."
 else
   echo "All changes are related to DOCS. ONLY_DOCS=$ONLY_DOCS."
 fi
 
 if are_paths_changed "${go_mod_changeset[@]}" ; then
-  GO_MOD_CHANGES="true"
+  export GO_MOD_CHANGES="true"
 fi
 
 if are_paths_changed "${packaging_changeset[@]}" ; then
-  PACKAGING_CHANGES="true"
+  export PACKAGING_CHANGES="true"
 fi
 
 check_and_set_beat_vars

.buildkite/scripts/generate_metricbeat_pipeline.sh

@@ -149,7 +149,7 @@ if are_conditions_met_packaging; then
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         env:
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          PLATFORMS: "${PACKAGING_PLATFORMS}"
 
       - label: ":linux: Packaging ARM"
         key: "packaging-arm"
@@ -159,7 +159,7 @@ if are_conditions_met_packaging; then
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         env:
-          PLATFORMS: "linux/arm64"
+          PLATFORMS: "${PACKAGING_ARM_PLATFORMS}"
           PACKAGES: "docker"
 
 YAML

.buildkite/scripts/generate_packetbeat_pipeline.sh

@@ -92,7 +92,7 @@ else
   exit 0
 fi
 
-if are_conditions_met_arm_tests && are_conditions_met_macos_tests; then
+if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then
   cat >> $pipelineName <<- YAML
 
   - group: "Extended Tests"
@@ -150,7 +150,7 @@ if are_conditions_met_packaging; then
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         env:
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          PLATFORMS: "${PACKAGING_PLATFORMS}"
 
       - label: ":linux: Packaging ARM"
         key: "packaging-arm"
@@ -160,7 +160,7 @@ if are_conditions_met_packaging; then
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         env:
-          PLATFORMS: "linux/arm64"
+          PLATFORMS: "${PACKAGING_ARM_PLATFORMS}"
           PACKAGES: "docker"
 
 YAML

.buildkite/scripts/generate_winlogbeat_pipeline.sh

@@ -96,7 +96,7 @@ if are_conditions_met_packaging; then
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         env:
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          PLATFORMS: "${PACKAGING_PLATFORMS}"
 
 
 YAML