Skip to content

Commit ac6d909

Browse files
jvpasinattojvpasinatto
authored
CLOUD-940 - Use authenticated calls to check backup existence (#2247)
* Use aws cli to check backup existence * Use cloud clis
1 parent 89ecaa8 commit ac6d909

File tree

4 files changed

+314
-35
lines changed

4 files changed

+314
-35
lines changed

‎Jenkinsfile‎

Lines changed: 35 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -218,22 +218,24 @@ void makeReport() {
218218
}
219219

220220
void clusterRunner(String cluster) {
221-
def clusterCreated=0
221+
withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
222+
def clusterCreated=0
222223

223-
for (int i=0; i<tests.size(); i++) {
224-
if (tests[i]["result"] == "skipped" && currentBuild.nextBuild == null) {
225-
tests[i]["result"] = "failure"
226-
tests[i]["cluster"] = cluster
227-
if (clusterCreated == 0) {
228-
createCluster(cluster)
229-
clusterCreated++
224+
for (int i=0; i<tests.size(); i++) {
225+
if (tests[i]["result"] == "skipped" && currentBuild.nextBuild == null) {
226+
tests[i]["result"] = "failure"
227+
tests[i]["cluster"] = cluster
228+
if (clusterCreated == 0) {
229+
createCluster(cluster)
230+
clusterCreated++
231+
}
232+
runTest(i)
230233
}
231-
runTest(i)
232234
}
233-
}
234235

235-
if (clusterCreated >= 1) {
236-
shutdownCluster(cluster)
236+
if (clusterCreated >= 1) {
237+
shutdownCluster(cluster)
238+
}
237239
}
238240
}
239241

@@ -313,6 +315,27 @@ EOF
313315
sudo percona-release enable-only tools
314316
sudo yum install -y percona-xtrabackup-80 | true
315317
"""
318+
installAzureCLI()
319+
azureAuth()
320+
}
321+
322+
void azureAuth() {
323+
withCredentials([azureServicePrincipal('PERCONA-OPERATORS-SP')]) {
324+
sh '''
325+
az login --service-principal -u "$AZURE_CLIENT_ID" -p "$AZURE_CLIENT_SECRET" -t "$AZURE_TENANT_ID" --allow-no-subscriptions
326+
az account set -s "$AZURE_SUBSCRIPTION_ID"
327+
'''
328+
}
329+
}
330+
331+
void installAzureCLI() {
332+
sh """
333+
if ! command -v az &>/dev/null; then
334+
curl -s -L https://azurecliprod.blob.core.windows.net/install.py -o install.py
335+
printf "/usr/azure-cli\\n/usr/bin" | sudo python3 install.py
336+
sudo /usr/azure-cli/bin/python -m pip install "urllib3<2.0.0" > /dev/null
337+
fi
338+
"""
316339
}
317340

318341
boolean isManualBuild() {

‎e2e-tests/demand-backup-cloud/run‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,25 @@ main() {
154154
echo "Skipping test because SKIP_REMOTE_BACKUPS variable is set!"
155155
exit 0
156156
else
157+
if command -v aws >/dev/null 2>&1; then
158+
echo "AWS CLI is installed"
159+
else
160+
echo "AWS CLI is not installed"
161+
exit 1
162+
fi
163+
if command -v gsutil >/dev/null 2>&1; then
164+
echo "gutil is installed"
165+
else
166+
echo "gsutil command is not installed"
167+
exit 1
168+
fi
169+
if command -v az >/dev/null 2>&1; then
170+
echo "Azure CLI is installed"
171+
else
172+
echo "Azure CLI is not installed"
173+
exit 1
174+
fi
175+
157176
create_infra $namespace
158177

159178
cluster="demand-backup-cloud"
@@ -211,24 +230,29 @@ main() {
211230
backup_dest_azure=$(kubectl_bin get pxc-backup "$backup_name_azure" -o jsonpath='{.status.destination}' | sed -e 's/.json$//' | cut -c 9-)
212231

213232
desc "Check backup existence"
214-
check_backup_existence "https://s3.amazonaws.com/${backup_dest_aws}.sst_info/sst_info.00000000000000000000" "aws-s3"
215-
check_backup_existence "https://storage.googleapis.com/${backup_dest_gcp}.sst_info/sst_info.00000000000000000000" "gcp-cs"
216-
check_backup_existence "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}.sst_info/sst_info.00000000000000000000" "azure-blob"
233+
setup_aws_credentials
234+
check_backup_existence_aws "$backup_dest_aws" ".sst_info/sst_info.00000000000000000000"
235+
236+
setup_gcs_credentials
237+
check_backup_existence_gcs "${backup_dest_gcp}"
238+
239+
setup_azure_credentials
240+
check_backup_existence_azure "${backup_dest_azure}" ".sst_info/sst_info.00000000000000000000"
217241

218242
kubectl_bin delete pxc-backup --all
219243

220244
desc "Check backup deletion"
221-
check_backup_deletion "https://s3.amazonaws.com/${backup_dest_aws}.sst_info/sst_info.00000000000000000000" "aws-s3"
222-
check_backup_deletion "https://storage.googleapis.com/${backup_dest_gcp}.sst_info/sst_info.00000000000000000000" "gcp-cs"
223-
check_backup_deletion "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}.sst_info/sst_info.00000000000000000000" "azure-blob"
245+
check_backup_deletion_aws "$backup_dest_aws" ".sst_info/sst_info.00000000000000000000"
246+
check_backup_deletion_gcs "${backup_dest_gcp}"
247+
check_backup_deletion_azure "${backup_dest_azure}" ".sst_info/sst_info.00000000000000000000"
224248

225249
if [ "$EKS" = 1 ]; then
226250
backup_name_aws_iam="on-demand-backup-aws-s3-iam"
227251
desc "Run backup ${backup_name_aws_iam} for $cluster cluster using IAM"
228252
run_backup_with_delete "${backup_name_aws_iam}"
229253
desc "Check backup existence for $backup_name_aws_iam"
230254
backup_dest_aws_iam=$(kubectl_bin get pxc-backup "$backup_name_aws_iam" -o jsonpath='{.status.destination}' | sed -e 's/.json$//' | cut -c 6-)
231-
check_backup_existence "https://s3.amazonaws.com/${backup_dest_aws_iam}.sst_info/sst_info.00000000000000000000" "aws-s3-iam"
255+
check_backup_existence_aws "${backup_dest_aws_iam}" ".sst_info/sst_info.00000000000000000000"
232256
fi
233257

234258
destroy $namespace

‎e2e-tests/functions‎

Lines changed: 218 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1604,11 +1604,127 @@ function kpatch_set_field() {
16041604
kubectl_bin patch $type $name --type=json -p "[{\"op\": \"replace\", \"path\": \"$path\", \"value\": $value}]" >/dev/null
16051605
}
16061606

1607-
function check_backup_existence() {
1608-
path=$1
1609-
storage_name=$2
1607+
function setup_aws_credentials() {
1608+
local secret_name="aws-s3-secret"
1609+
1610+
if [[ -n "$AWS_ACCESS_KEY_ID" ]] && [[ -n "$AWS_SECRET_ACCESS_KEY" ]]; then
1611+
echo "AWS credentials already set in environment"
1612+
return 0
1613+
fi
1614+
1615+
echo "Setting up AWS credentials from secret: $secret_name"
1616+
1617+
# Disable tracing for the entire credential section
1618+
local trace_was_on=0
1619+
if [[ $- == *x* ]]; then
1620+
trace_was_on=1
1621+
set +x
1622+
fi
1623+
1624+
AWS_ACCESS_KEY_ID=$(kubectl get secret "$secret_name" -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' 2>/dev/null | base64 -d 2>/dev/null)
1625+
AWS_SECRET_ACCESS_KEY=$(kubectl get secret "$secret_name" -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' 2>/dev/null | base64 -d 2>/dev/null)
1626+
1627+
if [[ -z "$AWS_ACCESS_KEY_ID" ]] || [[ -z "$AWS_SECRET_ACCESS_KEY" ]]; then
1628+
# Re-enable tracing before error message if it was on
1629+
[[ $trace_was_on -eq 1 ]] && set -x
1630+
echo "Failed to extract AWS credentials from secret"
1631+
return 1
1632+
fi
1633+
1634+
export AWS_ACCESS_KEY_ID
1635+
export AWS_SECRET_ACCESS_KEY
1636+
1637+
# Re-enable tracing if it was on
1638+
[[ $trace_was_on -eq 1 ]] && set -x
1639+
1640+
echo "AWS credentials configured successfully"
1641+
}
1642+
1643+
function setup_gcs_credentials() {
1644+
local secret_name="gcp-cs-secret"
1645+
1646+
if gsutil ls >/dev/null 2>&1; then
1647+
echo "GCS credentials already set in environment"
1648+
return 0
1649+
fi
1650+
1651+
echo "Setting up GCS credentials from K8s secret: $secret_name"
1652+
1653+
# Disable tracing for the entire credential section
1654+
local trace_was_on=0
1655+
if [[ $- == *x* ]]; then
1656+
trace_was_on=1
1657+
set +x
1658+
fi
1659+
1660+
ACCESS_KEY_ID=$(kubectl get secret "$secret_name" -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' 2>/dev/null | base64 -d 2>/dev/null)
1661+
SECRET_ACCESS_KEY=$(kubectl get secret "$secret_name" -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' 2>/dev/null | base64 -d 2>/dev/null)
1662+
1663+
if [[ -z "$ACCESS_KEY_ID" ]] || [[ -z "$SECRET_ACCESS_KEY" ]]; then
1664+
# Re-enable tracing before error message if it was on
1665+
[[ $trace_was_on -eq 1 ]] && set -x
1666+
echo "Failed to extract GCS credentials from secret" >&2
1667+
return 1
1668+
fi
1669+
1670+
boto_tmp=$(mktemp /tmp/boto.XXXXXX)
1671+
chmod 600 "$boto_tmp"
1672+
1673+
cat <<EOF >"$boto_tmp"
1674+
[Credentials]
1675+
gs_access_key_id = ${ACCESS_KEY_ID}
1676+
gs_secret_access_key = ${SECRET_ACCESS_KEY}
1677+
EOF
1678+
1679+
export BOTO_CONFIG="$boto_tmp"
1680+
1681+
unset ACCESS_KEY_ID
1682+
unset SECRET_ACCESS_KEY
1683+
1684+
[[ $trace_was_on -eq 1 ]] && set -x
1685+
1686+
echo "GCS credentials configured successfully"
1687+
}
1688+
1689+
function setup_azure_credentials() {
1690+
local secret_name="azure-secret"
1691+
1692+
echo "Setting up Azure credentials from K8s secret: $secret_name"
1693+
1694+
# Disable tracing for the entire credential section
1695+
local trace_was_on=0
1696+
if [[ $- == *x* ]]; then
1697+
trace_was_on=1
1698+
set +x
1699+
fi
1700+
1701+
AZURE_STORAGE_ACCOUNT=$(kubectl_bin get secret "$secret_name" -o jsonpath='{.data.AZURE_STORAGE_ACCOUNT_NAME}' 2>/dev/null | base64 -d 2>/dev/null)
1702+
AZURE_STORAGE_KEY=$(kubectl_bin get secret "$secret_name" -o jsonpath='{.data.AZURE_STORAGE_ACCOUNT_KEY}' 2>/dev/null | base64 -d 2>/dev/null)
1703+
1704+
if [[ -z "$AZURE_STORAGE_ACCOUNT" ]] || [[ -z "$AZURE_STORAGE_KEY" ]]; then
1705+
# Re-enable tracing before error message if it was on
1706+
[[ $trace_was_on -eq 1 ]] && set -x
1707+
echo "Failed to extract Azure credentials from secret" >&2
1708+
return 1
1709+
fi
1710+
1711+
export AZURE_STORAGE_ACCOUNT
1712+
export AZURE_STORAGE_KEY
1713+
1714+
# Re-enable tracing if it was on
1715+
[[ $trace_was_on -eq 1 ]] && set -x
1716+
1717+
echo "Azure credentials configured successfully"
1718+
}
1719+
1720+
function check_backup_existence_aws() {
1721+
bucket=$(echo "$1" | cut -d'/' -f1)
1722+
key_prefix=$(echo "$1" | cut -d'/' -f2-)
1723+
key=$2
1724+
storage_name="aws-s3"
16101725
retry=0
1611-
until [[ $(curl -sw '%{http_code}' -o /dev/null "$path") -eq 200 ]]; do
1726+
1727+
until aws s3api head-object --bucket "$bucket" --key "${key_prefix}${key}" &>/dev/null; do
16121728
if [ $retry -ge 30 ]; then
16131729
echo "max retry count $retry reached. something went wrong with operator or kubernetes cluster"
16141730
echo "Backup was not found in bucket -- $storage_name"
@@ -1618,22 +1734,114 @@ function check_backup_existence() {
16181734
sleep 10
16191735
((retry += 1))
16201736
done
1737+
1738+
echo "Backup ${key_prefix}${key} found in bucket $bucket in $storage_name"
16211739
}
16221740

1623-
function check_backup_deletion() {
1624-
path=$1
1625-
storage_name=$2
1741+
function check_backup_existence_gcs() {
1742+
backup_dest_gcp=$1
1743+
storage_name="gcp-cs"
16261744
retry=0
1627-
until [[ $(curl -sw '%{http_code}' -o /dev/null "$path") -eq 403 ]] || [[ $(curl -sw '%{http_code}' -o /dev/null "$path") -eq 404 ]] || [[ $(curl -sw '%{http_code}' -o /dev/null "$path") -eq 400 ]]; do
1745+
1746+
gcs_path="gs://${backup_dest_gcp}.sst_info/sst_info.00000000000000000000"
1747+
1748+
until gsutil ls "$gcs_path" >/dev/null 2>&1; do
16281749
if [ $retry -ge 30 ]; then
1750+
echo "Max retry count $retry reached. Something went wrong with operator or Kubernetes cluster."
1751+
echo "Backup was not found in bucket -- $storage_name"
1752+
exit 1
1753+
fi
1754+
echo "Waiting for backup in $storage_name ($gcs_path)..."
1755+
sleep 10
1756+
((retry += 1))
1757+
done
1758+
1759+
echo "Backup found in $storage_name: $gcs_path"
1760+
}
1761+
1762+
function check_backup_existence_azure() {
1763+
container=$(echo "$1" | cut -d'/' -f1)
1764+
blob_prefix=$(echo "$1" | cut -d'/' -f2-)
1765+
blob=$2
1766+
storage_name="azure-blob"
1767+
retry=0
1768+
blob_path="${blob_prefix}${blob}"
1769+
1770+
until az storage blob show --container-name "$container" --name "$blob_path" &>/dev/null; do
1771+
if [ $retry -ge 30 ]; then
1772+
echo "max retry count $retry reached. something went wrong with operator or kubernetes cluster"
1773+
echo "Backup was not found in container -- $storage_name"
1774+
exit 1
1775+
fi
1776+
echo "waiting for backup in $storage_name"
1777+
sleep 10
1778+
((retry += 1))
1779+
done
1780+
1781+
echo "Backup ${blob_path} found in container $container in $storage_name"
1782+
}
1783+
1784+
function check_backup_deletion_aws() {
1785+
bucket=$(echo "$1" | cut -d'/' -f1)
1786+
key_prefix=$(echo "$1" | cut -d'/' -f2-)
1787+
key=$2
1788+
storage_name="aws-s3"
1789+
retry=0
1790+
1791+
while aws s3api head-object --bucket "$bucket" --key "${key_prefix}${key}" &>/dev/null; do
1792+
if [ $retry -ge 15 ]; then
1793+
echo "max retry count $retry reached. something went wrong with operator or kubernetes cluster"
1794+
echo "Backup still exists in $storage_name (expected it to be deleted)"
1795+
exit 1
1796+
fi
1797+
echo "waiting for backup to be deleted from $storage_name"
1798+
sleep 10
1799+
((retry += 1))
1800+
done
1801+
1802+
echo "Backup ${key_prefix}${key} in bucket $bucket not found in $storage_name"
1803+
}
1804+
1805+
function check_backup_deletion_gcs() {
1806+
backup_dest_gcp=$1
1807+
storage_name="gcp-cs"
1808+
retry=0
1809+
gcs_path="gs://${backup_dest_gcp}.sst_info/sst_info.00000000000000000000"
1810+
1811+
while gsutil ls "$gcs_path" >/dev/null 2>&1; do
1812+
if [ $retry -ge 15 ]; then
1813+
echo "max retry count $retry reached. something went wrong with operator or kubernetes cluster"
1814+
echo "Backup $gcs_path still exists in $storage_name (expected it to be deleted)"
1815+
exit 1
1816+
fi
1817+
echo "waiting for backup to be deleted from $storage_name"
1818+
sleep 10
1819+
((retry += 1))
1820+
done
1821+
1822+
echo "Backup $gcs_path not found in $storage_name"
1823+
}
1824+
1825+
function check_backup_deletion_azure() {
1826+
container=$(echo "$1" | cut -d'/' -f1)
1827+
blob_prefix=$(echo "$1" | cut -d'/' -f2-)
1828+
blob=$2
1829+
storage_name="azure-blob"
1830+
retry=0
1831+
blob_path="${blob_prefix}${blob}"
1832+
1833+
while az storage blob show --container-name "$container" --name "$blob_path" &>/dev/null; do
1834+
if [ $retry -ge 15 ]; then
16291835
echo "max retry count $retry reached. something went wrong with operator or kubernetes cluster"
1630-
echo "Backup was not removed from bucket -- $storage_name"
1836+
echo "Backup still exists in $storage_name (expected it to be deleted)"
16311837
exit 1
16321838
fi
1633-
echo "waiting for backup deletion $storage_name"
1839+
echo "waiting for backup to be deleted from $storage_name"
16341840
sleep 10
16351841
((retry += 1))
16361842
done
1843+
1844+
echo "Backup ${blob_path} in container $container not found in $storage_name"
16371845
}
16381846

16391847
check_passwords_leak() {

0 commit comments

Comments
 (0)