Skip to content

Commit 6bf41ce

Browse files
jinrohjinroh
authored
cloudinit: enable retries when installing packages (#244)
Also let's use apt-get instead of apt. apt-get being more appropriate for script with a stable api. Mainly to avoid this warning: ``` WARNING : apt does not have a stable CLI interface. ```
1 parent 47a59d6 commit 6bf41ce

File tree

5 files changed

+18
-18
lines changed

5 files changed

+18
-18
lines changed

‎azure/arm/install.sh‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,12 @@ echo "nbd" > /etc/modules-load.d/nbd.conf
2020
echo "options nbd nbds_max=128" > /etc/modprobe.d/nbd.conf
2121

2222
# Install requirements
23-
apt update
24-
apt install -y curl
23+
apt-get update
24+
apt-get install -o Acquire::Retries="5" -y curl
2525

2626
# Remove uneeded packages
27-
apt remove -y libx11-6
28-
apt autoremove -y
27+
apt-get remove -y libx11-6
28+
apt-get autoremove -y
2929

3030
# Perform unattended upgrades
3131
unattended-upgrade -v
@@ -56,7 +56,7 @@ DD_INSTALL_ONLY=true \
5656

5757
# Install the agentless-scanner
5858
echo "deb [signed-by=/usr/share/keyrings/datadog-archive-keyring.gpg] https://apt.datadoghq.com/ $DD_AGENTLESS_CHANNEL agentless-scanner" >> /etc/apt/sources.list.d/datadog.list
59-
apt update
59+
apt-get update
6060
agentless_pkg_pattern="([[:digit:]]:)?$DD_AGENTLESS_VERSION(\.[[:digit:]]+){0,1}(~rc\.[[:digit:]]+)?(-[[:digit:]])?"
6161
agentless_version_custom="$(apt-cache madison datadog-agentless-scanner | grep -E "$agentless_pkg_pattern" -om1)" || true
6262
if [ -z "$agentless_version_custom" ]; then
@@ -66,7 +66,7 @@ fi
6666
# We mask/unmask because apt auto-starts the service, and we do
6767
# not want to start it before the configuration is in place.
6868
systemctl mask datadog-agentless-scanner.service
69-
apt install -y "datadog-agentless-scanner=$agentless_version_custom"
69+
apt-get install -o Acquire::Retries="5" -y "datadog-agentless-scanner=$agentless_version_custom"
7070
systemctl unmask datadog-agentless-scanner.service
7171

7272
# Adding automatic reboot on kernel updates

‎azure/arm/main.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
"_generator": {
77
"name": "bicep",
88
"version": "0.38.33.27573",
9-
"templateHash": "3834264689116102169"
9+
"templateHash": "10692215097073774100"
1010
}
1111
},
1212
"functions": [
@@ -156,7 +156,7 @@
156156
},
157157
"variables": {
158158
"$fxv#0": "function Set-AzureAgentlessOptions {\n [CmdletBinding()]\n param (\n [Parameter(Mandatory, ValueFromPipeline)]\n [Guid[]]$Subscriptions,\n [Parameter(Mandatory)]\n [string]$DatadogSite,\n [Parameter(Mandatory, HelpMessage = \"Datadog API Key\")]\n [ValidatePattern(\"^[0-9a-f]{32}$\")]\n [string]$APIKey,\n [Parameter(Mandatory, HelpMessage = \"Datadog Application Key\")]\n [ValidatePattern(\"^[0-9a-f]{40}$\")]\n [string]$ApplicationKey\n )\n begin {\n $url = \"https://api.${DatadogSite}/api/v2/agentless_scanning/accounts/azure\"\n $headers = @{\n \"Content-Type\" = \"application/vnd.api+json\"\n \"DD-API-KEY\" = $APIKey\n \"DD-APPLICATION-KEY\" = $ApplicationKey\n \"Dd-Call-Source\" = \"arm-agentless\"\n }\n }\n process {\n $subscription_id = $_.ToString()\n $body = @{\n \"data\" = @{\n \"id\" = $subscription_id\n \"type\" = \"azure_scan_options\"\n \"attributes\" = @{\n \"vuln_containers_os\" = $true\n \"vuln_host_os\" = $true\n }\n }\n } | ConvertTo-Json\n\n $result = Invoke-RestMethod -Method POST -Uri $url -Headers $headers -Body $body -SkipHttpErrorCheck -StatusCodeVariable status\n if ($status -eq 409) {\n # Subscription already exists; update it instead\n $result = Invoke-RestMethod -Method PATCH -Uri \"${url}/${subscription_id}\" -Headers $headers -Body $body -SkipHttpErrorCheck -StatusCodeVariable status\n }\n if ($status -ge 200 -and $status -lt 300) {\n Write-Output \"Successfully enabled Agentless Scanning for subscription ${subscription_id}\"\n }\n else {\n Write-Error \"Failed to enable Agentless Scanning for subscription ${subscription_id}: $(ConvertTo-Json -Compress $result)\"\n }\n }\n}\n\nfunction Convert-ScopeToSubscriptionId {\n [CmdletBinding()]\n param (\n [Parameter(Mandatory, ValueFromPipeline)]\n [string[]]$Scopes\n )\n process {\n $scope = $_.Trim()\n if ($scope -match '^/subscriptions/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(/|$)') {\n return $Matches[1]\n }\n Write-Warning \"Ignoring scope: $scope\"\n }\n}\n\n${env:SCAN_SCOPES} |\nConvertFrom-Json |\nConvert-ScopeToSubscriptionId |\nSort-Object |\nGet-Unique |\nSet-AzureAgentlessOptions -APIKey ${env:DD_API_KEY} -ApplicationKey ${env:DD_APP_KEY} -DatadogSite ${env:DD_SITE}\n",
159-
"$fxv#1": "#!/bin/bash\nset +x\nset -u\nset -e\nset -o pipefail\n\nfatal_error () {\n printf \"FATAL ERROR: shutting down\\n\"\n shutdown -h now\n}\n\ntrap 'fatal_error' ERR\n\n# Remove SSH mock public key\nsed -i '/.*${ssh_mock_public_key}.*/d' '${ssh_authorized_keys_file}'\n\n# Enable the nbd module\nmodprobe nbd nbds_max=128\necho \"nbd\" > /etc/modules-load.d/nbd.conf\necho \"options nbd nbds_max=128\" > /etc/modprobe.d/nbd.conf\n\n# Install requirements\napt update\napt install -y curl\n\n# Remove uneeded packages\napt remove -y libx11-6\napt autoremove -y\n\n# Perform unattended upgrades\nunattended-upgrade -v\n\nre='@Microsoft.KeyVault\\(SecretUri=(https://.*)\\)'\nif [[ \"${api_key}\" =~ $re ]]; then\n echo \"Datadog API key is a Key Vault reference\"\n DD_API_KEY=\"ENC[${api_key}]\"\nelse\n DD_API_KEY=\"${api_key}\"\nfi\n\n# Append the last 6 bytes of the VM UUID to prevent hostname collisions\nVM_ID=$(cat /sys/devices/virtual/dmi/id/product_uuid)\nDD_HOSTNAME=\"$(hostname)-${VM_ID:(-12)}\"\nDD_SITE=\"${site}\"\nDD_AGENTLESS_VERSION=\"${scanner_version}\"\nDD_AGENTLESS_CHANNEL=\"${scanner_channel}\"\n\nhostnamectl hostname \"$DD_HOSTNAME\"\n\n# Install the agent\nDD_INSTALL_ONLY=true \\\n DD_API_KEY=\"TBD\" \\\n DD_SITE=\"$DD_SITE\" \\\n DD_HOSTNAME=\"$DD_HOSTNAME\" \\\n bash -c \"$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_agent7.sh)\"\n\n# Install the agentless-scanner\necho \"deb [signed-by=/usr/share/keyrings/datadog-archive-keyring.gpg] https://apt.datadoghq.com/ $DD_AGENTLESS_CHANNEL agentless-scanner\" >> /etc/apt/sources.list.d/datadog.list\napt update\nagentless_pkg_pattern=\"([[:digit:]]:)?$DD_AGENTLESS_VERSION(\\.[[:digit:]]+){0,1}(~rc\\.[[:digit:]]+)?(-[[:digit:]])?\"\nagentless_version_custom=\"$(apt-cache madison datadog-agentless-scanner | grep -E \"$agentless_pkg_pattern\" -om1)\" || true\nif [ -z \"$agentless_version_custom\" ]; then\n printf \"Could not find a version of datadog-agentless-scanner from %s\" \"$DD_AGENTLESS_VERSION\"\n exit 1\nfi\n# We mask/unmask because apt auto-starts the service, and we do\n# not want to start it before the configuration is in place.\nsystemctl mask datadog-agentless-scanner.service\napt install -y \"datadog-agentless-scanner=$agentless_version_custom\"\nsystemctl unmask datadog-agentless-scanner.service\n\n# Adding automatic reboot on kernel updates\ncat << EOF >> /etc/apt/apt.conf.d/50unattended-upgrades\nUnattended-Upgrade::Automatic-Reboot \"true\";\nUnattended-Upgrade::Automatic-Reboot-WithUsers \"true\";\nUnattended-Upgrade::Automatic-Reboot-Time \"now\";\nEOF\n\n# Perform unattended upgrades 10 min after boot, then every 3 hours\ncat << EOF > /etc/systemd/system/apt-daily-upgrade.timer\n[Unit]\nDescription=Daily apt upgrade and clean activities\nAfter=apt-daily.timer\n\n[Timer]\nOnActiveSec=10min\nOnCalendar=0/3:00:00\nPersistent=true\n\n[Install]\nWantedBy=timers.target\nEOF\n\nsystemctl daemon-reload\nsystemctl restart apt-daily-upgrade.timer\n\n# Activate agentless scanner logging\nmkdir -p /etc/datadog-agent/conf.d/agentless-scanner.d\ncat <<EOF > /etc/datadog-agent/conf.d/agentless-scanner.d/conf.yaml\nlogs:\n - type: file\n path: \"/var/log/datadog/agentless-scanner.log\"\n service: \"agentless-scanner\"\n source: go\n sourcecategory: sourcecode\n start_position: beginning\nEOF\n\nchown -R dd-agent: /etc/datadog-agent/conf.d/agentless-scanner.d\n\n# Custom configuration for agent\ncat <<EOF > /etc/datadog-agent/datadog.yaml\napi_key: $DD_API_KEY\nsite: $DD_SITE\nhostname: $DD_HOSTNAME\nlogs_enabled: true\nec2_prefer_imdsv2: true\nsecret_backend_command: /usr/local/bin/dd-secret-backend\nEOF\n\ncat <<EOF > /usr/local/bin/dd-secret-backend\n#!/bin/bash\ndatadog-agentless-scanner secrets || exit 1\nEOF\nchown dd-agent: /usr/local/bin/dd-secret-backend\nchmod 700 /usr/local/bin/dd-secret-backend\n\ncat <<EOF > /etc/datadog-agent/agentless-scanner.yaml\nhostname: $DD_HOSTNAME\napi_key: $DD_API_KEY\nsite: $DD_SITE\nazure_client_id: ${azure_client_id}\ninstallation_mode: terraform\ninstallation_version: 0.11.6\nEOF\n\nchown dd-agent: /etc/datadog-agent/agentless-scanner.yaml\nchmod 600 /etc/datadog-agent/agentless-scanner.yaml\n\n# Restart the agent\nsystemctl restart datadog-agent\n\n# Stop the scanner after 24 hours. This will cause the health\n# probe to fail and trigger an automatic instance replacement.\nsystemd-run --on-boot=24h systemctl stop datadog-agentless-scanner\n\n# Enable and start datadog-agentless-scaner\nsystemctl enable --now datadog-agentless-scanner\n",
159+
"$fxv#1": "#!/bin/bash\nset +x\nset -u\nset -e\nset -o pipefail\n\nfatal_error () {\n printf \"FATAL ERROR: shutting down\\n\"\n shutdown -h now\n}\n\ntrap 'fatal_error' ERR\n\n# Remove SSH mock public key\nsed -i '/.*${ssh_mock_public_key}.*/d' '${ssh_authorized_keys_file}'\n\n# Enable the nbd module\nmodprobe nbd nbds_max=128\necho \"nbd\" > /etc/modules-load.d/nbd.conf\necho \"options nbd nbds_max=128\" > /etc/modprobe.d/nbd.conf\n\n# Install requirements\napt-get update\napt-get install -o Acquire::Retries=\"5\" -y curl\n\n# Remove uneeded packages\napt-get remove -y libx11-6\napt-get autoremove -y\n\n# Perform unattended upgrades\nunattended-upgrade -v\n\nre='@Microsoft.KeyVault\\(SecretUri=(https://.*)\\)'\nif [[ \"${api_key}\" =~ $re ]]; then\n echo \"Datadog API key is a Key Vault reference\"\n DD_API_KEY=\"ENC[${api_key}]\"\nelse\n DD_API_KEY=\"${api_key}\"\nfi\n\n# Append the last 6 bytes of the VM UUID to prevent hostname collisions\nVM_ID=$(cat /sys/devices/virtual/dmi/id/product_uuid)\nDD_HOSTNAME=\"$(hostname)-${VM_ID:(-12)}\"\nDD_SITE=\"${site}\"\nDD_AGENTLESS_VERSION=\"${scanner_version}\"\nDD_AGENTLESS_CHANNEL=\"${scanner_channel}\"\n\nhostnamectl hostname \"$DD_HOSTNAME\"\n\n# Install the agent\nDD_INSTALL_ONLY=true \\\n DD_API_KEY=\"TBD\" \\\n DD_SITE=\"$DD_SITE\" \\\n DD_HOSTNAME=\"$DD_HOSTNAME\" \\\n bash -c \"$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_agent7.sh)\"\n\n# Install the agentless-scanner\necho \"deb [signed-by=/usr/share/keyrings/datadog-archive-keyring.gpg] https://apt.datadoghq.com/ $DD_AGENTLESS_CHANNEL agentless-scanner\" >> /etc/apt/sources.list.d/datadog.list\napt-get update\nagentless_pkg_pattern=\"([[:digit:]]:)?$DD_AGENTLESS_VERSION(\\.[[:digit:]]+){0,1}(~rc\\.[[:digit:]]+)?(-[[:digit:]])?\"\nagentless_version_custom=\"$(apt-cache madison datadog-agentless-scanner | grep -E \"$agentless_pkg_pattern\" -om1)\" || true\nif [ -z \"$agentless_version_custom\" ]; then\n printf \"Could not find a version of datadog-agentless-scanner from %s\" \"$DD_AGENTLESS_VERSION\"\n exit 1\nfi\n# We mask/unmask because apt auto-starts the service, and we do\n# not want to start it before the configuration is in place.\nsystemctl mask datadog-agentless-scanner.service\napt-get install -o Acquire::Retries=\"5\" -y \"datadog-agentless-scanner=$agentless_version_custom\"\nsystemctl unmask datadog-agentless-scanner.service\n\n# Adding automatic reboot on kernel updates\ncat << EOF >> /etc/apt/apt.conf.d/50unattended-upgrades\nUnattended-Upgrade::Automatic-Reboot \"true\";\nUnattended-Upgrade::Automatic-Reboot-WithUsers \"true\";\nUnattended-Upgrade::Automatic-Reboot-Time \"now\";\nEOF\n\n# Perform unattended upgrades 10 min after boot, then every 3 hours\ncat << EOF > /etc/systemd/system/apt-daily-upgrade.timer\n[Unit]\nDescription=Daily apt upgrade and clean activities\nAfter=apt-daily.timer\n\n[Timer]\nOnActiveSec=10min\nOnCalendar=0/3:00:00\nPersistent=true\n\n[Install]\nWantedBy=timers.target\nEOF\n\nsystemctl daemon-reload\nsystemctl restart apt-daily-upgrade.timer\n\n# Activate agentless scanner logging\nmkdir -p /etc/datadog-agent/conf.d/agentless-scanner.d\ncat <<EOF > /etc/datadog-agent/conf.d/agentless-scanner.d/conf.yaml\nlogs:\n - type: file\n path: \"/var/log/datadog/agentless-scanner.log\"\n service: \"agentless-scanner\"\n source: go\n sourcecategory: sourcecode\n start_position: beginning\nEOF\n\nchown -R dd-agent: /etc/datadog-agent/conf.d/agentless-scanner.d\n\n# Custom configuration for agent\ncat <<EOF > /etc/datadog-agent/datadog.yaml\napi_key: $DD_API_KEY\nsite: $DD_SITE\nhostname: $DD_HOSTNAME\nlogs_enabled: true\nec2_prefer_imdsv2: true\nsecret_backend_command: /usr/local/bin/dd-secret-backend\nEOF\n\ncat <<EOF > /usr/local/bin/dd-secret-backend\n#!/bin/bash\ndatadog-agentless-scanner secrets || exit 1\nEOF\nchown dd-agent: /usr/local/bin/dd-secret-backend\nchmod 700 /usr/local/bin/dd-secret-backend\n\ncat <<EOF > /etc/datadog-agent/agentless-scanner.yaml\nhostname: $DD_HOSTNAME\napi_key: $DD_API_KEY\nsite: $DD_SITE\nazure_client_id: ${azure_client_id}\ninstallation_mode: terraform\ninstallation_version: 0.11.6\nEOF\n\nchown dd-agent: /etc/datadog-agent/agentless-scanner.yaml\nchmod 600 /etc/datadog-agent/agentless-scanner.yaml\n\n# Restart the agent\nsystemctl restart datadog-agent\n\n# Stop the scanner after 24 hours. This will cause the health\n# probe to fail and trigger an automatic instance replacement.\nsystemd-run --on-boot=24h systemctl stop datadog-agentless-scanner\n\n# Enable and start datadog-agentless-scaner\nsystemctl enable --now datadog-agentless-scanner\n",
160160
"sshMockPublicKey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJWFDAB+VRKsHvHjIyiEN9izvhaosXAUMG1jPMo9hcnE",
161161
"sshAuthorizedKeysFile": "[format('/home/{0}/.ssh/authorized_keys', parameters('adminUsername'))]",
162162
"tags": "[union(parameters('resourceTags'), createObject('Datadog', 'true', 'DatadogAgentlessScanner', 'true'))]",

‎azure/modules/custom-data/templates/install.sh.tftpl‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ echo "nbd" > /etc/modules-load.d/nbd.conf
1717
echo "options nbd nbds_max=128" > /etc/modprobe.d/nbd.conf
1818

1919
# Install requirements
20-
apt update
21-
apt install -y curl
20+
apt-get update
21+
apt-get install -o Acquire::Retries="5" -y curl
2222

2323
# Remove uneeded packages
2424
apt remove -y libx11-6
@@ -54,7 +54,7 @@ DD_INSTALL_ONLY=true \
5454

5555
# Install the agentless-scanner
5656
echo "deb [signed-by=/usr/share/keyrings/datadog-archive-keyring.gpg] $DD_AGENTLESS_REPOSITORY $DD_AGENTLESS_CHANNEL agentless-scanner" >> /etc/apt/sources.list.d/datadog.list
57-
apt update
57+
apt-get update
5858
agentless_pkg_pattern="([[:digit:]]:)?$DD_AGENTLESS_VERSION(\.[[:digit:]]+){0,1}(~rc\.[[:digit:]]+)?(-[[:digit:]])?"
5959
agentless_version_custom="$(apt-cache madison datadog-agentless-scanner | grep -E "$agentless_pkg_pattern" -om1)" || true
6060
if [ -z "$agentless_version_custom" ]; then
@@ -64,7 +64,7 @@ fi
6464
# We mask/unmask because apt auto-starts the service, and we do
6565
# not want to start it before the configuration is in place.
6666
systemctl mask datadog-agentless-scanner.service
67-
apt install -y "datadog-agentless-scanner=$agentless_version_custom"
67+
apt-get install -o Acquire::Retries="5" -y "datadog-agentless-scanner=$agentless_version_custom"
6868
systemctl unmask datadog-agentless-scanner.service
6969

7070
# Adding automatic reboot on kernel updates

‎gcp/modules/instance/startup-script.sh.tftpl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ DD_INSTALL_ONLY=true \
2323

2424
# Install the agentless-scanner
2525
echo "deb [signed-by=/usr/share/keyrings/datadog-archive-keyring.gpg] $DD_AGENTLESS_REPOSITORY $DD_AGENTLESS_CHANNEL agentless-scanner" >> /etc/apt/sources.list.d/datadog.list
26-
apt update
26+
apt-get update
2727
agentless_pkg_pattern="([[:digit:]]:)?$DD_AGENTLESS_VERSION(\.[[:digit:]]+){0,1}(~rc\.[[:digit:]]+)?(-[[:digit:]])?"
2828
agentless_version_custom="$(apt-cache madison datadog-agentless-scanner | grep -E "$agentless_pkg_pattern" -om1)" || true
2929
if [ -z "$agentless_version_custom" ]; then
@@ -33,7 +33,7 @@ fi
3333
# We mask/unmask because apt auto-starts the service, and we do
3434
# not want to start it before the configuration is in place.
3535
systemctl mask datadog-agentless-scanner.service
36-
apt install -y "datadog-agentless-scanner=$agentless_version_custom"
36+
apt-get install -o Acquire::Retries="5" -y "datadog-agentless-scanner=$agentless_version_custom"
3737
systemctl unmask datadog-agentless-scanner.service
3838

3939
# Adding automatic reboot on kernel updates

‎modules/user_data/templates/install.sh.tftpl‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ echo "nbd" > /etc/modules-load.d/nbd.conf
1717
echo "options nbd nbds_max=128" > /etc/modprobe.d/nbd.conf
1818

1919
# Install requirements
20-
apt update
21-
apt install -y curl
20+
apt-get update
21+
apt-get install -o Acquire::Retries="5" -y curl
2222

2323
# Remove uneeded packages
2424
apt remove -y libx11-6
@@ -51,7 +51,7 @@ DD_INSTALL_ONLY=true \
5151

5252
# Install the agentless-scanner
5353
echo "deb [signed-by=/usr/share/keyrings/datadog-archive-keyring.gpg] $DD_AGENTLESS_REPOSITORY $DD_AGENTLESS_CHANNEL agentless-scanner" >> /etc/apt/sources.list.d/datadog.list
54-
apt update
54+
apt-get update
5555
agentless_pkg_pattern="([[:digit:]]:)?$DD_AGENTLESS_VERSION(\.[[:digit:]]+){0,1}(~rc\.[[:digit:]]+)?(-[[:digit:]])?"
5656
agentless_version_custom="$(apt-cache madison datadog-agentless-scanner | grep -E "$agentless_pkg_pattern" -om1)" || true
5757
if [ -z "$agentless_version_custom" ]; then
@@ -61,7 +61,7 @@ fi
6161
# We mask/unmask because apt auto-starts the service, and we do
6262
# not want to start it before the configuration is in place.
6363
systemctl mask datadog-agentless-scanner.service
64-
apt install -y "datadog-agentless-scanner=$agentless_version_custom"
64+
apt-get install -o Acquire::Retries="5" -y "datadog-agentless-scanner=$agentless_version_custom"
6565
systemctl unmask datadog-agentless-scanner.service
6666

6767
# Adding automatic reboot on kernel updates

0 commit comments

Comments
 (0)