diff --git a/azure/arm/install.sh b/azure/arm/install.sh index fbfa5cec..09a656db 100644 --- a/azure/arm/install.sh +++ b/azure/arm/install.sh @@ -20,12 +20,12 @@ echo "nbd" > /etc/modules-load.d/nbd.conf echo "options nbd nbds_max=128" > /etc/modprobe.d/nbd.conf # Install requirements -apt update -apt install -y curl +apt-get update +apt-get install -o Acquire::Retries="5" -y curl # Remove uneeded packages -apt remove -y libx11-6 -apt autoremove -y +apt-get remove -y libx11-6 +apt-get autoremove -y # Perform unattended upgrades unattended-upgrade -v @@ -56,7 +56,7 @@ DD_INSTALL_ONLY=true \ # Install the agentless-scanner echo "deb [signed-by=/usr/share/keyrings/datadog-archive-keyring.gpg] https://apt.datadoghq.com/ $DD_AGENTLESS_CHANNEL agentless-scanner" >> /etc/apt/sources.list.d/datadog.list -apt update +apt-get update agentless_pkg_pattern="([[:digit:]]:)?$DD_AGENTLESS_VERSION(\.[[:digit:]]+){0,1}(~rc\.[[:digit:]]+)?(-[[:digit:]])?" agentless_version_custom="$(apt-cache madison datadog-agentless-scanner | grep -E "$agentless_pkg_pattern" -om1)" || true if [ -z "$agentless_version_custom" ]; then @@ -66,7 +66,7 @@ fi # We mask/unmask because apt auto-starts the service, and we do # not want to start it before the configuration is in place. systemctl mask datadog-agentless-scanner.service -apt install -y "datadog-agentless-scanner=$agentless_version_custom" +apt-get install -o Acquire::Retries="5" -y "datadog-agentless-scanner=$agentless_version_custom" systemctl unmask datadog-agentless-scanner.service # Adding automatic reboot on kernel updates diff --git a/azure/arm/main.json b/azure/arm/main.json index 19f06ebc..cee85b28 100644 --- a/azure/arm/main.json +++ b/azure/arm/main.json @@ -6,7 +6,7 @@ "_generator": { "name": "bicep", "version": "0.38.33.27573", - "templateHash": "3834264689116102169" + "templateHash": "10692215097073774100" } }, "functions": [ @@ -156,7 +156,7 @@ }, "variables": { "$fxv#0": "function Set-AzureAgentlessOptions {\n [CmdletBinding()]\n param (\n [Parameter(Mandatory, ValueFromPipeline)]\n [Guid[]]$Subscriptions,\n [Parameter(Mandatory)]\n [string]$DatadogSite,\n [Parameter(Mandatory, HelpMessage = \"Datadog API Key\")]\n [ValidatePattern(\"^[0-9a-f]{32}$\")]\n [string]$APIKey,\n [Parameter(Mandatory, HelpMessage = \"Datadog Application Key\")]\n [ValidatePattern(\"^[0-9a-f]{40}$\")]\n [string]$ApplicationKey\n )\n begin {\n $url = \"https://api.${DatadogSite}/api/v2/agentless_scanning/accounts/azure\"\n $headers = @{\n \"Content-Type\" = \"application/vnd.api+json\"\n \"DD-API-KEY\" = $APIKey\n \"DD-APPLICATION-KEY\" = $ApplicationKey\n \"Dd-Call-Source\" = \"arm-agentless\"\n }\n }\n process {\n $subscription_id = $_.ToString()\n $body = @{\n \"data\" = @{\n \"id\" = $subscription_id\n \"type\" = \"azure_scan_options\"\n \"attributes\" = @{\n \"vuln_containers_os\" = $true\n \"vuln_host_os\" = $true\n }\n }\n } | ConvertTo-Json\n\n $result = Invoke-RestMethod -Method POST -Uri $url -Headers $headers -Body $body -SkipHttpErrorCheck -StatusCodeVariable status\n if ($status -eq 409) {\n # Subscription already exists; update it instead\n $result = Invoke-RestMethod -Method PATCH -Uri \"${url}/${subscription_id}\" -Headers $headers -Body $body -SkipHttpErrorCheck -StatusCodeVariable status\n }\n if ($status -ge 200 -and $status -lt 300) {\n Write-Output \"Successfully enabled Agentless Scanning for subscription ${subscription_id}\"\n }\n else {\n Write-Error \"Failed to enable Agentless Scanning for subscription ${subscription_id}: $(ConvertTo-Json -Compress $result)\"\n }\n }\n}\n\nfunction Convert-ScopeToSubscriptionId {\n [CmdletBinding()]\n param (\n [Parameter(Mandatory, ValueFromPipeline)]\n [string[]]$Scopes\n )\n process {\n $scope = $_.Trim()\n if ($scope -match '^/subscriptions/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(/|$)') {\n return $Matches[1]\n }\n Write-Warning \"Ignoring scope: $scope\"\n }\n}\n\n${env:SCAN_SCOPES} |\nConvertFrom-Json |\nConvert-ScopeToSubscriptionId |\nSort-Object |\nGet-Unique |\nSet-AzureAgentlessOptions -APIKey ${env:DD_API_KEY} -ApplicationKey ${env:DD_APP_KEY} -DatadogSite ${env:DD_SITE}\n", - "$fxv#1": "#!/bin/bash\nset +x\nset -u\nset -e\nset -o pipefail\n\nfatal_error () {\n printf \"FATAL ERROR: shutting down\\n\"\n shutdown -h now\n}\n\ntrap 'fatal_error' ERR\n\n# Remove SSH mock public key\nsed -i '/.*${ssh_mock_public_key}.*/d' '${ssh_authorized_keys_file}'\n\n# Enable the nbd module\nmodprobe nbd nbds_max=128\necho \"nbd\" > /etc/modules-load.d/nbd.conf\necho \"options nbd nbds_max=128\" > /etc/modprobe.d/nbd.conf\n\n# Install requirements\napt update\napt install -y curl\n\n# Remove uneeded packages\napt remove -y libx11-6\napt autoremove -y\n\n# Perform unattended upgrades\nunattended-upgrade -v\n\nre='@Microsoft.KeyVault\\(SecretUri=(https://.*)\\)'\nif [[ \"${api_key}\" =~ $re ]]; then\n echo \"Datadog API key is a Key Vault reference\"\n DD_API_KEY=\"ENC[${api_key}]\"\nelse\n DD_API_KEY=\"${api_key}\"\nfi\n\n# Append the last 6 bytes of the VM UUID to prevent hostname collisions\nVM_ID=$(cat /sys/devices/virtual/dmi/id/product_uuid)\nDD_HOSTNAME=\"$(hostname)-${VM_ID:(-12)}\"\nDD_SITE=\"${site}\"\nDD_AGENTLESS_VERSION=\"${scanner_version}\"\nDD_AGENTLESS_CHANNEL=\"${scanner_channel}\"\n\nhostnamectl hostname \"$DD_HOSTNAME\"\n\n# Install the agent\nDD_INSTALL_ONLY=true \\\n DD_API_KEY=\"TBD\" \\\n DD_SITE=\"$DD_SITE\" \\\n DD_HOSTNAME=\"$DD_HOSTNAME\" \\\n bash -c \"$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_agent7.sh)\"\n\n# Install the agentless-scanner\necho \"deb [signed-by=/usr/share/keyrings/datadog-archive-keyring.gpg] https://apt.datadoghq.com/ $DD_AGENTLESS_CHANNEL agentless-scanner\" >> /etc/apt/sources.list.d/datadog.list\napt update\nagentless_pkg_pattern=\"([[:digit:]]:)?$DD_AGENTLESS_VERSION(\\.[[:digit:]]+){0,1}(~rc\\.[[:digit:]]+)?(-[[:digit:]])?\"\nagentless_version_custom=\"$(apt-cache madison datadog-agentless-scanner | grep -E \"$agentless_pkg_pattern\" -om1)\" || true\nif [ -z \"$agentless_version_custom\" ]; then\n printf \"Could not find a version of datadog-agentless-scanner from %s\" \"$DD_AGENTLESS_VERSION\"\n exit 1\nfi\n# We mask/unmask because apt auto-starts the service, and we do\n# not want to start it before the configuration is in place.\nsystemctl mask datadog-agentless-scanner.service\napt install -y \"datadog-agentless-scanner=$agentless_version_custom\"\nsystemctl unmask datadog-agentless-scanner.service\n\n# Adding automatic reboot on kernel updates\ncat << EOF >> /etc/apt/apt.conf.d/50unattended-upgrades\nUnattended-Upgrade::Automatic-Reboot \"true\";\nUnattended-Upgrade::Automatic-Reboot-WithUsers \"true\";\nUnattended-Upgrade::Automatic-Reboot-Time \"now\";\nEOF\n\n# Perform unattended upgrades 10 min after boot, then every 3 hours\ncat << EOF > /etc/systemd/system/apt-daily-upgrade.timer\n[Unit]\nDescription=Daily apt upgrade and clean activities\nAfter=apt-daily.timer\n\n[Timer]\nOnActiveSec=10min\nOnCalendar=0/3:00:00\nPersistent=true\n\n[Install]\nWantedBy=timers.target\nEOF\n\nsystemctl daemon-reload\nsystemctl restart apt-daily-upgrade.timer\n\n# Activate agentless scanner logging\nmkdir -p /etc/datadog-agent/conf.d/agentless-scanner.d\ncat < /etc/datadog-agent/conf.d/agentless-scanner.d/conf.yaml\nlogs:\n - type: file\n path: \"/var/log/datadog/agentless-scanner.log\"\n service: \"agentless-scanner\"\n source: go\n sourcecategory: sourcecode\n start_position: beginning\nEOF\n\nchown -R dd-agent: /etc/datadog-agent/conf.d/agentless-scanner.d\n\n# Custom configuration for agent\ncat < /etc/datadog-agent/datadog.yaml\napi_key: $DD_API_KEY\nsite: $DD_SITE\nhostname: $DD_HOSTNAME\nlogs_enabled: true\nec2_prefer_imdsv2: true\nsecret_backend_command: /usr/local/bin/dd-secret-backend\nEOF\n\ncat < /usr/local/bin/dd-secret-backend\n#!/bin/bash\ndatadog-agentless-scanner secrets || exit 1\nEOF\nchown dd-agent: /usr/local/bin/dd-secret-backend\nchmod 700 /usr/local/bin/dd-secret-backend\n\ncat < /etc/datadog-agent/agentless-scanner.yaml\nhostname: $DD_HOSTNAME\napi_key: $DD_API_KEY\nsite: $DD_SITE\nazure_client_id: ${azure_client_id}\ninstallation_mode: terraform\ninstallation_version: 0.11.6\nEOF\n\nchown dd-agent: /etc/datadog-agent/agentless-scanner.yaml\nchmod 600 /etc/datadog-agent/agentless-scanner.yaml\n\n# Restart the agent\nsystemctl restart datadog-agent\n\n# Stop the scanner after 24 hours. This will cause the health\n# probe to fail and trigger an automatic instance replacement.\nsystemd-run --on-boot=24h systemctl stop datadog-agentless-scanner\n\n# Enable and start datadog-agentless-scaner\nsystemctl enable --now datadog-agentless-scanner\n", + "$fxv#1": "#!/bin/bash\nset +x\nset -u\nset -e\nset -o pipefail\n\nfatal_error () {\n printf \"FATAL ERROR: shutting down\\n\"\n shutdown -h now\n}\n\ntrap 'fatal_error' ERR\n\n# Remove SSH mock public key\nsed -i '/.*${ssh_mock_public_key}.*/d' '${ssh_authorized_keys_file}'\n\n# Enable the nbd module\nmodprobe nbd nbds_max=128\necho \"nbd\" > /etc/modules-load.d/nbd.conf\necho \"options nbd nbds_max=128\" > /etc/modprobe.d/nbd.conf\n\n# Install requirements\napt-get update\napt-get install -o Acquire::Retries=\"5\" -y curl\n\n# Remove uneeded packages\napt-get remove -y libx11-6\napt-get autoremove -y\n\n# Perform unattended upgrades\nunattended-upgrade -v\n\nre='@Microsoft.KeyVault\\(SecretUri=(https://.*)\\)'\nif [[ \"${api_key}\" =~ $re ]]; then\n echo \"Datadog API key is a Key Vault reference\"\n DD_API_KEY=\"ENC[${api_key}]\"\nelse\n DD_API_KEY=\"${api_key}\"\nfi\n\n# Append the last 6 bytes of the VM UUID to prevent hostname collisions\nVM_ID=$(cat /sys/devices/virtual/dmi/id/product_uuid)\nDD_HOSTNAME=\"$(hostname)-${VM_ID:(-12)}\"\nDD_SITE=\"${site}\"\nDD_AGENTLESS_VERSION=\"${scanner_version}\"\nDD_AGENTLESS_CHANNEL=\"${scanner_channel}\"\n\nhostnamectl hostname \"$DD_HOSTNAME\"\n\n# Install the agent\nDD_INSTALL_ONLY=true \\\n DD_API_KEY=\"TBD\" \\\n DD_SITE=\"$DD_SITE\" \\\n DD_HOSTNAME=\"$DD_HOSTNAME\" \\\n bash -c \"$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_agent7.sh)\"\n\n# Install the agentless-scanner\necho \"deb [signed-by=/usr/share/keyrings/datadog-archive-keyring.gpg] https://apt.datadoghq.com/ $DD_AGENTLESS_CHANNEL agentless-scanner\" >> /etc/apt/sources.list.d/datadog.list\napt-get update\nagentless_pkg_pattern=\"([[:digit:]]:)?$DD_AGENTLESS_VERSION(\\.[[:digit:]]+){0,1}(~rc\\.[[:digit:]]+)?(-[[:digit:]])?\"\nagentless_version_custom=\"$(apt-cache madison datadog-agentless-scanner | grep -E \"$agentless_pkg_pattern\" -om1)\" || true\nif [ -z \"$agentless_version_custom\" ]; then\n printf \"Could not find a version of datadog-agentless-scanner from %s\" \"$DD_AGENTLESS_VERSION\"\n exit 1\nfi\n# We mask/unmask because apt auto-starts the service, and we do\n# not want to start it before the configuration is in place.\nsystemctl mask datadog-agentless-scanner.service\napt-get install -o Acquire::Retries=\"5\" -y \"datadog-agentless-scanner=$agentless_version_custom\"\nsystemctl unmask datadog-agentless-scanner.service\n\n# Adding automatic reboot on kernel updates\ncat << EOF >> /etc/apt/apt.conf.d/50unattended-upgrades\nUnattended-Upgrade::Automatic-Reboot \"true\";\nUnattended-Upgrade::Automatic-Reboot-WithUsers \"true\";\nUnattended-Upgrade::Automatic-Reboot-Time \"now\";\nEOF\n\n# Perform unattended upgrades 10 min after boot, then every 3 hours\ncat << EOF > /etc/systemd/system/apt-daily-upgrade.timer\n[Unit]\nDescription=Daily apt upgrade and clean activities\nAfter=apt-daily.timer\n\n[Timer]\nOnActiveSec=10min\nOnCalendar=0/3:00:00\nPersistent=true\n\n[Install]\nWantedBy=timers.target\nEOF\n\nsystemctl daemon-reload\nsystemctl restart apt-daily-upgrade.timer\n\n# Activate agentless scanner logging\nmkdir -p /etc/datadog-agent/conf.d/agentless-scanner.d\ncat < /etc/datadog-agent/conf.d/agentless-scanner.d/conf.yaml\nlogs:\n - type: file\n path: \"/var/log/datadog/agentless-scanner.log\"\n service: \"agentless-scanner\"\n source: go\n sourcecategory: sourcecode\n start_position: beginning\nEOF\n\nchown -R dd-agent: /etc/datadog-agent/conf.d/agentless-scanner.d\n\n# Custom configuration for agent\ncat < /etc/datadog-agent/datadog.yaml\napi_key: $DD_API_KEY\nsite: $DD_SITE\nhostname: $DD_HOSTNAME\nlogs_enabled: true\nec2_prefer_imdsv2: true\nsecret_backend_command: /usr/local/bin/dd-secret-backend\nEOF\n\ncat < /usr/local/bin/dd-secret-backend\n#!/bin/bash\ndatadog-agentless-scanner secrets || exit 1\nEOF\nchown dd-agent: /usr/local/bin/dd-secret-backend\nchmod 700 /usr/local/bin/dd-secret-backend\n\ncat < /etc/datadog-agent/agentless-scanner.yaml\nhostname: $DD_HOSTNAME\napi_key: $DD_API_KEY\nsite: $DD_SITE\nazure_client_id: ${azure_client_id}\ninstallation_mode: terraform\ninstallation_version: 0.11.6\nEOF\n\nchown dd-agent: /etc/datadog-agent/agentless-scanner.yaml\nchmod 600 /etc/datadog-agent/agentless-scanner.yaml\n\n# Restart the agent\nsystemctl restart datadog-agent\n\n# Stop the scanner after 24 hours. This will cause the health\n# probe to fail and trigger an automatic instance replacement.\nsystemd-run --on-boot=24h systemctl stop datadog-agentless-scanner\n\n# Enable and start datadog-agentless-scaner\nsystemctl enable --now datadog-agentless-scanner\n", "sshMockPublicKey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJWFDAB+VRKsHvHjIyiEN9izvhaosXAUMG1jPMo9hcnE", "sshAuthorizedKeysFile": "[format('/home/{0}/.ssh/authorized_keys', parameters('adminUsername'))]", "tags": "[union(parameters('resourceTags'), createObject('Datadog', 'true', 'DatadogAgentlessScanner', 'true'))]", diff --git a/azure/modules/custom-data/templates/install.sh.tftpl b/azure/modules/custom-data/templates/install.sh.tftpl index f46f40df..b665ac88 100644 --- a/azure/modules/custom-data/templates/install.sh.tftpl +++ b/azure/modules/custom-data/templates/install.sh.tftpl @@ -17,8 +17,8 @@ echo "nbd" > /etc/modules-load.d/nbd.conf echo "options nbd nbds_max=128" > /etc/modprobe.d/nbd.conf # Install requirements -apt update -apt install -y curl +apt-get update +apt-get install -o Acquire::Retries="5" -y curl # Remove uneeded packages apt remove -y libx11-6 @@ -54,7 +54,7 @@ DD_INSTALL_ONLY=true \ # Install the agentless-scanner echo "deb [signed-by=/usr/share/keyrings/datadog-archive-keyring.gpg] $DD_AGENTLESS_REPOSITORY $DD_AGENTLESS_CHANNEL agentless-scanner" >> /etc/apt/sources.list.d/datadog.list -apt update +apt-get update agentless_pkg_pattern="([[:digit:]]:)?$DD_AGENTLESS_VERSION(\.[[:digit:]]+){0,1}(~rc\.[[:digit:]]+)?(-[[:digit:]])?" agentless_version_custom="$(apt-cache madison datadog-agentless-scanner | grep -E "$agentless_pkg_pattern" -om1)" || true if [ -z "$agentless_version_custom" ]; then @@ -64,7 +64,7 @@ fi # We mask/unmask because apt auto-starts the service, and we do # not want to start it before the configuration is in place. systemctl mask datadog-agentless-scanner.service -apt install -y "datadog-agentless-scanner=$agentless_version_custom" +apt-get install -o Acquire::Retries="5" -y "datadog-agentless-scanner=$agentless_version_custom" systemctl unmask datadog-agentless-scanner.service # Adding automatic reboot on kernel updates diff --git a/gcp/modules/instance/startup-script.sh.tftpl b/gcp/modules/instance/startup-script.sh.tftpl index c70cd8c1..0c5905b2 100644 --- a/gcp/modules/instance/startup-script.sh.tftpl +++ b/gcp/modules/instance/startup-script.sh.tftpl @@ -23,7 +23,7 @@ DD_INSTALL_ONLY=true \ # Install the agentless-scanner echo "deb [signed-by=/usr/share/keyrings/datadog-archive-keyring.gpg] $DD_AGENTLESS_REPOSITORY $DD_AGENTLESS_CHANNEL agentless-scanner" >> /etc/apt/sources.list.d/datadog.list -apt update +apt-get update agentless_pkg_pattern="([[:digit:]]:)?$DD_AGENTLESS_VERSION(\.[[:digit:]]+){0,1}(~rc\.[[:digit:]]+)?(-[[:digit:]])?" agentless_version_custom="$(apt-cache madison datadog-agentless-scanner | grep -E "$agentless_pkg_pattern" -om1)" || true if [ -z "$agentless_version_custom" ]; then @@ -33,7 +33,7 @@ fi # We mask/unmask because apt auto-starts the service, and we do # not want to start it before the configuration is in place. systemctl mask datadog-agentless-scanner.service -apt install -y "datadog-agentless-scanner=$agentless_version_custom" +apt-get install -o Acquire::Retries="5" -y "datadog-agentless-scanner=$agentless_version_custom" systemctl unmask datadog-agentless-scanner.service # Adding automatic reboot on kernel updates diff --git a/modules/user_data/templates/install.sh.tftpl b/modules/user_data/templates/install.sh.tftpl index d6396b78..c9a98298 100644 --- a/modules/user_data/templates/install.sh.tftpl +++ b/modules/user_data/templates/install.sh.tftpl @@ -17,8 +17,8 @@ echo "nbd" > /etc/modules-load.d/nbd.conf echo "options nbd nbds_max=128" > /etc/modprobe.d/nbd.conf # Install requirements -apt update -apt install -y curl +apt-get update +apt-get install -o Acquire::Retries="5" -y curl # Remove uneeded packages apt remove -y libx11-6 @@ -51,7 +51,7 @@ DD_INSTALL_ONLY=true \ # Install the agentless-scanner echo "deb [signed-by=/usr/share/keyrings/datadog-archive-keyring.gpg] $DD_AGENTLESS_REPOSITORY $DD_AGENTLESS_CHANNEL agentless-scanner" >> /etc/apt/sources.list.d/datadog.list -apt update +apt-get update agentless_pkg_pattern="([[:digit:]]:)?$DD_AGENTLESS_VERSION(\.[[:digit:]]+){0,1}(~rc\.[[:digit:]]+)?(-[[:digit:]])?" agentless_version_custom="$(apt-cache madison datadog-agentless-scanner | grep -E "$agentless_pkg_pattern" -om1)" || true if [ -z "$agentless_version_custom" ]; then @@ -61,7 +61,7 @@ fi # We mask/unmask because apt auto-starts the service, and we do # not want to start it before the configuration is in place. systemctl mask datadog-agentless-scanner.service -apt install -y "datadog-agentless-scanner=$agentless_version_custom" +apt-get install -o Acquire::Retries="5" -y "datadog-agentless-scanner=$agentless_version_custom" systemctl unmask datadog-agentless-scanner.service # Adding automatic reboot on kernel updates