Sean simmons progress/env testing

.expeditor/build-packages.pipeline.yml

@@ -0,0 +1,72 @@
+expeditor:
+  secrets:
+    GITHUB_TOKEN:
+      account: github/chef
+      field: token
+    PIPELINE_HAB_AUTH_TOKEN:
+      path: account/static/habitat/chef-ci
+      field: auth_token # Production Builder
+      # acceptance_auth_token = acceptance
+  accounts:
+    - aws/chef-artifactory
+  defaults:
+    buildkite:
+      timeout_in_minutes: 45
+      env:
+        GITHUB_TOKEN:
+        account: github/chef
+        field: token
+        HAB_ORIGIN: "chef"
+        PIPELINE_HAB_BLDR_URL: "https://bldr.habitat.sh"
+        # Necessary to prevent old studios from poisoning builds after core plans refreshes
+        HAB_STUDIO_SECRET_HAB_PREFER_LOCAL_CHEF_DEPS: "true"
+        HAB_STUDIO_SECRET_HAB_REFRESH_CHANNEL: "unstable"
+
+steps:
+
+  - label: "[:linux: build hab-pkg-export-tar and upload to :amazon-s3:]"
+    command:
+      - ./.expeditor/scripts/export-infra-tar.sh
+    expeditor:
+      secrets:
+        GITHUB_TOKEN:
+          account: github/chef
+          field: token
+      accounts:
+        - aws/chef-artifactory
+      executor:
+        docker:
+          privileged: true
+          environment:
+            - BUILD_PKG_TARGET=x86_64-linux
+
+  - wait
+
+  - label: ":linux: Build RPM package"
+    commands:
+      - ./.expeditor/scripts/package-rpm.sh
+    expeditor:
+      secrets:
+        GITHUB_TOKEN:
+          account: github/chef
+          field: token
+      accounts:
+        - aws/chef-artifactory
+      executor:
+        linux:
+          privileged: true
+
+  - label: ":linux: Build Debian package"
+  # FYI - you must set a chmod 555 on .expeditor/scripts/package-deb.sh which allows it to be executed via docker - commit it back to source once permissions are changed
+    commands:
+      - .expeditor/scripts/package-deb.sh
+    expeditor:
+      secrets:
+        GITHUB_TOKEN:
+          account: github/chef
+          field: token
+      accounts:
+        - aws/chef-artifactory
+      executor:
+        docker:
+          privileged: true

.expeditor/config.yml

@@ -51,6 +51,8 @@ pipelines:
       definition: .expeditor/habitat-export.pipeline.yml
   - hab-export:
       definition: .expeditor/hab-export-pipeline.yml
+  - build-packages:
+      definition: .expeditor/build-packages.pipeline.yml
 
 subscriptions:
   - workload: pull_request_merged:{{github_repo}}:{{release_branch}}:*

.expeditor/hab-export-pipeline.yml

@@ -10,37 +10,24 @@ expeditor:
     buildkite:
       timeout_in_minutes: 45
       env:
-        HAB_ORIGIN: "chef-demo"
+        HAB_ORIGIN: "chef"
         PIPELINE_HAB_BLDR_URL: "https://bldr.habitat.sh"
         # Necessary to prevent old studios from poisoning builds after core plans refreshes
         HAB_STUDIO_SECRET_HAB_PREFER_LOCAL_CHEF_DEPS: "true"
-        HAB_STUDIO_SECRET_HAB_REFRESH_CHANNEL: "stable"
+        HAB_STUDIO_SECRET_HAB_REFRESH_CHANNEL: "unstable"
 
 steps:
 
-  - label: "[:linux: build hab-pkg-export-tar and uplaod to :amazon-s3:]"
+  - label: "[:linux: build hab-pkg-export-tar and upload to :amazon-s3:]"
     command:
-      - echo "--- testing hab export tar chef-demo/hello-omnibus-harmony"
-      - hab pkg export tar chef-demo/hello-omnibus-harmony
-      - echo "--- uploading tar.gz file to buldkite, artifactory, and s3" && buildkite-agent artifact upload "*.tar.gz"
-      - aws s3 cp $tar_filename $s3_bucket_uri/$tar_filename --content-type "application/gzip" --profile "<profile to use>"
-      # - curl [email protected]:<PASSWORD> -T <PATH_TO_FILE> "https://artifactory-internal.ps.chef.co/artifactory/unstable-habitat-tarbal/<TARGET_FILE_PATH>"
+      - echo "--- testing hab export tar chef/chef-infra-client"
+      - hab pkg export tar $EXPEDITOR_PKG_IDENTS_CHEFINFRACLIENTX86_64LINUX --channel unstable
+      - mkdir -p upload && find . -name "chef-chef-infra-client-*.tar.gz" | grep -E 'chef-chef-infra-client-[0-9]+\.[0-9]+\.[0-9]+-[0-9]+\.tar\.gz' | xargs -I {} cp {} upload/
+      - cd upload/ && buildkite-agent artifact upload "*.tar.gz" && aws s3 sync . s3://chef-hab-migration-tool-bucket/rc2_hab_pkg_chef_client/rc2_tar_folder/ --exclude "*" --include "*.tar.gz" --region us-west-2 --profile chef-cd
+
     expeditor:
-      accounts:
-        - aws/chef-cd
       executor:
         docker:
           privileged: true
           environment:
             - BUILD_PKG_TARGET=x86_64-linux
-
-  # - label: "[:windows: build hab-pkg-export-tar]"
-  #   command:
-  #     - powershell .expeditor/scripts/release_habitat/build_component.ps1 pkg-export-tar
-  #   expeditor:
-  #     executor:
-  #       docker:
-  #         host_os: windows
-  #         environment:
-  #           - BUILD_PKG_TARGET=x86_64-windows
-  #           - BUILDKITE_AGENT_ACCESS_TOKEN

.expeditor/scripts/build-infra-deb.sh

@@ -0,0 +1,234 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "-------DEBUG-------"
+echo "$GITHUB_TOKEN"
+
+validate_env_vars() {
+    if [ -z "${CHEF_INFRA_MIGRATE_TAR:-}" ] || [ -z "${CHEF_INFRA_HAB_TAR:-}" ]; then
+        echo "Environment variables CHEF_INFRA_MIGRATE_TAR and CHEF_INFRA_HAB_TAR must be set to the URLs of the respective tarball files."
+        echo "Usage: Set the following environment variables before running the script:"
+        echo "  export CHEF_INFRA_MIGRATE_TAR=<url_to_chef-migrate-tarball>"
+        echo "  export CHEF_INFRA_HAB_TAR=<url_to_chef-infra-client-tarball>"
+        echo "Example:"
+        echo   export CHEF_INFRA_MIGRATE_TAR=\"https://example.com/migration-tools_Linux_x86_64.tar.gz\"
+        echo   export CHEF_INFRA_HAB_TAR=\"https://example.com/chef-chef-infra-client-19.0.54-20241121145703.tar.gz\"
+        exit 1
+    fi
+    echo "Environment variables validated successfully."
+}
+
+initialize_vars() {
+    TAR_NAME=$(basename "$CHEF_INFRA_HAB_TAR")
+    VERSION=$(echo "$TAR_NAME" | cut -d '-' -f 5 )
+    RELEASE="1"
+    ARCH=$(dpkg --print-architecture)
+    DEB_NAME="chef-infra-client-${VERSION}-${RELEASE}_${ARCH}.deb"
+
+    if [[ -z "$VERSION" ]]; then
+        echo "Error: Failed to extract version from tarball name: $TAR_NAME"
+        exit 1
+    fi
+    TEMP_DIR="$HOME/temp_chef-infra-client_${VERSION}-${RELEASE}_${ARCH}"
+    CHEF_BIN_DIR="/hab/migration/bin"
+    CHEF_BUNDLE_DIR="/hab/migration/bundle"
+    DEB_PKG_NAME=chef-infra-client
+
+    echo "Variables initialized successfully."
+}
+
+create_temp_dir() {
+    mkdir -p "$TEMP_DIR" || { echo "Error: Failed to create temporary directory"; exit 1; }
+    echo "Temporary directory created at $TEMP_DIR."
+}
+
+download_migration_tool_from_github_releases() {
+    local output_path="$1"
+
+    echo "--- Downloading migration tools to $output_path.."
+
+    if [ -z "${GITHUB_TOKEN:-}" ]; then
+     echo "GITHUB_TOKEN is not set. Cannot download migration tool from"
+     exit 1
+    fi
+
+    echo "fetching latest release of migration tool"
+    if ! curl -fSL -H "Authorization: Bearer $GITHUB_TOKEN" "https://api.github.com/repos/chef/migration-tools/releases/latest" -o migration-tool-latest-release.json; then
+      echo "Error: Failed to fetch latest release information of migration tools from"
+      exit 1
+    fi
+    latest_version=$(cat migration-tool-latest-release.json | jq -r '.tag_name')
+
+    echo "requesting migration-tools_Linux_x86_64.tar.gz from '$latest_version' release"
+    if ! cat migration-tool-latest-release.json \
+        | jq '.assets[] | select (.name == "migration-tools_Linux_x86_64.tar.gz") | .url' \
+        | xargs curl -fSL -H "Authorization: Bearer $GITHUB_TOKEN" -H "Accept: application/octet-stream" -o "$output_path"; then
+      echo "Error: Failed to download latest release '$latest_version' of migration tools from"
+      exit 1
+    fi
+
+    file "$output_path"
+
+    echo "Downloaded to $output_path"
+}
+
+download_tarball_from_buildkite_artifactory() {
+    local tarball_name="$1"
+    local output_path="$2"
+
+    echo "Downloading $tarball_name to $output_path.."
+    if ! buildkite-agent artifact download "$tarball_name" "$output_path"; then
+        echo "Error: Failed to download $tarball_name"
+        exit 1
+    fi
+}
+
+download_files() {
+    echo "Downloading migration tool..."
+    # NOTE: this should be pulling from artifactory in the future along with some versioning. 
+    download_migration_tool_from_github_releases "$TEMP_DIR/$CHEF_INFRA_MIGRATE_TAR"
+
+    echo "Downloading Chef Infra tarball..."
+    download_tarball_from_buildkite_artifactory "$CHEF_INFRA_HAB_TAR" "$TEMP_DIR"
+
+    echo "Files downloaded successfully."
+}
+
+prepare_package() {
+    PACKAGE_DIR="$TEMP_DIR/infra-client-package"
+    mkdir -p "$PACKAGE_DIR/DEBIAN" || { echo "Error: Failed to create DEBIAN directory"; exit 1; }
+    mkdir -p "$PACKAGE_DIR$CHEF_BIN_DIR" || { echo "Error: Failed to create Chef bin directory"; exit 1; }
+    mkdir -p "$PACKAGE_DIR$CHEF_BUNDLE_DIR" || { echo "Error: Failed to create Chef bundle directory"; exit 1; }
+
+    echo "Unpacking migration tool..."
+    file "$TEMP_DIR/$CHEF_INFRA_MIGRATE_TAR"
+    tar -xf "$TEMP_DIR/$CHEF_INFRA_MIGRATE_TAR" -C "$PACKAGE_DIR$CHEF_BIN_DIR/" || { echo "Error: Failed to unpack migration tool"; exit 1; }
+
+    echo "Copying Chef Infra tarball..."
+    file "$TEMP_DIR/$TAR_NAME"
+    cp "$TEMP_DIR/$TAR_NAME" "$PACKAGE_DIR$CHEF_BUNDLE_DIR/" || { echo "Error: Failed to copy Chef Infra tarball"; exit 1; }
+
+    prepare_control_file
+    prepare_preinstall_script
+    prepare_postinstall_script
+    echo "Package structure prepared successfully."
+}
+
+prepare_control_file() {
+    cat <<EOL > "$PACKAGE_DIR/DEBIAN/control"
+Package: $DEB_PKG_NAME
+Version: $VERSION
+Architecture: $ARCH
+Maintainer: The Chef Maintainers <[email protected]>
+Description: Chef Infra Client
+ Chef Infra Client is an agent for configuration management.
+Conflicts: chef-workstation
+URL: 		    https://www.chef.io
+Packager: 	    Chef Software, Inc. <[email protected]>
+Group: 		    default
+License:        Chef EULA
+
+EOL
+    if [ $? -ne 0 ]; then
+        echo "Error: Failed to create control file"; exit 1
+    fi
+    echo "Control file prepared successfully."
+}
+
+prepare_preinstall_script() {
+    cat <<EOL > "$PACKAGE_DIR/DEBIAN/preinst"
+#!/bin/bash
+
+BACKUP_DIR="/opt/chef_backup"
+echo "LICENSE_KEY=\$LICENSE_KEY" > /tmp/chef_env
+echo "LICENSE_SERVER=\$LICENSE_SERVER" >> /tmp/chef_env
+chmod 600 /tmp/chef_env
+
+if [ -z "\${LICENSE_KEY:-}" ]; then
+    LICENSE_KEY=$(env | grep -m 1 '^LICENSE_KEY=' | cut -d '=' -f 2 | xargs)
+
+    if [ -z "\$LICENSE_KEY" ]; then
+        echo -e "\nError: LICENSE_KEY environment variable is required."
+        echo "Usage: sudo LICENSE_KEY=\"<license-key>\" dpkg -i <deb-file>"
+        exit 1
+    fi
+fi
+
+EOL
+    chmod +x "$PACKAGE_DIR/DEBIAN/preinst" || { echo "Error: Failed to make preinst script executable"; exit 1; }
+    echo "Pre-install script prepared successfully."
+}
+
+prepare_postinstall_script() {
+    cat <<EOL > "$PACKAGE_DIR/DEBIAN/postinst"
+#!/bin/bash
+
+CHEF_BIN_DIR="/hab/migration/bin"
+CHEF_BUNDLE_DIR="/hab/migration/bundle"
+BACKUP_DIR="/opt/chef_backup"
+FRESH_INSTALL_FLAG=""
+LICENSE_KEY_FLAG=""
+LICENSE_SERVER_FLAG=""
+
+LICENSE_SERVER=\${CHEF_INFRA_LICENSE_SERVER:-}
+LICENSE_KEY=\${CHEF_INFRA_LICENSE_KEY:-}
+
+if [ -f /tmp/chef_env ]; then
+    source /tmp/chef_env
+    rm -f /tmp/chef_env
+fi
+
+if [ ! -d "/opt/chef/" ]; then
+    FRESH_INSTALL_FLAG="--fresh_install"
+    echo "Postinstall: Detected fresh installation."
+else
+    echo "Postinstall: Detected upgrade installation."
+fi
+
+if [ -f "\$CHEF_BIN_DIR/chef-migrate" ]; then
+    echo "Running post-install tasks..."
+
+     if [ -n "\$LICENSE_KEY" ]; then
+        LICENSE_KEY_FLAG="--license.key \$LICENSE_KEY"
+    fi
+
+    if [ -n "\$LICENSE_SERVER" ]; then
+        LICENSE_SERVER_FLAG="--license.server \$LICENSE_SERVER"
+    fi
+
+    MIGRATE_CMD="\$CHEF_BIN_DIR/chef-migrate apply airgap \$FRESH_INSTALL_FLAG \$CHEF_BUNDLE_DIR/$TAR_NAME \$LICENSE_KEY_FLAG \$LICENSE_SERVER_FLAG"
+
+    echo "Executing: \$MIGRATE_CMD"
+    eval \$MIGRATE_CMD || { echo "Error: Post-installation failed."; exit 1; }
+
+    cp /hab/chef/bin/* \$CHEF_BIN_DIR || { echo "Error: Failed to copy binaries to \$CHEF_BIN_DIR"; exit 1; }
+else
+    echo "Error: chef-migrate tool not found in \$CHEF_BIN_DIR"
+    exit 1
+fi
+EOL
+    chmod +x "$PACKAGE_DIR/DEBIAN/postinst" || { echo "Error: Failed to make postinst script executable"; exit 1; }
+    echo "Post-install script prepared successfully."
+}
+
+build_package() {
+    dpkg-deb --build "$PACKAGE_DIR" "$DEB_NAME" || { echo "Error: Failed to build .deb package"; exit 1; }
+
+    echo "Package built successfully: $DEB_NAME."
+    echo "$DEB_NAME" > "DEB_PKG_NAME"
+}
+
+cleanup() {
+    rm -rf "$TEMP_DIR" || { echo "Error: Failed to clean up temporary directory"; exit 1; }
+    echo "Temporary directory cleaned up."
+}
+
+trap cleanup EXIT
+
+validate_env_vars
+initialize_vars
+create_temp_dir
+download_files
+prepare_package
+build_package