From fefa96e89b8a3317fb55049d53a229077a1e096f Mon Sep 17 00:00:00 2001 From: Kyle Laker Date: Sun, 17 Jul 2022 13:00:27 -0400 Subject: [PATCH] Convert Packer definition to HCL2 This rewrites the packer JSON files to use the HCL2 language support added in 1.5 and stabilized in 1.7. The initial conversion was performed using the `hcl2_upgrade` command but quite a lot of custom work has been done on top of that. A base shared definition for the VM is created that defines all the shared options (CPU, memory, SSH stuff, etc) and then that is specialized in the `build` block for each `source`, this is where the differences between Ubuntu and Mint are specified. Additionally, we now store version information as complex objects that meet our needs a bit more closely. This is required because some of the interpolation that we did in JSON is no longer available. As I was reviewing the VirtualBox builder config I also moved a few things from manual `VBoxManage` commands to actual packer definitions but some of those could have been done in the JSON format as well. --- .github/workflows/lint.yml | 21 ++-- .gitignore | 3 +- README.md | 26 ++++- packer/beta-vars.json | 10 -- packer/main.pkr.hcl | 123 ++++++++++++++++++++++ packer/mint-beta.pkrvars.hcl | 7 ++ packer/mint-build.json | 119 --------------------- packer/ubuntu-build.json | 198 ----------------------------------- packer/variables.pkr.hcl | 91 ++++++++++++++++ 9 files changed, 251 insertions(+), 347 deletions(-) delete mode 100644 packer/beta-vars.json create mode 100644 packer/main.pkr.hcl create mode 100644 packer/mint-beta.pkrvars.hcl delete mode 100644 packer/mint-build.json delete mode 100644 packer/ubuntu-build.json create mode 100644 packer/variables.pkr.hcl diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2d998278..31585cdf 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,16 +30,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - name: Set up Python 3.x - uses: actions/setup-python@v4 - with: - python-version: 3.x - - name: Install dependencies - run: | - pip3 install demjson3 - - name: Run jsonlint - run: | - jsonlint packer/*json - name: Install Hashicorp repo run: > wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | @@ -53,10 +43,15 @@ jobs: sudo apt-get install -y -qq packer - name: Packer validate run: | - packer validate packer/ubuntu-build.json - packer validate packer/mint-build.json # Allow beta/test builds to fail validation - packer validate -var-file=packer/beta-vars.json packer/mint-build.json || true + packer validate -var-file=mint-beta.pkrvars.hcl . || true + # This runs last to ensure it's exit status is captured + packer validate . + working-directory: packer + - name: Check Packer HCL formatting + run: | + packer fmt -diff -check . + working-directory: packer Python: name: Run Python lint tests runs-on: ubuntu-20.04 diff --git a/.gitignore b/.gitignore index a05e2a0e..00720ed7 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,4 @@ # Packer-related binaries packer/packer_cache/* -packer/artifacts_mint/* -packer/artifacts_ubuntu/* +packer/artifacts*/* diff --git a/README.md b/README.md index e4a8086e..420c0851 100644 --- a/README.md +++ b/README.md @@ -143,10 +143,10 @@ Linux and Windows hosts, but feedback on other platforms is always welcome. Due to difficulties with Packer packaging, this VM is frequently built with the latest version of Packer available directly from Hashicorp. Check the -`mint-build.json` file for the current minimum version required. +`main.pkr.hcl` file for the current minimum version required. Once the prerequisites are installed, change into the `cs-vm-build/packer` -directory and execute `packer build mint-build.json`. This will take a +directory and execute `packer build -only "*.mint" .`. This will take a considerable amount of time, depending on host resources available, but should output progress indicators along the way. @@ -164,15 +164,31 @@ specify `dsound` for Windows, `coreaudio` for Mac. ### Building beta images A large number of variables can be overridden at once by passing a `var-file` -to Packer. An example of this is provided as `beta-vars.json`, and can be used +to Packer. An example of this is provided as `mint-beta.pkrvars.hcl`, and can be used like this: -`packer build -var-file=beta-vars.json mint-build.json` +`packer build -var-file=mint-beta.pkrvars.hcl -only "*.mint" .` Packer allows further overrides, with precedence given to the last option in the command. For example, to build a beta image on Windows, use this command: -`packer build -var-file=beta-vars.json -var 'audio=dsound' mint-build.json` +`packer build -var-file=mint-beta.pkrvars.hcl -var 'audio=dsound' -only "*.mint" .` + +### Building Ubuntu images + +Native support is available for creating an Ubuntu variant of the image (and in +fact, all previous commands have specifically excluded Ubuntu image builds). By +default, the `packer` configuration will build both a Mint and Ubuntu VM. Try it +with: + +`packer build .` + +Much as with the previous commands, you can build only an ubuntu-based image by +running: + +`packer build -only "*.ubuntu" .` + +Support for building beta variants of Ubuntu images is not currently supported. ## Contributing diff --git a/packer/beta-vars.json b/packer/beta-vars.json deleted file mode 100644 index 6074359b..00000000 --- a/packer/beta-vars.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "semester": "Fa22", - "version": "21", - - "git_repo": "https://github.com/jmunixusers/cs-vm-build", - "git_branch": "main", - - "mirror_url": "http://mirror.cs.jmu.edu/pub/linuxmint/images/testing", - "iso_file": "linuxmint-{{user `version`}}-cinnamon-64bit-beta.iso" -} diff --git a/packer/main.pkr.hcl b/packer/main.pkr.hcl new file mode 100644 index 00000000..61fcba2c --- /dev/null +++ b/packer/main.pkr.hcl @@ -0,0 +1,123 @@ +packer { + required_version = ">= 1.7.0" +} + +source "virtualbox-iso" "base-build" { + cpus = 2 + memory = 4096 + disk_size = 20480 + guest_os_type = "Ubuntu_64" + gfx_vram_size = 64 + + format = "ova" + gfx_controller = "vmsvga" + gfx_accelerate_3d = true + hard_drive_discard = true + hard_drive_interface = "sata" + hard_drive_nonrotational = true + headless = "${var.headless}" + http_directory = "http" + iso_interface = "sata" + rtc_time_base = "UTC" + sata_port_count = 2 + sound = "${var.audio}" + ssh_username = "${var.ssh_user}" + ssh_password = "${var.ssh_pass}" + ssh_timeout = "100m" + + boot_wait = "5s" + boot_command = [ + "", + "c", + "linux /casper/vmlinuz fsck.mode=skip noprompt", + " auto url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/oem-preseed.cfg", + " automatic-ubiquity debug-ubiquity keymap=us", + # Different distributions name (and compress) the initrd differently. Fortunately, + # GRUB is mostly smart and if the file doesn't exist, it just won't apply that directive. + # So to prevent duplication, we specify both and let GRUB ignore the wrong one. + "initrd /casper/initrd", + "initrd /casper/initrd.lz", + "boot" + ] + shutdown_command = "echo -e \"${var.ssh_pass}\\n\" | sudo -S poweroff" + + vboxmanage = [ + ["modifyvm", "{{ .Name }}", "--audioin", "off"], + ["modifyvm", "{{ .Name }}", "--clipboard-mode", "bidirectional"], + ["modifyvm", "{{ .Name }}", "--mouse", "usbtablet"], + ["modifyvm", "{{ .Name }}", "--pae", "on"], + ["modifyvm", "{{ .Name }}", "--usb", "on", "--usbehci", "off", "--usbxhci", "off"], + ["modifyvm", "{{ .Name }}", "--vrde", "off"], + ["storagectl", "{{ .Name }}", "--name", "IDE Controller", "--remove"], + ["storagectl", "{{ .Name }}", "--name", "SATA Controller", "--hostiocache", "on"] + ] + vboxmanage_post = [ + ["storageattach", "{{ .Name }}", "--storagectl", "SATA Controller", "--port", "1", "--type", "dvddrive", "--medium", "emptydrive"] + ] +} + +build { + source "source.virtualbox-iso.base-build" { + name = "mint" + vm_name = "${var.vm_name} Linux Mint ${var.semester}" + iso_url = "${local.mint_info.mirror_url}/${local.mint_info.iso_file}" + iso_checksum = "file:${local.mint_info.mirror_url}/sha256sum.txt" + output_filename = "image-${lower(var.semester)}-mint" + output_directory = "${local.artifact_dir_prefix}mint" + export_opts = [ + "--manifest", + "--vsys", "0", + "--description", "Build date: ${local.build_id}\nPacker version: ${packer.version}", + "--product", "${var.vm_name} Linux Mint ${var.semester}", + "--producturl", "https://linuxmint.com/", + "--vendor", "JMU Unix Users Group", + "--vendorurl", "${var.git_repo}", + "--version", "${var.mint_version.version}" + ] + } + + source "source.virtualbox-iso.base-build" { + name = "ubuntu" + vm_name = "${var.vm_name} Ubuntu ${var.semester}" + iso_url = "${local.ubuntu_info.mirror_url}/${local.ubuntu_info.iso_file}" + iso_checksum = "file:${local.ubuntu_info.mirror_url}/SHA256SUMS" + output_filename = "image-${lower(var.semester)}-ubuntu" + output_directory = "${local.artifact_dir_prefix}ubuntu" + export_opts = [ + "--manifest", + "--vsys", "0", + "--description", "Build date: ${local.build_id}\nPacker version: ${packer.version}", + "--product", "${var.vm_name} Ubuntu ${var.semester}", + "--producturl", "https://ubuntu.com/", + "--vendor", "JMU Unix Users Group", + "--vendorurl", "${var.git_repo}", + "--version", "${var.ubuntu_version.version}" + ] + } + + provisioner "shell" { + execute_command = "echo 'oem' | sudo -S sh -c '{{ .Vars }} {{ .Path }}'" + environment_vars = [ + "DEBIAN_FRONTEND=noninteractive" + ] + inline = [ + "echo STOPPING APT", + "systemctl stop unattended-upgrades.service || true", + "while(pgrep -a apt-get); do sleep 1; done", + "echo UPDATE", + "apt-get update", + "echo INSTALL", + "apt-get install -V -y git ansible aptitude", + "git clone -b ${var.git_branch} ${var.git_repo}", + "cd /home/oem/cs-vm-build/scripts", + "./oem-build", + "/usr/sbin/oem-config-prepare" + ] + } + + post-processor "checksum" { + checksum_types = ["sha256"] + keep_input_artifact = true + output = "${local.artifact_dir_prefix}${source.name}/image-${lower(var.semester)}-${source.name}.{{ .ChecksumType }}sum" + } +} diff --git a/packer/mint-beta.pkrvars.hcl b/packer/mint-beta.pkrvars.hcl new file mode 100644 index 00000000..12626e2d --- /dev/null +++ b/packer/mint-beta.pkrvars.hcl @@ -0,0 +1,7 @@ +semester = "Fa22" + +mint_version = { + version = "21" + name = "vanessa" + beta = true +} diff --git a/packer/mint-build.json b/packer/mint-build.json deleted file mode 100644 index de2e45b9..00000000 --- a/packer/mint-build.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "min_packer_version": "1.6.0", - "variables": { - "vm_name": "JMU Linux Mint", - "semester": "Sp22", - "version": "20.3", - - "build_id": "{{isotime \"2006-01-02\"}}", - - "git_repo": "https://github.com/jmunixusers/cs-vm-build", - "git_branch": "main", - - "headless": "true", - "audio": "pulse", - - "output_dir": "{{pwd}}/artifacts_mint", - "mirror_url": "https://mirror.cs.jmu.edu/pub/linuxmint/images/stable/{{user `version`}}", - "iso_file": "linuxmint-{{user `version`}}-cinnamon-64bit.iso", - - "ssh_user": "oem", - "ssh_pass": "oem" - }, - "builders": [ - { - "type": "virtualbox-iso", - "headless": "{{user `headless`}}", - "http_directory": "http", - "vm_name": "{{user `vm_name`}} {{user `semester`}}", - - "cpus": 2, - "memory": 4096, - "sound": "{{user `audio`}}", - - "vboxmanage": [ - ["modifyvm", "{{.Name}}", "--audioin", "off" ], - ["modifyvm", "{{.Name}}", "--clipboard-mode", "bidirectional" ], - ["modifyvm", "{{.Name}}", "--graphicscontroller", "vmsvga" ], - ["modifyvm", "{{.Name}}", "--mouse", "usbtablet"], - ["modifyvm", "{{.Name}}", "--pae", "on"], - ["modifyvm", "{{.Name}}", "--rtcuseutc", "on"], - ["modifyvm", "{{.Name}}", "--usb", "on", "--usbehci", "off", "--usbxhci", "off" ], - ["modifyvm", "{{.Name}}", "--vram", "64" ], - ["modifyvm", "{{.Name}}", "--vrde", "off" ], - ["storagectl", "{{.Name}}", "--name", "IDE Controller", "--remove" ], - ["storagectl", "{{.Name}}", "--name", "SATA Controller", "--hostiocache", "on" ] - ], - - "vboxmanage_post": [ - ["modifyvm", "{{.Name}}", "--accelerate3d", "on" ], - ["storageattach", "{{.Name}}", "--storagectl", "SATA Controller", - "--port", "1", "--type", "dvddrive", "--medium", "emptydrive"] - ], - - "iso_url": "{{user `mirror_url`}}/{{user `iso_file`}}", - "iso_checksum": "file:{{user `mirror_url`}}/sha256sum.txt", - "iso_interface": "sata", - - "hard_drive_discard": "true", - "hard_drive_interface": "sata", - "hard_drive_nonrotational": true, - "sata_port_count": 2, - "disk_size": "20480", - - "guest_os_type": "Ubuntu_64", - "boot_wait": "5s", - "boot_command": [ - "", - "c", - "linux /casper/vmlinuz fsck.mode=skip noprompt", - " auto url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/oem-preseed.cfg", - " automatic-ubiquity debug-ubiquity keymap=us", - "initrd /casper/initrd.lz", - "boot" - ], - - "ssh_username": "{{user `ssh_user`}}", - "ssh_password": "{{user `ssh_pass`}}", - "ssh_timeout": "100m", - "shutdown_command": "echo -e \"{{user `ssh_pass`}}\\n\" | sudo -S poweroff", - - "output_directory": "{{user `output_dir`}}", - "output_filename": "image-{{user `semester` | lower }}", - "format": "ova", - - "export_opts": - [ - "--manifest", - "--vsys", "0", - "--description", "Build date: {{user `build_id`}}\nPacker version: {{packer_version}}", - "--product", "{{user `vm_name`}} {{user `semester`}}", - "--producturl", "https://linuxmint.com/", - "--vendor", "JMU Unix Users Group", - "--vendorurl", "{{user `git_repo`}}", - "--version", "{{user `version`}}" - ] - } - ], - "provisioners": [{ - "type": "shell", - "execute_command": "echo 'oem' | sudo -S sh -c '{{ .Vars }} {{ .Path }}'", - "inline": [ - "while(pgrep -a apt-get); do sleep 1; done", - "export DEBIAN_FRONTEND=noninteractive", - "apt-get update", - "apt-get install -y git ansible aptitude", - "git clone -b {{user `git_branch`}} {{user `git_repo`}}", - "cd /home/oem/cs-vm-build/scripts", - "./oem-build", - "/usr/sbin/oem-config-prepare" - ] - }], - "post-processors": [ - { - "type": "checksum", - "checksum_types": "sha256", - "output": "{{user `output_dir`}}/image-{{user `semester` | lower }}.{{.ChecksumType}}sum" - } - ] -} diff --git a/packer/ubuntu-build.json b/packer/ubuntu-build.json deleted file mode 100644 index e4e9dc65..00000000 --- a/packer/ubuntu-build.json +++ /dev/null @@ -1,198 +0,0 @@ -{ - "min_packer_version": "1.6.0", - "variables": { - "vm_name": "JMU Ubuntu", - "semester": "Fa20", - "version": "20.04", - "build_id": "{{isotime \"2006-01-02\"}}", - "git_repo": "https://github.com/jmunixusers/cs-vm-build", - "git_branch": "main", - "headless": "true", - "audio": "pulse", - "output_dir": "{{pwd}}/artifacts_ubuntu", - "mirror_url": "https://mirror.cs.jmu.edu/pub/ubuntu-iso/focal", - "iso_file": "ubuntu-20.04.4-desktop-amd64.iso", - "ssh_user": "oem", - "ssh_pass": "oem" - }, - "builders": [ - { - "type": "virtualbox-iso", - "headless": "{{user `headless`}}", - "http_directory": "http", - "vm_name": "{{user `vm_name`}} {{user `semester`}}", - "cpus": 2, - "memory": 4096, - "sound": "{{user `audio`}}", - "vboxmanage": [ - [ - "modifyvm", - "{{.Name}}", - "--audioin", - "off" - ], - [ - "modifyvm", - "{{.Name}}", - "--clipboard-mode", - "bidirectional" - ], - [ - "modifyvm", - "{{.Name}}", - "--graphicscontroller", - "vmsvga" - ], - [ - "modifyvm", - "{{.Name}}", - "--mouse", - "usbtablet" - ], - [ - "modifyvm", - "{{.Name}}", - "--pae", - "on" - ], - [ - "modifyvm", - "{{.Name}}", - "--rtcuseutc", - "on" - ], - [ - "modifyvm", - "{{.Name}}", - "--usb", - "on", - "--usbehci", - "off", - "--usbxhci", - "off" - ], - [ - "modifyvm", - "{{.Name}}", - "--vram", - "64" - ], - [ - "modifyvm", - "{{.Name}}", - "--vrde", - "off" - ], - [ - "storagectl", - "{{.Name}}", - "--name", - "IDE Controller", - "--remove" - ], - [ - "storagectl", - "{{.Name}}", - "--name", - "SATA Controller", - "--hostiocache", - "on" - ] - ], - "vboxmanage_post": [ - [ - "modifyvm", - "{{.Name}}", - "--accelerate3d", - "on" - ], - [ - "storageattach", - "{{.Name}}", - "--storagectl", - "SATA Controller", - "--port", - "1", - "--type", - "dvddrive", - "--medium", - "emptydrive" - ] - ], - "iso_url": "{{user `mirror_url`}}/{{user `iso_file`}}", - "iso_checksum": "file:{{user `mirror_url`}}/SHA256SUMS", - "iso_interface": "sata", - "hard_drive_discard": "true", - "hard_drive_interface": "sata", - "hard_drive_nonrotational": true, - "sata_port_count": 2, - "disk_size": "20480", - "guest_os_type": "Ubuntu_64", - "boot_wait": "5s", - "boot_command": [ - "", - "", - "", - "", - "/casper/vmlinuz", - " auto url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/oem-preseed.cfg", - " automatic-ubiquity", - " debug-ubiquity", - " keymap=us", - " boot=casper initrd=/casper/initrd noprompt --", - "" - ], - "ssh_username": "{{user `ssh_user`}}", - "ssh_password": "{{user `ssh_pass`}}", - "ssh_timeout": "100m", - "shutdown_command": "echo -e \"{{user `ssh_pass`}}\\n\" | sudo -S poweroff", - "output_directory": "{{user `output_dir`}}", - "output_filename": "image-{{user `semester` | lower }}", - "format": "ova", - "export_opts": [ - "--manifest", - "--vsys", - "0", - "--description", - "Build date: {{user `build_id`}}\nPacker version: {{packer_version}}", - "--product", - "{{user `vm_name`}} {{user `semester`}}", - "--producturl", - "https://ubuntu.com/", - "--vendor", - "JMU Unix Users Group", - "--vendorurl", - "{{user `git_repo`}}", - "--version", - "{{user `version`}}" - ] - } - ], - "provisioners": [ - { - "type": "shell", - "execute_command": "echo 'oem' | sudo -S sh -c '{{ .Vars }} {{ .Path }}'", - "inline": [ - "echo STOPPING APT", - "systemctl stop unattended-upgrades.service", - "while(pgrep -a apt-get); do sleep 1; done", - "export DEBIAN_FRONTEND=noninteractive", - "echo UPDATE", - "apt-get update", - "echo INSTALL", - "apt-get install -V -y git ansible aptitude", - "git clone -b {{user `git_branch`}} {{user `git_repo`}}", - "cd /home/oem/cs-vm-build/scripts", - "./oem-build", - "/usr/sbin/oem-config-prepare" - ] - } - ], - "post-processors": [ - { - "type": "checksum", - "checksum_types": "sha256", - "output": "{{user `output_dir`}}/image-{{user `semester` | lower }}.{{.ChecksumType}}sum" - } - ] -} diff --git a/packer/variables.pkr.hcl b/packer/variables.pkr.hcl new file mode 100644 index 00000000..083172ea --- /dev/null +++ b/packer/variables.pkr.hcl @@ -0,0 +1,91 @@ +variable "mint_version" { + type = object({ + name = string + version = string + beta = bool + }) + default = { + name = "Una" + version = "20.3" + beta = false + } +} + +variable "ubuntu_version" { + type = object({ + name = string + version = string + minor_version = string + }) + default = { + name = "jammy" + version = "22.04" + minor_version = "" + } +} + +variable "mirror" { + type = object({ + base = string + ubuntu_path = string + mint_path = string + }) + default = { + base = "https://mirror.cs.jmu.edu/pub" + ubuntu_path = "ubuntu-iso" + mint_path = "linuxmint/images" + } +} + +variable "audio" { + type = string + default = "pulse" +} + +variable "git_branch" { + type = string + default = "main" +} + +variable "git_repo" { + type = string + default = "https://github.com/jmunixusers/cs-vm-build" +} + +variable "headless" { + type = bool + default = true +} + +variable "semester" { + type = string + default = "Sp22" +} + +variable "ssh_pass" { + type = string + default = "oem" +} + +variable "ssh_user" { + type = string + default = "oem" +} + +variable "vm_name" { + type = string + default = "JMU CS" +} + +locals { + build_id = "${legacy_isotime("2006-01-02")}" + ubuntu_info = { + mirror_url = "${var.mirror.base}/${var.mirror.ubuntu_path}/${var.ubuntu_version.name}" + iso_file = "ubuntu-${var.ubuntu_version.version}${var.ubuntu_version.minor_version}-desktop-amd64.iso" + } + mint_info = { + mirror_url = "${var.mirror.base}/${var.mirror.mint_path}/${var.mint_version.beta ? "testing" : "stable/${var.mint_version.version}"}" + iso_file = "linuxmint-${var.mint_version.version}-cinnamon-64bit${var.mint_version.beta ? "-beta" : ""}.iso" + } + artifact_dir_prefix = "${path.cwd}/artifacts_" +}