diff --git a/.gitignore b/.gitignore index 3fe11ee..56b474c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,8 @@ /.yardoc /_yardoc/ /coverage/ -/doc/ /pkg/ /spec/reports/ /tmp/ .idea -Gemfile.lock \ No newline at end of file +Gemfile.lock diff --git a/doc/podman.md b/doc/podman.md new file mode 100644 index 0000000..dd36c44 --- /dev/null +++ b/doc/podman.md @@ -0,0 +1,44 @@ +Known podman issues +=================== + +"Unable to start container" error +--------------------------------- +When you first build a package, podman downloads the base image and attempts +to create a local image with correct version PostgreSQL installed into it. +However on some systems, running podman may initially result in the following +error: + +``` +Error: unable to start container "[CONTAINER_ID]": container create failed (no logs from conmon): conmon bytes "": readObjectStart: expect { or n, but found , error found in #0 byte of ...||..., bigger context ...||... +``` + +To fix this issue the following steps need to be performed: + +1. Wipe all podman runtime and configuration: + + pkill -9 -f 'conmon|podman' + rm -rf ~/.local/share/containers/ + rm -rf ~/.config/containers/ + rm -rf /run/user/$(id -u)/libpod/ + podman system reset --force + +2. Reinstall podman and its components: + + sudo apt reinstall podman conmon runc crun + +3. Create storage settings conf file: + + mkdir -p ~/.config/containers/ + touch ~/.config/containers/storage.conf + + # Add the following lines to the newly + # created ~/.config/containers/storage.conf: + # + [storage] + driver = "overlay" + graphroot = "/home/$USER/.local/share/containers/storage" + runroot = "/run/user/$(id -u)/containers" + +4. Restart the service (if applicable): + + service podman restart diff --git a/lib/pgpm/deb/Dockerfile b/lib/pgpm/deb/Dockerfile index 46a9940..9f07dd2 100644 --- a/lib/pgpm/deb/Dockerfile +++ b/lib/pgpm/deb/Dockerfile @@ -7,7 +7,7 @@ # { "builder": {"Entitlements": {"security-insecure": true }} } # ``` # ``` -# DOCKER_BUILDKIT=1 docker build --allow security.insecure -t IMAGE_NAME /path/to/pgpm +# DOCKER_BUILDKIT=1 docker build --allow security.insecure -t IMAGE_NAME . # ``` # This Dockerfile is used to build a Debian image, which includes pbuilder and @@ -22,10 +22,12 @@ VOLUME /proc ARG DEBIAN_FRONTEND=noninteractive RUN apt update RUN apt install -y build-essential pbuilder fakeroot fakechroot +RUN apt install -y vim ripgrep # for ease of debugging RUN echo 'MIRRORSITE=http://deb.debian.org/debian' > /etc/pbuilderrc RUN echo 'AUTO_DEBSIGN=${AUTO_DEBSIGN:-no}' > /root/.pbuilderrc RUN echo 'HOOKDIR=/var/cache/pbuilder/hooks' >> /root/.pbuilderrc -RUN --security=insecure pbuilder create - -COPY pbuilder_install_script.sh /root/pbuilder_install_script.sh -RUN --security=insecure pbuilder execute --save-after-exec /root/pbuilder_install_script.sh +COPY scripts/faketar /usr/bin/ +RUN chmod +x /usr/bin/faketar +RUN sed -E -i "s/local TAR=tar/local TAR=faketar/" /usr/lib/pbuilder/pbuilder-modules +RUN sed -E -i "s/if [!] tar -c --use-compress-program/if ! faketar -c --use-compress-program/" /usr/lib/pbuilder/pbuilder-modules +RUN --security=insecure fakeroot pbuilder create diff --git a/lib/pgpm/deb/builder.rb b/lib/pgpm/deb/builder.rb index a036c8c..5e0a927 100644 --- a/lib/pgpm/deb/builder.rb +++ b/lib/pgpm/deb/builder.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "English" -require "debug" module Pgpm module Deb @@ -13,9 +12,8 @@ def initialize(spec) end def build - pull_image + prepare_image start_container - patch_pbuilder prepare_versioned_source generate_deb_src_files(:versioned) @@ -32,9 +30,16 @@ def build private - # Depends on postgres version and arch + # Only depends on the arch -- this is the image being pulled from + # a remote repo, which is then used to build a local image with correct + # postgres version installed inside its pbuilder's chroot. + def base_image_name + "quay.io/qount25/pgpm-debian-#{@spec.arch}" + end + + # Locally built image with correct chroot installed. def image_name - "quay.io/qount25/pgpm-debian-pg#{@spec.package.postgres_major_version}-#{@spec.arch}" + "pgpm-debian-pg#{@spec.package.postgres_version}-#{@spec.arch}" end def prepare_versioned_source @@ -88,7 +93,7 @@ def prepare_default_source # 2. Determine the name of the .control file inside the versioned build deb_dir = "#{pbuilds_dir}/#{build_dir}/build/#{@spec.deb_pkg_name(:versioned)}-0/debian/#{@spec.deb_pkg_name(:versioned)}" - control_fn = "#{deb_dir}/usr/share/postgresql/#{@spec.package.postgres_major_version}/extension/#{@spec.package.extension_name}--#{@spec.package.version}.control" + control_fn = "#{deb_dir}/usr/share/postgresql/#{@spec.package.postgres_version}/extension/#{@spec.package.extension_name}--#{@spec.package.version}.control" # 3. Copy .control file to the source-default dir puts "Copying #{control_fn} into /root/pgpm/source-default/" @@ -102,18 +107,59 @@ def prepare_default_source end end - def pull_image + def prepare_image puts "Checking if podman image exists..." # Check if image exists system("podman image exists #{image_name}") - if $CHILD_STATUS.to_i.positive? # image doesn't exist -- pull image from a remote repository - puts " No. Pulling image #{image_name}..." - system("podman pull #{image_name}") + if $CHILD_STATUS.to_i.positive? + puts " Image for the specific pg version doesn't exist. Will build." + system("podman image exists #{base_image_name}") + if $CHILD_STATUS.to_i.positive? + puts " Base image doesn't exist. Pulling it..." + system("podman pull #{base_image_name}") + end + build_local_image else puts " Yes, image #{image_name} already exists! OK" end end + def build_local_image + puts " Building local #{image_name}..." + container_opts = "-it --privileged --tmpfs /tmp --name pgpm-deb-tmp" + system("podman create #{container_opts} #{base_image_name}") + system("podman start pgpm-deb-tmp") + + patch_pbuilder + + # Generate pbuilder_install script.sh, copy it inside the image + pbuild_install_script_path = "#{@pgpm_dir}/pbuilder_install_script.sh" + puts " Generating #{pbuild_install_script_path}..." + File.write pbuild_install_script_path.to_s, @spec.generate("pbuilder_install_script.sh") + system("podman container cp #{pbuild_install_script_path} pgpm-deb-tmp:/root/") + + # This command installs relevant postgresql packages into the chroot + # base image inside the container (along with some other necessary + # packages) and saves chroot base image with these changes. + puts " Updating chroot image..." + system("podman exec -w /root pgpm-deb-tmp /bin/bash -c 'fakeroot pbuilder execute --save-after-exec ./pbuilder_install_script.sh'") + + # Exiting -- most likely error occurred because we cannot find the same + # postgresql version in the Debian repository. The bash script + # will do error reporting for us, so we just exit. + if $CHILD_STATUS.to_i.positive? + stop_and_remove_deb_tmp_image + exit 1 + end + system("podman commit pgpm-deb-tmp #{image_name}") + stop_and_remove_deb_tmp_image + end + + def stop_and_remove_deb_tmp_image + system("podman stop pgpm-deb-tmp") + system("podman container rm pgpm-deb-tmp") + end + def generate_deb_src_files(pkg_type = :versioned) puts "Generating debian files..." Dir.mkdir "#{@pgpm_dir}/source-#{pkg_type}/debian" @@ -150,7 +196,9 @@ def start_container # a result. def patch_pbuilder cmd = "sed -E -i \"s/(^function clean_subdirectories.*$)/\\1\\n return/g\" /usr/lib/pbuilder/pbuilder-modules" - system("podman exec #{@container_name} /bin/bash -c '#{cmd}'") + system("podman exec pgpm-deb-tmp /bin/bash -c '#{cmd}'") + cmd = "sed -E -i \"s/if [[] [!] -f [\\\"]([$]BASETGZ)/if [ ! -d \\\"\\1/\" /usr/lib/pbuilder/pbuilder-modules" + system("podman exec pgpm-deb-tmp /bin/bash -c '#{cmd}'") end def run_build(pkg_type = :versioned) diff --git a/lib/pgpm/deb/pbuilder_install_script.sh b/lib/pgpm/deb/pbuilder_install_script.sh deleted file mode 100644 index 484850f..0000000 --- a/lib/pgpm/deb/pbuilder_install_script.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -apt update -DEBIAN_FRONTEND=noninteractive apt -y install build-essential curl lsb-release ca-certificates - -### PostgreSQL installation -# -install -d /usr/share/postgresql-common/pgdg -curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc - -# Create the repository configuration file: -sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' - -# Update the package lists: -apt update - -# Install the latest version of PostgreSQL: -# If you want a specific version, use 'postgresql-16' or similar instead of 'postgresql' -apt -y install postgresql-17 postgresql-server-dev-17 postgresql-common -# -### END OF PostgreSQL installation - diff --git a/lib/pgpm/deb/scripts/faketar b/lib/pgpm/deb/scripts/faketar new file mode 100755 index 0000000..1e0e748 --- /dev/null +++ b/lib/pgpm/deb/scripts/faketar @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +ARGS="$@" +echo "faketar ${ARGS[@]}" + +if [[ " ${ARGS[*]} " =~ [[:space:]]-x[[:space:]] ]]; then # compress + # Replacing this command: + # tar -x -p -f "$BASETGZ" + src="$4" + echo "Copying $src/* to $(pwd)" + cp -pur $src/* ./ +elif [[ " ${ARGS[*]} " =~ [[:space:]]-c[[:space:]] ]]; then # extract + # Replacing this command: + # tar -c --use-compress-program "$COMPRESSPROG" -f "${BASETGZ}.tmp" ./* + target="$5" + src="." + mkdir $target + echo "Moving $src/* to $target/" + mv $src/* $target/ + # Remove existing directory into which we move the contents. + # Otherwise, pbuilder (when it calls `mv`) will move $target inside it, + # instead of copying $target/* into it. + if [[ -d "${target%".tmp"}" ]]; then + rm -rf "${target%".tmp"}" + fi +fi diff --git a/lib/pgpm/deb/spec.rb b/lib/pgpm/deb/spec.rb index c3356f6..f5702c7 100644 --- a/lib/pgpm/deb/spec.rb +++ b/lib/pgpm/deb/spec.rb @@ -7,12 +7,10 @@ module Pgpm module Deb class Spec - attr_reader :package, :release, :postgres_version, :postgres_distribution + attr_reader :package, :release def initialize(package) - @postgres_distribution = Pgpm::Postgres::Distribution.in_scope @package = package - @package.postgres_major_version = @postgres_distribution.major_version @release = 1 end @@ -38,9 +36,9 @@ def source_version def deb_pkg_name(type = :versioned) if type == :versioned - "#{@package.name.gsub("_", "-")}+#{source_version}-pg#{@package.postgres_major_version}" + "#{@package.name.gsub("_", "-")}+#{source_version}-pg#{@package.postgres_version}" else - "#{@package.name.gsub("_", "-")}-pg#{@package.postgres_major_version}" + "#{@package.name.gsub("_", "-")}-pg#{@package.postgres_version}" end end diff --git a/lib/pgpm/deb/templates/pbuilder_install_script.sh.erb b/lib/pgpm/deb/templates/pbuilder_install_script.sh.erb new file mode 100644 index 0000000..0fe612d --- /dev/null +++ b/lib/pgpm/deb/templates/pbuilder_install_script.sh.erb @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# Run this script with: +# fakeroot pbuilder execute --save-after-exec path/to/pbuilder_install_script.sh + + +# This helps us avoid almost all instances of a very annoying: +# +# ERROR: ld.so: object 'libfakeroot-sysv.so' from LD_PRELOAD cannot be preloaded... +# +# which isn't critical, but is polluting the output and makes debugging or +# simply reading the output more difficult. +export LD_PRELOAD="$(find / -name libfakeroot-sysv.so 2>/dev/null)" + +# These packages are universally required by pbuilder, so we pre-install them +# inside our chroot. +apt update +DEBIAN_FRONTEND=noninteractive apt -y install \ + build-essential curl lsb-release ca-certificates automake autopoint \ + autotools-dev bsdextrautils debhelper dh-autoreconf dh-strip-nondeterminism \ + dwz file gettext gettext-base groff-base intltool-debian libarchive-zip-perl \ + libdebhelper-perl libelf1t64 libfile-stripnondeterminism-perl libmagic-mgc \ + libmagic1t64 libpipeline1 libtool libuchardet0 man-db po-debconf + +### PostgreSQL installation +# +install -d /usr/share/postgresql-common/pgdg +curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc + +# Create the repository configuration file: +sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + +# Add archive repos containing all postgres versions deleted from +# apt.postgresql.org. For example, when version 17.4 came out, version 17.3 +# was moved to the archive-apt source. We want all of them, because we don't know +# ahead of time which version is going to be required by the user. +apt_sources_fn="/etc/apt/sources.list.d/pgdg.list" +cat $apt_sources_fn | grep 's|https://apt|https://apt-archive' >> $apt_sources_fn + +# Update the package lists: +apt update + +PG_VERSION=<%= @package.postgres_version %> +PG_MAJOR_VERSION=<%= @package.postgres_version(:major) %> + +# Let's check if the version we're trying to install actually exists. +pkg="$(apt-cache madison postgresql-$PG_MAJOR_VERSION | grep " | $PG_VERSION" | head -n1)" +if [[ "$pkg" == "" ]]; then + # If it doesn't we exit with status 1 and an explanation of an error. This will + # trigger the caller (spec.rb) to also exit with status one and, thus, will + # not build a local image -- because user-requested version of postgresql + # cannot be found in the repository. + >&2 echo "ERROR:" + >&2 echo " Couldn't find postgresql-$PG_MAJOR_VERSION package with version $PG_VERSION." + exit 1 +else + pkg_version="$(echo "$pkg" | cut -d "|" -f 2 | xargs)" + apt -y install postgresql-$PG_MAJOR_VERSION=$pkg_version \ + postgresql-server-dev-$PG_MAJOR_VERSION=$pkg_version \ + postgresql-common +fi diff --git a/lib/pgpm/package/dependencies.rb b/lib/pgpm/package/dependencies.rb index 8869156..fe66296 100644 --- a/lib/pgpm/package/dependencies.rb +++ b/lib/pgpm/package/dependencies.rb @@ -5,14 +5,12 @@ module Pgpm class Package module Dependencies - attr_accessor :postgres_major_version - def build_dependencies case Pgpm::OS.in_scope.class.name when "debian", "ubuntu" deps = [ - "postgresql-#{postgres_major_version}", - "postgresql-server-dev-#{postgres_major_version}", + "postgresql-#{postgres_version(:major)} (>= #{postgres_version})", + "postgresql-server-dev-#{postgres_version(:major)} (>= #{postgres_version})", "postgresql-common" ] if native? @@ -26,7 +24,7 @@ def build_dependencies def dependencies case Pgpm::OS.in_scope.class.name when "debian", "ubuntu" - ["postgresql-#{postgres_major_version}"] + ["postgresql-#{postgres_version(:major)} (>= #{postgres_version})"] when "rocky+epel-9", "redhat", "fedora" [] end @@ -44,6 +42,14 @@ def topologically_ordered_with_dependencies TopologicalPackageSorter.new([self, *all_requirements]).sorted_packages end + def postgres_version(version_type = :major_minor) + v = Pgpm::Postgres::Distribution.in_scope.version + if version_type == :major + v = v.split(".").first + end + v + end + class TopologicalPackageSorter include TSort diff --git a/lib/pgpm/rpm/spec.rb b/lib/pgpm/rpm/spec.rb index fbc8aef..05e26f7 100644 --- a/lib/pgpm/rpm/spec.rb +++ b/lib/pgpm/rpm/spec.rb @@ -6,16 +6,12 @@ module Pgpm module RPM class Spec - attr_reader :package, :release, :postgres_version, :postgres_distribution + attr_reader :package, :release def initialize(package) @postgres_distribution = Pgpm::Postgres::Distribution.in_scope @package = package @release = 1 - - # Needed in order to return correct dependencies for the selected - # version of postgres and selected OS. - @package.postgres_major_version = @postgres_distribution.major_version end def versionless diff --git a/packages/timescale/timescaledb.rb b/packages/timescale/timescaledb.rb index 59f0d55..5a7747d 100644 --- a/packages/timescale/timescaledb.rb +++ b/packages/timescale/timescaledb.rb @@ -28,6 +28,8 @@ def dependencies deps = case Pgpm::OS.in_scope.class.name when "rocky+epel-9", "redhat", "fedora" ["openssl"] + else + [] end super + deps end