operation/apt: update to use .sources file

Author: maisim

src/pyinfra/facts/apt.py

@@ -348,7 +348,11 @@ def flush():
 
 class AptKeys(GpgFactBase):
     """
-    Returns information on GPG keys apt has in its keychain:
+    Returns information on GPG keys available to APT.
+    
+    This fact searches in APT's modern keyring directories instead of using
+    the deprecated apt-key command. It provides compatibility with the old
+    AptKeys interface while using the modern GPG infrastructure.
 
     .. code:: python
 
@@ -360,14 +364,23 @@ class AptKeys(GpgFactBase):
         }
     """
 
-    # This requires both apt-key *and* apt-key itself requires gpg
     @override
     def command(self) -> str:
-        return "! command -v gpg || apt-key list --with-colons"
+        # Use APT's standard keyring directories - this generates a complex command
+        # that finds and processes all keyrings in APT directories
+        apt_directories = ["/etc/apt/trusted.gpg.d", "/etc/apt/keyrings", "/usr/share/keyrings"]
+        search_locations = " ".join(f'"{d}"' for d in apt_directories)
+        
+        # Find all GPG keyring files and process them
+        return (
+            f"for keyring in $(find {search_locations} -type f \\( -name '*.gpg' -o -name '*.asc' -o -name '*.kbx' \\) 2>/dev/null); do "
+            f"gpg --batch --no-default-keyring --keyring \"$keyring\" --list-keys --with-colons 2>/dev/null || true; "
+            f"done"
+        )
 
     @override
     def requires_command(self) -> str:
-        return "apt-key"
+        return "gpg"
 
 
 class AptSimulationDict(TypedDict):

src/pyinfra/operations/apt.py

@@ -20,7 +20,7 @@
 )
 from pyinfra.facts.deb import DebPackage, DebPackages
 from pyinfra.facts.files import File
-from pyinfra.facts.gpg import GpgKey
+from pyinfra.facts.gpg import GpgKey, GpgKeyrings
 from pyinfra.facts.server import Date
 from pyinfra.operations import files, gpg
 
@@ -98,28 +98,57 @@ def _derive_dest_from_src_and_keyids(
     return f"/etc/apt/keyrings/{base}.gpg"
 
 
+def _get_apt_keys_comprehensive() -> dict[str, str]:
+    """
+    Get all GPG keys available in APT directories using the GpgKeyrings fact.
+    This provides more comprehensive coverage than AptKeys fact.
+    Falls back gracefully if GpgKeyrings data is not available.
+    
+    Returns:
+        dict: Key ID -> keyring file path mapping
+    """
+    try:
+        apt_directories = ["/etc/apt/trusted.gpg.d", "/etc/apt/keyrings", "/usr/share/keyrings"]
+        keyrings_info = host.get_fact(GpgKeyrings, directories=apt_directories)
+        
+        all_keys = {}
+        for keyring_path, keyring_data in keyrings_info.items():
+            keys = keyring_data.get("keys", {})
+            for key_id in keys.keys():
+                all_keys[key_id] = keyring_path
+        
+        return all_keys
+    except (KeyError, AttributeError):
+        # Fallback to empty dict if GpgKeyrings fact is not available (e.g., in tests)
+        return {}
+
+
 @operation()
 def key(
     src: str | None = None,
     keyserver: str | None = None,
     keyid: str | list[str] | None = None,
     dest: str | None = None,
+    present: bool = True,
 ):
     """
-    Add apt GPG keys *without* apt-key:
-      - Keys are stored under /etc/apt/keyrings/<name>.gpg (binary, dearmored if needed).
-      - You must reference the resulting file in your apt source via `signed-by=...`.
+    Add or remove apt GPG keys using modern keyring management.
+    
+    This operation manages GPG keys for APT repositories without using the deprecated apt-key command.
+    Keys are stored in /etc/apt/keyrings/ and can be referenced in source lists via signed-by=.
 
     Args:
         src: filename or URL to a key (ASCII .asc or binary .gpg)
         keyserver: keyserver URL for fetching keys by ID
-        keyid: key ID or list of key IDs (required with keyserver)
+        keyid: key ID or list of key IDs (required with keyserver, optional for removal)
         dest: optional keyring filename/path ('.gpg' will be enforced, defaults under /etc/apt/keyrings)
+        present: whether the key should be present (True) or absent (False)
 
     Behavior:
-        - Idempotent via AptKeys: if the key IDs are already present in any apt keyring, nothing is changed.
-        - If src is ASCII (.asc), it will be dearmored; if binary (.gpg), it's copied as-is.
-        - Keyserver flow uses a temporary GNUPGHOME, then exports and dearmors to the destination keyring.
+        - Installation: Idempotent via AptKeys - if key IDs are already present, nothing changes
+        - Removal: Uses GpgKeyrings fact to find and remove keys from APT directories
+        - If src is ASCII (.asc), it will be dearmored; if binary (.gpg), it's copied as-is
+        - Keyserver flow uses temporary GNUPGHOME, then exports to destination keyring
 
     Examples:
         apt.key(
@@ -129,9 +158,15 @@ def key(
         )
 
         apt.key(
-            name="Install VirtualBox key",
-            src="https://www.virtualbox.org/download/oracle_vbox_2016.asc",
-            dest="oracle-virtualbox.gpg",
+            name="Remove specific keyring file",
+            dest="old-vendor.gpg",
+            present=False,
+        )
+
+        apt.key(
+            name="Remove key by ID from all APT keyrings",
+            keyid="0xCOMPROMISED123",
+            present=False,
         )
 
         apt.key(
@@ -142,8 +177,26 @@ def key(
         )
     """
 
-    # Gather currently installed keys (across trusted.gpg.d/, keyrings/, etc.)
+    # Handle removal operations using the GPG infrastructure
+    if present is False:
+        # Use the GPG operation for removal, but restrict to APT directories
+        apt_working_dirs = ["/etc/apt/trusted.gpg.d", "/etc/apt/keyrings", "/usr/share/keyrings"]
+        yield from gpg.key._inner(
+            dest=dest,
+            keyid=keyid,
+            present=False,
+            working_dirs=apt_working_dirs,
+        )
+        return
+
+    # Installation logic (existing code)
+    # Get comprehensive view of all keys in APT directories
+    existing_keys_comprehensive = _get_apt_keys_comprehensive()
+    # Also get the legacy AptKeys fact for compatibility
     existing_keys = host.get_fact(AptKeys)
+    
+    # Combine both sources of key information for complete coverage
+    all_available_keys = set(existing_keys_comprehensive.keys()) | set(existing_keys.keys())
 
     # Check idempotency for src branch
     if src:
@@ -152,7 +205,7 @@ def key(
 
         # If we don't know the IDs (eg. unreachable URL), we cannot determine idempotency -> try to install.
         # Otherwise, skip if all key IDs are already present.
-        if keyids_from_src and all(kid in existing_keys for kid in keyids_from_src):
+        if keyids_from_src and all(kid in all_available_keys for kid in keyids_from_src):
             host.noop(f"All keys from {src} are already available in the apt keychain")
             return
 
@@ -166,7 +219,7 @@ def key(
         if isinstance(keyid, str):
             keyid = [keyid]
 
-        needed_keys = sorted(set(keyid) - set(existing_keys.keys()))
+        needed_keys = sorted(set(keyid) - all_available_keys)
         if not needed_keys:
             host.noop(f"Keys {', '.join(keyid)} are already available in the apt keychain")
             return

src/pyinfra/operations/gpg.py

@@ -21,6 +21,7 @@ def key(
     dearmor: bool = True,
     mode: str = "0644",
     present: bool = True,
+    working_dirs: list[str] | None = None,
 ):
     """
     Install or remove GPG keys from various sources.
@@ -33,8 +34,9 @@ def key(
         dearmor: whether to convert ASCII armored keys to binary format
         mode: file permissions for the installed key
         present: whether the key should be present (True) or absent (False)
+        working_dirs: directories to search for existing keyrings (required for removal without dest)
                 When False: if dest is provided, removes from specific keyring;
-                           if dest is None, removes from all APT keyrings;
+                           if dest is None, removes from keyrings found in working_dirs;
                            if keyid is provided, removes specific key(s);
                            if keyid is None, removes entire keyring file(s)
 
@@ -59,10 +61,10 @@ def key(
         )
 
         gpg.key(
-            name="Remove key from all APT keyrings",
+            name="Remove key from specific directories",
             keyid="0xCOMPROMISED123",
             present=False,
-            # dest=None means search in all keyrings
+            working_dirs=["/etc/apt/keyrings", "/usr/share/keyrings"],
         )
 
         gpg.key(
@@ -79,20 +81,24 @@ def key(
         if not dest:
             raise OperationError("`dest` must be provided for installation")
     elif present is False:
-        # For removal, either dest or keyid must be provided
-        if not dest and not keyid:
-            raise OperationError("For removal, either `dest` or `keyid` must be provided")
+        # For removal, either dest or (keyid and working_dirs) must be provided
+        if not dest and not (keyid and working_dirs):
+            raise OperationError(
+                "For removal, either `dest` or both `keyid` and `working_dirs` must be provided"
+            )
 
     # For removal, handle different scenarios
     if present is False:
         if not dest and keyid:
-            # Remove key(s) from all keyrings found in APT directories
+            # Remove key(s) from all keyrings found in specified directories
             if isinstance(keyid, str):
                 keyid = [keyid]
 
-            # Use the GpgKeyrings fact to find all keyrings with default APT directories
-            apt_directories = ["/etc/apt/trusted.gpg.d", "/etc/apt/keyrings", "/usr/share/keyrings"]
-            keyrings_info = host.get_fact(GpgKeyrings, directories=apt_directories)
+            if not working_dirs:
+                raise OperationError("`working_dirs` must be provided when removing keys without `dest`")
+
+            # Use the GpgKeyrings fact to find all keyrings in specified directories
+            keyrings_info = host.get_fact(GpgKeyrings, directories=working_dirs)
 
             for keyring_path, keyring_data in keyrings_info.items():
                 # Get the keys from the GpgKeyrings fact data
@@ -112,8 +118,8 @@ def key(
                             keys_to_remove.append(existing_key_id)
 
                 if keys_to_remove:
-                    # For APT keyrings, remove the entire keyring file if any target keys are found
-                    # This is the safest approach for APT key management
+                    # Remove the entire keyring file if any target keys are found
+                    # This is the safest approach for keyring management
                     yield from files.file._inner(
                         path=keyring_path,
                         present=False,
@@ -148,7 +154,7 @@ def key(
                         break
 
                 if keys_found:
-                    # Remove the entire keyring file - safest approach for APT
+                    # Remove the entire keyring file - safest approach for keyring management
                     yield from files.file._inner(
                         path=dest,
                         present=False,
@@ -234,16 +240,9 @@ def key(
         # Export GNUPGHOME and fetch keys
         yield f'export GNUPGHOME="{temp_dir}" && gpg --batch --keyserver "{keyserver}" --recv-keys {joined}'  # noqa: E501
 
-        # Export keys to destination
-        if dearmor:
-            # For binary output (dearmored), use --export without --armor
-            yield (f'export GNUPGHOME="{temp_dir}" && ' f'gpg --batch --export {joined} > "{dest}"')
-        else:
-            # For ASCII output, use --export --armor
-            yield (
-                f'export GNUPGHOME="{temp_dir}" && '
-                f'gpg --batch --export --armor {joined} > "{dest}"'
-            )
+        # Export keys to destination - always use direct binary export
+        # gpg --export produces binary format by default, no dearmoring needed
+        yield (f'export GNUPGHOME="{temp_dir}" && ' f'gpg --batch --export {joined} > "{dest}"')
 
         # Clean up temporary directory
         yield from files.directory._inner(

tests/facts/apt.AptKeys/keys.json

@@ -1,5 +1,5 @@
 {
-    "command": "for f in   /etc/apt/trusted.gpg   /etc/apt/trusted.gpg.d/*.gpg /etc/apt/trusted.gpg.d/*.asc   /etc/apt/keyrings/*.gpg /etc/apt/keyrings/*.asc   /usr/share/keyrings/*.gpg /usr/share/keyrings/*.asc ; do   [ -e \"$f\" ] || continue;   case \"$f\" in     *.asc)       gpg --batch --show-keys --with-colons --keyid-format LONG \"$f\"       ;;     *)       gpg --batch --no-default-keyring --keyring \"$f\"           --list-keys --with-colons --keyid-format LONG       ;;   esac; done",
+    "command": "for keyring in $(find \"/etc/apt/trusted.gpg.d\" \"/etc/apt/keyrings\" \"/usr/share/keyrings\" -type f \\( -name '*.gpg' -o -name '*.asc' -o -name '*.kbx' \\) 2>/dev/null); do gpg --batch --no-default-keyring --keyring \"$keyring\" --list-keys --with-colons 2>/dev/null || true; done",
     "requires_command": "gpg",
     "output": [
         "tru:t:1:1601454628:0:3:1:5",

tests/operations/apt.key/add_keyserver.json

@@ -19,7 +19,7 @@
         "mkdir -p /tmp/pyinfra-gpg-empfile_",
         "chmod 700 /tmp/pyinfra-gpg-empfile_",
         "export GNUPGHOME=\"/tmp/pyinfra-gpg-empfile_\" && gpg --batch --keyserver \"key-server.net\" --recv-keys abc",
-        "export GNUPGHOME=\"/tmp/pyinfra-gpg-empfile_\" && gpg --batch --export abc | gpg --batch --dearmor -o \"/etc/apt/keyrings/keyserver-abc.gpg\"",
+        "export GNUPGHOME=\"/tmp/pyinfra-gpg-empfile_\" && gpg --batch --export abc > \"/etc/apt/keyrings/keyserver-abc.gpg\"",
         "mkdir -p /etc/apt/keyrings",
         "touch /etc/apt/keyrings/keyserver-abc.gpg",
         "chmod 644 /etc/apt/keyrings/keyserver-abc.gpg"

tests/operations/apt.key/add_keyserver_multiple.json

@@ -19,7 +19,7 @@
         "mkdir -p /tmp/pyinfra-gpg-empfile_",
         "chmod 700 /tmp/pyinfra-gpg-empfile_",
         "export GNUPGHOME=\"/tmp/pyinfra-gpg-empfile_\" && gpg --batch --keyserver \"key-server.net\" --recv-keys abc def",
-        "export GNUPGHOME=\"/tmp/pyinfra-gpg-empfile_\" && gpg --batch --export abc def | gpg --batch --dearmor -o \"/etc/apt/keyrings/keyserver-abc-def.gpg\"",
+        "export GNUPGHOME=\"/tmp/pyinfra-gpg-empfile_\" && gpg --batch --export abc def > \"/etc/apt/keyrings/keyserver-abc-def.gpg\"",
         "mkdir -p /etc/apt/keyrings",
         "touch /etc/apt/keyrings/keyserver-abc-def.gpg",
         "chmod 644 /etc/apt/keyrings/keyserver-abc-def.gpg"

tests/operations/gpg.key/keyserver_multiple.json

@@ -20,7 +20,7 @@
         "mkdir -p /tmp/pyinfra-gpg-empfile_",
         "chmod 700 /tmp/pyinfra-gpg-empfile_",
         "export GNUPGHOME=\"/tmp/pyinfra-gpg-empfile_\" && gpg --batch --keyserver \"hkps://keyserver.ubuntu.com\" --recv-keys 0xD88E42B4 0x7EA0A9C3",
-        "export GNUPGHOME=\"/tmp/pyinfra-gpg-empfile_\" && gpg --batch --export 0xD88E42B4 0x7EA0A9C3 | gpg --batch --dearmor -o \"/etc/apt/keyrings/vendor.gpg\"",
+        "export GNUPGHOME=\"/tmp/pyinfra-gpg-empfile_\" && gpg --batch --export 0xD88E42B4 0x7EA0A9C3 > \"/etc/apt/keyrings/vendor.gpg\"",
         "mkdir -p /etc/apt/keyrings",
         "touch /etc/apt/keyrings/vendor.gpg",
         "chmod 644 /etc/apt/keyrings/vendor.gpg"

tests/operations/gpg.key/keyserver_single.json

@@ -20,7 +20,7 @@
         "mkdir -p /tmp/pyinfra-gpg-empfile_",
         "chmod 700 /tmp/pyinfra-gpg-empfile_",
         "export GNUPGHOME=\"/tmp/pyinfra-gpg-empfile_\" && gpg --batch --keyserver \"hkps://keyserver.ubuntu.com\" --recv-keys 0xD88E42B4",
-        "export GNUPGHOME=\"/tmp/pyinfra-gpg-empfile_\" && gpg --batch --export 0xD88E42B4 | gpg --batch --dearmor -o \"/etc/apt/keyrings/vendor.gpg\"",
+        "export GNUPGHOME=\"/tmp/pyinfra-gpg-empfile_\" && gpg --batch --export 0xD88E42B4 > \"/etc/apt/keyrings/vendor.gpg\"",
         "mkdir -p /etc/apt/keyrings",
         "touch /etc/apt/keyrings/vendor.gpg",
         "chmod 644 /etc/apt/keyrings/vendor.gpg"

tests/operations/gpg.key/remove_by_id.json

@@ -5,12 +5,28 @@
         "present": false
     },
     "facts": {
+        "gpg.GpgKeyrings": {
+            "directories=['/etc/apt/keyrings']": {
+                "/etc/apt/keyrings/vendor.gpg": {
+                    "format": "gpg",
+                    "keys": {
+                        "ABCDEF1234567890": {
+                            "validity": "-",
+                            "length": 4096,
+                            "subkeys": {},
+                            "fingerprint": "ABCDEF1234567890FEDCBA0987654321ABCDEF12",
+                            "uid_hash": "ABC123DEF456",
+                            "uid": "Vendor Key <[email protected]>"
+                        }
+                    }
+                }
+            }
+        },
         "files.File": {
             "path=/etc/apt/keyrings/vendor.gpg": {"mode": 644}
         }
     },
     "commands": [
-        "gpg --batch --no-default-keyring --keyring \"/etc/apt/keyrings/vendor.gpg\" --delete-keys 0xABCDEF12 2>/dev/null || true",
-        "if ! gpg --batch --no-default-keyring --keyring \"/etc/apt/keyrings/vendor.gpg\" --list-keys 2>/dev/null | grep -q \"pub\"; then rm -f \"/etc/apt/keyrings/vendor.gpg\"; fi"
+        "rm -f /etc/apt/keyrings/vendor.gpg"
     ]
 }

tests/operations/gpg.key/remove_from_all_keyrings.json

@@ -1,15 +1,32 @@
 {
     "kwargs": {
         "keyid": "0xCOMPROMISED123",
-        "present": false
+        "present": false,
+        "working_dirs": ["/etc/apt/trusted.gpg.d", "/etc/apt/keyrings", "/usr/share/keyrings"]
+    },
+    "facts": {
+        "gpg.GpgKeyrings": {
+            "directories=['/etc/apt/trusted.gpg.d', '/etc/apt/keyrings', '/usr/share/keyrings']": {
+                "/etc/apt/trusted.gpg.d/compromised.gpg": {
+                    "format": "gpg",
+                    "keys": {
+                        "COMPROMISED123567890": {
+                            "validity": "-",
+                            "length": 4096,
+                            "subkeys": {},
+                            "fingerprint": "COMPROMISED123567890FEDCBA0987654321COMPROMISED123",
+                            "uid_hash": "ABC123DEF456",
+                            "uid": "Compromised Key <[email protected]>"
+                        }
+                    }
+                }
+            }
+        },
+        "files.File": {
+            "path=/etc/apt/trusted.gpg.d/compromised.gpg": {"mode": 644}
+        }
     },
-    "facts": {},
     "commands": [
-        "for keyring in /etc/apt/trusted.gpg.d/*.gpg; do [ -e \"$keyring\" ] && gpg --batch --no-default-keyring --keyring \"$keyring\" --delete-keys 0xCOMPROMISED123 2>/dev/null || true; done",
-        "for keyring in /etc/apt/trusted.gpg.d/*.gpg; do [ -e \"$keyring\" ] && ! gpg --batch --no-default-keyring --keyring \"$keyring\" --list-keys 2>/dev/null | grep -q \"pub\" && rm -f \"$keyring\" || true; done",
-        "for keyring in /etc/apt/keyrings/*.gpg; do [ -e \"$keyring\" ] && gpg --batch --no-default-keyring --keyring \"$keyring\" --delete-keys 0xCOMPROMISED123 2>/dev/null || true; done",
-        "for keyring in /etc/apt/keyrings/*.gpg; do [ -e \"$keyring\" ] && ! gpg --batch --no-default-keyring --keyring \"$keyring\" --list-keys 2>/dev/null | grep -q \"pub\" && rm -f \"$keyring\" || true; done",
-        "for keyring in /usr/share/keyrings/*.gpg; do [ -e \"$keyring\" ] && gpg --batch --no-default-keyring --keyring \"$keyring\" --delete-keys 0xCOMPROMISED123 2>/dev/null || true; done",
-        "for keyring in /usr/share/keyrings/*.gpg; do [ -e \"$keyring\" ] && ! gpg --batch --no-default-keyring --keyring \"$keyring\" --list-keys 2>/dev/null | grep -q \"pub\" && rm -f \"$keyring\" || true; done"
+        "rm -f /etc/apt/trusted.gpg.d/compromised.gpg"
     ]
 }