diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..849278d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,121 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + # Pin to the same toolchain as rust-toolchain.toml (blob-decoder's app/CLI + # is built and tested with the pinned stable). Do not float `stable`. + - uses: dtolnay/rust-toolchain@1.96.0 + with: + components: clippy, rustfmt + - run: cargo test --all-features + - run: cargo clippy --all-targets --all-features -- -D warnings + - run: cargo fmt --check + + msrv: + # Dedicated MSRV job: the LIBRARY must build on its declared floor (1.88.0, + # forced by plist 1.10 -> time 0.3.53). Builds the lib without default + # features (the downstream-library surface; the CLI's clap is not part of it). + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: dtolnay/rust-toolchain@1.88.0 + - name: Build library on MSRV (no default features) + run: cargo build --lib --no-default-features + + lean-build: + # Gate: a library consumer doing `default-features = false` must NOT pull in + # clap (a CLI-only dep behind the `cli` feature). + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: dtolnay/rust-toolchain@1.96.0 + - name: Lean lib build (no default features) + run: cargo build --no-default-features + - name: Assert clap absent from the lean tree + run: | + cargo tree --no-default-features -e no-dev --prefix none > /tmp/lean-tree.txt + if grep -qE '^clap ' /tmp/lean-tree.txt; then + echo "FAIL: clap present in the lean (no-default-features) build:" + grep -E '^clap ' /tmp/lean-tree.txt + exit 1 + fi + + coverage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: dtolnay/rust-toolchain@1.96.0 + with: + components: llvm-tools-preview + - uses: taiki-e/install-action@a402910a723481c4c80d006d75298c796a9c8695 # cargo-llvm-cov + with: + tool: cargo-llvm-cov + # All test targets, all features. The gate requires 100% function coverage + # of the LIBRARY, honoring `// cov:unreachable` markers on provably-dead + # defensive arms; main.rs (the Humble CLI shell) is excluded by the script. + - name: Coverage + 100% function gate + run: | + cargo llvm-cov --all-features --json --output-path cov.json + python3 scripts/coverage-gate.py cov.json + - name: Emit lcov + run: cargo llvm-cov report --lcov --output-path lcov.info + - name: Upload to Codecov + uses: codecov/codecov-action@0f8570b1a125f4937846a11fcfa3bcd548bd8c97 # v4.6.0 + with: + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false + + secrets: + name: Secret Scan (gitleaks) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 0 + - name: Install gitleaks + run: | + VERSION=8.30.1 + curl -sSfL "https://github.com/gitleaks/gitleaks/releases/download/v${VERSION}/gitleaks_${VERSION}_linux_x64.tar.gz" \ + | tar xz -C /tmp gitleaks + - name: Run gitleaks + run: /tmp/gitleaks detect --source . --config .gitleaks.toml + + deny: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + - name: Install cargo-deny + uses: taiki-e/install-action@a402910a723481c4c80d006d75298c796a9c8695 + with: + tool: cargo-deny + - name: Check advisories, licenses, bans, sources + run: cargo deny check + + freshness: + # Advisory dependency-freshness gate: fails if the committed Cargo.lock is + # stale relative to the requirements (see CLAUDE.md "Dependency Freshness"). + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + - run: cargo update --locked diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..3a033e7 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,20 @@ +name: Docs + +on: + push: + branches: [main] + paths: ["docs/**", "mkdocs.yml", ".github/workflows/docs.yml"] + +permissions: + contents: write + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.x" + - run: pip install mkdocs-material + - run: mkdocs gh-deploy --force diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml new file mode 100644 index 0000000..cc9b305 --- /dev/null +++ b/.github/workflows/fuzz.yml @@ -0,0 +1,35 @@ +name: Fuzz + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: "0 3 * * 1" # weekly deeper run + +env: + CARGO_TERM_COLOR: always + +jobs: + fuzz: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + # cargo-fuzz needs nightly (`-Z` flags). rust-toolchain.toml pins a stable + # version that would win over the @nightly channel, so force `cargo +nightly` + # on every fuzz command (see CLAUDE.md "Rust MSRV & Toolchain Policy"). + - uses: dtolnay/rust-toolchain@nightly + with: + # ASan needs a dynamically-linked target; the musl-built cargo-fuzz + # otherwise defaults to musl (crt-static) and fails with E0463. + targets: x86_64-unknown-linux-gnu + - uses: taiki-e/install-action@a402910a723481c4c80d006d75298c796a9c8695 # cargo-fuzz + with: + tool: cargo-fuzz + - name: Build fuzz target + run: cargo +nightly fuzz build --target x86_64-unknown-linux-gnu + - name: Smoke-run (no-panic / bounded-memory invariant) + run: | + cargo +nightly fuzz run identify --target x86_64-unknown-linux-gnu \ + -- -max_total_time=60 -runs=500000 -rss_limit_mb=2048 diff --git a/.gitignore b/.gitignore index 00d9f88..931e9db 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,9 @@ cov.json lcov.info +# mkdocs build output (docs.yml builds + deploys in CI; local `mkdocs build` output) +/site + # Claude Code session state — ephemeral, never tracked /.claude/ diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..a8d97e8 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,12 @@ +# gitleaks configuration for blob-decoder. Extends the bundled default ruleset; +# allowlists generated / build-output trees (all gitignored, never source) so a +# full `gitleaks dir .` stays clean and fast. The pre-commit hook scans only +# staged files regardless. +[extend] +useDefault = true + +[allowlist] +description = "Project-specific allowlist" +paths = [ + '''(^|/)target/''', +] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..75d0aa8 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,21 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-toml + - id: check-merge-conflict + + - repo: https://github.com/doublify/pre-commit-rust + rev: v1.0 + hooks: + - id: fmt + - id: clippy + args: ["--all-targets", "--all-features", "--", "-D", "warnings"] + + - repo: https://github.com/gitleaks/gitleaks + rev: v8.21.2 + hooks: + - id: gitleaks diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..69bad42 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,18 @@ +# Changelog + +All notable changes to this project are documented here. The format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this project adheres to +[Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- Initial `blob-decoder` library + `blob-decode` CLI: identify opaque blobs of + unknown type, decode them, and report scored, cited candidates, recursively + unwrapping nested wrappers (base64 → gzip → binary-plist). +- Recognises binary/XML plist, gzip, zlib, Snappy, JSON, UUID, base64, hex, + UTF-16LE, and UTF-8 text, dispatching to `plist`/`flate2`/`snap`/`base64`/ + `hex`/`uuid`/`serde_json`. +- Bomb/DoS guards: size-capped decompression and depth-capped recursion via + `Limits`; a `cargo-fuzz` `identify` target for the no-panic invariant. diff --git a/LICENSE b/LICENSE index e92b680..d645695 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,4 @@ + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -32,33 +33,36 @@ not limited to compiled object code, generated documentation, and conversions to other media types. - "Work" shall mean the work of authorship made available under - the License, as indicated by a copyright notice that is included in - or attached to the work (an example is provided in the Appendix below). + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other - transformations represent, as a whole, an original work of authorship. - For the purposes of this License, Derivative Works shall not include - works that remain separable from, or merely link (or bind by name) - to the interfaces of, the Work and Derivative Works thereof. - - "Contribution" shall mean, as submitted to the Licensor for inclusion - in the Work by the copyright owner or by an individual or Legal Entity - authorized to submit on behalf of the copyright owner. For the purposes - of this definition, "submitted" means any form of electronic, verbal, - or written communication sent to the Licensor or its representatives, - including but not limited to communication on electronic mailing lists, - source code control systems, and issue tracking systems that are managed - by, or on behalf of, the Licensor for the purpose of discussing and - improving the Work, but excluding communication that is conspicuously - marked or designated in writing by the copyright owner as "Not a - Contribution." - - "Contributor" shall mean Licensor and any Legal Entity on behalf of - whom a Contribution has been received by the Licensor and included - within the Work. + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, @@ -74,10 +78,9 @@ use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their - Contribution(s) alone or by the combined work (as a Contributor - alone or by the combined work as a Contributor) of their - Contribution(s) alone or by the combined work (as a Contributor). - If You institute patent litigation against any entity (including a + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses @@ -89,8 +92,8 @@ modifications, and in Source or Object form, provided that You meet the following conditions: - (a) You must give any other recipients of the Work or Derivative - Works a copy of this License; and + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and @@ -102,28 +105,28 @@ the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its - distribution, You must include a readable copy of the - attribution notices contained within such NOTICE file, in - at least one of the following places: within a NOTICE text - file distributed as part of the Derivative Works; within - the Source form or documentation, if provided along with the - Derivative Works; or, within a display generated by the - Derivative Works, if and wherever such third-party notices - normally appear. The contents of the NOTICE file are for - informational purposes only and do not modify the License. - You may add Your own attribution notices within Derivative - Works that You distribute, alongside or in addition to the - NOTICE text from the Work, provided that such additional - attribution notices cannot be construed as modifying the - License. - - You may add Your own license statement for Your modifications and - may provide additional grant of rights to use, reproduce, modify, - prepare Derivative Works of, convert to Source form, and distribute - such Modifications, as a separate work, provided Your use, - reproduction, modification, preparation of Derivative Works, - conversion to Source form, and distribution of such Modifications - complies with the terms and conditions of this License. + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work @@ -145,7 +148,7 @@ implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or reproducing the Work and assume any + appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, @@ -153,27 +156,38 @@ unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, - incidental, or exemplary damages of any character arising as a + incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or all other - commercial damages or losses), even if such Contributor has been - advised of the possibility of such damages. - - 9. Accepting Warranty or Liability. While redistributing the Work or - Derivative Works thereof, You may choose to offer, and charge a fee - for, acceptance of support, warranty, indemnity, or other liability - obligations and/or rights consistent with this License. However, in - accepting such obligations, You may offer such obligations only on - Your own behalf and on Your sole responsibility, not on behalf of - any other Contributor, and only if You agree to indemnify, defend, - and hold each Contributor harmless for any liability incurred by, - or claims asserted against, such Contributor by reason of your - accepting any such warranty or additional liability. + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS - Copyright 2026 Security Ronin Ltd. + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index ebfbf29..f7793d1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,87 @@ # blob-decoder -Identify and decode opaque forensic blobs of unknown type — scored, cited -candidates, recursively unwrapping nested wrappers. +[![Crates.io](https://img.shields.io/crates/v/blob-decoder.svg)](https://crates.io/crates/blob-decoder) +[![Docs.rs](https://img.shields.io/docsrs/blob-decoder)](https://docs.rs/blob-decoder) +[![License: Apache-2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE) +[![CI](https://github.com/SecurityRonin/blob-decoder/actions/workflows/ci.yml/badge.svg)](https://github.com/SecurityRonin/blob-decoder/actions/workflows/ci.yml) +[![Fuzz](https://github.com/SecurityRonin/blob-decoder/actions/workflows/fuzz.yml/badge.svg)](https://github.com/SecurityRonin/blob-decoder/actions/workflows/fuzz.yml) +[![Sponsor](https://img.shields.io/badge/sponsor-h4x0r-ea4aaa?logo=github-sponsors)](https://github.com/sponsors/h4x0r) -Scaffold in progress. See the `feat/blob-decoder` branch for the engine. +**What is this blob? Hand it the bytes — get back what they are, decoded.** + +Every examination turns up an opaque value — a column full of gibberish, a +property-list `Data` field, a config string that is obviously encoded but you are +not sure how. `blob-decoder` reads those bytes, **identifies** the type, +**decodes** it, and — the differentiator — **recursively unwraps nested +wrappers**, so a base64'd, gzip'd binary-plist comes back as the whole chain, +each link scored and cited. It never guesses one answer for an ambiguous blob; it +ranks every plausible reading by honest confidence. + +```console +$ B64=$(printf '{"user":"alice"}' | gzip | base64 | tr -d '\n') +$ blob-decode --string "$B64" +blob-decode: 48 bytes; 2 candidate reading(s), best first: + [MED ] base64 text — base64 text; decodes to 36 bytes + cite: RFC 4648 §4-5 (Base 64 / Base64url) + decodes to 36 bytes: + [HIGH] gzip stream — gzip stream; 16 bytes decompressed + cite: RFC 1952 (GZIP file format) + decodes to 16 bytes: + [HIGH] JSON — JSON object with 1 keys + cite: RFC 8259 (JSON) + [LOW ] UTF-8 text — UTF-8 text preview: "H4sIAD4mUWoAA6tWKi1OLVKyUkrMyUxOVaoFAPcasFUQAAAA" + cite: RFC 3629 (UTF-8) +``` + +The base64 is also *technically* UTF-8, so that reading is offered too — at Low +confidence, ranked last. Nothing is hidden; everything is scored. + +**[Full documentation →](https://securityronin.github.io/blob-decoder/)** + +## What it recognises + +| Kind | How | Confidence | +|---|---|---| +| binary plist, XML plist | `bplist00` magic / ` int: + cov_path = sys.argv[1] if len(sys.argv) > 1 else "cov.json" + data = json.load(open(cov_path, encoding="utf-8")) + funcs = data["data"][0]["functions"] + + # Merge instantiations by SOURCE LOCATION, not symbol name: llvm-cov embeds + # the crate-disambiguator hash in each mangled name, so the same source + # function compiled into different test binaries has different names but the + # same (file, line, col). Covered if ANY instantiation at that location ran. + merged = defaultdict(lambda: {"count": 0, "file": None, "line": None}) + for f in funcs: + files = f.get("filenames") or ["?"] + file = files[0] + regions = f.get("regions", []) + if not regions: + continue + start = min(regions, key=lambda r: (r[0], r[1])) + line, col = start[0], start[1] + key = (file, line, col) + m = merged[key] + m["count"] += f["count"] + m["file"] = file + m["line"] = line + + src_cache: dict[str, list[str]] = {} + + def marked(file: str, line: int) -> bool: + # The marker may sit on the function/closure line or the line above. + if file not in src_cache: + try: + src_cache[file] = open(file, encoding="utf-8").read().splitlines() + except OSError: + src_cache[file] = [] + lines = src_cache[file] + # Check the closure/function line and up to two lines above it, so a + # short multi-line `// cov:unreachable: …` comment block counts. + for ln in (line - 2, line - 1, line): + if 0 <= ln - 1 < len(lines) and MARKER in lines[ln - 1]: + return True + return False + + failing, exempted = [], [] + for (file, line, _col), m in merged.items(): + if m["count"] > 0: + continue + # Only gate the crate's own LIBRARY source (skip tests/ and deps). main.rs + # is the Humble-Object CLI shell — exercised by tests/cli.rs but not gated + # to 100% (Coverage discipline: gate the testable library, not thin glue). + if "/src/" not in file or "/tests/" in file or file.endswith("/main.rs"): + continue + short = file.split("/src/")[-1] + if marked(file, line): + exempted.append(f"{short}:{line}") + else: + failing.append(f"{short}:{line}") + + if exempted: + print(f"cov:unreachable exemptions ({len(exempted)}):") + for e in sorted(exempted): + print(f" - {e}") + if failing: + print(f"\nUNCOVERED functions without a `// cov:unreachable` marker ({len(failing)}):") + for x in sorted(failing): + print(f" ✗ {x}") + print("\nFAIL: cover these, or annotate provably-dead arms with `// cov:unreachable: `.") + return 1 + print("\nOK: every uncovered function is an annotated, provably-unreachable arm.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/identify.rs b/src/identify.rs new file mode 100644 index 0000000..c291c18 --- /dev/null +++ b/src/identify.rs @@ -0,0 +1,544 @@ +//! The orchestration engine: identify → dispatch → score → recursively unwrap. +//! +//! Every detector here is a thin dispatcher over a mature crate (`plist`, +//! `flate2`, `snap`, `base64`, `hex`, `uuid`, `serde_json`) — this module owns +//! only the *identification*, *scoring*, and *recursive unwrap*, never the codec +//! itself. All input is attacker-controllable, so the invariant is: no panic, no +//! OOM (every decompression is size-capped and the recursion is depth-capped), +//! and every failure degrades to a lower-confidence or absent reading. + +use std::io::{Cursor, Read}; + +use base64::Engine as _; + +use crate::{BlobKind, Candidate, Confidence, DecodedChain, Limits}; + +/// Identify a blob with default [`Limits`]. Returns scored candidates, best +/// (highest [`Confidence`]) first. Always returns at least one candidate (an +/// [`BlobKind::Unknown`] reading that surfaces the raw head bytes when nothing +/// else matched). +#[must_use] +pub fn identify(bytes: &[u8]) -> Vec { + identify_with_limits(bytes, Limits::default(), 0) +} + +/// Identify a blob with explicit resource [`Limits`] and a starting recursion +/// `depth` — the entry point the recursive unwrap calls into. +#[must_use] +pub fn identify_with_limits(bytes: &[u8], limits: Limits, depth: usize) -> Vec { + let mut out: Vec = Vec::new(); + + // Strong magic / full-parse detectors always run (cheap, high-signal). + push(&mut out, detect_binary_plist(bytes)); + push(&mut out, detect_xml_plist(bytes)); + push(&mut out, detect_gzip(bytes, limits, depth)); + push(&mut out, detect_zlib(bytes, limits, depth)); + push(&mut out, detect_snappy(bytes, limits, depth)); + push(&mut out, detect_json(bytes)); + push(&mut out, detect_uuid_string(bytes)); + + // Heuristic detectors are bounded by max_input (they scan / decode the whole + // blob), so huge inputs get magic-only identification. + if bytes.len() <= limits.max_input { + push(&mut out, detect_base64(bytes, limits, depth)); + push(&mut out, detect_hex(bytes, limits, depth)); + push(&mut out, detect_uuid_bytes(bytes)); + push(&mut out, detect_utf16le(bytes)); + push(&mut out, detect_utf8_text(bytes)); + } + + if out.is_empty() { + out.push(unknown(bytes)); + } + + out.sort_by(|a, b| { + b.score + .cmp(&a.score) + .then_with(|| kind_rank(b.kind).cmp(&kind_rank(a.kind))) + .then_with(|| a.kind.label().cmp(b.kind.label())) + }); + out +} + +fn push(out: &mut Vec, c: Option) { + if let Some(c) = c { + out.push(c); + } +} + +/// Specificity tiebreak when two candidates share a [`Confidence`]: a concrete +/// magic-identified type outranks a generic wrapper, which outranks bare text. +fn kind_rank(kind: BlobKind) -> u8 { + match kind { + BlobKind::BinaryPlist + | BlobKind::XmlPlist + | BlobKind::Gzip + | BlobKind::Zlib + | BlobKind::Snappy + | BlobKind::Json + | BlobKind::Uuid => 3, + BlobKind::Base64 | BlobKind::Hex => 2, + BlobKind::Utf16Le | BlobKind::Utf8Text => 1, + BlobKind::Unknown => 0, + } +} + +// --------------------------------------------------------------------------- +// Wrapper payload recursion +// --------------------------------------------------------------------------- + +/// Build the [`DecodedChain`] for a wrapper's decoded payload: recurse to +/// identify it, unless the depth cap is reached (the DoS backstop for +/// infinitely-nested wrappers). +fn build_chain(data: &[u8], capped: bool, limits: Limits, depth: usize) -> DecodedChain { + let best = if depth + 1 >= limits.max_depth { + Box::new(depth_capped(data)) + } else { + identify_with_limits(data, limits, depth + 1) + .into_iter() + .next() + // cov:unreachable: identify_with_limits never returns empty (pushes Unknown), kept defensive fallback. + .map_or_else(|| Box::new(unknown(data)), Box::new) + }; + DecodedChain { + decoded_len: data.len(), + capped, + best, + } +} + +fn depth_capped(data: &[u8]) -> Candidate { + Candidate { + kind: BlobKind::Unknown, + score: Confidence::Low, + summary: format!( + "recursion depth cap reached; {} bytes not further decoded (head: {})", + data.len(), + head_hex(data) + ), + citation: BlobKind::Unknown.citation(), + inner: None, + } +} + +// --------------------------------------------------------------------------- +// Bounded decompression (the decompression-bomb guard) +// --------------------------------------------------------------------------- + +/// Read at most `cap` bytes from `r`; return the bytes and whether the stream +/// was *capped* (had more to give). Bounds memory to `cap`, so a decompression +/// bomb can never exhaust it. +fn bounded_read(r: R, cap: usize) -> std::io::Result<(Vec, bool)> { + let mut out = Vec::new(); + // take(cap+1): if we get cap+1 bytes the stream had more → it was capped. + r.take(cap as u64 + 1).read_to_end(&mut out)?; + let capped = out.len() > cap; + if capped { + out.truncate(cap); + } + Ok((out, capped)) +} + +// --------------------------------------------------------------------------- +// Detectors — strong magic / full parse +// --------------------------------------------------------------------------- + +fn detect_binary_plist(bytes: &[u8]) -> Option { + if !bytes.starts_with(b"bplist") { + return None; + } + Some(match plist::Value::from_reader(Cursor::new(bytes)) { + Ok(v) => leaf( + BlobKind::BinaryPlist, + Confidence::High, + format!("binary plist: {}", describe_plist(&v)), + ), + Err(e) => leaf( + BlobKind::BinaryPlist, + Confidence::Medium, + format!("bplist magic but parse failed: {e}"), + ), + }) +} + +fn detect_xml_plist(bytes: &[u8]) -> Option { + let head = bytes.trim_ascii_start(); + // Bounded prefix scan for the plist markers, so a huge non-plist XML body is + // rejected cheaply. Require the `plist` token to distinguish it from any XML. + let probe = &head[..head.len().min(1024)]; + let starts_xml = probe.starts_with(b" leaf( + BlobKind::XmlPlist, + Confidence::High, + format!("XML plist: {}", describe_plist(&v)), + ), + Err(e) => leaf( + BlobKind::XmlPlist, + Confidence::Medium, + format!("XML plist markup but parse failed: {e}"), + ), + }) +} + +fn detect_gzip(bytes: &[u8], limits: Limits, depth: usize) -> Option { + // RFC 1952: magic 1f 8b. A 2-byte magic is specific enough that a decode + // failure is still worth reporting (as a truncated/corrupt gzip). + if !bytes.starts_with(&[0x1f, 0x8b]) { + return None; + } + match bounded_read(flate2::read::GzDecoder::new(bytes), limits.max_output) { + Ok((data, capped)) => Some(wrapper( + BlobKind::Gzip, + Confidence::High, + format!( + "gzip stream; {} bytes decompressed{}", + data.len(), + if capped { " (capped at limit)" } else { "" } + ), + build_chain(&data, capped, limits, depth), + )), + Err(e) => Some(leaf( + BlobKind::Gzip, + Confidence::Medium, + format!("gzip magic but decompression failed: {e}"), + )), + } +} + +fn detect_zlib(bytes: &[u8], limits: Limits, depth: usize) -> Option { + // RFC 1950 header has no unique magic — only CM=deflate + the FCHECK mod-31 + // constraint (a weak ~1/500 filter). So we claim zlib ONLY on a SUCCESSFUL + // decompress; a header match that fails to inflate is treated as coincidence + // (returns None), never a false Medium on random bytes. + if bytes.len() < 2 { + return None; + } + let (cmf, flg) = (bytes[0], bytes[1]); + if cmf & 0x0f != 0x08 || cmf >> 4 > 7 { + return None; + } + if !((u16::from(cmf) << 8) | u16::from(flg)).is_multiple_of(31) { + return None; + } + let (data, capped) = + bounded_read(flate2::read::ZlibDecoder::new(bytes), limits.max_output).ok()?; + Some(wrapper( + BlobKind::Zlib, + Confidence::High, + format!( + "zlib stream; {} bytes decompressed{}", + data.len(), + if capped { " (capped at limit)" } else { "" } + ), + build_chain(&data, capped, limits, depth), + )) +} + +fn detect_snappy(bytes: &[u8], limits: Limits, depth: usize) -> Option { + // Snappy framing format: the stream identifier chunk (0xff + "sNaPpY"). + const MAGIC: &[u8] = &[0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59]; + if !bytes.starts_with(MAGIC) { + return None; + } + match bounded_read(snap::read::FrameDecoder::new(bytes), limits.max_output) { + Ok((data, capped)) => Some(wrapper( + BlobKind::Snappy, + Confidence::High, + format!( + "Snappy framed stream; {} bytes decompressed{}", + data.len(), + if capped { " (capped at limit)" } else { "" } + ), + build_chain(&data, capped, limits, depth), + )), + Err(e) => Some(leaf( + BlobKind::Snappy, + Confidence::Medium, + format!("Snappy magic but decompression failed: {e}"), + )), + } +} + +fn detect_json(bytes: &[u8]) -> Option { + let trimmed = bytes.trim_ascii(); + // Only object/array roots are claimed as JSON: a bare `123` or `true` is + // technically JSON but too ambiguous to assert. + if !matches!(trimmed.first(), Some(b'{' | b'[')) { + return None; + } + let value: serde_json::Value = serde_json::from_slice(trimmed).ok()?; + Some(leaf( + BlobKind::Json, + Confidence::High, + describe_json(&value), + )) +} + +fn detect_uuid_string(bytes: &[u8]) -> Option { + let s = std::str::from_utf8(bytes).ok()?.trim(); + // Require the hyphenated (or braced/urn) canonical form: a bare 32-hex run is + // better read as hex, so we do not claim it here. + if !s.contains('-') { + return None; + } + let u = uuid::Uuid::try_parse(s).ok()?; + Some(leaf( + BlobKind::Uuid, + Confidence::High, + format!( + "UUID {u} (version {}, variant {:?})", + u.get_version_num(), + u.get_variant() + ), + )) +} + +// --------------------------------------------------------------------------- +// Detectors — heuristic (coincidence-prone, scored Low unless the payload is +// itself recognised) +// --------------------------------------------------------------------------- + +fn detect_base64(bytes: &[u8], limits: Limits, depth: usize) -> Option { + let decoded = try_base64(bytes)?; + let chain = build_chain(&decoded, false, limits, depth); + let score = wrapper_score(chain.best.kind); + Some(wrapper( + BlobKind::Base64, + score, + format!("base64 text; decodes to {} bytes", chain.decoded_len), + chain, + )) +} + +fn detect_hex(bytes: &[u8], limits: Limits, depth: usize) -> Option { + let s = bytes.trim_ascii(); + if s.len() < 4 || !s.len().is_multiple_of(2) || !s.iter().all(u8::is_ascii_hexdigit) { + return None; + } + let decoded = hex::decode(s).ok()?; + let chain = build_chain(&decoded, false, limits, depth); + let score = wrapper_score(chain.best.kind); + Some(wrapper( + BlobKind::Hex, + score, + format!("hexadecimal text; decodes to {} bytes", chain.decoded_len), + chain, + )) +} + +fn detect_uuid_bytes(bytes: &[u8]) -> Option { + let arr: [u8; 16] = bytes.try_into().ok()?; + let u = uuid::Uuid::from_bytes(arr); + Some(leaf( + BlobKind::Uuid, + // Any 16 bytes form a syntactically valid UUID — never over-claim. + Confidence::Low, + format!("if a UUID: {u} (note: any 16 bytes form a valid UUID)"), + )) +} + +fn detect_utf16le(bytes: &[u8]) -> Option { + if bytes.len() < 4 || !bytes.len().is_multiple_of(2) { + return None; + } + let has_bom = bytes.starts_with(&[0xff, 0xfe]); + let body = if has_bom { &bytes[2..] } else { bytes }; + let units: Vec = body + .chunks_exact(2) + .map(|c| u16::from_le_bytes([c[0], c[1]])) + .collect(); + if units.is_empty() { + return None; + } + let text = String::from_utf16(&units).ok()?; + if !mostly_printable(&text) { + return None; + } + // Without a BOM, require the ASCII-plane dominance that real UTF-16LE text + // shows (high byte zero), else even-length binary would masquerade as text. + let ascii_plane = body.chunks_exact(2).filter(|c| c[1] == 0).count(); + if !has_bom && (ascii_plane * 2) < units.len() { + return None; + } + Some(leaf( + BlobKind::Utf16Le, + if has_bom { + Confidence::Medium + } else { + Confidence::Low + }, + format!("UTF-16LE text preview: \"{}\"", preview(&text)), + )) +} + +fn detect_utf8_text(bytes: &[u8]) -> Option { + let s = std::str::from_utf8(bytes).ok()?; + if s.is_empty() || !mostly_printable(s) { + return None; + } + Some(leaf( + BlobKind::Utf8Text, + Confidence::Low, + format!("UTF-8 text preview: \"{}\"", preview(s)), + )) +} + +// --------------------------------------------------------------------------- +// Scoring / construction helpers +// --------------------------------------------------------------------------- + +/// A wrapper (base64/hex) whose decoded payload is itself a concrete recognised +/// type is Medium; one that decodes only to opaque bytes or plain text is Low +/// (the decode was probably coincidental). +fn wrapper_score(inner: BlobKind) -> Confidence { + match inner { + BlobKind::Unknown | BlobKind::Utf8Text | BlobKind::Utf16Le => Confidence::Low, + _ => Confidence::Medium, + } +} + +fn leaf(kind: BlobKind, score: Confidence, summary: String) -> Candidate { + Candidate { + kind, + score, + summary, + citation: kind.citation(), + inner: None, + } +} + +fn wrapper(kind: BlobKind, score: Confidence, summary: String, chain: DecodedChain) -> Candidate { + Candidate { + kind, + score, + summary, + citation: kind.citation(), + inner: Some(Box::new(chain)), + } +} + +fn unknown(bytes: &[u8]) -> Candidate { + Candidate { + kind: BlobKind::Unknown, + score: Confidence::Low, + summary: if bytes.is_empty() { + "unrecognized: empty input".to_owned() + } else { + format!( + "unrecognized; {} bytes (head: {})", + bytes.len(), + head_hex(bytes) + ) + }, + citation: BlobKind::Unknown.citation(), + inner: None, + } +} + +/// Validate + decode base64 (standard or URL-safe, whitespace-tolerant). Returns +/// the decoded bytes, or `None` if the input is not clean, correctly-padded +/// base64 — delegating the decode itself to the `base64` crate. +fn try_base64(bytes: &[u8]) -> Option> { + let cleaned: Vec = bytes + .iter() + .copied() + .filter(|b| !b.is_ascii_whitespace()) + .collect(); + if cleaned.len() < 8 || !cleaned.len().is_multiple_of(4) { + return None; + } + let eq = cleaned + .iter() + .position(|&b| b == b'=') + .unwrap_or(cleaned.len()); + let (body, padding) = cleaned.split_at(eq); + if padding.len() > 2 || padding.iter().any(|&b| b != b'=') || body.is_empty() { + return None; + } + let is_std = body + .iter() + .all(|&b| b.is_ascii_alphanumeric() || b == b'+' || b == b'/'); + let is_url = body + .iter() + .all(|&b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_'); + if is_std { + base64::engine::general_purpose::STANDARD + .decode(&cleaned) + .ok() + } else if is_url { + base64::engine::general_purpose::URL_SAFE + .decode(&cleaned) + .ok() + } else { + None + } +} + +fn describe_plist(v: &plist::Value) -> String { + match v { + plist::Value::Array(a) => format!("array with {} items", a.len()), + plist::Value::Dictionary(d) => format!("dict with {} entries", d.len()), + plist::Value::Boolean(_) => "boolean".to_owned(), + plist::Value::Data(d) => format!("data ({} bytes)", d.len()), + plist::Value::Date(_) => "date".to_owned(), + plist::Value::Real(_) => "real".to_owned(), + plist::Value::Integer(_) => "integer".to_owned(), + plist::Value::String(_) => "string".to_owned(), + plist::Value::Uid(_) => "uid".to_owned(), + _ => "value".to_owned(), + } +} + +fn describe_json(v: &serde_json::Value) -> String { + match v { + serde_json::Value::Object(m) => format!("JSON object with {} keys", m.len()), + serde_json::Value::Array(a) => format!("JSON array with {} elements", a.len()), + // Unreachable: detect_json only enters on `{`/`[` roots. + _ => "JSON value".to_owned(), + } +} + +fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack.windows(needle.len()).any(|w| w == needle) +} + +fn mostly_printable(s: &str) -> bool { + let total = s.chars().count(); + if total == 0 { + return false; + } + let printable = s + .chars() + .filter(|c| !c.is_control() || matches!(c, '\t' | '\n' | '\r')) + .count(); + (printable * 100) >= (total * 90) +} + +fn head_hex(bytes: &[u8]) -> String { + let n = bytes.len().min(16); + let mut s = hex::encode(&bytes[..n]); + if bytes.len() > n { + s = format!("{s} (+{} more)", bytes.len() - n); + } + s +} + +fn preview(s: &str) -> String { + const MAX: usize = 48; + let flat: String = s + .chars() + .map(|c| if c.is_control() { ' ' } else { c }) + .collect(); + if flat.chars().count() <= MAX { + flat + } else { + let cut: String = flat.chars().take(MAX).collect(); + format!("{cut}… ({} chars total)", s.chars().count()) + } +} diff --git a/src/lib.rs b/src/lib.rs index 03eb2a2..e1973a9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,4 +9,195 @@ //! `uuid`, `flate2`, `snap`, `serde_json`); this crate adds only the //! orchestration layer: identify → dispatch → score → recursively unwrap, plus a //! clean forensic result type. +//! +//! # Epistemics +//! +//! A blob is often underdetermined: bytes that are valid hex are frequently also +//! valid base64, and a run of ASCII is *technically* decodable as base64 to +//! gibberish. `blob-decoder` never asserts a single verdict — it returns every +//! plausible reading with an honest [`Confidence`], and a low-confidence reading +//! lowers the rank, never hides the finding. +//! +//! # Example +//! +//! ``` +//! // gzip magic (0x1f 0x8b) → identified as a Gzip wrapper. +//! let gz = b"\x1f\x8b\x08\x00\x00\x00\x00\x00"; +//! let cands = blob_decoder::identify(gz); +//! assert_eq!(cands[0].kind, blob_decoder::BlobKind::Gzip); +//! ``` #![forbid(unsafe_code)] +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))] + +pub mod identify; + +pub use identify::{identify, identify_with_limits}; + +/// The engine's version (`CARGO_PKG_VERSION`), for callers that surface it. +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); + +/// A recognised (or unrecognised) blob type. +/// +/// The `citation` and `label` are carried per-kind so a reading is traceable to +/// the authoritative format definition it was matched against. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum BlobKind { + /// Apple binary property list (`bplist00` magic). + BinaryPlist, + /// Apple XML property list. + XmlPlist, + /// gzip member (`1f 8b` magic). + Gzip, + /// zlib stream (RFC 1950 header). + Zlib, + /// Snappy framed stream. + Snappy, + /// base64 text (standard or URL-safe alphabet). + Base64, + /// Hexadecimal text. + Hex, + /// A UUID / GUID (16 raw bytes or the canonical hyphenated string). + Uuid, + /// JSON (object or array root). + Json, + /// UTF-16LE text. + Utf16Le, + /// UTF-8 text (printable). + Utf8Text, + /// No known type matched — the raw head bytes are reported for the analyst. + Unknown, +} + +impl BlobKind { + /// A short human label. + #[must_use] + pub fn label(self) -> &'static str { + match self { + Self::BinaryPlist => "Apple binary property list", + Self::XmlPlist => "Apple XML property list", + Self::Gzip => "gzip stream", + Self::Zlib => "zlib stream", + Self::Snappy => "Snappy framed stream", + Self::Base64 => "base64 text", + Self::Hex => "hexadecimal text", + Self::Uuid => "UUID / GUID", + Self::Json => "JSON", + Self::Utf16Le => "UTF-16LE text", + Self::Utf8Text => "UTF-8 text", + Self::Unknown => "unknown", + } + } + + /// The authoritative spec / reference this kind is matched against. + #[must_use] + pub fn citation(self) -> &'static str { + match self { + Self::BinaryPlist | Self::XmlPlist => { + "Apple CoreFoundation CFBinaryPList.c; `man 5 plist`" + } + Self::Gzip => "RFC 1952 (GZIP file format)", + Self::Zlib => "RFC 1950 (ZLIB compressed data format)", + Self::Snappy => "google/snappy framing_format.txt", + Self::Base64 => "RFC 4648 §4-5 (Base 64 / Base64url)", + Self::Hex => "RFC 4648 §8 (Base 16)", + Self::Uuid => "RFC 9562 (UUID)", + Self::Json => "RFC 8259 (JSON)", + Self::Utf16Le => "The Unicode Standard; RFC 2781 (UTF-16LE)", + Self::Utf8Text => "RFC 3629 (UTF-8)", + Self::Unknown => "no matching format", + } + } + + /// True when this kind is a *wrapper* whose payload is itself another blob + /// (base64/hex text, or a compression stream) — the recursion drivers. + #[must_use] + pub fn is_wrapper(self) -> bool { + matches!( + self, + Self::Gzip | Self::Zlib | Self::Snappy | Self::Base64 | Self::Hex + ) + } +} + +/// How strongly the evidence supports a reading. Ordered `Low < Medium < High` +/// so candidates sort best-first by *descending* confidence. +/// +/// - [`Confidence::High`] — a strong, near-unique MAGIC signature or a full +/// successful structural parse (`bplist00`, `1f 8b`, a valid RFC 1950 header, +/// a parseable JSON object, a canonical hyphenated UUID string). +/// - [`Confidence::Medium`] — a magic matched but the payload failed to fully +/// decode, or a heuristic wrapper (base64/hex) whose decoded payload was +/// itself recognised as a concrete type. +/// - [`Confidence::Low`] — a purely structural heuristic that a random blob +/// could satisfy by coincidence (16 arbitrary bytes as a UUID; base64/hex text +/// decoding only to more opaque bytes; plain printable text). +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Confidence { + /// A coincidence-prone structural heuristic. + Low, + /// A magic matched but payload decode was partial, or a heuristic wrapper + /// whose payload was recognised. + Medium, + /// A strong magic signature or a full successful parse. + High, +} + +/// One scored, cited candidate reading of a blob. A wrapper candidate nests the +/// identification of its decoded payload in [`Candidate::inner`], so a +/// `base64 → gzip → binary-plist` blob reports the whole chain. +#[derive(Debug, Clone, serde::Serialize)] +pub struct Candidate { + /// The identified (or unidentified) kind. + pub kind: BlobKind, + /// How strongly the evidence supports this reading. + pub score: Confidence, + /// A human summary of *what was found* (root type, byte counts, decoded + /// text prefix, or — for [`BlobKind::Unknown`] — the raw head bytes). + pub summary: String, + /// The authoritative spec citation for [`Candidate::kind`]. + pub citation: &'static str, + /// For a wrapper kind, the identification of the decoded/decompressed + /// payload — the next link in the chain. `None` for a leaf reading, a failed + /// decode, or when the recursion depth cap was reached. + #[serde(skip_serializing_if = "Option::is_none")] + pub inner: Option>, +} + +/// The decoded payload of a wrapper [`Candidate`]: how many bytes it produced +/// and the best reading of those bytes. +#[derive(Debug, Clone, serde::Serialize)] +pub struct DecodedChain { + /// Number of bytes the wrapper decoded/decompressed to (after any cap). + pub decoded_len: usize, + /// True when the decoded output was truncated at the size cap (a possible + /// decompression bomb) — the payload reading is of the capped prefix. + pub capped: bool, + /// The best (highest-confidence) reading of the decoded payload. + pub best: Box, +} + +/// Resource bounds for [`identify_with_limits`] — the guard against +/// decompression bombs and infinitely-nested wrappers on untrusted input. +#[derive(Debug, Clone, Copy)] +pub struct Limits { + /// Maximum recursion depth through nested wrappers. + pub max_depth: usize, + /// Maximum bytes to hold from a single decompression/decode step (a + /// decompression bomb is capped here, never allowed to exhaust memory). + pub max_output: usize, + /// Inputs larger than this skip the *heuristic* decoders (base64/hex/text); + /// magic-signature detection still runs. Bounds worst-case work. + pub max_input: usize, +} + +impl Default for Limits { + fn default() -> Self { + Self { + max_depth: 8, + max_output: 64 * 1024 * 1024, + max_input: 128 * 1024 * 1024, + } + } +} diff --git a/src/main.rs b/src/main.rs index fb1e31c..32094df 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,141 @@ //! `blob-decode` CLI — a thin Humble-Object shell over the `blob-decoder` engine. +//! +//! Hand it a blob (from a file, stdin, or inline as `--string`/`--hex`/ +//! `--base64`) and it prints the scored, cited candidate readings, recursively +//! unwrapping any nested wrappers. Exit codes are pipeline-safe: `0` a concrete +//! type was identified, `2` nothing matched (Unknown), `1` an input error. #![forbid(unsafe_code)] +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))] -fn main() { - // Scaffold placeholder; the CLI is implemented on the feat/blob-decoder branch. +use std::io::Read; +use std::process::ExitCode; + +use base64::Engine as _; +use blob_decoder::{identify, BlobKind, Candidate, Confidence}; +use clap::Parser; + +/// Identify and decode opaque blobs of unknown type. +#[derive(Parser, Debug)] +#[command( + name = "blob-decode", + version, + about = "Identify and decode opaque blobs of unknown type — scored, cited candidates." +)] +struct Cli { + /// File to read the blob from ('-' or omitted reads stdin). + file: Option, + /// Treat this literal string's UTF-8 bytes as the blob. + #[arg(long, conflicts_with_all = ["hex", "base64", "file"])] + string: Option, + /// Decode this hex string into the blob's bytes. + #[arg(long, conflicts_with_all = ["string", "base64", "file"])] + hex: Option, + /// Decode this base64 string into the blob's bytes. + #[arg(long = "base64", conflicts_with_all = ["string", "hex", "file"])] + base64: Option, + /// Emit JSON instead of the human-readable tree. + #[arg(long)] + json: bool, + /// Show at most N readings (default: all). + #[arg(long, value_name = "N")] + top: Option, +} + +fn main() -> ExitCode { + let cli = Cli::parse(); + + let bytes = match load(&cli) { + Ok(b) => b, + Err(e) => { + eprintln!("blob-decode: {e}"); + return ExitCode::from(1); + } + }; + + let mut cands = identify(&bytes); + if let Some(n) = cli.top { + cands.truncate(n); + } + + if cli.json { + match serde_json::to_string_pretty(&cands) { + Ok(s) => println!("{s}"), + Err(e) => { + eprintln!("blob-decode: serialization failed: {e}"); + return ExitCode::from(1); + } + } + } else { + print_human(&bytes, &cands); + } + + // Pipeline signal: 2 when nothing concrete was identified (best is Unknown). + if cands.first().is_none_or(|c| c.kind == BlobKind::Unknown) { + ExitCode::from(2) + } else { + ExitCode::SUCCESS + } +} + +/// Resolve the blob bytes from whichever input source was given. +fn load(cli: &Cli) -> Result, String> { + if let Some(s) = &cli.string { + return Ok(s.clone().into_bytes()); + } + if let Some(h) = &cli.hex { + return hex::decode(h.trim()).map_err(|e| format!("invalid hex input: {e}")); + } + if let Some(b) = &cli.base64 { + return base64::engine::general_purpose::STANDARD + .decode(b.trim()) + .map_err(|e| format!("invalid base64 input: {e}")); + } + match cli.file.as_deref() { + None | Some("-") => { + let mut buf = Vec::new(); + std::io::stdin() + .read_to_end(&mut buf) + .map_err(|e| format!("reading stdin: {e}"))?; + Ok(buf) + } + Some(path) => std::fs::read(path).map_err(|e| format!("{path}: {e}")), + } +} + +fn print_human(bytes: &[u8], cands: &[Candidate]) { + println!( + "blob-decode: {} bytes; {} candidate reading(s), best first:", + bytes.len(), + cands.len() + ); + for c in cands { + print_candidate(c, 1); + } +} + +fn print_candidate(c: &Candidate, indent: usize) { + let pad = " ".repeat(indent); + println!( + "{pad}[{}] {} — {}", + conf_str(c.score), + c.kind.label(), + c.summary + ); + println!("{pad} cite: {}", c.citation); + if let Some(chain) = &c.inner { + println!( + "{pad} decodes to {} bytes{}:", + chain.decoded_len, + if chain.capped { " (capped)" } else { "" } + ); + print_candidate(&chain.best, indent + 1); + } +} + +fn conf_str(c: Confidence) -> &'static str { + match c { + Confidence::High => "HIGH", + Confidence::Medium => "MED ", + Confidence::Low => "LOW ", + } } diff --git a/tests/api.rs b/tests/api.rs new file mode 100644 index 0000000..f014877 --- /dev/null +++ b/tests/api.rs @@ -0,0 +1,95 @@ +#![allow(clippy::unwrap_used, clippy::expect_used)] +//! Core API contract: confidence ordering, best-first sorting, the always-present +//! Unknown fallback, honest scoring on junk, and lossless serialization. + +use blob_decoder::{identify, BlobKind, Confidence}; + +#[test] +fn confidence_orders_high_above_medium_above_low() { + assert!(Confidence::High > Confidence::Medium); + assert!(Confidence::Medium > Confidence::Low); +} + +#[test] +fn empty_input_yields_unknown() { + let cands = identify(b""); + assert!( + !cands.is_empty(), + "must always return at least one candidate" + ); + assert_eq!(cands[0].kind, BlobKind::Unknown); +} + +#[test] +fn unknown_summary_shows_the_raw_head_bytes() { + // Robustness: an "unknown" finding must surface the actual bytes, in hex. + let junk = [0xDEu8, 0xAD, 0xBE, 0xEF]; + let cands = identify(&junk); + let unknown = cands + .iter() + .find(|c| c.kind == BlobKind::Unknown) + .expect("unknown candidate present"); + assert!( + unknown.summary.to_lowercase().contains("dead"), + "unknown summary should include the head bytes in hex, got: {}", + unknown.summary + ); +} + +#[test] +fn random_bytes_never_claim_high_confidence() { + // 21 arbitrary bytes: not a UUID (not 16), not valid magic — must degrade to + // Low/Unknown, never a confident verdict. + let junk = [ + 0x37u8, 0x9a, 0x02, 0xf1, 0x13, 0x88, 0xcc, 0x01, 0x77, 0x42, 0x9e, 0xde, 0x05, 0xba, 0xad, + 0xf3, 0x0d, 0x11, 0x22, 0x33, 0x44, + ]; + let cands = identify(&junk); + assert!( + cands.iter().all(|c| c.score < Confidence::High), + "random bytes must not produce a High-confidence reading" + ); +} + +#[test] +fn candidates_are_sorted_best_first() { + let gz = b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00"; + let cands = identify(gz); + for w in cands.windows(2) { + assert!(w[0].score >= w[1].score, "candidates must be sorted desc"); + } +} + +#[test] +fn output_serializes_to_json() { + let cands = identify(b"{\"a\":1}"); + let json = serde_json::to_string(&cands).expect("candidates serialize"); + assert!(json.contains("json")); +} + +#[test] +fn blobkind_wrapper_classification() { + assert!(BlobKind::Gzip.is_wrapper()); + assert!(BlobKind::Base64.is_wrapper()); + assert!(BlobKind::Hex.is_wrapper()); + assert!(!BlobKind::Json.is_wrapper()); + assert!(!BlobKind::Unknown.is_wrapper()); + // label + citation are populated for every kind. + assert!(!BlobKind::BinaryPlist.label().is_empty()); + assert!(BlobKind::Uuid.citation().contains("9562")); +} + +#[test] +fn hex_and_base64_readings_coexist_and_sort_deterministically() { + // "deadbeefdeadbeef" is BOTH valid hex (16 nibbles) and valid base64 (len%4=0), + // each decoding to opaque bytes → two Low, equal-rank readings. Exercises the + // final label tiebreak in the sort. + let cands = identify(b"deadbeefdeadbeef"); + assert!(cands.iter().any(|c| c.kind == BlobKind::Hex)); + assert!(cands.iter().any(|c| c.kind == BlobKind::Base64)); + // Deterministic order across runs. + let again = identify(b"deadbeefdeadbeef"); + let k1: Vec<_> = cands.iter().map(|c| c.kind).collect(); + let k2: Vec<_> = again.iter().map(|c| c.kind).collect(); + assert_eq!(k1, k2); +} diff --git a/tests/cli.rs b/tests/cli.rs new file mode 100644 index 0000000..ce88ee0 --- /dev/null +++ b/tests/cli.rs @@ -0,0 +1,61 @@ +#![allow(clippy::unwrap_used, clippy::expect_used)] +//! CLI contract for `blob-decode` (a thin Humble-Object shell): input sources +//! (--string / --hex / --base64 / file / stdin), human + JSON output, and +//! pipeline-safe exit codes (0 identified, 2 unknown, 1 error). + +use std::io::Write; +use std::process::{Command, Stdio}; + +fn bin() -> Command { + Command::new(env!("CARGO_BIN_EXE_blob-decode")) +} + +#[test] +fn string_input_identifies_json_human() { + let out = bin().args(["--string", "{\"a\":1}"]).output().unwrap(); + assert!(out.status.success()); + let s = String::from_utf8_lossy(&out.stdout).to_lowercase(); + assert!(s.contains("json"), "stdout: {s}"); +} + +#[test] +fn json_output_is_valid_json_array() { + let out = bin() + .args(["--string", "[1,2,3]", "--json"]) + .output() + .unwrap(); + assert!(out.status.success()); + let v: serde_json::Value = serde_json::from_slice(&out.stdout).expect("valid JSON"); + assert!(v.is_array()); +} + +#[test] +fn stdin_is_read_when_no_source_given() { + let mut child = bin() + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(b"{\"a\":1}").unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success()); + assert!(String::from_utf8_lossy(&out.stdout) + .to_lowercase() + .contains("json")); +} + +#[test] +fn unknown_input_exits_2() { + // 0xDEADBEEF: four bytes, no known type → Unknown → exit 2. + let out = bin().args(["--hex", "deadbeef"]).output().unwrap(); + assert_eq!(out.status.code(), Some(2)); +} + +#[test] +fn invalid_hex_exits_1() { + let out = bin().args(["--hex", "zzzz"]).output().unwrap(); + assert_eq!(out.status.code(), Some(1)); + assert!(String::from_utf8_lossy(&out.stderr) + .to_lowercase() + .contains("hex")); +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000..7884ce0 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,88 @@ +//! Shared helpers for producing REAL blob inputs with independent, standard +//! tools (python3 `plistlib`/`zlib`, system `gzip`/`base64`/`uuidgen`). Every +//! producer is env-gated: when the tool is absent the test SKIPs cleanly rather +//! than failing (Test-Data Provenance: prefer real, tool-produced inputs over +//! self-authored fixtures — the producer is independent of the crate under test). + +#![allow(dead_code)] + +use std::io::Write; +use std::process::{Command, Stdio}; + +/// Run `program args…`, feed `input` on stdin, return stdout bytes. `None` if the +/// program is missing or exits non-zero (caller SKIPs the test). +pub fn run_piped(program: &str, args: &[&str], input: &[u8]) -> Option> { + let mut child = Command::new(program) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .ok()?; + child.stdin.take()?.write_all(input).ok()?; + let out = child.wait_with_output().ok()?; + if out.status.success() { + Some(out.stdout) + } else { + None + } +} + +/// Run `program args…` with no stdin, return stdout bytes. +pub fn run(program: &str, args: &[&str]) -> Option> { + let out = Command::new(program).args(args).output().ok()?; + if out.status.success() { + Some(out.stdout) + } else { + None + } +} + +/// A binary plist of `{"a": 1, "b": [1, 2]}` produced by python3 `plistlib`. +pub fn bplist_dict() -> Option> { + run( + "python3", + &[ + "-c", + "import plistlib,sys; sys.stdout.buffer.write(plistlib.dumps({'a':1,'b':[1,2]}, fmt=plistlib.FMT_BINARY))", + ], + ) +} + +/// An XML plist of the same dict produced by python3 `plistlib`. +pub fn xml_plist_dict() -> Option> { + run( + "python3", + &[ + "-c", + "import plistlib,sys; sys.stdout.buffer.write(plistlib.dumps({'a':1,'b':[1,2]}, fmt=plistlib.FMT_XML))", + ], + ) +} + +/// zlib-compress `data` via python3 `zlib` (independent of flate2). +pub fn zlib_compress(data: &[u8]) -> Option> { + let mut child = Command::new("python3") + .args([ + "-c", + "import sys,zlib; sys.stdout.buffer.write(zlib.compress(sys.stdin.buffer.read()))", + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .ok()?; + child.stdin.take()?.write_all(data).ok()?; + let out = child.wait_with_output().ok()?; + out.status.success().then_some(out.stdout) +} + +/// gzip-compress `data` via the system `gzip` (independent of flate2). +pub fn gzip(data: &[u8]) -> Option> { + run_piped("gzip", &["-c"], data) +} + +/// base64-encode `data` via the system `base64` (independent of the base64 crate). +pub fn base64(data: &[u8]) -> Option> { + run_piped("base64", &[], data) +} diff --git a/tests/data/README.md b/tests/data/README.md new file mode 100644 index 0000000..b93b990 --- /dev/null +++ b/tests/data/README.md @@ -0,0 +1,24 @@ +# Test data provenance + +Small, self-produced fixtures committed so the plist decode paths run everywhere +(not only where `python3` is installed). Larger, tool-produced inputs are +generated at test time by `tests/common/mod.rs` and are not committed. + +| File | Source | Contents | MD5 | License | Used by | +|---|---|---|---|---|---| +| `sample.bplist` | Generated with CPython `plistlib.dumps(..., fmt=FMT_BINARY)` (Python 3.11) | Apple binary plist of `{"name":"blob","ints":[1,2,3],"ok":true}` | `d9b5d192fc830e705aecc6586571d2fa` | CC0-1.0 (self-authored, no third-party content) | `tests/fixtures.rs` | +| `sample.plist` | Generated with CPython `plistlib.dumps(..., fmt=FMT_XML)` (Python 3.11) | Apple XML plist of the same dict | `65852730501a0fd8f0dc0a0079b5d197` | CC0-1.0 (self-authored) | `tests/fixtures.rs` | + +Regenerate with: + +```bash +python3 -c "import plistlib; open('tests/data/sample.bplist','wb').write(plistlib.dumps({'name':'blob','ints':[1,2,3],'ok':True}, fmt=plistlib.FMT_BINARY))" +python3 -c "import plistlib; open('tests/data/sample.plist','wb').write(plistlib.dumps({'name':'blob','ints':[1,2,3],'ok':True}, fmt=plistlib.FMT_XML))" +``` + +These are structural fixtures for identification tests, not a ground-truth +correctness oracle for the `plist` crate itself (the `plist` crate is the +established reference for the format; `blob-decoder` only dispatches to it). The +tier-2 validation — real inputs produced by independent tools (system `gzip` / +`base64`, python3 `zlib`) and decoded back — lives in `tests/magic.rs`, +`tests/nested.rs`, and `docs/validation.md`. diff --git a/tests/data/sample.bplist b/tests/data/sample.bplist new file mode 100644 index 0000000..8446fd5 Binary files /dev/null and b/tests/data/sample.bplist differ diff --git a/tests/data/sample.plist b/tests/data/sample.plist new file mode 100644 index 0000000..0c3b795 --- /dev/null +++ b/tests/data/sample.plist @@ -0,0 +1,16 @@ + + + + + ints + + 1 + 2 + 3 + + name + blob + ok + + + diff --git a/tests/fixtures.rs b/tests/fixtures.rs new file mode 100644 index 0000000..3e90da8 --- /dev/null +++ b/tests/fixtures.rs @@ -0,0 +1,47 @@ +#![allow(clippy::unwrap_used, clippy::expect_used)] +//! Identification against committed, real plist fixtures (produced by python3 +//! `plistlib`; see tests/data/README.md). Unlike the tool-gated tests these run +//! everywhere, so the plist decode paths are always exercised. + +use blob_decoder::{identify, BlobKind, Confidence}; + +const BPLIST: &[u8] = include_bytes!("data/sample.bplist"); +const XMLPLIST: &[u8] = include_bytes!("data/sample.plist"); + +#[test] +fn committed_binary_plist_is_identified() { + let cands = identify(BPLIST); + assert_eq!(cands[0].kind, BlobKind::BinaryPlist); + assert_eq!(cands[0].score, Confidence::High); + assert!(cands[0].summary.contains("dict"), "{}", cands[0].summary); +} + +#[test] +fn committed_xml_plist_is_identified() { + let cands = identify(XMLPLIST); + assert_eq!(cands[0].kind, BlobKind::XmlPlist); + assert_eq!(cands[0].score, Confidence::High); +} + +#[test] +fn bplist_magic_with_garbage_body_is_medium_not_high() { + // Magic present, structure invalid → reported, but downgraded and payloadless. + let mut junk = b"bplist00".to_vec(); + junk.extend_from_slice(&[0xff, 0x00, 0x13, 0x37, 0xab]); + let cands = identify(&junk); + let bp = cands + .iter() + .find(|c| c.kind == BlobKind::BinaryPlist) + .expect("bplist reading present"); + assert!(bp.score < Confidence::High); + assert!(bp.summary.to_lowercase().contains("fail"), "{}", bp.summary); +} + +#[test] +fn snappy_magic_with_garbage_body_does_not_panic() { + let mut junk = vec![0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59]; + junk.extend_from_slice(&[0xde, 0xad, 0xff, 0x00, 0x99]); + let cands = identify(&junk); + // A reading is produced (Snappy or Unknown); the point is: no panic. + assert!(!cands.is_empty()); +} diff --git a/tests/identifiers.rs b/tests/identifiers.rs new file mode 100644 index 0000000..7c8ed55 --- /dev/null +++ b/tests/identifiers.rs @@ -0,0 +1,56 @@ +#![allow(clippy::unwrap_used, clippy::expect_used)] +//! UUID (string + raw bytes) and hex identification, incl. the honest scoring +//! gap: a canonical UUID *string* is High; 16 arbitrary *bytes* are only Low. + +mod common; + +use blob_decoder::{identify, BlobKind, Confidence}; + +#[test] +fn canonical_uuid_string_is_high_confidence() { + let Some(out) = common::run("uuidgen", &[]) else { + eprintln!("SKIP: uuidgen unavailable"); + return; + }; + let s = String::from_utf8_lossy(&out); + let uuid = s.trim(); + let cands = identify(uuid.as_bytes()); + let top = &cands[0]; + assert_eq!(top.kind, BlobKind::Uuid); + assert_eq!(top.score, Confidence::High); + assert!( + top.summary.contains(uuid) || top.summary.to_lowercase().contains(&uuid.to_lowercase()) + ); +} + +#[test] +fn raw_16_bytes_are_only_a_low_confidence_uuid() { + // Any 16 bytes form a syntactically valid UUID, so this must NOT be High — + // over-claiming here would fabricate certainty. + let sixteen = [ + 0x55u8, 0x0e, 0x84, 0x00, 0xe2, 0x9b, 0x41, 0xd4, 0xa7, 0x16, 0x44, 0x66, 0x55, 0x44, 0x00, + 0x00, + ]; + let cands = identify(&sixteen); + let uuid = cands + .iter() + .find(|c| c.kind == BlobKind::Uuid) + .expect("a UUID reading is offered for 16 bytes"); + assert_eq!(uuid.score, Confidence::Low); +} + +#[test] +fn hex_text_wrapping_json_is_recovered() { + let payload = b"{\"x\":1}"; + let hexed = hex::encode(payload); + let cands = identify(hexed.as_bytes()); + let hex_cand = cands + .iter() + .find(|c| c.kind == BlobKind::Hex) + .expect("hex reading present"); + let inner = hex_cand.inner.as_ref().expect("hex payload identified"); + assert_eq!(inner.best.kind, BlobKind::Json); + // A hex wrapper whose payload is a concrete type is Medium (better than a + // bare hex-charset coincidence). + assert_eq!(hex_cand.score, Confidence::Medium); +} diff --git a/tests/magic.rs b/tests/magic.rs new file mode 100644 index 0000000..9ae1e6e --- /dev/null +++ b/tests/magic.rs @@ -0,0 +1,86 @@ +#![allow(clippy::unwrap_used, clippy::expect_used)] +//! Magic-signature identification against REAL inputs produced by independent +//! tools (python3 plistlib/zlib, system gzip, the snap crate). Each is env-gated: +//! a missing producer SKIPs, never fails. + +mod common; + +use blob_decoder::{identify, BlobKind, Confidence}; + +#[test] +fn binary_plist_is_identified() { + let Some(bytes) = common::bplist_dict() else { + eprintln!("SKIP: python3 plistlib unavailable"); + return; + }; + let cands = identify(&bytes); + assert_eq!(cands[0].kind, BlobKind::BinaryPlist); + assert_eq!(cands[0].score, Confidence::High); + assert!( + cands[0].summary.to_lowercase().contains("dict"), + "summary should describe the root dict, got: {}", + cands[0].summary + ); +} + +#[test] +fn xml_plist_is_identified() { + let Some(bytes) = common::xml_plist_dict() else { + eprintln!("SKIP: python3 plistlib unavailable"); + return; + }; + let cands = identify(&bytes); + assert_eq!(cands[0].kind, BlobKind::XmlPlist); + assert_eq!(cands[0].score, Confidence::High); +} + +#[test] +fn gzip_wrapping_json_is_recovered() { + let Some(bytes) = common::gzip(b"{\"x\":1}") else { + eprintln!("SKIP: gzip unavailable"); + return; + }; + let cands = identify(&bytes); + assert_eq!(cands[0].kind, BlobKind::Gzip); + assert_eq!(cands[0].score, Confidence::High); + let inner = cands[0].inner.as_ref().expect("gzip payload identified"); + assert_eq!(inner.best.kind, BlobKind::Json); +} + +#[test] +fn zlib_wrapping_json_is_recovered() { + let Some(bytes) = common::zlib_compress(b"{\"x\":1}") else { + eprintln!("SKIP: python3 zlib unavailable"); + return; + }; + let cands = identify(&bytes); + assert_eq!(cands[0].kind, BlobKind::Zlib); + assert_eq!(cands[0].score, Confidence::High); + let inner = cands[0].inner.as_ref().expect("zlib payload identified"); + assert_eq!(inner.best.kind, BlobKind::Json); +} + +#[test] +fn snappy_framed_is_identified() { + use std::io::Write; + let mut enc = snap::write::FrameEncoder::new(Vec::new()); + enc.write_all(b"{\"x\":1}").unwrap(); + let bytes = enc.into_inner().unwrap(); + let cands = identify(&bytes); + assert_eq!(cands[0].kind, BlobKind::Snappy); + assert_eq!(cands[0].score, Confidence::High); +} + +#[test] +fn json_object_is_identified() { + let cands = identify(b"{\"a\":[1,2,3],\"b\":true}"); + assert_eq!(cands[0].kind, BlobKind::Json); + assert_eq!(cands[0].score, Confidence::High); +} + +#[test] +fn json_array_is_identified() { + let cands = identify(b"[1, 2, 3]"); + assert_eq!(cands[0].kind, BlobKind::Json); + assert_eq!(cands[0].score, Confidence::High); +} diff --git a/tests/nested.rs b/tests/nested.rs new file mode 100644 index 0000000..f9a18ac --- /dev/null +++ b/tests/nested.rs @@ -0,0 +1,35 @@ +#![allow(clippy::unwrap_used, clippy::expect_used)] +//! Recursive unwrapping: a base64 → gzip → JSON blob must report the whole chain. + +mod common; + +use blob_decoder::{identify, BlobKind}; + +#[test] +fn base64_of_gzip_of_json_reports_full_chain() { + let Some(gz) = common::gzip(b"{\"nested\":true}") else { + eprintln!("SKIP: gzip unavailable"); + return; + }; + let Some(b64) = common::base64(&gz) else { + eprintln!("SKIP: base64 unavailable"); + return; + }; + let cands = identify(&b64); + + // Top reading: base64 wrapper. + let top = &cands[0]; + assert_eq!(top.kind, BlobKind::Base64); + + // → gzip + let gzip_chain = top.inner.as_ref().expect("base64 payload identified"); + assert_eq!(gzip_chain.best.kind, BlobKind::Gzip); + + // → json + let json_chain = gzip_chain + .best + .inner + .as_ref() + .expect("gzip payload identified"); + assert_eq!(json_chain.best.kind, BlobKind::Json); +} diff --git a/tests/robustness.rs b/tests/robustness.rs new file mode 100644 index 0000000..3497bda --- /dev/null +++ b/tests/robustness.rs @@ -0,0 +1,94 @@ +#![allow(clippy::unwrap_used, clippy::expect_used)] +//! Adversarial inputs: truncated streams, a decompression bomb, invalid base64, +//! and random bytes must degrade gracefully (never panic, never OOM). + +mod common; + +use blob_decoder::{identify, identify_with_limits, BlobKind, Confidence, Limits}; + +#[test] +fn truncated_gzip_does_not_panic_and_is_not_high() { + let Some(full) = common::gzip(b"this is some content that will be cut in half midstream") + else { + eprintln!("SKIP: gzip unavailable"); + return; + }; + let truncated = &full[..full.len() / 2]; + let cands = identify(truncated); + // gzip magic is still present, so a Gzip reading is offered — but decode + // failed, so it must NOT be High, and the payload chain is absent. + let gz = cands.iter().find(|c| c.kind == BlobKind::Gzip); + if let Some(gz) = gz { + assert!(gz.score < Confidence::High); + assert!(gz.inner.is_none()); + assert!( + gz.summary.to_lowercase().contains("fail") + || gz.summary.to_lowercase().contains("error"), + "a failed decode must say so: {}", + gz.summary + ); + } +} + +#[test] +fn zlib_bomb_is_capped_not_oom() { + // 64 MiB of zeros compresses to a few KiB; with a 1 MiB output cap the + // decoder must stop at the cap, flag it, and never allocate the full bomb. + use std::io::Write; + let mut enc = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all(&vec![0u8; 64 * 1024 * 1024]).unwrap(); + let bomb = enc.finish().unwrap(); + + let limits = Limits { + max_depth: 4, + max_output: 1024 * 1024, + max_input: 128 * 1024 * 1024, + }; + let cands = identify_with_limits(&bomb, limits, 0); + let zlib = cands + .iter() + .find(|c| c.kind == BlobKind::Zlib) + .expect("zlib reading present"); + let inner = zlib.inner.as_ref().expect("payload present (capped)"); + assert!(inner.capped, "the bomb must be reported as capped"); + assert!( + inner.decoded_len <= limits.max_output, + "decoded output must not exceed the cap" + ); +} + +#[test] +fn invalid_base64_does_not_claim_base64() { + // Contains characters outside the base64 alphabet. + let cands = identify(b"not!!valid!!base64!!@@##"); + assert!( + cands.iter().all(|c| c.kind != BlobKind::Base64), + "junk with non-alphabet chars must not be read as base64" + ); +} + +#[test] +fn random_bytes_degrade_gracefully() { + let junk: Vec = (0u16..500) + .map(|i| (i.wrapping_mul(37) ^ 0xA5) as u8) + .collect(); + let cands = identify(&junk); + assert!(!cands.is_empty()); + assert!(cands.iter().all(|c| c.score < Confidence::High)); +} + +#[test] +fn deeply_nested_wrappers_terminate() { + // base64 of base64 of base64 … must stop at the depth cap without looping. + let mut data = b"{\"x\":1}".to_vec(); + for _ in 0..20 { + let Some(b) = common::base64(&data) else { + eprintln!("SKIP: base64 unavailable"); + return; + }; + // strip the trailing newline base64(1) adds so the next layer is clean + data = b.iter().copied().filter(|c| *c != b'\n').collect(); + } + let cands = identify(&data); + assert!(!cands.is_empty(), "must terminate and return candidates"); +}