diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..a380c9a --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# Enables the "Sponsor" button on the repo. +github: [Mrbaeksang] +custom: ["https://baeksang.dev"] diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..680ff1f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,44 @@ +name: ๐Ÿ› Bug report +description: Something didn't work โ€” a fetch, an Escalation, a Bypass, or a report +labels: ["bug", "investigate"] +body: + - type: markdown + attributes: + value: Thanks for helping make DeepCloak better. Please fill in what you can. + - type: textarea + id: what-happened + attributes: + label: What happened? + description: What did you expect, and what did you get instead? + placeholder: I ran `deepcloak "..."` and the Bypass failed on ... + validations: + required: true + - type: textarea + id: repro + attributes: + label: Steps to reproduce + description: The exact command(s). Redact anything private. + render: shell + placeholder: | + deepcloak "How does Cloudflare Turnstile detect bots?" --depth detailed + validations: + required: true + - type: textarea + id: evidence + attributes: + label: Evidence Record (if any) + description: Paste the `*.evidence.json` sidecar or the ๐Ÿ›ก๏ธ section of the report โ€” it tells us which Bot Wall was hit and whether the Escalation ran. + render: json + - type: input + id: provider + attributes: + label: LLM provider / model + placeholder: openai / anthropic / ollama / openai-endpoint (local) + - type: input + id: versions + attributes: + label: Versions + description: "`deepcloak --version`, Python, OS" + placeholder: deepcloak 0.1.0 ยท Python 3.12 ยท Linux + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..0f3a776 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: ๐Ÿ’ฌ Questions & ideas (Discussions) + url: https://github.com/Mrbaeksang/deepcloak/discussions + about: Ask how to use DeepCloak, share a run, or float an idea before opening an issue. + - name: ๐Ÿ” Report a security issue + url: https://github.com/Mrbaeksang/deepcloak/security/policy + about: Please report vulnerabilities privately โ€” see our security policy. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..b24b3fe --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,22 @@ +name: ๐Ÿ’ก Feature request +description: Suggest a capability or improvement +labels: ["enhancement", "spec"] +body: + - type: textarea + id: problem + attributes: + label: What's the problem? + description: The use case from your side, not a solution yet. + placeholder: When a site uses , DeepCloak can't ... + validations: + required: true + - type: textarea + id: idea + attributes: + label: What would you like it to do? + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives you've considered diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..ea85ced --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,15 @@ + + +## What & why + + + +## How it was verified + + + +- [ ] `pytest` passes +- [ ] `ruff check .` is clean +- [ ] Touches only what the change needs (no drive-by refactors) +- [ ] If it contradicts an ADR in `docs/adr/`, I flagged it instead of silently overriding diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..29a241f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,31 @@ +name: release + +# Publishes to PyPI when a GitHub Release is published. +# One-time setup on pypi.org โ†’ Publishing โ†’ add a Trusted Publisher: +# PyPI project: deepcloak ยท owner: Mrbaeksang ยท repo: deepcloak +# workflow: release.yml ยท environment: pypi +# No API token needed (OIDC trusted publishing). + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + pypi: + name: Build and publish to PyPI + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write # required for trusted publishing + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install --upgrade build + - run: python -m build + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1fd332a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,29 @@ +# Changelog + +All notable changes to DeepCloak are documented here. Format follows +[Keep a Changelog](https://keepachangelog.com/); this project uses [SemVer](https://semver.org/). + +## [Unreleased] + +## [0.1.0] โ€” 2026-06-05 + +First public release. + +### Added +- **Deep Research that reads Bot-walled pages.** Plain fetch first; on a Bot Wall + (Cloudflare / Datadome / Turnstile / reCAPTCHA) it Escalates one URL to a Stealth Fetch + and Bypasses it โ€” recovering content other agents drop. +- **StealthRetriever** โ€” handed to `local-deep-research` via its `retrievers=` API so the + research loop synthesises over Bypassed full pages, not snippets. +- **Evidence Records** โ€” every fetch is recorded (Bot Wall kind, Escalation, Bypass, + plain status, timing); reports end with a `๐Ÿ›ก๏ธ Bypassed N bot-walled sources` badge and a + `*.evidence.json` sidecar. +- Three surfaces over one core: **CLI**, **MCP server** (`deep_research` / `quick_summary` + / `get_evidence`), and a **Claude skill**. +- Local-first: works with `ollama` or any OpenAI-compatible local endpoint โ€” no API key + required. Default search is DuckDuckGo (keyless); SearXNG is opt-in. +- `--respect-robots` to honor robots.txt (ignored by default โ€” see ADR-0002). +- Per-document content cap so small-context local models can finish a report. + +[Unreleased]: https://github.com/Mrbaeksang/deepcloak/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/Mrbaeksang/deepcloak/releases/tag/v0.1.0 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..f5f6789 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,29 @@ +# Code of Conduct + +## Our pledge + +We want DeepCloak to be a welcoming, harassment-free project for everyone, regardless of +experience, background, or identity. + +## Our standards + +**Encouraged:** being respectful, giving and accepting constructive feedback, focusing on +what's best for the project and community. + +**Not tolerated:** harassment, insults, personal or political attacks, publishing others' +private information, or other conduct that would be inappropriate in a professional +setting. + +## Scope + +This applies in all project spaces (issues, PRs, Discussions) and when representing the +project in public. + +## Enforcement + +Report unacceptable behavior to **contact@baeksang.dev**. Reports are reviewed privately +and acted on as needed โ€” from a warning to a ban. Maintainers who don't follow the code +in good faith may face consequences too. + +This is a condensed adaptation of the [Contributor Covenant](https://www.contributor-covenant.org/), +version 2.1. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..9c27a77 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,26 @@ +# Security Policy + +## Reporting a vulnerability + +Please report security issues **privately** โ€” do not open a public issue. + +- Email: **contact@baeksang.dev** +- Or use GitHub's [private vulnerability reporting](https://github.com/Mrbaeksang/deepcloak/security/advisories/new). + +Include what you found, how to reproduce it, and the impact. We aim to acknowledge +within a few days and will credit you in the fix unless you prefer otherwise. + +## Supported versions + +DeepCloak is pre-1.0; fixes land on the latest release. Pin a version in production. + +## Scope & responsible use + +DeepCloak **Bypasses** bot-detection (Cloudflare, Datadome, Turnstile, reCAPTCHA) so a +research agent can read pages a person with a browser could read. **You are responsible +for having the right to access whatever you fetch** โ€” don't use it to violate a site's +terms or the law. robots.txt is ignored by default; pass `--respect-robots` to honor it. + +This is a thin orchestrator over [`local-deep-research`](https://github.com/LearningCircuit/local-deep-research) +and [`CloakBrowser`](https://github.com/CloakHQ/CloakBrowser); vulnerabilities in those +upstreams should also be reported to them. diff --git a/docs/media/og.png b/docs/media/og.png new file mode 100644 index 0000000..5f69bb4 Binary files /dev/null and b/docs/media/og.png differ diff --git a/pyproject.toml b/pyproject.toml index cdb5496..e57fe72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,10 @@ Author = "https://baeksang.dev" [tool.hatch.build.targets.wheel] packages = ["src/deepcloak"] +# Keep the PyPI sdist lean โ€” ship code + docs, not the demo media or showcase site. +[tool.hatch.build.targets.sdist] +include = ["src/deepcloak", "README.md", "LICENSE", "NOTICE", "CHANGELOG.md", "pyproject.toml"] + [tool.ruff] line-length = 100 src = ["src", "tests"] diff --git a/showcase/app/layout.tsx b/showcase/app/layout.tsx index 6becc7b..c1569b9 100644 --- a/showcase/app/layout.tsx +++ b/showcase/app/layout.tsx @@ -11,9 +11,18 @@ export const metadata: Metadata = { "A local-first deep research agent that bypasses Cloudflare, Datadome, Turnstile & reCAPTCHA to read the whole web.", metadataBase: new URL("https://deepcloak.vercel.app"), openGraph: { - title: "DeepCloak", + title: "DeepCloak โ€” deep research that reads bot-walled pages", description: "Deep research that reads pages other tools can't โ€” even behind a Bot Wall.", type: "website", + url: "https://deepcloak.vercel.app", + siteName: "DeepCloak", + images: [{ url: "/og.png", width: 1280, height: 640, alt: "DeepCloak bypassing Cloudflare/Turnstile in a real run" }], + }, + twitter: { + card: "summary_large_image", + title: "DeepCloak โ€” deep research that reads bot-walled pages", + description: "Deep research that reads pages other tools can't โ€” even behind a Bot Wall.", + images: ["/og.png"], }, }; diff --git a/showcase/public/og.png b/showcase/public/og.png new file mode 100644 index 0000000..5f69bb4 Binary files /dev/null and b/showcase/public/og.png differ