diff --git a/.gitignore b/.gitignore index 4ffb5bb..8e6fc2c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,10 @@ dist/ build/ *.egg-info/ +.venv/ +uv.lock + +.tmp_artifacts/ + +# Example output files generated by running scripts under `examples/`. +examples/output.csv diff --git a/README.md b/README.md index 6d75070..92a30c0 100644 --- a/README.md +++ b/README.md @@ -1,59 +1,155 @@ # usegolib -**usegolib** lets Python call Go libraries by loading a Go-built shared library directly into the Python process. +[![CI](https://github.com/TimLai666/usegolib/actions/workflows/ci.yml/badge.svg)](https://github.com/TimLai666/usegolib/actions/workflows/ci.yml) +[![PyPI](https://img.shields.io/pypi/v/usegolib.svg)](https://pypi.org/project/usegolib/) +[![Python Versions](https://img.shields.io/pypi/pyversions/usegolib.svg)](https://pypi.org/project/usegolib/) +[![License](https://img.shields.io/github/license/TimLai666/usegolib.svg)](LICENSE) + +Call Go libraries from Python by loading a Go-built shared library directly into the Python process. - No C/C++ glue code written by you - No background Go process (no daemon / RPC / subprocess) -- MessagePack-based ABI +- MessagePack-based ABI with manifest-driven schema validation -## What problem does this solve? +## Status -- You have a Go library -- You want Python users to call it like a native Python module -- You do NOT want: - - C/C++ glue code - - gopy-style heavy bindings - - running a Go server or subprocess +`usegolib` is early-stage software (v0.x). Expect sharp edges and evolving interfaces. -## Model +## How It Works -- **Build time (CI / build machine)**: use Go toolchain + an auto-provisioned C compiler (Zig) to build a Go module into a shared library + `manifest.json`. -- **Runtime (end-user / production)**: Python loads the shared library and calls it via MessagePack. Go is not running as a service. +- Build time (CI / build machine): use the Go toolchain plus an auto-provisioned C compiler (Zig) to build a Go module into a shared library and emit `manifest.json`. +- Runtime (end-user / production): Python loads the shared library and calls it via MessagePack. No daemon, no RPC, no subprocess. ## Requirements - Python 3.10+ -- Go toolchain (build-time only; not required for runtime if you ship prebuilt artifacts/wheels) +- Go toolchain + - required for building artifacts (CI / build machine) + - required in dev if you rely on auto-build (missing artifact triggers a build) + - not required for end-user runtime if you ship prebuilt artifacts or wheels + +## Installation + +```bash +python -m pip install usegolib +``` -## Quickstart (local build + call) +From source (dev): ```bash -# Build an artifact from a local Go module directory -python -m usegolib build --module path/to/go/module --out out/artifact +python -m pip install -e ".[dev]" ``` +## Quickstart + +Build an artifact from a local Go module directory: + +```bash +usegolib build --module path/to/go/module --out out/artifact +``` + +Import from the artifact root and call exported functions: + ```python import usegolib -# Import from an artifact *root* (may contain many modules/versions/platforms) h = usegolib.import_("example.com/mod", artifact_dir="out/artifact") print(h.AddInt(1, 2)) ``` -Typed wrappers (when schema is present in the artifact manifest): +Dev convenience: omit `artifact_dir` to use the default artifact cache root. If the artifact +is missing, `usegolib.import_` will build it into the default cache when a Go toolchain is +available. + +```python +import usegolib + +h = usegolib.import_("example.com/mod") +print(h.AddInt(1, 2)) +``` + +To enforce "no build" behavior, pass an explicit artifact root and set `build_if_missing=False`: + +```python +import usegolib + +h = usegolib.import_("example.com/mod", artifact_dir="out/artifact", build_if_missing=False) +``` + +## Example: Calling Insyra From Python + +Insyra is a Go data-analysis library: `github.com/HazelnutParadise/insyra`. + +Auto-build on import (developer convenience; downloads the module + builds into usegolib's default artifact root): + +```python +import usegolib + +insyra = usegolib.import_("github.com/HazelnutParadise/insyra", version="v0.2.14") + +# DataList (object handle + variadic any) +dl = insyra.NewDataList() +dl.Append(1, 2, 3, 4, 5) +print(dl.Sum(), dl.Mean()) + +# DataTable (variadic map[string]any + variadic bool) +dt = insyra.NewDataTable() +dt.AppendRowsByColIndex({"A": 1, "B": 2}, {"A": 3, "B": 4}) +print(dt.NumRows(), dt.NumCols()) +dt.Show() +``` + +Notes: + +- `usegolib` can only call functions/methods whose parameter and return types are supported by the current type bridge (see `docs/abi.md`). +- This example requires a Go toolchain to be installed. (At the time of writing, `insyra@v0.2.14` requires `go >= 1.25` and may cause Go to auto-download a newer toolchain.) +- If you want to disable auto-build, pass `build_if_missing=False` (and usually an explicit `artifact_dir`). +- For end-users without Go, ship prebuilt artifacts/wheels (see Packaging). + +## Methods (Object Handles) + +For exported methods, `usegolib` supports Go-side object handles (opaque ids) to avoid serializing the receiver on every call: + +```python +with h.object("Counter", {"n": 1}) as c: + print(c.Inc(2)) +``` + +Typed object handles (schema required): ```python th = h.typed() -print(th.AddInt(1, 2)) +types = th.types + +with th.object("Counter", types.Counter(N=10)) as c: + snap = c.Snapshot() + print(snap) +``` + +## Generic Functions (Build-Time Instantiation) + +Generic functions require explicit build-time instantiation: + +```bash +usegolib build --module example.com/mod --out out/artifact --generics generics.json +``` + +At runtime (schema required), use the helper: + +```python +fn = h.generic("Id", ["int64"]) +print(fn(123)) ``` -## Quickstart (generate a distributable Python package project) +## Packaging (Ship Wheels Without Requiring Go) + +Generate a distributable Python package project embedding artifacts: ```bash -python -m usegolib package --module path/to/go/module --python-package-name mypkg --out out/ +usegolib package --module path/to/go/module --python-package-name mypkg --out out/ ``` -Then install the generated project: +Install the generated project: ```bash python -m pip install -e out/mypkg @@ -61,22 +157,28 @@ python -m pip install -e out/mypkg ```python import mypkg + print(mypkg.AddInt(1, 2)) ``` -## Version Rules (important) +## Version Rules - One Go module = one version per Python process - If a module is already loaded, all subpackages must use the same version -- If `version=None` and multiple artifact versions exist under `artifact_dir`, import fails with `AmbiguousArtifactError` - -## Policies - -- `docs/versioning.md`: ABI + manifest versioning and compatibility rules -- `docs/compatibility.md`: supported platforms and toolchain pins -- `docs/security.md`: security boundaries and current hardening -- `docs/reproducible-builds.md`: reproducible build guidance +- If `version=None` and multiple artifact versions exist under `artifact_dir`, import fails with `AmbiguousArtifactError` (unless that module is already loaded in the current process, in which case `version=None` follows the already-loaded version) + +## Documentation + +- `docs/roadmap.md` +- `docs/abi.md` +- `docs/cli.md` +- `docs/versioning.md` +- `docs/compatibility.md` +- `docs/testing.md` +- `docs/security.md` +- `docs/troubleshooting.md` +- `docs/releasing.md` -## Releasing +## License -- `docs/releasing.md` +MIT. See `LICENSE`. diff --git a/docs/abi.md b/docs/abi.md index a80bbe3..ee49007 100644 --- a/docs/abi.md +++ b/docs/abi.md @@ -23,7 +23,10 @@ The ABI is intentionally small: the wire format carries only generic MessagePack All requests are MessagePack maps: - `abi`: integer ABI version (v0 == `0`) -- `op`: operation name (v0 supports only `call`) +- `op`: operation name (v0 supports: `call`, `obj_new`, `obj_call`, `obj_free`) + +### `op = "call"` + - `pkg`: Go package import path (string) - `fn`: exported function name (string) - `args`: list of arguments (see Type Bridge below) @@ -40,6 +43,66 @@ Example (conceptual): } ``` +### `op = "obj_new"` + +Create a Go-side object instance and return an opaque id. + +- `pkg`: Go package import path (string) +- `type`: receiver struct type name (string, exported; no leading `*`) +- `init`: optional record-struct value used to initialize the struct (Level 3); if omitted, the zero value is used + +Example (conceptual): + +```text +{ + "abi": 0, + "op": "obj_new", + "pkg": "example.com/mod", + "type": "Counter", + "init": {"n": 1} +} +``` + +### `op = "obj_call"` + +Call a method on a previously created object. + +- `pkg`: Go package import path (string) +- `type`: receiver struct type name (string, exported; no leading `*`) +- `id`: object id returned by `obj_new` +- `method`: exported method name (string) +- `args`: list of arguments (see Type Bridge below) + +Example (conceptual): + +```text +{ + "abi": 0, + "op": "obj_call", + "pkg": "example.com/mod", + "type": "Counter", + "id": 1, + "method": "Inc", + "args": [2] +} +``` + +### `op = "obj_free"` + +Free a Go-side object id. + +- `id`: object id returned by `obj_new` + +Example (conceptual): + +```text +{ + "abi": 0, + "op": "obj_free", + "id": 1 +} +``` + ## Response Format All responses are MessagePack maps: @@ -68,6 +131,9 @@ Supported values crossing the ABI: - `string` - `bytes` - slices/lists of supported values (including lists of bytes) +- `any`: + - `any` is represented on the wire as a plain MessagePack value (no extra envelope) + - `[]any` and `map[string]any` are supported recursively - Level 2: - `map[string]T` where `T` is a Level 1 scalar or a slice of Level 1 scalars - Encoded as a MessagePack map from string keys to values @@ -77,6 +143,19 @@ Supported values crossing the ABI: Unsupported values MUST fail with `UnsupportedTypeError`. +### Variadic Parameters (`...T`) + +Go variadic parameters (`...T`) are represented in the ABI as a single final argument whose value is a list. + +Examples: +- Go: `Append(values ...any)` + - Python: `obj.Append(1, 2, 3)` sends `args = [[1, 2, 3]]` + - Python: `obj.Append()` sends `args = [[]]` +- Go: `Data(useNamesAsKeys ...bool)` + - Python: `dt.Data(True)` sends `args = [[True]]` + +When manifest schema is present, the Python runtime packs varargs automatically. + ### Canonical Keys (Tags) For record structs, canonical keys follow: @@ -102,6 +181,7 @@ These are interpreted using the manifest schema; on the wire they are just strin Artifacts embed a schema in `manifest.json` describing: - callable symbols and their parameter/return types +- callable methods (receiver type + method name) and their parameter/return types - named struct types and their fields (including keys, aliases, required/omitempty) When schema is present, the runtime validates call arguments and successful results against the schema diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..5560bfb --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,71 @@ +# CLI + +`usegolib` installs a `usegolib` command (and also supports `python -m usegolib`). + +## Installation + +```bash +python -m pip install usegolib +usegolib version +``` + +## Build Artifacts + +Build from a local Go module directory: + +```bash +usegolib build --module path/to/go/module --out out/artifacts +``` + +Build from a remote Go module (downloads via `go mod download`): + +```bash +usegolib build --module github.com/HazelnutParadise/insyra@v0.2.14 --out out/artifacts +``` + +Force rebuild: + +```bash +usegolib build --module github.com/HazelnutParadise/insyra@v0.2.14 --out out/artifacts --force +``` + +Re-download modules before rebuilding (implies `--force`): + +```bash +usegolib build --module github.com/HazelnutParadise/insyra@v0.2.14 --out out/artifacts --redownload +``` + +Notes: +- `--redownload` uses an isolated Go module cache (sets `GOMODCACHE`) under the output root unless `--gomodcache` is provided. +- Building requires a Go toolchain on `PATH` (`go`). Zig is bootstrapped automatically for cgo. +- If Go module downloads fail due to proxy/network issues, retry. If `proxy.golang.org` is blocked/unreliable in your environment, try `GOPROXY=direct`. + +## Manage The Artifact Cache + +Artifacts are stored in an artifact root directory. By default this is: +- overridden by `USEGOLIB_ARTIFACT_DIR` when set +- otherwise OS cache (see `src/usegolib/paths.py`) + +Delete an artifact (dry-run first): + +```bash +usegolib artifact rm --module github.com/HazelnutParadise/insyra@v0.2.14 +``` + +Actually delete: + +```bash +usegolib artifact rm --module github.com/HazelnutParadise/insyra@v0.2.14 --yes +``` + +Delete all versions of a module/package for the current platform: + +```bash +usegolib artifact rm --module github.com/HazelnutParadise/insyra --all-versions --yes +``` + +Rebuild into the artifact root (optionally cleaning existing artifacts and re-downloading sources): + +```bash +usegolib artifact rebuild --module github.com/HazelnutParadise/insyra@v0.2.14 --clean --redownload +``` diff --git a/docs/roadmap.md b/docs/roadmap.md index 42bde5a..eafc0f3 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -11,7 +11,7 @@ The project has two independent but compatible tracks: ## Milestones -### Milestone 0 (Project Scaffold + Specs) -> v0.0.1 +### Milestone 0 (Project Scaffold + Specs) - `openspec/specs/usegolib-core/spec.md` (current truth) - `docs/roadmap.md` (this file) @@ -22,7 +22,7 @@ OpenSpec changes (recommended): - `add-project-scaffold` - `add-abi-v0-spec` -### Milestone 1 (Runtime Loader + Calls) -> v0.1.0 +### Milestone 1 (Runtime Loader + Calls) - Python package `usegolib` (runtime loader + ABI client) - `ModuleHandle` proxy: `usegolib.import_(...) -> ModuleHandle`, `handle.FuncName(...)` via ABI `call` @@ -36,7 +36,7 @@ Tests: OpenSpec change: - `implement-runtime-loader-v0` -### Milestone 2 (Builder + Manifest + Zig Bootstrap) -> v0.2.0 +### Milestone 2 (Builder + Manifest + Zig Bootstrap) - `usegolib build` CLI: - Resolve module/version (`@latest` or pinned) @@ -54,7 +54,7 @@ OpenSpec changes: - `implement-builder-v0` - `add-zig-toolchain-bootstrap` -### Milestone 3 (Packager -> Wheel Layout) -> v0.3.0 +### Milestone 3 (Packager -> Wheel Layout) - `usegolib package` to generate a Python package skeleton embedding artifacts + manifest - Wheel build flow where end-user `pip install` does not require Go @@ -62,15 +62,31 @@ OpenSpec changes: OpenSpec change: - `add-packager-and-wheel-layout` -### Milestone 4 (Hardening + Type Levels) -> v0.4 - v0.6 +### Milestone 4 (Hardening + Type Levels) - v0.4: caching + file locking + concurrency safety - v0.5: Type bridge Level 2 (`map[string]T`) - v0.6: Type bridge Level 3 Phase A (record structs) -### Milestone 5 (ABI Stability + Compatibility Policy) -> v1.0 +### Milestone 5 (ABI Stability + Compatibility Policy) - ABI/manfiest versioning policy - security hardening (hash verification, restricted downloads) - production documentation (troubleshooting, compatibility matrix) +### Milestone 6 (API Expansion: Methods + Generics) + +This is planned after Milestone 5 (v1.0 readiness) unless explicitly reprioritized. + +Current v0.x behavior: generic functions are ignored unless instantiated via `usegolib build --generics`. + +- Exported method support (implemented) + - Go-side object handles (opaque ids) to avoid serializing receivers on every call + - ABI ops: `obj_new`, `obj_call`, `obj_free` + - Python API: `PackageHandle.object(type_name, init=None) -> GoObject` and `TypedPackageHandle.object(...) -> TypedGoObject` + - Schema/manifest: `schema.methods` (receiver + method signature) for runtime validation and typed decoding +- Generic function support (Phase A) + - support calling exported generic functions by generating concrete instantiations at build time for an explicit set of type arguments (implemented via `usegolib build --generics `) + - avoid implicit type inference across the ABI (type arguments are explicit in manifest schema) +- Generic function support (Phase B) + - improve Python ergonomics via `handle.generic(name, type_args)` helper and bindgen output for instantiated symbols diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..bf6bbdf --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,104 @@ +# Troubleshooting + +Common issues when importing, building, or loading `usegolib` artifacts. + +## Import Errors + +### `ArtifactNotFoundError` + +Meaning: no matching artifact for the requested module/package/version was found under the artifact root. + +Fix: +- If you are an end-user: install a wheel that embeds artifacts (via `usegolib package`) or point `artifact_dir` to a directory containing prebuilt artifacts. +- If you are a developer: omit `artifact_dir` (use default cache) or set `build_if_missing=True` to allow building into your artifact root. + +### `AmbiguousArtifactError` + +Meaning: `version=None` but multiple versions exist for the same module under the artifact root (and the module is not already loaded in the current Python process). + +Fix: +- Pass an explicit `version="vX.Y.Z"` (or `"local"` for local module builds), or +- Remove/relocate the extra version(s) under the artifact root. + +Note: +- If the module is already loaded in the current process, `usegolib.import_(..., version=None)` follows the already-loaded module version (including for subpackage imports), and does not fail with ambiguity. + +Tip: you can delete cached artifacts via the CLI: + +```bash +usegolib artifact rm --module example.com/mod@vX.Y.Z --yes +usegolib artifact rm --module example.com/mod --all-versions --yes +``` + +### `VersionConflictError` + +Meaning: the current Python process already loaded one version of a module, and you're trying to load a different version (possibly via a subpackage import). + +Fix: +- Ensure all imports use the same module version in a single process. +- Restart the Python process if you need to switch versions. + +## Build Errors + +### Go toolchain missing / `go` not on PATH + +Auto-build (import triggers download/build) requires a Go toolchain. + +Fix: +- Install Go, or +- Avoid auto-build: ship prebuilt artifacts/wheels and import with an explicit `artifact_dir` and `build_if_missing=False`. + +### Go module download fails (network / proxy) + +Symptoms: build logs mention `proxy.golang.org`, `sum.golang.org`, `i/o timeout`, `wsarecv`, or similar network/proxy errors. + +Fix: +- Retry (transient failures are common), and +- If `proxy.golang.org` is blocked/unreliable in your environment, try `GOPROXY=direct` and rebuild. + +```powershell +$env:GOPROXY="direct" +usegolib artifact rebuild --module example.com/mod@vX.Y.Z --clean --redownload +``` + +### Go auto-downloads a newer toolchain + +If the target module requires a newer Go version, Go may print messages like `go: downloading go1.25.0` during builds. +This requires network access. If you want to avoid auto toolchain downloads, install a suitable Go version up front or configure Go toolchain selection (for example via `GOTOOLCHAIN`). + +### Zig bootstrap failures + +Meaning: `usegolib` could not download/verify/extract Zig for cgo. + +Fix: +- Ensure network access to `https://ziglang.org/`. +- Override Zig binary: set `USEGOLIB_ZIG` (or `ZIG`) to a local Zig executable path. +- If you are pinning: verify `USEGOLIB_ZIG_VERSION` matches an entry in Zig's `index.json`. + +## Load Errors + +### `LoadError` for SHA256 mismatch + +Meaning: the shared library file does not match the hash recorded in `manifest.json`. + +Fix: +- Rebuild the artifact (or re-download the artifact root). +- Ensure the artifact root is not modified after build. + +### `LoadError` for platform mismatch + +Meaning: the artifact was built for a different `goos/goarch` than the current machine. + +Fix: +- Build artifacts for your target platform, or install a wheel that includes your platform's artifact. + +## WSL (Linux) Local Verification + +If you're developing on Windows, run Linux tests via: + +```powershell +.\tools\wsl_linux_tests.ps1 +.\tools\wsl_linux_tests.ps1 -Integration +``` + +If integration tests fail because Go is missing in WSL, the integration runner will bootstrap Go automatically (no sudo). diff --git a/examples/insyra_basic.py b/examples/insyra_basic.py new file mode 100644 index 0000000..2604fc5 --- /dev/null +++ b/examples/insyra_basic.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import usegolib + + +def main() -> None: + # Auto-build on import (developer convenience): + # - Downloads the Go module + # - Builds a shared library artifact into usegolib's default artifact root + # + # Requirements (for auto-build): + # - Go toolchain installed (insyra@v0.2.14 requires go >= 1.25) + # - Network access (module download) + # + # If you want to disable auto-build, pass `build_if_missing=False` (and usually an explicit `artifact_dir`). + # + # For end-users without Go, ship a prebuilt wheel produced by `usegolib package`. + insyra = usegolib.import_("github.com/HazelnutParadise/insyra", version="v0.2.14") + isr = usegolib.import_("github.com/HazelnutParadise/insyra/isr") + # --- DataList (object handle + variadic any) --- + dl = isr.DL.Of(1,2,3) # Go: NewDataList(values ...any) -> *DataList + dl.Append(1, 2, 3, 4, 5) # Go: Append(values ...any) + print("DataList.Sum() ->", dl.Sum()) + print("DataList.Mean() ->", dl.Mean()) + print("DataList.Data() ->", dl.Data()) + print("DataList.Show() ->") + dl.Show() + + # --- DataTable (variadic map[string]any + variadic bool) --- + dt = insyra.NewDataTable() + dt.AppendRowsByColIndex({"A": 1, "B": 2}, {"A": 3, "B": 4}) + print("DataTable.NumRows() ->", dt.NumRows()) + print("DataTable.NumCols() ->", dt.NumCols()) + print("DataTable.Data(True) ->", dt.Data(True)) + print("DataTable.Show() ->") + dt.Show() + dt.ToCSV("output.csv", False, False, False) + +if __name__ == "__main__": + main() diff --git a/openspec/changes/archive/2026-02-09-update-import-auto-build/.openspec.yaml b/openspec/changes/archive/2026-02-09-update-import-auto-build/.openspec.yaml new file mode 100644 index 0000000..9bc4ae2 --- /dev/null +++ b/openspec/changes/archive/2026-02-09-update-import-auto-build/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-09 diff --git a/openspec/changes/archive/2026-02-09-update-import-auto-build/README.md b/openspec/changes/archive/2026-02-09-update-import-auto-build/README.md new file mode 100644 index 0000000..485d459 --- /dev/null +++ b/openspec/changes/archive/2026-02-09-update-import-auto-build/README.md @@ -0,0 +1,3 @@ +# update-import-auto-build + +Allow usegolib.import_ to use a default artifact cache dir and auto-build missing artifacts (including subpackage imports). diff --git a/openspec/changes/archive/2026-02-09-update-import-auto-build/design.md b/openspec/changes/archive/2026-02-09-update-import-auto-build/design.md new file mode 100644 index 0000000..5adcdb1 --- /dev/null +++ b/openspec/changes/archive/2026-02-09-update-import-auto-build/design.md @@ -0,0 +1,32 @@ +# Design: Default Artifact Root + Auto-Build Import + +## Default Artifact Root + +Introduce a runtime default artifact root for `usegolib.import_`: + +- If `artifact_dir` is passed: use it. +- Else if `USEGOLIB_ARTIFACT_DIR` is set: use it. +- Else use OS defaults: + - Windows: `%LOCALAPPDATA%/usegolib/artifacts` + - macOS/Linux: `~/.cache/usegolib/artifacts` + +The directory is created on demand. + +## Auto-Build Behavior + +`build_if_missing` becomes tri-state: + +- `True`: build missing artifacts into the chosen artifact root. +- `False`: never build; missing artifacts raise `ArtifactNotFoundError`. +- `None` (auto): + - if `artifact_dir` is omitted: behave like `True` + - if `artifact_dir` is provided: behave like `False` + +## Local Directory Imports (Subpackages) + +If the `module` argument is a directory: +- Find the module root directory by walking up to the nearest `go.mod`. +- Parse module path from that `go.mod`. +- Compute the package import path as `module_path + "/" + ` (if any). +- Build target is always the module root directory (not the subdirectory). + diff --git a/openspec/changes/archive/2026-02-09-update-import-auto-build/proposal.md b/openspec/changes/archive/2026-02-09-update-import-auto-build/proposal.md new file mode 100644 index 0000000..2b71ba1 --- /dev/null +++ b/openspec/changes/archive/2026-02-09-update-import-auto-build/proposal.md @@ -0,0 +1,29 @@ +## Why + +For day-to-day usage, requiring users to manually pre-build artifacts (and pass `artifact_dir`) adds friction. We want a single call (`usegolib.import_`) to be enough: it should use a default artifact cache directory and, when appropriate, build missing artifacts automatically (including when importing a subpackage of a module). + +## What Changes + +- `usegolib.import_`: + - no longer requires `artifact_dir` + - uses a default artifact cache directory when `artifact_dir` is omitted + - supports `build_if_missing=None` (auto mode): + - if `artifact_dir` is omitted: build missing artifacts into the default cache + - if `artifact_dir` is provided: do not build unless `build_if_missing=True` + - supports local subpackage directory imports by mapping the directory to the correct Go import path +- Add an environment override: `USEGOLIB_ARTIFACT_DIR` to set the default artifact root. + +## Capabilities + +### New Capabilities + + +### Modified Capabilities + +- `usegolib-core`: import defaults and auto-build behavior + +## Impact + +- Affected code: `src/usegolib/importer.py`, `src/usegolib/builder/resolve.py` (local module root discovery), new `src/usegolib/paths.py` +- Affected docs: `README.md`, `docs/testing.md` (optional), `docs/roadmap.md` (optional) + diff --git a/openspec/changes/archive/2026-02-09-update-import-auto-build/specs/usegolib-core/spec.md b/openspec/changes/archive/2026-02-09-update-import-auto-build/specs/usegolib-core/spec.md new file mode 100644 index 0000000..f182994 --- /dev/null +++ b/openspec/changes/archive/2026-02-09-update-import-auto-build/specs/usegolib-core/spec.md @@ -0,0 +1,61 @@ +## MODIFIED Requirements + +### Requirement: Python Import API +The system SHALL expose a Python API that imports a Go module or subpackage and returns a handle that can be used to call exported Go identifiers. + +#### Scenario: Import root module from the default artifact root +- **WHEN** Python calls `usegolib.import_("example.com/mod", version=None)` +- **THEN** the system searches the default artifact root for a matching artifact for the current OS/arch +- **AND THEN** the system loads the matching shared library into the current Python process +- **AND THEN** the call returns a handle bound to package `example.com/mod` + +#### Scenario: Import root module from an explicit artifact root +- **WHEN** Python calls `usegolib.import_("example.com/mod", version=None, artifact_dir="out/")` +- **THEN** the system searches `artifact_dir` for a matching artifact for the current OS/arch +- **AND THEN** the system loads the matching shared library into the current Python process +- **AND THEN** the call returns a handle bound to package `example.com/mod` + +#### Scenario: Import subpackage returns a handle bound to that package +- **WHEN** Python calls `usegolib.import_("example.com/mod/subpkg", version=None, artifact_dir="out/")` +- **THEN** the system loads a matching artifact that contains package `example.com/mod/subpkg` +- **AND THEN** the call returns a handle bound to package `example.com/mod/subpkg` + +#### Scenario: Import chooses a specific version when provided +- **WHEN** Python calls `usegolib.import_("example.com/mod", version="v1.2.3", artifact_dir="out/")` +- **THEN** the system MUST load version `v1.2.3` for that module/package (if present) + +#### Scenario: Import fails when version is omitted but ambiguous +- **WHEN** Python calls `usegolib.import_("example.com/mod", version=None, artifact_dir="out/")` +- **AND WHEN** `artifact_dir` contains more than one version for `example.com/mod` for the current OS/arch +- **THEN** the import MUST fail +- **AND THEN** the system SHALL raise `AmbiguousArtifactError` + +### Requirement: Import Builds Missing Artifacts (Dev Mode) +When `build_if_missing=True`, the system SHALL build a missing artifact into the artifact root and then load it. + +When `build_if_missing=None` (auto), the system SHALL: +- build missing artifacts when `artifact_dir` is omitted (using the default artifact root) +- NOT build missing artifacts when `artifact_dir` is explicitly provided + +#### Scenario: Import triggers build when artifact is missing +- **WHEN** Python calls `usegolib.import_("example.com/mod", version="vX.Y.Z", artifact_dir="out/", build_if_missing=True)` +- **AND WHEN** no matching artifact exists under `artifact_dir` for the current OS/arch +- **THEN** the system builds the artifact into `artifact_dir` +- **AND THEN** the system loads the newly built artifact and returns a handle + +#### Scenario: Import auto-builds when artifact_dir is omitted +- **WHEN** Python calls `usegolib.import_("example.com/mod", version="vX.Y.Z", build_if_missing=None)` +- **AND WHEN** no matching artifact exists under the default artifact root for the current OS/arch +- **THEN** the system builds the artifact into the default artifact root +- **AND THEN** the system loads the newly built artifact and returns a handle + +#### Scenario: Import does not build by default when artifact_dir is provided +- **WHEN** Python calls `usegolib.import_("example.com/mod", version="vX.Y.Z", artifact_dir="out/", build_if_missing=None)` +- **AND WHEN** no matching artifact exists under `artifact_dir` for the current OS/arch +- **THEN** the call fails with `ArtifactNotFoundError` + +#### Scenario: Import does not rebuild when artifact exists +- **WHEN** Python calls `usegolib.import_("example.com/mod", version="vX.Y.Z", artifact_dir="out/", build_if_missing=True)` +- **AND WHEN** a matching artifact already exists under `artifact_dir` for the current OS/arch +- **THEN** the system loads the existing artifact without rebuilding + diff --git a/openspec/changes/archive/2026-02-09-update-import-auto-build/tasks.md b/openspec/changes/archive/2026-02-09-update-import-auto-build/tasks.md new file mode 100644 index 0000000..83cfcf6 --- /dev/null +++ b/openspec/changes/archive/2026-02-09-update-import-auto-build/tasks.md @@ -0,0 +1,23 @@ +## 1. Specs And Validation + +- [x] 1.1 Run `openspec validate update-import-auto-build --strict --no-interactive` + +## 2. Implementation + +- [x] 2.1 Add `USEGOLIB_ARTIFACT_DIR` default artifact root resolution (`src/usegolib/paths.py`) +- [x] 2.2 Update `usegolib.import_` to allow `artifact_dir=None` (use default root) +- [x] 2.3 Update `usegolib.import_` to support `build_if_missing=None` auto behavior +- [x] 2.4 Update `usegolib.import_` to support local subpackage directory imports (map to correct package import path) +- [x] 2.5 Update builder local resolution to accept subdirectories (walk up to find `go.mod`) + +## 3. Tests + +- [x] 3.1 Add unit test for module-root discovery when target is a subdirectory +- [x] 3.2 Add integration test: `import_` without `artifact_dir` auto-builds into `USEGOLIB_ARTIFACT_DIR` +- [x] 3.3 Add integration test: importing a subpackage auto-builds and binds to the subpackage + +## 4. Verification + +- [x] 4.1 Run `python -m pytest -q` +- [x] 4.2 Run `tools/wsl_linux_tests.ps1` +- [x] 4.3 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/.openspec.yaml b/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/README.md b/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/README.md new file mode 100644 index 0000000..f25727d --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/README.md @@ -0,0 +1,3 @@ +# add-any-and-variadic-support + +Support Go 'any' and variadic parameters for practical libraries like insyra DataList/DataTable. diff --git a/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/design.md b/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/design.md new file mode 100644 index 0000000..13d5da4 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/design.md @@ -0,0 +1,30 @@ +## Context + +Current scanner/type-bridge intentionally excluded `any` and variadic parameters. That made the system safe and simple, but it also excludes a large class of practical APIs (especially data libraries). + +## Goals / Non-Goals + +**Goals:** +- Recognize variadic parameters in scanning and bridge generation. +- Support `any` values (and `[]any`, `map[string]any`) across the ABI. +- Preserve existing ABI v0 and existing behavior for non-variadic, non-`any` symbols. + +**Non-Goals:** +- Support arbitrary map key types (e.g. `map[any]any`). +- Provide strict schema validation for `any` beyond "MessagePack-compatible". + +## Decisions + +- Represent variadic parameters in schema as `...T`. + - Rationale: keeps the information in the manifest without needing a separate boolean. +- Transport shape for variadic: + - Runtime packs Python varargs into a single list value for the last parameter. + - Bridge wrapper expands that slice when calling Go (`arg...`). +- Validation: + - `any` is treated as permissive; when schema says `any`, validation accepts any value. + +## Risks / Trade-offs + +- [More permissive typing] → Mitigation: keep `any` opt-in by signature; callers still get validation for non-`any` types. +- [Ambiguity for already-packed list] → Mitigation: runtime treats "exact arity and last arg is list/tuple" as already-packed. + diff --git a/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/proposal.md b/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/proposal.md new file mode 100644 index 0000000..c7e6e93 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/proposal.md @@ -0,0 +1,30 @@ +## Why + +Many real-world Go libraries (including Insyra) model data as `any` and use variadic parameters (e.g. `values ...any`). Today those symbols are excluded by the scanner/build filter, which blocks practical interop use cases like `DataList` and `DataTable`. + +## What Changes + +- Builder scanner recognizes variadic parameters and records them in a stable type form (`...T`). +- Type bridge supports `any` (Go `interface{}`) and common containers (`[]any`, `map[string]any`) for parameters and results. +- Bridge wrappers correctly call variadic functions/methods using slice expansion (`arg...`) and can encode/decode `any` values safely. +- Runtime packs variadic Python arguments into the ABI form and validates them against schema when present. + +## Capabilities + +### New Capabilities + +- (none) + +### Modified Capabilities + +- `usegolib-core`: extend the supported type bridge to include `any` and variadic parameters + +## Impact + +- Affected code: + - `src/usegolib/builder/scan.py` (variadic scanning) + - `src/usegolib/builder/build.py` (signature support filter) + - `src/usegolib/builder/gobridge.py` (arg conversion + export for `any`, variadic call sites) + - `src/usegolib/schema.py` (parse/validate `...T` and `any`) + - `src/usegolib/handle.py` (runtime packing for variadic calls) + - docs + examples diff --git a/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/specs/usegolib-core/spec.md b/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/specs/usegolib-core/spec.md new file mode 100644 index 0000000..0916b96 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/specs/usegolib-core/spec.md @@ -0,0 +1,27 @@ +## ADDED Requirements + +### Requirement: Support Type `any` (V0.x) +The type bridge SHALL support Go `any` (`interface{}`) as a dynamic value that can carry any MessagePack-compatible value. + +Supported containers SHALL include at least: +- `[]any` +- `map[string]any` + +#### Scenario: Call symbol with any parameter +- **WHEN** a callable symbol accepts an `any` parameter +- **AND WHEN** Python passes a MessagePack-compatible value +- **THEN** the call succeeds + +#### Scenario: Return any result +- **WHEN** a callable symbol returns `any` +- **THEN** Python receives the dynamic value + +### Requirement: Variadic Parameters (V0.x) +The builder scanner SHALL detect variadic parameters and record them as `...T` in symbol signatures. + +When manifest schema is present, the runtime SHALL allow Python calls to pass variadic arguments naturally and SHALL pack them into the ABI form for encoding. + +#### Scenario: Call variadic function from Python +- **WHEN** a callable symbol has a variadic parameter (`...T`) +- **AND WHEN** Python passes multiple arguments for that parameter +- **THEN** the runtime packs the arguments and the call succeeds diff --git a/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/tasks.md b/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/tasks.md new file mode 100644 index 0000000..b632092 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-any-and-variadic-support/tasks.md @@ -0,0 +1,36 @@ +## 1. Specs And Validation + +- [x] 1.1 Run `openspec validate add-any-and-variadic-support --strict --no-interactive` + +## 2. Scanner + +- [x] 2.1 Extend Go scanner to detect variadic parameters and render `...T` + +## 3. Builder Support Filter + +- [x] 3.1 Treat `any` as supported type +- [x] 3.2 Treat `...T` as supported when `T` is supported + +## 4. Bridge Generator (Go) + +- [x] 4.1 Support `any`, `[]any`, `map[string]any`, and `[]map[string]any` in arg conversion +- [x] 4.2 Support returning `any` by exporting through `exportAny` (including interface unwrapping) +- [x] 4.3 Call variadic symbols using slice expansion for the last argument (`arg...`) + +## 5. Runtime + Schema + +- [x] 5.1 Extend schema parsing/validation to understand `...T` and `any` +- [x] 5.2 Pack Python varargs for variadic symbols before ABI encoding + +## 6. Examples + Docs + +- [x] 6.1 Update `examples/insyra_basic.py` to use auto-download (`import_` with `version="v0.2.14"`) and demonstrate DataList/DataTable +- [x] 6.2 Update `docs/abi.md` (type bridge section) for `any` and variadic behavior + +## 7. Verification + +- [x] 7.1 Run `python -m pytest -q` +- [x] 7.2 Run integration tests: `USEGOLIB_INTEGRATION=1 python -m pytest -q -k integration` +- [x] 7.3 Run WSL unit tests: `tools/wsl_linux_tests.ps1` +- [x] 7.4 Run WSL integration tests: `tools/wsl_linux_tests.ps1 -Integration` +- [x] 7.5 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-add-cli-artifact-management/.openspec.yaml b/openspec/changes/archive/2026-02-10-add-cli-artifact-management/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-cli-artifact-management/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-add-cli-artifact-management/README.md b/openspec/changes/archive/2026-02-10-add-cli-artifact-management/README.md new file mode 100644 index 0000000..a8a2b85 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-cli-artifact-management/README.md @@ -0,0 +1,3 @@ +# add-cli-artifact-management + +CLI: remote module build + artifact rebuild/delete + docs diff --git a/openspec/changes/archive/2026-02-10-add-cli-artifact-management/proposal.md b/openspec/changes/archive/2026-02-10-add-cli-artifact-management/proposal.md new file mode 100644 index 0000000..c8005b7 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-cli-artifact-management/proposal.md @@ -0,0 +1,18 @@ +# Proposal: CLI Remote Build + Artifact Cache Management + +## Summary +Add developer-facing CLI commands to: +- build artifacts from remote Go module import paths (via `go mod download`) +- force rebuild and optionally re-download module sources +- delete or rebuild cached artifacts in the default artifact root + +Also add documentation describing CLI installation and usage. + +## Motivation +Users relying on auto-build or local artifact caching need a supported way to: +- rebuild a cached artifact (including re-downloading the Go module inputs) +- delete stale/ambiguous cached versions without manually spelunking the cache directory + +## Non-Goals +- Publishing prebuilt artifacts to a public registry (handled by wheel packaging flows) +- Deleting Go's global module cache (`go clean -modcache`) system-wide diff --git a/openspec/changes/archive/2026-02-10-add-cli-artifact-management/specs/usegolib-dev/spec.md b/openspec/changes/archive/2026-02-10-add-cli-artifact-management/specs/usegolib-dev/spec.md new file mode 100644 index 0000000..31c5d98 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-cli-artifact-management/specs/usegolib-dev/spec.md @@ -0,0 +1,36 @@ +## ADDED Requirements + +### Requirement: CLI Supports Artifact Cache Management +The repository SHALL provide CLI commands to manage locally cached artifacts for the current platform. + +At minimum, the CLI SHALL support: +- deleting cached artifacts by module/package and version +- deleting all cached versions for a module/package +- rebuilding cached artifacts + +#### Scenario: Delete a specific cached version +- **WHEN** a user runs `usegolib artifact rm --module --version --yes` +- **THEN** the matching artifact directory for the current platform is deleted + +#### Scenario: Delete all cached versions +- **WHEN** a user runs `usegolib artifact rm --module --all-versions --yes` +- **THEN** all cached versions for the current platform are deleted + +#### Scenario: Rebuild cached artifacts +- **WHEN** a user runs `usegolib artifact rebuild --module --version ` +- **THEN** the artifact is rebuilt into the selected artifact root + +### Requirement: CLI Can Re-Download Go Modules Before Rebuild +The CLI SHALL support rebuilding an artifact while forcing a fresh module download without mutating the user's global Go module cache. + +#### Scenario: Re-download before build +- **WHEN** a user runs `usegolib build --module --version --out --redownload` +- **THEN** the build uses an isolated `GOMODCACHE` and clears that cache before building + +### Requirement: CLI Usage Documentation +The repository SHALL document CLI installation and usage, including artifact cache deletion and rebuild workflows. + +#### Scenario: CLI docs exist +- **WHEN** a user reads the repository documentation +- **THEN** `docs/cli.md` exists and documents `usegolib build` and `usegolib artifact` commands + diff --git a/openspec/changes/archive/2026-02-10-add-cli-artifact-management/tasks.md b/openspec/changes/archive/2026-02-10-add-cli-artifact-management/tasks.md new file mode 100644 index 0000000..6bbe359 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-cli-artifact-management/tasks.md @@ -0,0 +1,22 @@ +## 1. Specs And Validation + +- [x] 1.1 Add spec deltas for CLI artifact management + redownload support +- [x] 1.2 Run `openspec validate add-cli-artifact-management --type change --strict --no-interactive` + +## 2. Implementation + +- [x] 2.1 Builder: allow passing `GOMODCACHE` through resolve/scan/build steps +- [x] 2.2 CLI: `usegolib build --redownload/--gomodcache` +- [x] 2.3 CLI: `usegolib artifact rm` and `usegolib artifact rebuild` +- [x] 2.4 Artifact module: implement delete helpers and index rebuild +- [x] 2.5 Docs: add `docs/cli.md` and link from README / troubleshooting + +## 3. Tests + +- [x] 3.1 Unit: artifact delete helpers +- [x] 3.2 Unit: builder resolve tests updated for env plumbing + +## 4. Verification + +- [x] 4.1 Run `python -m pytest -q` +- [x] 4.2 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-add-generic-functions/.openspec.yaml b/openspec/changes/archive/2026-02-10-add-generic-functions/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-generic-functions/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-add-generic-functions/README.md b/openspec/changes/archive/2026-02-10-add-generic-functions/README.md new file mode 100644 index 0000000..988d412 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-generic-functions/README.md @@ -0,0 +1,3 @@ +# add-generic-functions + +Add build-time instantiation + runtime helpers for exported generic Go functions. diff --git a/openspec/changes/archive/2026-02-10-add-generic-functions/design.md b/openspec/changes/archive/2026-02-10-add-generic-functions/design.md new file mode 100644 index 0000000..63103af --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-generic-functions/design.md @@ -0,0 +1,52 @@ +## Context + +Go generic (type parameter) functions cannot be called through the current v0 bridge because: +- the scanner intentionally skips generic declarations +- the bridge registers only concrete (non-generic) symbols + +However, many generic functions become callable if we choose a concrete instantiation at build time. + +## Goals / Non-Goals + +**Goals:** +- Scan exported top-level generic functions and capture their (generic) signature shape. +- Allow build-time instantiation via an explicit config file to avoid implicit inference. +- Generate concrete bridge wrappers that call `Fn[T1,T2,...](...)`. +- Record the mapping in `manifest.json` schema so runtime can resolve and validate calls. +- Provide a runtime helper API for calling generic instantiations. + +**Non-Goals:** +- Auto-infer generic type arguments at runtime. +- Support generic methods (receiver + type params) in this change. +- Support arbitrary constraint analysis; compilation is the final authority. + +## Decisions + +- **Explicit configuration**: generic instantiations are selected by `usegolib build --generics `. + - Rationale: avoids guesswork and unstable heuristics; keeps ABI unchanged. +- **Concrete symbol export**: each instantiation becomes a normal callable symbol with a stable, mangled name. + - Rationale: reuses existing `call` op and schema validation; no ABI changes. +- **Schema mapping**: store `schema.generics` as a list of mapping entries so runtime can resolve `generic(name, type_args)`. + - Rationale: avoids relying on name-mangling in user code and allows future bindgen improvements. + +## Config Format + +JSON file with an `instantiations` list: + +```json +{ + "instantiations": [ + {"pkg": "example.com/mod", "name": "Id", "type_args": ["int64"]}, + {"pkg": "example.com/mod", "name": "Id", "type_args": ["Person"], "symbol": "Id__Person"} + ] +} +``` + +If `symbol` is omitted, the builder generates a stable name based on type arguments. + +## Risks / Trade-offs + +- [Many instantiations] → Mitigation: require explicit config, do not default to “instantiate everything”. +- [Type argument mismatch / build failure] → Mitigation: validate arity and fail fast with actionable BuildError. +- [Schema missing] → Mitigation: runtime generic helper requires schema; otherwise users can only call concrete symbol names directly. + diff --git a/openspec/changes/archive/2026-02-10-add-generic-functions/proposal.md b/openspec/changes/archive/2026-02-10-add-generic-functions/proposal.md new file mode 100644 index 0000000..0a983e3 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-generic-functions/proposal.md @@ -0,0 +1,33 @@ +## Why + +Go 1.18+ generic (type parameter) functions are currently ignored by the builder scanner, so they cannot be called from Python even when their concrete instantiations would be safe under the v0 type bridge. + +## What Changes + +- Builder scans exported generic functions (top-level) and allows **explicit build-time instantiation** into callable bridge symbols. +- `usegolib build` accepts a generics instantiation config file to select concrete type arguments. +- Artifacts embed a schema mapping from generic function + type arguments to the generated concrete symbol name. +- Runtime exposes a helper API to call generic instantiations without relying on ad-hoc name mangling. +- Bindgen includes generated functions for instantiated generic symbols (Phase A/B usability). + +## Capabilities + +### New Capabilities + +- (none) + +### Modified Capabilities + +- `usegolib-core`: add requirements for build-time generic instantiation + runtime calling support + +## Impact + +- Affected code: + - builder scanner (`src/usegolib/builder/scan.py`) + - builder artifact build + manifest schema (`src/usegolib/builder/build.py`) + - Go bridge generator (`src/usegolib/builder/gobridge.py`) + - runtime API (`src/usegolib/handle.py`, `src/usegolib/schema.py`) + - CLI surface (`src/usegolib/cli.py`) + - bindgen (`src/usegolib/bindgen.py`) +- CLI: new optional `usegolib build --generics ` flag +- Manifest schema: adds `schema.generics` mapping (non-breaking; additive) diff --git a/openspec/changes/archive/2026-02-10-add-generic-functions/specs/usegolib-core/spec.md b/openspec/changes/archive/2026-02-10-add-generic-functions/specs/usegolib-core/spec.md new file mode 100644 index 0000000..41d6005 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-generic-functions/specs/usegolib-core/spec.md @@ -0,0 +1,20 @@ +## ADDED Requirements + +### Requirement: Generic Function Instantiation (V0.x) +The builder SHALL support exported top-level generic Go functions by generating concrete instantiations at build time for an explicit set of type arguments. + +The manifest schema SHALL record the mapping from: +- generic function name + type arguments +- to the concrete callable symbol name exported by the bridge + +The runtime SHALL provide a Python API to call a generic function instantiation by specifying the generic function name and type arguments. + +#### Scenario: Build instantiates a generic function and Python calls it +- **WHEN** the builder is configured to instantiate an exported generic Go function with concrete type arguments +- **AND WHEN** Python calls that instantiation through the runtime +- **THEN** the call succeeds and returns the correct result + +#### Scenario: Calling a non-configured generic instantiation fails +- **WHEN** Python requests a generic instantiation that is not present in the manifest schema +- **THEN** the runtime fails with an error + diff --git a/openspec/changes/archive/2026-02-10-add-generic-functions/tasks.md b/openspec/changes/archive/2026-02-10-add-generic-functions/tasks.md new file mode 100644 index 0000000..566e859 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-generic-functions/tasks.md @@ -0,0 +1,46 @@ +## 1. Specs And Validation + +- [x] 1.1 Run `openspec validate add-generic-functions --strict --no-interactive` + +## 2. CLI Surface + +- [x] 2.1 Add `usegolib build --generics ` flag and plumb to builder + +## 3. Builder Scan + +- [x] 3.1 Extend Go scanner to emit exported generic function definitions +- [x] 3.2 Parse generic definitions into `ModuleScan` + +## 4. Builder Instantiation + Manifest + +- [x] 4.1 Parse generics config JSON and validate entries +- [x] 4.2 Instantiate generic signatures (type param substitution) and filter supported instantiations +- [x] 4.3 Include instantiated symbols in `symbols` and `schema.symbols` +- [x] 4.4 Add `schema.generics` mapping to manifest schema + +## 5. Bridge Generator (Go) + +- [x] 5.1 Generate wrapper functions that call `Fn[T...](...)` for configured instantiations + +## 6. Runtime API + Schema + +- [x] 6.1 Extend `Schema` to parse `schema.generics` mapping +- [x] 6.2 Add `PackageHandle.generic(name, type_args) -> Callable` (and typed variant) + +## 7. Bindgen + +- [x] 7.1 Include instantiated generic symbols (and mapping) in generated bindings output + +## 8. Docs + Roadmap + +- [x] 8.1 Update `docs/roadmap.md` to reflect generic support implementation +- [x] 8.2 Update `README.md` with a generic usage example + +## 9. Tests + Verification + +- [x] 9.1 Add integration test exercising generic instantiation build + runtime call +- [x] 9.2 Run `python -m pytest -q` +- [x] 9.3 Run integration tests: `USEGOLIB_INTEGRATION=1 python -m pytest -q -k integration` +- [x] 9.4 Run WSL unit tests: `tools/wsl_linux_tests.ps1` +- [x] 9.5 Run WSL integration tests: `tools/wsl_linux_tests.ps1 -Integration` +- [x] 9.6 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-add-go-object-handles/.openspec.yaml b/openspec/changes/archive/2026-02-10-add-go-object-handles/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-go-object-handles/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-add-go-object-handles/README.md b/openspec/changes/archive/2026-02-10-add-go-object-handles/README.md new file mode 100644 index 0000000..6b49e6a --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-go-object-handles/README.md @@ -0,0 +1,3 @@ +# add-go-object-handles + +Add Go-side object handles (opaque IDs) and exported method calls without serializing receiver each time. diff --git a/openspec/changes/archive/2026-02-10-add-go-object-handles/design.md b/openspec/changes/archive/2026-02-10-add-go-object-handles/design.md new file mode 100644 index 0000000..1f63a5a --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-go-object-handles/design.md @@ -0,0 +1,54 @@ +# Design: Go-Side Object Handles + Method Dispatch + +## Goal + +Enable calling exported methods without serializing the receiver struct each time: +- create object once (`obj_new`) from record-struct input +- subsequent method calls pass only `(id, args...)` +- free explicitly (`obj_free`) with best-effort Python finalizer + +## ABI Extensions (v0) + +All requests remain MessagePack maps with `abi=0`. + +- `op="obj_new"`: + - `pkg`: package import path + - `type`: receiver struct type name (in that package) + - `init`: record-struct map (optional; omitted means zero value) +- `op="obj_call"`: + - `pkg`: package import path + - `type`: receiver struct type name + - `id`: int (opaque handle) + - `method`: exported method name + - `args`: list +- `op="obj_free"`: + - `id`: int + +Responses reuse the existing envelope (`ok/result/error`). + +## Go Bridge Implementation + +- Maintain an object registry: `map[uint64]any` protected by a mutex. +- Generate a `typeByKey` map to get `reflect.Type` for allowed struct types (`pkg.Type`). +- `obj_new`: + - Convert `init` into the struct type using existing `convertToType` + - Store a pointer to the struct in the registry + - Return the generated ID (int) +- `obj_call`: + - Dispatch to a generated wrapper function keyed by `pkg:type:method` + - Wrapper: + - load `*T` from registry + - convert args + - call `recv.Method(...)` + - encode results using existing export helpers +- `obj_free`: delete from registry + +## Python Runtime + +- Add `PackageHandle.object(type_name, init=None) -> GoObject`. +- `GoObject` holds `(handle, pkg, type, id)` and supports: + - `__getattr__` to call methods + - `close()` / context manager + - `__del__` best-effort `obj_free` +- When schema is present, validate `type_name` exists and validate method args/results. + diff --git a/openspec/changes/archive/2026-02-10-add-go-object-handles/proposal.md b/openspec/changes/archive/2026-02-10-add-go-object-handles/proposal.md new file mode 100644 index 0000000..454105b --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-go-object-handles/proposal.md @@ -0,0 +1,33 @@ +## Why + +For performance and ergonomics, we want to call Go exported methods without re-serializing the entire receiver struct on every call. This requires Go-side state with an opaque handle ID, plus Python proxy objects that hold the handle and call methods through the ABI. + +## What Changes + +- Extend ABI v0 with additional operations: + - `obj_new`: create a Go object from an initial record-struct value and return an opaque ID + - `obj_call`: call an exported method on a stored object by ID + - `obj_free`: free an object ID +- Builder: + - scan exported methods (non-generic) and include them in manifest schema + - generate bridge code that supports object registry + method dispatch +- Runtime: + - add a Python `GoObject` proxy with `obj.MethodName(...)` calls + - add `PackageHandle.object(type_name, init=...) -> GoObject` + +## Capabilities + +### New Capabilities + + +### Modified Capabilities + +- `usegolib-core`: ABI operations + method support + object handles + +## Impact + +- Affected code: `src/usegolib/abi.py`, `src/usegolib/handle.py`, `src/usegolib/schema.py`, + `src/usegolib/builder/scan.py`, `src/usegolib/builder/build.py`, `src/usegolib/builder/gobridge.py` +- Affected docs: `docs/abi.md` +- Affected tests: add new integration tests for object handles/method calls + diff --git a/openspec/changes/archive/2026-02-10-add-go-object-handles/specs/usegolib-core/spec.md b/openspec/changes/archive/2026-02-10-add-go-object-handles/specs/usegolib-core/spec.md new file mode 100644 index 0000000..b25d693 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-go-object-handles/specs/usegolib-core/spec.md @@ -0,0 +1,20 @@ +## ADDED Requirements + +### Requirement: Go Object Handles And Exported Methods (V0.x) +The system SHALL support calling exported Go methods without serializing the receiver value on every call by using Go-side object handles. + +The runtime SHALL expose a Python API to: +- create a Go-side object handle for a named struct type (`obj_new`) +- call exported methods on that handle (`obj_call`) +- free the handle (`obj_free`) + +#### Scenario: Create object handle and call method +- **WHEN** Python creates a Go object handle for a named struct type +- **AND WHEN** Python calls an exported method on the object handle +- **THEN** the call succeeds and returns the method result + +#### Scenario: Freeing an object handle makes it unusable +- **WHEN** Python frees a Go object handle +- **AND WHEN** Python calls a method on the freed handle +- **THEN** the call fails with an error + diff --git a/openspec/changes/archive/2026-02-10-add-go-object-handles/tasks.md b/openspec/changes/archive/2026-02-10-add-go-object-handles/tasks.md new file mode 100644 index 0000000..23fb735 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-go-object-handles/tasks.md @@ -0,0 +1,40 @@ +## 1. Specs And Validation + +- [x] 1.1 Run `openspec validate add-go-object-handles --strict --no-interactive` + +## 2. ABI And Docs + +- [x] 2.1 Extend `src/usegolib/abi.py` with request encoders for `obj_new/obj_call/obj_free` +- [x] 2.2 Update `docs/abi.md` to document new ops and object-handle semantics + +## 3. Builder (Scan + Manifest) + +- [x] 3.1 Extend Go scanner to emit exported methods with receiver type +- [x] 3.2 Update `src/usegolib/builder/scan.py` to parse methods +- [x] 3.3 Include methods in manifest schema and/or top-level symbols list + +## 4. Bridge Runtime (Go) + +- [x] 4.1 Extend `src/usegolib/builder/gobridge.py` to generate: + - object registry (id -> *T) + - type map for struct constructors + - method dispatch wrappers + - new ops in `usegolib_call` + +## 5. Python Runtime API + +- [x] 5.1 Add `PackageHandle.object(...) -> GoObject` +- [x] 5.2 Add `GoObject` proxy class with `close()` and context manager +- [x] 5.3 Validate type/methods against schema when available + +## 6. Tests + +- [x] 6.1 Add integration test module with a struct + methods and verify `obj_new/obj_call/obj_free` + +## 7. Verification + +- [x] 7.1 Run `python -m pytest -q` +- [x] 7.2 Run integration tests: `USEGOLIB_INTEGRATION=1 python -m pytest -q -k integration` +- [x] 7.3 Run WSL unit tests: `tools/wsl_linux_tests.ps1` +- [x] 7.4 Run WSL integration tests: `tools/wsl_linux_tests.ps1 -Integration` +- [x] 7.5 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-add-godoc-docstrings/.openspec.yaml b/openspec/changes/archive/2026-02-10-add-godoc-docstrings/.openspec.yaml new file mode 100644 index 0000000..5b134f3 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-godoc-docstrings/.openspec.yaml @@ -0,0 +1,2 @@ +id: add-godoc-docstrings + diff --git a/openspec/changes/archive/2026-02-10-add-godoc-docstrings/README.md b/openspec/changes/archive/2026-02-10-add-godoc-docstrings/README.md new file mode 100644 index 0000000..939394d --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-godoc-docstrings/README.md @@ -0,0 +1,4 @@ +# add-godoc-docstrings + +Embed Go doc comments in the artifact schema and surface them as Python docstrings (runtime and bindgen). + diff --git a/openspec/changes/archive/2026-02-10-add-godoc-docstrings/proposal.md b/openspec/changes/archive/2026-02-10-add-godoc-docstrings/proposal.md new file mode 100644 index 0000000..6225edf --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-godoc-docstrings/proposal.md @@ -0,0 +1,15 @@ +# Change: GoDoc To Python Docstrings + +## Why +When calling Go modules from Python, users often rely on IDE help / `help()` output to understand APIs. Today, `usegolib` exposes callable symbols but drops Go doc comments, so Python callables have no docstrings. + +## What Changes +- The builder scanner extracts Go doc comments for exported functions/methods (and generic funcs) from source AST comments (not `go doc` text output). +- The artifact manifest schema includes a `doc` field for symbols and methods. +- The Python runtime attaches docstrings to dynamically created callables (best-effort when schema is present). +- The bindgen generator emits docstrings into the generated Python module. + +## Impact +- Affected specs: `usegolib-core` +- Affected code: `src/usegolib/builder/scan.py`, `src/usegolib/builder/symbols.py`, `src/usegolib/builder/build.py`, `src/usegolib/schema.py`, `src/usegolib/handle.py`, `src/usegolib/bindgen.py` + diff --git a/openspec/changes/archive/2026-02-10-add-godoc-docstrings/specs/usegolib-core/spec.md b/openspec/changes/archive/2026-02-10-add-godoc-docstrings/specs/usegolib-core/spec.md new file mode 100644 index 0000000..9f6625e --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-godoc-docstrings/specs/usegolib-core/spec.md @@ -0,0 +1,14 @@ +## ADDED Requirements + +### Requirement: GoDoc To Python Docstrings (V0.x) +When building an artifact, the system SHALL extract Go doc comments for exported functions and methods and include them in the artifact schema. + +When schema is present, the Python runtime SHOULD attach these docs as docstrings on the dynamic callables it returns. + +The bindgen generator SHOULD emit these docs as Python docstrings in generated modules. + +#### Scenario: Bindgen includes docstrings +- **WHEN** an artifact manifest schema includes `doc` for a symbol +- **AND WHEN** the user runs `usegolib bindgen` +- **THEN** the generated Python method includes a docstring derived from the Go doc comment + diff --git a/openspec/changes/archive/2026-02-10-add-godoc-docstrings/tasks.md b/openspec/changes/archive/2026-02-10-add-godoc-docstrings/tasks.md new file mode 100644 index 0000000..d146bda --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-godoc-docstrings/tasks.md @@ -0,0 +1,23 @@ +## 1. Specs And Validation + +- [x] 1.1 Run `openspec validate add-godoc-docstrings --strict --no-interactive` + +## 2. Scanner + Manifest + +- [x] 2.1 Extract doc comments for exported functions and methods via Go AST comments (`parser.ParseComments`) +- [x] 2.2 Emit `doc` fields into manifest `schema.symbols` and `schema.methods` + +## 3. Runtime + Bindgen + +- [x] 3.1 Parse `doc` fields into schema doc maps +- [x] 3.2 Attach docstrings to dynamic callables returned by `PackageHandle.__getattr__` and `GoObject.__getattr__` +- [x] 3.3 Emit docstrings in `usegolib bindgen` generated modules + +## 4. Tests + +- [x] 4.1 Unit test: schema parses docs + bindgen output includes docstring text + +## 5. Verification + +- [x] 5.1 Run `python -m pytest -q` +- [x] 5.2 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-add-opaque-pointer-return-handles/.openspec.yaml b/openspec/changes/archive/2026-02-10-add-opaque-pointer-return-handles/.openspec.yaml new file mode 100644 index 0000000..7da09e3 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-opaque-pointer-return-handles/.openspec.yaml @@ -0,0 +1,2 @@ +id: add-opaque-pointer-return-handles + diff --git a/openspec/changes/archive/2026-02-10-add-opaque-pointer-return-handles/README.md b/openspec/changes/archive/2026-02-10-add-opaque-pointer-return-handles/README.md new file mode 100644 index 0000000..545052e --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-opaque-pointer-return-handles/README.md @@ -0,0 +1,4 @@ +# add-opaque-pointer-return-handles + +Return `*OpaqueStruct` results as object handles instead of record-struct dicts, so fluent Go APIs can be used from Python. + diff --git a/openspec/changes/archive/2026-02-10-add-opaque-pointer-return-handles/proposal.md b/openspec/changes/archive/2026-02-10-add-opaque-pointer-return-handles/proposal.md new file mode 100644 index 0000000..619d71b --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-opaque-pointer-return-handles/proposal.md @@ -0,0 +1,13 @@ +# Change: Return Opaque Pointer Results As Object Handles + +## Why +Many Go libraries (e.g. Insyra) use fluent APIs that return pointers to structs whose fields are unexported (opaque). When `usegolib` exports `*T` as a record-struct dict, the Python result becomes `{}` and users cannot call methods on it. + +## What Changes +- If a function/method returns `*T` where `T` is a struct type with **no exported fields** in the manifest schema, the bridge returns an **object id** instead of exporting a dict. +- The Python runtime wraps that id as a `GoObject` so users can call methods naturally. + +## Impact +- Affected specs: `usegolib-core` +- Affected code: bridge generator, schema validation, runtime handle conversion, integration tests + diff --git a/openspec/changes/archive/2026-02-10-add-opaque-pointer-return-handles/specs/usegolib-core/spec.md b/openspec/changes/archive/2026-02-10-add-opaque-pointer-return-handles/specs/usegolib-core/spec.md new file mode 100644 index 0000000..e6f100e --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-opaque-pointer-return-handles/specs/usegolib-core/spec.md @@ -0,0 +1,9 @@ +## ADDED Requirements + +### Requirement: Opaque Pointer Results Use Object Handles (V0.x) +When a callable symbol or method returns `*T` and the struct type `T` has no exported fields in the manifest schema, the system SHALL represent that value as an object handle rather than a record-struct dict. + +#### Scenario: Function returns *Opaque handle +- **WHEN** a Go function returns `*T` +- **AND WHEN** `T` has no exported fields +- **THEN** Python receives a `GoObject` that can be used to call exported methods on `T` diff --git a/openspec/changes/archive/2026-02-10-add-opaque-pointer-return-handles/tasks.md b/openspec/changes/archive/2026-02-10-add-opaque-pointer-return-handles/tasks.md new file mode 100644 index 0000000..0ec1702 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-add-opaque-pointer-return-handles/tasks.md @@ -0,0 +1,20 @@ +## 1. Specs And Validation + +- [x] 1.1 Run `openspec validate add-opaque-pointer-return-handles --strict --no-interactive` + +## 2. Bridge + Runtime + +- [x] 2.1 Detect "opaque" struct types (no exported fields) during build +- [x] 2.2 For results `*T` where `T` is opaque, return object id from `call`/`obj_call` handlers +- [x] 2.3 Runtime: convert returned object ids into `GoObject` for `*T` opaque results +- [x] 2.4 Schema validation: allow int object ids as valid values for `*T` opaque results + +## 3. Tests + +- [x] 3.1 Integration test: exported function returning `*Opaque` returns a handle usable for method calls + +## 4. Verification + +- [x] 4.1 Run `python -m pytest -q` +- [x] 4.2 Run integration tests (USEGOLIB_INTEGRATION=1) +- [x] 4.3 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-complete-milestone5-docs/.openspec.yaml b/openspec/changes/archive/2026-02-10-complete-milestone5-docs/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-complete-milestone5-docs/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-complete-milestone5-docs/README.md b/openspec/changes/archive/2026-02-10-complete-milestone5-docs/README.md new file mode 100644 index 0000000..972012a --- /dev/null +++ b/openspec/changes/archive/2026-02-10-complete-milestone5-docs/README.md @@ -0,0 +1,3 @@ +# complete-milestone5-docs + +Remove misleading internal version ranges from roadmap and add production troubleshooting docs. diff --git a/openspec/changes/archive/2026-02-10-complete-milestone5-docs/design.md b/openspec/changes/archive/2026-02-10-complete-milestone5-docs/design.md new file mode 100644 index 0000000..8d421e7 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-complete-milestone5-docs/design.md @@ -0,0 +1,8 @@ +# Design: Milestone 5 Docs Completion + +Doc-only change focused on: +- making the roadmap version-agnostic (avoid misleading internal version ranges) +- providing an explicit troubleshooting guide for production and dev workflows + +No runtime/build behavior changes. + diff --git a/openspec/changes/archive/2026-02-10-complete-milestone5-docs/proposal.md b/openspec/changes/archive/2026-02-10-complete-milestone5-docs/proposal.md new file mode 100644 index 0000000..b1b465a --- /dev/null +++ b/openspec/changes/archive/2026-02-10-complete-milestone5-docs/proposal.md @@ -0,0 +1,24 @@ +## Why + +The roadmap currently includes internal version ranges (e.g. `v0.7 - v0.9`) that can be misleading. Also, Milestone 5 calls out production documentation (troubleshooting) which is not yet present. + +## What Changes + +- Update `docs/roadmap.md` to remove misleading internal version numbers/ranges and keep milestones version-agnostic. +- Add `docs/troubleshooting.md` covering common build/load/import failures and recommended fixes. +- Link troubleshooting docs from `README.md`. + +## Capabilities + +### New Capabilities + + +### Modified Capabilities + +- `usegolib-dev`: developer-facing docs (roadmap clarity + troubleshooting docs) + +## Impact + +- Affected docs: `docs/roadmap.md`, `docs/troubleshooting.md`, `README.md` +- Affected spec: `openspec/specs/usegolib-dev/spec.md` + diff --git a/openspec/changes/archive/2026-02-10-complete-milestone5-docs/specs/usegolib-dev/spec.md b/openspec/changes/archive/2026-02-10-complete-milestone5-docs/specs/usegolib-dev/spec.md new file mode 100644 index 0000000..333a249 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-complete-milestone5-docs/specs/usegolib-dev/spec.md @@ -0,0 +1,16 @@ +## ADDED Requirements + +### Requirement: Troubleshooting Documentation +The repository SHALL include `docs/troubleshooting.md` covering common failures and remedies for building/loading/importing artifacts. + +#### Scenario: Troubleshooting docs exist +- **WHEN** a developer looks for common error remediation steps +- **THEN** `docs/troubleshooting.md` exists and covers common failure modes + +### Requirement: Roadmap Avoids Misleading Internal Version Numbers +The repository SHALL keep `docs/roadmap.md` version-agnostic for unreleased work and MUST NOT assign misleading internal version ranges to future milestones. + +#### Scenario: Roadmap does not claim internal version ranges +- **WHEN** a developer reads `docs/roadmap.md` +- **THEN** future milestones do not include internal version ranges like `v0.X - v0.Y` + diff --git a/openspec/changes/archive/2026-02-10-complete-milestone5-docs/tasks.md b/openspec/changes/archive/2026-02-10-complete-milestone5-docs/tasks.md new file mode 100644 index 0000000..c18cb80 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-complete-milestone5-docs/tasks.md @@ -0,0 +1,15 @@ +## 1. Specs And Validation + +- [x] 1.1 Run `openspec validate complete-milestone5-docs --strict --no-interactive` + +## 2. Documentation + +- [x] 2.1 Update `docs/roadmap.md` to remove misleading internal version numbers/ranges +- [x] 2.2 Add `docs/troubleshooting.md` +- [x] 2.3 Link troubleshooting docs from `README.md` + +## 3. Verification + +- [x] 3.1 Run `python -m pytest -q` +- [x] 3.2 Run `tools/wsl_linux_tests.ps1` +- [x] 3.3 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-fix-error-only-results/.openspec.yaml b/openspec/changes/archive/2026-02-10-fix-error-only-results/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-error-only-results/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-fix-error-only-results/README.md b/openspec/changes/archive/2026-02-10-fix-error-only-results/README.md new file mode 100644 index 0000000..86d2b28 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-error-only-results/README.md @@ -0,0 +1,3 @@ +# fix-error-only-results + +Support error-only returns (validate + GoError) diff --git a/openspec/changes/archive/2026-02-10-fix-error-only-results/proposal.md b/openspec/changes/archive/2026-02-10-fix-error-only-results/proposal.md new file mode 100644 index 0000000..f70d90b --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-error-only-results/proposal.md @@ -0,0 +1,13 @@ +# Proposal: Support `error`-Only Returns + +## Why +Some Go APIs return only `error` to signal success/failure (for example `ToCSV(...) error`). +For a successful call, the value is `nil` and there is no meaningful return payload. + +Without explicit support, schema validation rejects `error` as an "unknown type", and the bridge +may attempt to serialize non-nil error interfaces as values. + +## What Changes +- `error` is treated as a special result type: on success it is represented as `nil`. +- If an `error`-only return is non-nil, the bridge reports it as `GoError` via the ABI error envelope. + diff --git a/openspec/changes/archive/2026-02-10-fix-error-only-results/specs/usegolib-core/spec.md b/openspec/changes/archive/2026-02-10-fix-error-only-results/specs/usegolib-core/spec.md new file mode 100644 index 0000000..03a0544 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-error-only-results/specs/usegolib-core/spec.md @@ -0,0 +1,17 @@ +## ADDED Requirements + +### Requirement: Error-Only Results Are Treated As Nil On Success +When a callable symbol or method returns only `error`, the system SHALL represent successful results as `nil` (Python `None`). + +If the returned `error` is non-nil, the system SHALL report the failure via the ABI error envelope as `GoError`. + +#### Scenario: Method returns only error and succeeds +- **WHEN** a Go method returns `error` +- **AND WHEN** the call succeeds (`error` is `nil`) +- **THEN** the Python result is `None` + +#### Scenario: Method returns only error and fails +- **WHEN** a Go method returns `error` +- **AND WHEN** the call fails (`error` is non-nil) +- **THEN** the Python call raises `GoError` + diff --git a/openspec/changes/archive/2026-02-10-fix-error-only-results/tasks.md b/openspec/changes/archive/2026-02-10-fix-error-only-results/tasks.md new file mode 100644 index 0000000..e4083b0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-error-only-results/tasks.md @@ -0,0 +1,20 @@ +## 1. Specs And Validation + +- [x] 1.1 Add spec delta for error-only results +- [x] 1.2 Run `openspec validate fix-error-only-results --type change --strict --no-interactive` + +## 2. Implementation + +- [x] 2.1 Schema validation: accept `error` results as nil +- [x] 2.2 Go bridge: treat `error`-only returns as GoError on non-nil + +## 3. Tests + +- [x] 3.1 Unit: schema validates error-only result +- [x] 3.2 Integration: error-only return works + +## 4. Verification + +- [x] 4.1 Run `python -m pytest -q` +- [x] 4.2 Run integration tests (USEGOLIB_INTEGRATION=1) +- [x] 4.3 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-fix-remote-module-latest/.openspec.yaml b/openspec/changes/archive/2026-02-10-fix-remote-module-latest/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-remote-module-latest/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-fix-remote-module-latest/README.md b/openspec/changes/archive/2026-02-10-fix-remote-module-latest/README.md new file mode 100644 index 0000000..d41e998 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-remote-module-latest/README.md @@ -0,0 +1,3 @@ +# fix-remote-module-latest + +Fix remote module resolution to use @latest by default and document/verify remote auto-download behavior. diff --git a/openspec/changes/archive/2026-02-10-fix-remote-module-latest/design.md b/openspec/changes/archive/2026-02-10-fix-remote-module-latest/design.md new file mode 100644 index 0000000..7909444 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-remote-module-latest/design.md @@ -0,0 +1,13 @@ +# Design: Remote Version Defaulting + +## Rule + +- If `version is None`: use `@latest`. +- If `version == "latest"`: treat as `@latest`. +- If `version` already starts with `@`: pass through. +- Otherwise: pass through as-is (e.g. `v1.2.3`, pseudo-versions). + +## Testing + +Add unit tests that monkeypatch the internal `_go_mod_download_json` to capture the argument string passed to `go mod download -json`, verifying the `@latest` default without requiring `go` in unit tests. + diff --git a/openspec/changes/archive/2026-02-10-fix-remote-module-latest/proposal.md b/openspec/changes/archive/2026-02-10-fix-remote-module-latest/proposal.md new file mode 100644 index 0000000..19f0d3c --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-remote-module-latest/proposal.md @@ -0,0 +1,25 @@ +## Why + +The spec says remote module builds default to `@latest` when no version is provided. The current resolver uses `latest` (without `@`), which can break `go mod download` and prevents the "auto-download Go module" workflow from working reliably. + +## What Changes + +- Fix remote module version defaulting: `None` -> `@latest` (and accept `latest` by mapping to `@latest`). +- Add unit tests to enforce resolver behavior without requiring a Go toolchain in unit test runs. +- Update README to explicitly show importing a remote module/package triggers auto-download/build (dev mode). + +## Capabilities + +### New Capabilities + + +### Modified Capabilities + +- `usegolib-core`: remote module version resolution behavior (bug fix) + +## Impact + +- Affected code: `src/usegolib/builder/resolve.py` +- Affected docs: `README.md` +- Affected tests: `tests/test_builder_resolve.py` + diff --git a/openspec/changes/archive/2026-02-10-fix-remote-module-latest/specs/usegolib-core/spec.md b/openspec/changes/archive/2026-02-10-fix-remote-module-latest/specs/usegolib-core/spec.md new file mode 100644 index 0000000..9f57836 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-remote-module-latest/specs/usegolib-core/spec.md @@ -0,0 +1,19 @@ +## MODIFIED Requirements + +### Requirement: Module And Version Resolution (Build) +When building artifacts from a Go import path, the system SHALL resolve a concrete module version and record it in the artifact manifest. + +#### Scenario: Version omitted defaults to @latest for remote modules +- **WHEN** `usegolib build` is invoked with a Go module import path and `version=None` +- **THEN** the system resolves the version query to `@latest` +- **AND THEN** the manifest `version` field contains the resolved concrete version + +#### Scenario: Pinned version is used as-is for remote modules +- **WHEN** `usegolib build` is invoked with a Go module import path and `version="vX.Y.Z"` +- **THEN** the system builds using `vX.Y.Z` +- **AND THEN** the manifest `version` field MUST equal `vX.Y.Z` + +#### Scenario: Local module directories use version "local" +- **WHEN** `usegolib build` is invoked with a local module directory +- **THEN** the manifest `version` field MUST be `"local"` + diff --git a/openspec/changes/archive/2026-02-10-fix-remote-module-latest/tasks.md b/openspec/changes/archive/2026-02-10-fix-remote-module-latest/tasks.md new file mode 100644 index 0000000..ed6ff68 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-remote-module-latest/tasks.md @@ -0,0 +1,18 @@ +## 1. Specs And Validation + +- [x] 1.1 Run `openspec validate fix-remote-module-latest --strict --no-interactive` + +## 2. Implementation + +- [x] 2.1 Fix remote version default to `@latest` in `src/usegolib/builder/resolve.py` +- [x] 2.2 Update `README.md` to show remote module import auto-download/build behavior + +## 3. Tests + +- [x] 3.1 Add unit tests for remote resolver default and package-path trimming + +## 4. Verification + +- [x] 4.1 Run `python -m pytest -q` +- [x] 4.2 Run `tools/wsl_linux_tests.ps1` +- [x] 4.3 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/.openspec.yaml b/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/README.md b/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/README.md new file mode 100644 index 0000000..07a8a68 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/README.md @@ -0,0 +1,3 @@ +# fix-roadmap-milestone-order + +Reorder docs/roadmap.md milestones so numbering is increasing (5 before 6) and clarify placement. diff --git a/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/design.md b/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/design.md new file mode 100644 index 0000000..0a7ce08 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/design.md @@ -0,0 +1,6 @@ +# Design: Roadmap Ordering + +Doc-only change: +- Move the Milestone 6 section below Milestone 5. +- Add a brief sequencing note (Milestone 6 is post-v1.0 unless reprioritized). + diff --git a/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/proposal.md b/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/proposal.md new file mode 100644 index 0000000..f2ef76e --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/proposal.md @@ -0,0 +1,22 @@ +## Why + +`docs/roadmap.md` currently lists Milestone 6 before Milestone 5, which is confusing and looks like a sequencing decision. We should reorder milestones so numbering is increasing and clarify how post-v1.0 feature expansion fits into the roadmap. + +## What Changes + +- Reorder `docs/roadmap.md` so Milestone 5 appears before Milestone 6. +- Add a short note clarifying that Milestone 6 (methods/generics) is planned after v1.0 readiness work unless explicitly prioritized earlier. + +## Capabilities + +### New Capabilities + + +### Modified Capabilities + +- `usegolib-dev`: roadmap clarity and ordering + +## Impact + +- Affected docs: `docs/roadmap.md` + diff --git a/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/specs/usegolib-dev/spec.md b/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/specs/usegolib-dev/spec.md new file mode 100644 index 0000000..5fa5d3f --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/specs/usegolib-dev/spec.md @@ -0,0 +1,9 @@ +## ADDED Requirements + +### Requirement: Roadmap Milestone Ordering Is Consistent +The repository SHALL keep `docs/roadmap.md` milestones ordered by increasing milestone number to avoid confusion about sequencing. + +#### Scenario: Milestones are ordered +- **WHEN** a developer reads `docs/roadmap.md` +- **THEN** Milestone sections appear in increasing numerical order (e.g. 4, 5, 6) + diff --git a/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/tasks.md b/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/tasks.md new file mode 100644 index 0000000..6d0269c --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-roadmap-milestone-order/tasks.md @@ -0,0 +1,17 @@ +## 1. Specs And Design + +- [x] 1.1 Create `proposal.md` +- [x] 1.2 Create `design.md` +- [x] 1.3 Create delta spec `specs/usegolib-dev/spec.md` +- [x] 1.4 Run `openspec validate fix-roadmap-milestone-order --strict --no-interactive` + +## 2. Documentation + +- [x] 2.1 Reorder milestones in `docs/roadmap.md` so 5 is before 6 +- [x] 2.2 Add a sequencing note about Milestone 6 placement + +## 3. Verification + +- [x] 3.1 Run `python -m pytest -q` +- [x] 3.2 Run `tools/wsl_linux_tests.ps1` +- [x] 3.3 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-fix-windows-cp950-decode/.openspec.yaml b/openspec/changes/archive/2026-02-10-fix-windows-cp950-decode/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-windows-cp950-decode/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-fix-windows-cp950-decode/README.md b/openspec/changes/archive/2026-02-10-fix-windows-cp950-decode/README.md new file mode 100644 index 0000000..6da3eea --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-windows-cp950-decode/README.md @@ -0,0 +1,3 @@ +# fix-windows-cp950-decode + +Fix Windows cp950 UnicodeDecodeError when capturing Go subprocess output diff --git a/openspec/changes/archive/2026-02-10-fix-windows-cp950-decode/proposal.md b/openspec/changes/archive/2026-02-10-fix-windows-cp950-decode/proposal.md new file mode 100644 index 0000000..a7f8539 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-windows-cp950-decode/proposal.md @@ -0,0 +1,13 @@ +# Proposal: Fix Windows Locale Decode Errors In Builder + +## Why +On Windows, Python may default to a non-UTF8 locale encoding (for example `cp950`). +Go tools typically emit UTF-8 output. When `subprocess.run(..., text=True)` is used, +Python attempts to decode stdout using the locale encoding and can raise `UnicodeDecodeError`. + +This breaks auto-build and scanning on some Windows environments. + +## What Changes +- Capture Go tool output as bytes and decode as UTF-8 with `errors="replace"` to avoid crashes. +- When JSON output is expected, tolerate non-JSON prefix lines by extracting the first JSON object. + diff --git a/openspec/changes/archive/2026-02-10-fix-windows-cp950-decode/specs/usegolib-dev/spec.md b/openspec/changes/archive/2026-02-10-fix-windows-cp950-decode/specs/usegolib-dev/spec.md new file mode 100644 index 0000000..35c96ee --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-windows-cp950-decode/specs/usegolib-dev/spec.md @@ -0,0 +1,11 @@ +## ADDED Requirements + +### Requirement: Windows Builder Output Decoding Is Locale-Safe +The builder SHALL not fail with `UnicodeDecodeError` on Windows due to non-UTF8 locale encodings when capturing Go tool output. + +#### Scenario: Go tool output contains non-UTF8 bytes under Windows locale +- **WHEN** the builder captures stdout/stderr from Go tools on Windows +- **AND WHEN** the system locale encoding cannot decode those bytes (for example `cp950`) +- **THEN** the build/scan process does not crash with `UnicodeDecodeError` +- **AND THEN** errors are reported as `BuildError` when the underlying Go command fails + diff --git a/openspec/changes/archive/2026-02-10-fix-windows-cp950-decode/tasks.md b/openspec/changes/archive/2026-02-10-fix-windows-cp950-decode/tasks.md new file mode 100644 index 0000000..57d32ec --- /dev/null +++ b/openspec/changes/archive/2026-02-10-fix-windows-cp950-decode/tasks.md @@ -0,0 +1,18 @@ +## 1. Specs And Validation + +- [x] 1.1 Add spec delta for Windows locale-safe Go subprocess output decoding +- [x] 1.2 Run `openspec validate fix-windows-cp950-decode --type change --strict --no-interactive` + +## 2. Implementation + +- [x] 2.1 Decode Go subprocess output as UTF-8 bytes (avoid cp950 UnicodeDecodeError) +- [x] 2.2 Make JSON parsing robust to non-JSON prefix lines + +## 3. Tests + +- [x] 3.1 Unit: non-UTF8 prefix does not crash resolve/scan + +## 4. Verification + +- [x] 4.1 Run `python -m pytest -q` +- [x] 4.2 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-follow-loaded-version-on-import/.openspec.yaml b/openspec/changes/archive/2026-02-10-follow-loaded-version-on-import/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-follow-loaded-version-on-import/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-follow-loaded-version-on-import/README.md b/openspec/changes/archive/2026-02-10-follow-loaded-version-on-import/README.md new file mode 100644 index 0000000..6aaddfc --- /dev/null +++ b/openspec/changes/archive/2026-02-10-follow-loaded-version-on-import/README.md @@ -0,0 +1,3 @@ +# follow-loaded-version-on-import + +import_ version=None follows already-loaded module version to avoid ambiguity diff --git a/openspec/changes/archive/2026-02-10-follow-loaded-version-on-import/proposal.md b/openspec/changes/archive/2026-02-10-follow-loaded-version-on-import/proposal.md new file mode 100644 index 0000000..c67f790 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-follow-loaded-version-on-import/proposal.md @@ -0,0 +1,11 @@ +# Proposal: Follow Loaded Module Version When Importing Subpackages + +## Why +Users commonly import a root module with an explicit version and then import subpackages without repeating the version. +If multiple artifact versions exist on disk, omitting `version=` currently raises `AmbiguousArtifactError` before the runtime can +apply the per-process "single module version" rule. + +## What Changes +- If a module has already been loaded in the current Python process, `usegolib.import_(..., version=None)` will follow that loaded version. +- This applies to both the root module import path and any subpackage import paths under that module. + diff --git a/openspec/changes/archive/2026-02-10-follow-loaded-version-on-import/specs/usegolib-core/spec.md b/openspec/changes/archive/2026-02-10-follow-loaded-version-on-import/specs/usegolib-core/spec.md new file mode 100644 index 0000000..90553f9 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-follow-loaded-version-on-import/specs/usegolib-core/spec.md @@ -0,0 +1,10 @@ +## MODIFIED Requirements + +### Requirement: Python Import API +If a Go module is already loaded in the current Python process, then `usegolib.import_(..., version=None)` SHALL follow the already-loaded module version for that module and its subpackages. + +#### Scenario: Import omits version but follows already-loaded module version +- **WHEN** Python imports `example.com/mod` at version `v1.2.0` +- **AND WHEN** Python imports `example.com/mod/subpkg` with `version=None` +- **THEN** the second import resolves to version `v1.2.0` (the already-loaded module version) +- **AND THEN** the import does not fail with `AmbiguousArtifactError` even if multiple artifact versions exist on disk diff --git a/openspec/changes/archive/2026-02-10-follow-loaded-version-on-import/tasks.md b/openspec/changes/archive/2026-02-10-follow-loaded-version-on-import/tasks.md new file mode 100644 index 0000000..26d7a41 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-follow-loaded-version-on-import/tasks.md @@ -0,0 +1,17 @@ +## 1. Specs And Validation + +- [x] 1.1 Add spec delta: version=None follows already loaded module version +- [x] 1.2 Run `openspec validate follow-loaded-version-on-import --type change --strict --no-interactive` + +## 2. Implementation + +- [x] 2.1 Importer: if module already loaded and version omitted, use loaded version + +## 3. Tests + +- [x] 3.1 Unit: import_ avoids ambiguity by following loaded version + +## 4. Verification + +- [x] 4.1 Run `python -m pytest -q` +- [x] 4.2 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-improve-builder-network-resilience/.openspec.yaml b/openspec/changes/archive/2026-02-10-improve-builder-network-resilience/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-improve-builder-network-resilience/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-improve-builder-network-resilience/README.md b/openspec/changes/archive/2026-02-10-improve-builder-network-resilience/README.md new file mode 100644 index 0000000..2c9c749 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-improve-builder-network-resilience/README.md @@ -0,0 +1,3 @@ +# improve-builder-network-resilience + +Retry Go commands and provide actionable hints/fallbacks for transient module download failures (proxy issues). diff --git a/openspec/changes/archive/2026-02-10-improve-builder-network-resilience/specs/usegolib-core/spec.md b/openspec/changes/archive/2026-02-10-improve-builder-network-resilience/specs/usegolib-core/spec.md new file mode 100644 index 0000000..f290ef5 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-improve-builder-network-resilience/specs/usegolib-core/spec.md @@ -0,0 +1,16 @@ +## ADDED Requirements + +### Requirement: Builder Retries Transient Go Network Failures +When building or rebuilding artifacts for remote modules, the builder SHALL retry transient Go module download failures and provide actionable remediation hints. + +#### Scenario: Transient proxy failures are retried and may fall back to GOPROXY=direct +- **WHEN** a build executes Go commands that may download modules (for example `go mod download`, `go list`, or `go build`) +- **AND WHEN** the command fails due to a transient network/proxy error (for example `proxy.golang.org` timeouts) +- **THEN** the builder retries the command with a short backoff +- **AND THEN** if `GOPROXY` is not explicitly set, the builder MAY retry with `GOPROXY=direct` + +#### Scenario: Failures include a hint for common network/proxy remediation +- **WHEN** a build fails due to a network/proxy error downloading Go modules +- **THEN** the builder raises `BuildError` +- **AND THEN** the error message includes a hint mentioning `GOPROXY=direct` + diff --git a/openspec/changes/archive/2026-02-10-improve-godoc-visibility/.openspec.yaml b/openspec/changes/archive/2026-02-10-improve-godoc-visibility/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-improve-godoc-visibility/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-improve-godoc-visibility/README.md b/openspec/changes/archive/2026-02-10-improve-godoc-visibility/README.md new file mode 100644 index 0000000..a61f7ba --- /dev/null +++ b/openspec/changes/archive/2026-02-10-improve-godoc-visibility/README.md @@ -0,0 +1,3 @@ +# improve-godoc-visibility + +Ensure all dynamic callables (including typed wrappers) have GoDoc/signature docstrings diff --git a/openspec/changes/archive/2026-02-10-improve-godoc-visibility/proposal.md b/openspec/changes/archive/2026-02-10-improve-godoc-visibility/proposal.md new file mode 100644 index 0000000..e89e0fb --- /dev/null +++ b/openspec/changes/archive/2026-02-10-improve-godoc-visibility/proposal.md @@ -0,0 +1,13 @@ +# Proposal: Ensure GoDoc Is Visible In Python For All Callables + +## Why +Users rely on Python introspection (`help()`, IDE hover, `.__doc__`) to discover APIs. +When importing Go modules dynamically, docstrings should be consistently available for every exported function/method. + +## What Changes +- When schema is present, the runtime always sets `__doc__` for exported functions and methods: + - prefers extracted GoDoc + - falls back to a Go signature string when GoDoc is missing +- Typed wrappers preserve the underlying docstrings. +- Generic instantiation symbols include doc metadata when available. + diff --git a/openspec/changes/archive/2026-02-10-improve-godoc-visibility/specs/usegolib-core/spec.md b/openspec/changes/archive/2026-02-10-improve-godoc-visibility/specs/usegolib-core/spec.md new file mode 100644 index 0000000..9557b8a --- /dev/null +++ b/openspec/changes/archive/2026-02-10-improve-godoc-visibility/specs/usegolib-core/spec.md @@ -0,0 +1,32 @@ +## MODIFIED Requirements + +### Requirement: GoDoc To Python Docstrings (V0.x) +When building an artifact, the system SHALL extract Go doc comments for exported functions and methods and include them in the artifact schema. + +When schema is present, the Python runtime SHALL attach docstrings to the dynamic callables it returns for exported functions and methods. + +If GoDoc is missing for a callable, the runtime SHOULD still attach a signature-based docstring (so `help()` is informative). + +Typed wrappers (`handle.typed()` and typed objects) SHALL preserve the docstrings of the underlying dynamic callables. + +The bindgen generator SHOULD emit these docs as Python docstrings in generated modules. + +#### Scenario: Runtime exposes GoDoc for a function +- **WHEN** an artifact manifest schema includes `doc` for a symbol +- **AND WHEN** the user imports the package and accesses the function +- **THEN** the returned callable has `__doc__` containing the Go doc comment + +#### Scenario: Runtime provides signature docstring fallback +- **WHEN** an artifact manifest schema includes a symbol signature but has no `doc` +- **AND WHEN** the user accesses the callable +- **THEN** the returned callable has a non-empty `__doc__` containing a Go signature string + +#### Scenario: Typed wrappers preserve docstrings +- **WHEN** a user accesses a callable via `handle.typed()` +- **THEN** the typed callable has the same `__doc__` content as the untyped callable + +#### Scenario: Bindgen includes docstrings +- **WHEN** an artifact manifest schema includes `doc` for a symbol +- **AND WHEN** the user runs `usegolib bindgen` +- **THEN** the generated Python method includes a docstring derived from the Go doc comment + diff --git a/openspec/changes/archive/2026-02-10-improve-godoc-visibility/tasks.md b/openspec/changes/archive/2026-02-10-improve-godoc-visibility/tasks.md new file mode 100644 index 0000000..d52b191 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-improve-godoc-visibility/tasks.md @@ -0,0 +1,18 @@ +## 1. Specs And Validation + +- [x] 1.1 Update usegolib-core spec to require docstrings for all dynamic callables +- [x] 1.2 Run `openspec validate improve-godoc-visibility --type change --strict --no-interactive` + +## 2. Implementation + +- [x] 2.1 Runtime: attach GoDoc or signature fallback for all functions and methods (handles + typed wrappers) +- [x] 2.2 Builder: include doc for concrete generic instantiation symbols + +## 3. Tests + +- [x] 3.1 Unit: runtime attaches docs for functions/methods and preserves in typed wrappers + +## 4. Verification + +- [x] 4.1 Run `python -m pytest -q` +- [x] 4.2 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-support-package-vars/.openspec.yaml b/openspec/changes/archive/2026-02-10-support-package-vars/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-support-package-vars/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-support-package-vars/README.md b/openspec/changes/archive/2026-02-10-support-package-vars/README.md new file mode 100644 index 0000000..a36a5e9 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-support-package-vars/README.md @@ -0,0 +1,3 @@ +# support-package-vars + +Support exported package variables as callable namespace objects (e.g. isr.DL.Of(...)) via object handles + reflective method dispatch. diff --git a/openspec/changes/archive/2026-02-10-support-package-vars/specs/usegolib-core/spec.md b/openspec/changes/archive/2026-02-10-support-package-vars/specs/usegolib-core/spec.md new file mode 100644 index 0000000..60a0e6f --- /dev/null +++ b/openspec/changes/archive/2026-02-10-support-package-vars/specs/usegolib-core/spec.md @@ -0,0 +1,13 @@ +## ADDED Requirements + +### Requirement: Exported Package Variables Are Accessible For Method Calls +The runtime SHALL allow accessing exported Go package variables as Python attributes when they are used as "namespace" objects to call exported methods (common in Go fluent APIs). + +#### Scenario: Access exported package variable and call exported method +- **GIVEN** a Go package defines an exported package variable `DL` of a (possibly unexported) struct type +- **AND GIVEN** that type defines an exported method `Of(...any) *dl` (or similar) +- **WHEN** Python evaluates `pkg.DL.Of(1, 2, 3)` +- **THEN** the runtime resolves `pkg.DL` as an object handle bound to that variable +- **AND THEN** the runtime calls method `Of` on that object handle +- **AND THEN** if the result is a pointer to an opaque struct type, the result is represented in Python as a callable object handle (not a dict) + diff --git a/openspec/changes/archive/2026-02-10-update-cli-at-version-syntax/.openspec.yaml b/openspec/changes/archive/2026-02-10-update-cli-at-version-syntax/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-cli-at-version-syntax/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-update-cli-at-version-syntax/README.md b/openspec/changes/archive/2026-02-10-update-cli-at-version-syntax/README.md new file mode 100644 index 0000000..f513f16 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-cli-at-version-syntax/README.md @@ -0,0 +1,3 @@ +# update-cli-at-version-syntax + +Use module@version syntax in CLI (remove --version flags) diff --git a/openspec/changes/archive/2026-02-10-update-cli-at-version-syntax/proposal.md b/openspec/changes/archive/2026-02-10-update-cli-at-version-syntax/proposal.md new file mode 100644 index 0000000..39dfdaf --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-cli-at-version-syntax/proposal.md @@ -0,0 +1,10 @@ +# Proposal: Use `module@version` Syntax In CLI + +## Why +The Go ecosystem commonly specifies versions inline using `@` (for example `go get module@vX.Y.Z`). +Using the same convention in `usegolib` reduces friction and avoids an extra `--version` flag across commands. + +## What Changes +- CLI commands accept `@version` as part of the `--module` / `--package` values. +- CLI no longer exposes a separate `--version` flag for those commands. + diff --git a/openspec/changes/archive/2026-02-10-update-cli-at-version-syntax/specs/usegolib-dev/spec.md b/openspec/changes/archive/2026-02-10-update-cli-at-version-syntax/specs/usegolib-dev/spec.md new file mode 100644 index 0000000..67abde7 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-cli-at-version-syntax/specs/usegolib-dev/spec.md @@ -0,0 +1,19 @@ +## MODIFIED Requirements + +### Requirement: CLI Supports Artifact Cache Management +The CLI SHALL accept `@version` syntax when targeting a specific module/package version for artifact cache operations. + +#### Scenario: Delete a specific cached version +- **WHEN** a user runs `usegolib artifact rm --module @ --yes` +- **THEN** the matching artifact directory for the current platform is deleted + +#### Scenario: Rebuild cached artifacts +- **WHEN** a user runs `usegolib artifact rebuild --module @` +- **THEN** the artifact is rebuilt into the selected artifact root + +### Requirement: CLI Can Re-Download Go Modules Before Rebuild +The CLI SHALL accept `@version` syntax for remote module builds when re-downloading sources. + +#### Scenario: Re-download before build +- **WHEN** a user runs `usegolib build --module @ --out --redownload` +- **THEN** the build uses an isolated `GOMODCACHE` and clears that cache before building diff --git a/openspec/changes/archive/2026-02-10-update-cli-at-version-syntax/tasks.md b/openspec/changes/archive/2026-02-10-update-cli-at-version-syntax/tasks.md new file mode 100644 index 0000000..0d6ed7f --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-cli-at-version-syntax/tasks.md @@ -0,0 +1,19 @@ +## 1. Specs And Validation + +- [x] 1.1 Update usegolib-dev spec to reflect module@version CLI syntax +- [x] 1.2 Run `openspec validate update-cli-at-version-syntax --type change --strict --no-interactive` + +## 2. Implementation + +- [x] 2.1 Builder: parse `target@version` in resolve logic +- [x] 2.2 CLI: remove `--version` flags and accept `@version` for build/package/artifact/gen +- [x] 2.3 Docs: update CLI docs/examples + +## 3. Tests + +- [x] 3.1 Unit: resolve parses inline @version + +## 4. Verification + +- [x] 4.1 Run `python -m pytest -q` +- [x] 4.2 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-update-docs-troubleshooting-ambiguity-network/.openspec.yaml b/openspec/changes/archive/2026-02-10-update-docs-troubleshooting-ambiguity-network/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-docs-troubleshooting-ambiguity-network/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-update-docs-troubleshooting-ambiguity-network/README.md b/openspec/changes/archive/2026-02-10-update-docs-troubleshooting-ambiguity-network/README.md new file mode 100644 index 0000000..91d4eb2 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-docs-troubleshooting-ambiguity-network/README.md @@ -0,0 +1,3 @@ +# update-docs-troubleshooting-ambiguity-network + +Sync README/CLI/troubleshooting docs with import version-follow behavior and Go proxy/network remediation hints. diff --git a/openspec/changes/archive/2026-02-10-update-docs-troubleshooting-ambiguity-network/specs/usegolib-dev/spec.md b/openspec/changes/archive/2026-02-10-update-docs-troubleshooting-ambiguity-network/specs/usegolib-dev/spec.md new file mode 100644 index 0000000..0a69a18 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-docs-troubleshooting-ambiguity-network/specs/usegolib-dev/spec.md @@ -0,0 +1,15 @@ +## MODIFIED Requirements + +### Requirement: Troubleshooting Documentation +The repository SHALL include `docs/troubleshooting.md` covering common failures and remedies for building/loading/importing artifacts. + +#### Scenario: Ambiguous artifacts mention already-loaded version behavior +- **WHEN** a developer reads the `AmbiguousArtifactError` section in `docs/troubleshooting.md` +- **THEN** it explains that ambiguity applies when the module is not already loaded +- **AND THEN** it notes that `import_(..., version=None)` follows the already-loaded version within a process + +#### Scenario: Network/proxy failures mention GOPROXY remediation +- **WHEN** a developer reads the build-related troubleshooting for Go module download failures +- **THEN** it mentions proxy/network errors can be transient +- **AND THEN** it includes a remediation hint mentioning `GOPROXY=direct` + diff --git a/openspec/changes/archive/2026-02-10-update-go-not-found-hint/.openspec.yaml b/openspec/changes/archive/2026-02-10-update-go-not-found-hint/.openspec.yaml new file mode 100644 index 0000000..eef47e4 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-go-not-found-hint/.openspec.yaml @@ -0,0 +1,2 @@ +id: update-go-not-found-hint + diff --git a/openspec/changes/archive/2026-02-10-update-go-not-found-hint/README.md b/openspec/changes/archive/2026-02-10-update-go-not-found-hint/README.md new file mode 100644 index 0000000..b76bf79 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-go-not-found-hint/README.md @@ -0,0 +1,4 @@ +# update-go-not-found-hint + +Improve error messaging when auto-build requires the Go toolchain but `go` is not available on `PATH`. + diff --git a/openspec/changes/archive/2026-02-10-update-go-not-found-hint/proposal.md b/openspec/changes/archive/2026-02-10-update-go-not-found-hint/proposal.md new file mode 100644 index 0000000..0c4819d --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-go-not-found-hint/proposal.md @@ -0,0 +1,14 @@ +# Change: Improve Missing Go Toolchain Error + +## Why +When `usegolib` needs to build artifacts (auto-build on import or `usegolib build`) and the Go toolchain is not installed or not on `PATH`, users currently get a raw `FileNotFoundError` traceback. This is confusing and non-actionable. + +## What Changes +- Detect "missing `go` executable" failures and raise `BuildError` with an actionable message: + - install Go / ensure `go` is on `PATH` + - optionally disable auto-build using `build_if_missing=False` and use prebuilt artifacts/wheels + +## Impact +- Affected specs: `usegolib-core` +- Affected code: `src/usegolib/builder/build.py`, `src/usegolib/builder/resolve.py`, `src/usegolib/builder/scan.py` + diff --git a/openspec/changes/archive/2026-02-10-update-go-not-found-hint/specs/usegolib-core/spec.md b/openspec/changes/archive/2026-02-10-update-go-not-found-hint/specs/usegolib-core/spec.md new file mode 100644 index 0000000..eb5fff3 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-go-not-found-hint/specs/usegolib-core/spec.md @@ -0,0 +1,16 @@ +## ADDED Requirements + +### Requirement: Missing Go Toolchain Hint (V0.x) +When the system needs to build artifacts using the Go toolchain and the `go` executable cannot be found, the system SHALL raise a `BuildError` with an actionable hint. + +The hint SHOULD mention: +- installing Go and ensuring `go` is on `PATH` +- disabling auto-build via `build_if_missing=False` and using prebuilt artifacts/wheels + +#### Scenario: Import auto-build fails because `go` is missing +- **WHEN** Python calls `usegolib.import_("example.com/mod", version="vX.Y.Z")` +- **AND WHEN** the artifact is missing and auto-build is attempted +- **AND WHEN** `go` is not available on `PATH` +- **THEN** the call fails with `BuildError` +- **AND THEN** the error message includes a hint to install Go or disable auto-build + diff --git a/openspec/changes/archive/2026-02-10-update-go-not-found-hint/tasks.md b/openspec/changes/archive/2026-02-10-update-go-not-found-hint/tasks.md new file mode 100644 index 0000000..46834e2 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-go-not-found-hint/tasks.md @@ -0,0 +1,18 @@ +## 1. Specs And Validation + +- [x] 1.1 Run `openspec validate update-go-not-found-hint --strict --no-interactive` + +## 2. Builder Errors + +- [x] 2.1 Wrap missing `go` executable in a `BuildError` with an actionable hint (builder command runner) +- [x] 2.2 Wrap missing `go` executable in a `BuildError` for module resolution (`go mod download`) +- [x] 2.3 Wrap missing `go` executable in a `BuildError` for scanning (`go run` scanner) + +## 3. Tests + +- [x] 3.1 Add unit tests that simulate missing `go` and assert `BuildError` message contains guidance + +## 4. Verification + +- [x] 4.1 Run `python -m pytest -q` +- [x] 4.2 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-update-readme-go-requirements/.openspec.yaml b/openspec/changes/archive/2026-02-10-update-readme-go-requirements/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-readme-go-requirements/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-update-readme-go-requirements/README.md b/openspec/changes/archive/2026-02-10-update-readme-go-requirements/README.md new file mode 100644 index 0000000..104cd09 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-readme-go-requirements/README.md @@ -0,0 +1,3 @@ +# update-readme-go-requirements + +Clarify in README when Go toolchain is required (auto-build) vs not required (prebuilt artifacts/wheels). diff --git a/openspec/changes/archive/2026-02-10-update-readme-go-requirements/design.md b/openspec/changes/archive/2026-02-10-update-readme-go-requirements/design.md new file mode 100644 index 0000000..ed8d65d --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-readme-go-requirements/design.md @@ -0,0 +1,9 @@ +# Design: README Clarity For Go Requirement + +This is a documentation-only change. + +Key points to capture: +- `import_` can auto-build when `artifact_dir` is omitted. +- Auto-build requires a Go toolchain (Zig is bootstrapped automatically for cgo). +- End-users can avoid Go by using prebuilt artifacts or wheels embedding artifacts. + diff --git a/openspec/changes/archive/2026-02-10-update-readme-go-requirements/proposal.md b/openspec/changes/archive/2026-02-10-update-readme-go-requirements/proposal.md new file mode 100644 index 0000000..0d79651 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-readme-go-requirements/proposal.md @@ -0,0 +1,24 @@ +## Why + +`usegolib.import_` now supports auto-build and remote module downloads in dev workflows, which changes when Go is required on the user's machine. README should clearly distinguish end-user runtime usage (no Go) from developer convenience auto-build (needs Go). + +## What Changes + +- Update `README.md` to explicitly document: + - end-user runtime usage that does not require Go (prebuilt artifacts / packaged wheels) + - dev convenience auto-build behavior that requires Go toolchain + - how to disable building by using explicit `artifact_dir` and/or `build_if_missing=False` + +## Capabilities + +### New Capabilities + + +### Modified Capabilities + +- `usegolib-dev`: developer-facing documentation clarity + +## Impact + +- Affected docs: `README.md` + diff --git a/openspec/changes/archive/2026-02-10-update-readme-go-requirements/specs/usegolib-dev/spec.md b/openspec/changes/archive/2026-02-10-update-readme-go-requirements/specs/usegolib-dev/spec.md new file mode 100644 index 0000000..b36b854 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-readme-go-requirements/specs/usegolib-dev/spec.md @@ -0,0 +1,10 @@ +## ADDED Requirements + +### Requirement: README Clarifies Go Toolchain Requirement +The repository SHALL document in `README.md` when the Go toolchain is required (auto-build/build workflows) versus when it is not required (runtime with prebuilt artifacts/wheels). + +#### Scenario: README includes a "Do users need Go?" section +- **WHEN** a developer reads `README.md` +- **THEN** it contains a section explaining that end-users do not need Go when using prebuilt artifacts/wheels +- **AND THEN** it explains that auto-build (downloading/building missing artifacts) requires a Go toolchain + diff --git a/openspec/changes/archive/2026-02-10-update-readme-go-requirements/tasks.md b/openspec/changes/archive/2026-02-10-update-readme-go-requirements/tasks.md new file mode 100644 index 0000000..65b50df --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-readme-go-requirements/tasks.md @@ -0,0 +1,15 @@ +## 1. Specs And Design + +- [x] 1.1 Create `proposal.md` +- [x] 1.2 Create `design.md` +- [x] 1.3 Run `openspec validate update-readme-go-requirements --strict --no-interactive` + +## 2. Documentation + +- [x] 2.1 Update `README.md` with a clear "Do users need Go?" section and examples + +## 3. Verification + +- [x] 3.1 Run `python -m pytest -q` +- [x] 3.2 Run `tools/wsl_linux_tests.ps1` +- [x] 3.3 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/.openspec.yaml b/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/.openspec.yaml new file mode 100644 index 0000000..70eb9e0 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-10 diff --git a/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/README.md b/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/README.md new file mode 100644 index 0000000..b1df57b --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/README.md @@ -0,0 +1,3 @@ +# update-roadmap-methods-generics + +Update roadmap to plan future methods/generics support; clarify CLI/docs that remote module import paths are supported. diff --git a/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/design.md b/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/design.md new file mode 100644 index 0000000..984aa68 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/design.md @@ -0,0 +1,8 @@ +# Design: Roadmap Extension For Methods + Generics + +This change is documentation-only, clarifying: +- current behavior: methods/generics are ignored by the scanner (v0.x) +- planned future milestones and likely approach + +Implementation of method/generic support is out-of-scope for this change; only the roadmap is updated. + diff --git a/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/proposal.md b/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/proposal.md new file mode 100644 index 0000000..75999e9 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/proposal.md @@ -0,0 +1,25 @@ +## Why + +Users want `usegolib` to work with remote Go modules (auto-download) and to have a clear plan for future support of exported methods and generics. The roadmap and CLI/docs should reflect current capabilities and the intended upgrade path. + +## What Changes + +- Update `docs/roadmap.md` to add explicit future milestones for: + - exported method support + - generic function support +- Clarify CLI help text and README examples to state that `--module` may be a local module directory or a Go import path. + +## Capabilities + +### New Capabilities + + +### Modified Capabilities + +- `usegolib-dev`: roadmap and developer-facing docs for planned features + +## Impact + +- Affected docs: `docs/roadmap.md`, `README.md` +- Affected CLI help: `src/usegolib/cli.py` + diff --git a/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/specs/usegolib-dev/spec.md b/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/specs/usegolib-dev/spec.md new file mode 100644 index 0000000..64fad50 --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/specs/usegolib-dev/spec.md @@ -0,0 +1,9 @@ +## ADDED Requirements + +### Requirement: Roadmap Includes Planned Method And Generics Support +The repository SHALL maintain `docs/roadmap.md` to include planned milestones for future support of exported Go methods and generic functions. + +#### Scenario: Roadmap describes future method and generics milestones +- **WHEN** a developer reviews `docs/roadmap.md` +- **THEN** it includes explicit milestones describing planned support for exported methods and generic functions + diff --git a/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/tasks.md b/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/tasks.md new file mode 100644 index 0000000..cdcc26e --- /dev/null +++ b/openspec/changes/archive/2026-02-10-update-roadmap-methods-generics/tasks.md @@ -0,0 +1,15 @@ +## 1. Specs And Validation + +- [x] 1.1 Run `openspec validate update-roadmap-methods-generics --strict --no-interactive` + +## 2. Documentation + +- [x] 2.1 Update `docs/roadmap.md` with milestones for methods + generics support +- [x] 2.2 Clarify `README.md` examples for remote import paths +- [x] 2.3 Clarify CLI help for `usegolib build/package --module` (dir or import path) + +## 3. Verification + +- [x] 3.1 Run `python -m pytest -q` +- [x] 3.2 Run `tools/wsl_linux_tests.ps1` +- [x] 3.3 Run `openspec validate --all --strict --no-interactive` diff --git a/openspec/specs/usegolib-core/spec.md b/openspec/specs/usegolib-core/spec.md index 3ec3811..d4ecaa9 100644 --- a/openspec/specs/usegolib-core/spec.md +++ b/openspec/specs/usegolib-core/spec.md @@ -5,7 +5,13 @@ Define the end-to-end behavior of `usegolib`: importing Go modules/packages from ### Requirement: Python Import API The system SHALL expose a Python API that imports a Go module or subpackage and returns a handle that can be used to call exported Go identifiers. -#### Scenario: Import root module from an artifact root +#### Scenario: Import root module from the default artifact root +- **WHEN** Python calls `usegolib.import_("example.com/mod", version=None)` +- **THEN** the system searches the default artifact root for a matching artifact for the current OS/arch +- **AND THEN** the system loads the matching shared library into the current Python process +- **AND THEN** the call returns a handle bound to package `example.com/mod` + +#### Scenario: Import root module from an explicit artifact root - **WHEN** Python calls `usegolib.import_("example.com/mod", version=None, artifact_dir="out/")` - **THEN** the system searches `artifact_dir` for a matching artifact for the current OS/arch - **AND THEN** the system loads the matching shared library into the current Python process @@ -20,9 +26,16 @@ The system SHALL expose a Python API that imports a Go module or subpackage and - **WHEN** Python calls `usegolib.import_("example.com/mod", version="v1.2.3", artifact_dir="out/")` - **THEN** the system MUST load version `v1.2.3` for that module/package (if present) +#### Scenario: Import omits version but follows already-loaded module version +- **WHEN** Python imports `example.com/mod` at version `v1.2.0` +- **AND WHEN** Python imports `example.com/mod/subpkg` with `version=None` +- **THEN** the second import resolves to version `v1.2.0` (the already-loaded module version) +- **AND THEN** the import does not fail with `AmbiguousArtifactError` even if multiple artifact versions exist on disk + #### Scenario: Import fails when version is omitted but ambiguous - **WHEN** Python calls `usegolib.import_("example.com/mod", version=None, artifact_dir="out/")` - **AND WHEN** `artifact_dir` contains more than one version for `example.com/mod` for the current OS/arch +- **AND WHEN** `example.com/mod` is not already loaded in the current Python process - **THEN** the import MUST fail - **AND THEN** the system SHALL raise `AmbiguousArtifactError` @@ -80,7 +93,7 @@ When building artifacts from a Go import path, the system SHALL resolve a concre #### Scenario: Version omitted defaults to @latest for remote modules - **WHEN** `usegolib build` is invoked with a Go module import path and `version=None` -- **THEN** the system resolves the version to `@latest` +- **THEN** the system resolves the version query to `@latest` - **AND THEN** the manifest `version` field contains the resolved concrete version #### Scenario: Pinned version is used as-is for remote modules @@ -95,12 +108,27 @@ When building artifacts from a Go import path, the system SHALL resolve a concre ### Requirement: Import Builds Missing Artifacts (Dev Mode) When `build_if_missing=True`, the system SHALL build a missing artifact into the artifact root and then load it. +When `build_if_missing=None` (auto), the system SHALL: +- build missing artifacts when `artifact_dir` is omitted (using the default artifact root) +- NOT build missing artifacts when `artifact_dir` is explicitly provided + #### Scenario: Import triggers build when artifact is missing - **WHEN** Python calls `usegolib.import_("example.com/mod", version="vX.Y.Z", artifact_dir="out/", build_if_missing=True)` - **AND WHEN** no matching artifact exists under `artifact_dir` for the current OS/arch - **THEN** the system builds the artifact into `artifact_dir` - **AND THEN** the system loads the newly built artifact and returns a handle +#### Scenario: Import auto-builds when artifact_dir is omitted +- **WHEN** Python calls `usegolib.import_("example.com/mod", version="vX.Y.Z", build_if_missing=None)` +- **AND WHEN** no matching artifact exists under the default artifact root for the current OS/arch +- **THEN** the system builds the artifact into the default artifact root +- **AND THEN** the system loads the newly built artifact and returns a handle + +#### Scenario: Import does not build by default when artifact_dir is provided +- **WHEN** Python calls `usegolib.import_("example.com/mod", version="vX.Y.Z", artifact_dir="out/", build_if_missing=None)` +- **AND WHEN** no matching artifact exists under `artifact_dir` for the current OS/arch +- **THEN** the call fails with `ArtifactNotFoundError` + #### Scenario: Import does not rebuild when artifact exists - **WHEN** Python calls `usegolib.import_("example.com/mod", version="vX.Y.Z", artifact_dir="out/", build_if_missing=True)` - **AND WHEN** a matching artifact already exists under `artifact_dir` for the current OS/arch @@ -141,6 +169,32 @@ The system SHALL support passing and returning Level 2 values across the Python/ - **WHEN** a Go function returns `map[string][]byte` - **THEN** Python receives it as a dict mapping `str -> bytes` +### Requirement: Support Type `any` (V0.x) +The type bridge SHALL support Go `any` (`interface{}`) as a dynamic value that can carry any MessagePack-compatible value. + +Supported containers SHALL include at least: +- `[]any` +- `map[string]any` + +#### Scenario: Call symbol with any parameter +- **WHEN** a callable symbol accepts an `any` parameter +- **AND WHEN** Python passes a MessagePack-compatible value +- **THEN** the call succeeds + +#### Scenario: Return any result +- **WHEN** a callable symbol returns `any` +- **THEN** Python receives the dynamic value + +### Requirement: Variadic Parameters (V0.x) +The builder scanner SHALL detect variadic parameters and record them as `...T` in symbol signatures. + +When manifest schema is present, the runtime SHALL allow Python calls to pass variadic arguments naturally and SHALL pack them into the ABI form for encoding. + +#### Scenario: Call variadic function from Python +- **WHEN** a callable symbol has a variadic parameter (`...T`) +- **AND WHEN** Python passes multiple arguments for that parameter +- **THEN** the runtime packs the arguments and the call succeeds + ### Requirement: Export Scan Correctness (Build) When building artifacts, the system SHALL discover exported top-level functions and their parameter/return types from Go source in a way that is robust to valid Go syntax and formatting (including grouped parameters). @@ -389,3 +443,131 @@ The builder MUST: - **WHEN** a Zig archive contains an unsafe path (absolute or path traversal) - **THEN** the builder SHALL raise `BuildError` and MUST NOT extract the unsafe entry +### Requirement: Go Object Handles And Exported Methods (V0.x) +The system SHALL support calling exported Go methods without serializing the receiver value on every call by using Go-side object handles. + +The runtime SHALL expose a Python API to: +- create a Go-side object handle for a named struct type (`obj_new`) +- call exported methods on that handle (`obj_call`) +- free the handle (`obj_free`) + +#### Scenario: Create object handle and call method +- **WHEN** Python creates a Go object handle for a named struct type +- **AND WHEN** Python calls an exported method on the object handle +- **THEN** the call succeeds and returns the method result + +#### Scenario: Freeing an object handle makes it unusable +- **WHEN** Python frees a Go object handle +- **AND WHEN** Python calls a method on the freed handle +- **THEN** the call fails with an error + +### Requirement: Generic Function Instantiation (V0.x) +The builder SHALL support exported top-level generic Go functions by generating concrete instantiations at build time for an explicit set of type arguments. + +The manifest schema SHALL record the mapping from: +- generic function name + type arguments +- to the concrete callable symbol name exported by the bridge + +The runtime SHALL provide a Python API to call a generic function instantiation by specifying the generic function name and type arguments. + +#### Scenario: Build instantiates a generic function and Python calls it +- **WHEN** the builder is configured to instantiate an exported generic Go function with concrete type arguments +- **AND WHEN** Python calls that instantiation through the runtime +- **THEN** the call succeeds and returns the correct result + +#### Scenario: Calling a non-configured generic instantiation fails +- **WHEN** Python requests a generic instantiation that is not present in the manifest schema +- **THEN** the runtime fails with an error + +### Requirement: Missing Go Toolchain Hint (V0.x) +When the system needs to build artifacts using the Go toolchain and the `go` executable cannot be found, the system SHALL raise a `BuildError` with an actionable hint. + +The hint SHOULD mention: +- installing Go and ensuring `go` is on `PATH` +- disabling auto-build via `build_if_missing=False` and using prebuilt artifacts/wheels + +#### Scenario: Import auto-build fails because `go` is missing +- **WHEN** Python calls `usegolib.import_("example.com/mod", version="vX.Y.Z")` +- **AND WHEN** the artifact is missing and auto-build is attempted +- **AND WHEN** `go` is not available on `PATH` +- **THEN** the call fails with `BuildError` +- **AND THEN** the error message includes a hint to install Go or disable auto-build + +### Requirement: GoDoc To Python Docstrings (V0.x) +When building an artifact, the system SHALL extract Go doc comments for exported functions and methods and include them in the artifact schema. + +When schema is present, the Python runtime SHALL attach docstrings to the dynamic callables it returns for exported functions and methods. + +If GoDoc is missing for a callable, the runtime SHOULD still attach a signature-based docstring (so `help()` is informative). + +Typed wrappers (`handle.typed()` and typed objects) SHALL preserve the docstrings of the underlying dynamic callables. + +The bindgen generator SHOULD emit these docs as Python docstrings in generated modules. + +#### Scenario: Runtime exposes GoDoc for a function +- **WHEN** an artifact manifest schema includes `doc` for a symbol +- **AND WHEN** the user imports the package and accesses the function +- **THEN** the returned callable has `__doc__` containing the Go doc comment + +#### Scenario: Runtime provides signature docstring fallback +- **WHEN** an artifact manifest schema includes a symbol signature but has no `doc` +- **AND WHEN** the user accesses the callable +- **THEN** the returned callable has a non-empty `__doc__` containing a Go signature string + +#### Scenario: Typed wrappers preserve docstrings +- **WHEN** a user accesses a callable via `handle.typed()` +- **THEN** the typed callable has the same `__doc__` content as the untyped callable + +#### Scenario: Bindgen includes docstrings +- **WHEN** an artifact manifest schema includes `doc` for a symbol +- **AND WHEN** the user runs `usegolib bindgen` +- **THEN** the generated Python method includes a docstring derived from the Go doc comment + +### Requirement: Opaque Pointer Results Use Object Handles (V0.x) +When a callable symbol or method returns `*T` and the struct type `T` has no exported fields in the manifest schema, the system SHALL represent that value as an object handle rather than a record-struct dict. + +#### Scenario: Function returns *Opaque handle +- **WHEN** a Go function returns `*T` +- **AND WHEN** `T` has no exported fields +- **THEN** Python receives a `GoObject` that can be used to call exported methods on `T` + +### Requirement: Error-Only Results Are Treated As Nil On Success +When a callable symbol or method returns only `error`, the system SHALL represent successful results as `nil` (Python `None`). + +If the returned `error` is non-nil, the system SHALL report the failure via the ABI error envelope as `GoError`. + +#### Scenario: Method returns only error and succeeds +- **WHEN** a Go method returns `error` +- **AND WHEN** the call succeeds (`error` is `nil`) +- **THEN** the Python result is `None` + +#### Scenario: Method returns only error and fails +- **WHEN** a Go method returns `error` +- **AND WHEN** the call fails (`error` is non-nil) +- **THEN** the Python call raises `GoError` + +### Requirement: Builder Retries Transient Go Network Failures +When building or rebuilding artifacts for remote modules, the builder SHALL retry transient Go module download failures and provide actionable remediation hints. + +#### Scenario: Transient proxy failures are retried and may fall back to GOPROXY=direct +- **WHEN** a build executes Go commands that may download modules (for example `go mod download`, `go list`, or `go build`) +- **AND WHEN** the command fails due to a transient network/proxy error (for example `proxy.golang.org` timeouts) +- **THEN** the builder retries the command with a short backoff +- **AND THEN** if `GOPROXY` is not explicitly set, the builder MAY retry with `GOPROXY=direct` + +#### Scenario: Failures include a hint for common network/proxy remediation +- **WHEN** a build fails due to a network/proxy error downloading Go modules +- **THEN** the builder raises `BuildError` +- **AND THEN** the error message includes a hint mentioning `GOPROXY=direct` + +### Requirement: Exported Package Variables Are Accessible For Method Calls +The runtime SHALL allow accessing exported Go package variables as Python attributes when they are used as "namespace" objects to call exported methods (common in Go fluent APIs). + +#### Scenario: Access exported package variable and call exported method +- **GIVEN** a Go package defines an exported package variable `DL` of a (possibly unexported) struct type +- **AND GIVEN** that type defines an exported method `Of(...any) *dl` (or similar) +- **WHEN** Python evaluates `pkg.DL.Of(1, 2, 3)` +- **THEN** the runtime resolves `pkg.DL` as an object handle bound to that variable +- **AND THEN** the runtime calls method `Of` on that object handle +- **AND THEN** if the result is a pointer to an opaque struct type, the result is represented in Python as a callable object handle (not a dict) + diff --git a/openspec/specs/usegolib-dev/spec.md b/openspec/specs/usegolib-dev/spec.md index 729a185..2dd7a7f 100644 --- a/openspec/specs/usegolib-dev/spec.md +++ b/openspec/specs/usegolib-dev/spec.md @@ -90,3 +90,79 @@ The workflow MUST: - **WHEN** a maintainer runs the PyPI publish workflow - **THEN** the workflow publishes the built distributions using trusted publishing +### Requirement: Roadmap Includes Planned Method And Generics Support +The repository SHALL maintain `docs/roadmap.md` to include planned milestones for future support of exported Go methods and generic functions. + +#### Scenario: Roadmap describes future method and generics milestones +- **WHEN** a developer reviews `docs/roadmap.md` +- **THEN** it includes explicit milestones describing planned support for exported methods and generic functions + +### Requirement: README Clarifies Go Toolchain Requirement +The repository SHALL document in `README.md` when the Go toolchain is required (auto-build/build workflows) versus when it is not required (runtime with prebuilt artifacts/wheels). + +#### Scenario: README includes a "Do users need Go?" section +- **WHEN** a developer reads `README.md` +- **THEN** it contains a section explaining that end-users do not need Go when using prebuilt artifacts/wheels +- **AND THEN** it explains that auto-build (downloading/building missing artifacts) requires a Go toolchain + +### Requirement: Roadmap Milestone Ordering Is Consistent +The repository SHALL keep `docs/roadmap.md` milestones ordered by increasing milestone number to avoid confusion about sequencing. + +#### Scenario: Milestones are ordered +- **WHEN** a developer reads `docs/roadmap.md` +- **THEN** Milestone sections appear in increasing numerical order (e.g. 4, 5, 6) + +### Requirement: Troubleshooting Documentation +The repository SHALL include `docs/troubleshooting.md` covering common failures and remedies for building/loading/importing artifacts. + +#### Scenario: Ambiguous artifacts mention already-loaded version behavior +- **WHEN** a developer reads the `AmbiguousArtifactError` section in `docs/troubleshooting.md` +- **THEN** it explains that ambiguity applies when the module is not already loaded +- **AND THEN** it notes that `import_(..., version=None)` follows the already-loaded version within a process + +#### Scenario: Network/proxy failures mention GOPROXY remediation +- **WHEN** a developer reads the build-related troubleshooting for Go module download failures +- **THEN** it mentions proxy/network errors can be transient +- **AND THEN** it includes a remediation hint mentioning `GOPROXY=direct` + +### Requirement: Roadmap Avoids Misleading Internal Version Numbers +The repository SHALL keep `docs/roadmap.md` version-agnostic for unreleased work and MUST NOT assign misleading internal version ranges to future milestones. + +#### Scenario: Roadmap does not claim internal version ranges +- **WHEN** a developer reads `docs/roadmap.md` +- **THEN** future milestones do not include internal version ranges like `v0.X - v0.Y` + +### Requirement: CLI Supports Artifact Cache Management +The CLI SHALL accept `@version` syntax when targeting a specific module/package version for artifact cache operations. + +#### Scenario: Delete a specific cached version +- **WHEN** a user runs `usegolib artifact rm --module @ --yes` +- **THEN** the matching artifact directory for the current platform is deleted + +#### Scenario: Rebuild cached artifacts +- **WHEN** a user runs `usegolib artifact rebuild --module @` +- **THEN** the artifact is rebuilt into the selected artifact root + +### Requirement: CLI Can Re-Download Go Modules Before Rebuild +The CLI SHALL accept `@version` syntax for remote module builds when re-downloading sources. + +#### Scenario: Re-download before build +- **WHEN** a user runs `usegolib build --module @ --out --redownload` +- **THEN** the build uses an isolated `GOMODCACHE` and clears that cache before building + +### Requirement: CLI Usage Documentation +The repository SHALL document CLI installation and usage, including artifact cache deletion and rebuild workflows. + +#### Scenario: CLI docs exist +- **WHEN** a user reads the repository documentation +- **THEN** `docs/cli.md` exists and documents `usegolib build` and `usegolib artifact` commands + +### Requirement: Windows Builder Output Decoding Is Locale-Safe +The builder SHALL not fail with `UnicodeDecodeError` on Windows due to non-UTF8 locale encodings when capturing Go tool output. + +#### Scenario: Go tool output contains non-UTF8 bytes under Windows locale +- **WHEN** the builder captures stdout/stderr from Go tools on Windows +- **AND WHEN** the system locale encoding cannot decode those bytes (for example `cp950`) +- **THEN** the build/scan process does not crash with `UnicodeDecodeError` +- **AND THEN** errors are reported as `BuildError` when the underlying Go command fails + diff --git a/src/usegolib/abi.py b/src/usegolib/abi.py index 6afe194..e0292a0 100644 --- a/src/usegolib/abi.py +++ b/src/usegolib/abi.py @@ -37,6 +37,40 @@ def encode_call_request(*, pkg: str, fn: str, args: list[Any]) -> bytes: return msgpack.packb(payload, use_bin_type=True) +def encode_obj_new_request(*, pkg: str, type_name: str, init: Any | None) -> bytes: + payload = { + "abi": ABI_VERSION, + "op": "obj_new", + "pkg": pkg, + "type": type_name, + } + if init is not None: + payload["init"] = init + return msgpack.packb(payload, use_bin_type=True) + + +def encode_obj_call_request(*, pkg: str, type_name: str, obj_id: int, method: str, args: list[Any]) -> bytes: + payload = { + "abi": ABI_VERSION, + "op": "obj_call", + "pkg": pkg, + "type": type_name, + "id": obj_id, + "method": method, + "args": args, + } + return msgpack.packb(payload, use_bin_type=True) + + +def encode_obj_free_request(*, obj_id: int) -> bytes: + payload = { + "abi": ABI_VERSION, + "op": "obj_free", + "id": obj_id, + } + return msgpack.packb(payload, use_bin_type=True) + + def decode_response(payload: bytes) -> ABIResponse: try: obj = msgpack.unpackb(payload, raw=False) @@ -63,4 +97,3 @@ def decode_response(payload: bytes) -> ABIResponse: detail=err.get("detail") if isinstance(err.get("detail"), dict) else None, ), ) - diff --git a/src/usegolib/artifact.py b/src/usegolib/artifact.py index 1d349fc..8346a29 100644 --- a/src/usegolib/artifact.py +++ b/src/usegolib/artifact.py @@ -4,6 +4,7 @@ import json import os +import shutil import tempfile from dataclasses import dataclass from pathlib import Path @@ -262,3 +263,81 @@ def resolve_manifest( # version specified: take first (should be unique in a well-formed artifact dir) return candidates[0] + + +def find_manifest_dirs( + artifact_root: Path, + *, + package: str, + version: str | None, + all_versions: bool = False, + goos: str | None = None, + goarch: str | None = None, +) -> list[Path]: + """Locate artifact manifest directories matching the filters. + + This is primarily used for cache management commands (delete/rebuild). + """ + if version is None and not all_versions: + raise ValueError("version must be provided unless all_versions=True") + + goos = goos or host_goos() + goarch = goarch or host_goarch() + + root = Path(artifact_root) + out: list[Path] = [] + for d in _scan_manifest_dirs(root): + try: + m = read_manifest(d) + except Exception: + continue + if m.goos != goos or m.goarch != goarch: + continue + if package not in m.packages: + continue + if version is not None and m.version != version: + continue + out.append(Path(d)) + return out + + +def delete_artifacts( + artifact_root: Path, + *, + package: str, + version: str | None, + all_versions: bool = False, + goos: str | None = None, + goarch: str | None = None, +) -> list[Path]: + """Delete cached artifact directories and rebuild the index. + + Returns the list of deleted manifest directories. + """ + root = Path(artifact_root) + dirs = find_manifest_dirs( + root, + package=package, + version=version, + all_versions=all_versions, + goos=goos, + goarch=goarch, + ) + deleted: list[Path] = [] + for d in dirs: + try: + # Delete the platform leaf dir containing manifest.json + shared lib. + shutil.rmtree(d, ignore_errors=False) + deleted.append(d) + except FileNotFoundError: + continue + except OSError: + # Best-effort deletion: if it fails, continue to avoid partial cleanup blocking. + continue + + try: + _write_index_atomic(root, _build_index(root)) + except Exception: + pass + + return deleted diff --git a/src/usegolib/bindgen.py b/src/usegolib/bindgen.py index b34857d..5749ae2 100644 --- a/src/usegolib/bindgen.py +++ b/src/usegolib/bindgen.py @@ -53,6 +53,7 @@ def generate_python_bindings(*, schema: Schema, pkg: str, out_file: Path, opts: """Generate a static Python module from manifest schema for a package.""" structs = schema.structs_by_pkg.get(pkg, {}) symbols = schema.symbols_by_pkg.get(pkg, {}) + generics = schema.generics_by_pkg.get(pkg, {}) if not symbols: raise ValueError(f"no symbols found for package {pkg}") @@ -68,6 +69,15 @@ def generate_python_bindings(*, schema: Schema, pkg: str, out_file: Path, opts: lines.append("from usegolib.handle import PackageHandle") lines.append("") + def _emit_docstring(doc: str, *, indent: str) -> None: + doc = doc.replace('"""', '\\"\\"\\"').strip() + if not doc: + return + lines.append(f'{indent}"""') + for ln in doc.splitlines(): + lines.append(f"{indent}{ln}".rstrip()) + lines.append(f'{indent}"""') + # Dataclasses for named structs for struct_name, st in structs.items(): lines.append("@dataclass(frozen=True)") @@ -101,6 +111,14 @@ def generate_python_bindings(*, schema: Schema, pkg: str, out_file: Path, opts: lines.append("}") lines.append("") + if generics: + lines.append("_GENERICS: dict[tuple[str, tuple[str, ...]], str] = {") + for gname, insts in generics.items(): + for type_args, sym in insts.items(): + lines.append(f" ({gname!r}, {tuple(type_args)!r}): {sym!r},") + lines.append("}") + lines.append("") + # API wrapper api = opts.api_class_name lines.append(f"class {api}:") @@ -115,6 +133,24 @@ def generate_python_bindings(*, schema: Schema, pkg: str, out_file: Path, opts: ) lines.append("") + if generics: + lines.append(" def call_generic(self, name: str, type_args: list[str], *args: Any) -> Any:") + lines.append(" key = (name, tuple(type_args))") + lines.append(" sym = _GENERICS.get(key)") + lines.append(" if sym is None:") + lines.append(' raise KeyError(f\"generic instantiation not found: {key!r}\")') + lines.append(" r = getattr(self._h, sym)(*args)") + lines.append(" sig = self._schema.symbols_by_pkg.get(self._h.package, {}).get(sym)") + lines.append(" if sig is None:") + lines.append(" return r") + lines.append(" _params, results = sig") + lines.append(" if not results:") + lines.append(" return None") + lines.append( + " return usegolib.typed.decode_value(types=self._types, go_type=results[0], v=r)" + ) + lines.append("") + for fn_name, (params, results) in symbols.items(): # (T, error) is represented as a single successful result. ret_go = "nil" @@ -130,6 +166,9 @@ def generate_python_bindings(*, schema: Schema, pkg: str, out_file: Path, opts: call_args.append(arg_name) args_sig = ", ".join(["self", *arg_parts]) lines.append(f" def {fn_name}({args_sig}) -> {ret_py}:") + doc = schema.symbol_docs_by_pkg.get(pkg, {}).get(fn_name) + if doc: + _emit_docstring(doc, indent=" ") lines.append(f" r = self._h.{fn_name}({', '.join(call_args)})") if results: lines.append( diff --git a/src/usegolib/builder/build.py b/src/usegolib/builder/build.py index f987a58..c6785ed 100644 --- a/src/usegolib/builder/build.py +++ b/src/usegolib/builder/build.py @@ -3,10 +3,14 @@ import hashlib import json import os +import re +import shutil import subprocess import sys import tempfile +import time from pathlib import Path +from typing import Any from ..errors import BuildError from .fingerprint import fingerprint_local_module_dir @@ -14,23 +18,111 @@ from .resolve import resolve_module_target from .reuse import artifact_ready from .scan import scan_module -from .symbols import ExportedFunc +from .symbols import ExportedFunc, ExportedMethod, ExportedVar, GenericFuncDef, GenericInstantiation from .zig import ensure_zig -def _run(cmd: list[str], *, cwd: Path, env: dict[str, str] | None = None) -> str: - proc = subprocess.run( - cmd, - cwd=str(cwd), - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - check=False, +_GO_TRANSIENT_NET_RE = re.compile( + r"(" + r"proxy\.golang\.org" + r"|sum\.golang\.org" + r"|wsarecv" + r"|connection (?:attempt failed|reset)" + r"|i/o timeout" + r"|tls handshake timeout" + r"|unexpected eof" + r"|temporary failure" + r"|no such host" + r"|502 bad gateway" + r"|503 service unavailable" + r"|504 gateway timeout" + r")", + re.IGNORECASE, +) + + +def _go_network_hint(out: str) -> str | None: + if not _GO_TRANSIENT_NET_RE.search(out): + return None + return ( + "\n\nHint: Go module download failed due to a network/proxy error. " + "Try re-running the command. If `proxy.golang.org` is blocked/unreliable " + "in your environment, try setting `GOPROXY=direct` (or another reachable proxy) " + "and retry." ) - if proc.returncode != 0: - raise BuildError(f"command failed: {' '.join(cmd)}\n{proc.stdout}") - return proc.stdout + + +def _run(cmd: list[str], *, cwd: Path, env: dict[str, str] | None = None) -> str: + prog = cmd[0] if cmd else "" + + # Go commands that touch the module graph frequently fail due to transient + # proxy/network issues. Retry with a short backoff and optionally fall back + # to GOPROXY=direct if the proxy is the failing hop. + max_attempts = 1 + if prog == "go": + max_attempts = 3 + + base_env = env + cur_env = env + backoff_s = 0.5 + last_out = "" + + for attempt in range(max_attempts): + try: + proc = subprocess.run( + cmd, + cwd=str(cwd), + env=cur_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=False, + check=False, + ) + except FileNotFoundError as e: + if prog == "go": + raise BuildError( + "Go toolchain not found (`go` is missing from PATH). " + "Install Go and ensure `go` is available on PATH. " + "If you do not want auto-build on import, pass `build_if_missing=False` " + "(and use prebuilt artifacts/wheels)." + ) from e + raise BuildError(f"command not found: {prog}") from e + + stdout = (proc.stdout or b"").decode("utf-8", errors="replace") + stderr = (proc.stderr or b"").decode("utf-8", errors="replace") + out = "\n".join([s for s in [stdout.strip("\n"), stderr.strip("\n")] if s]) + "\n" + last_out = out + + if proc.returncode == 0: + return stdout + + # Retry only for `go` and only when we see a likely transient network error. + if prog == "go" and attempt < max_attempts - 1 and _GO_TRANSIENT_NET_RE.search(out): + # First retry: if the proxy seems to be the failing hop and GOPROXY is not + # explicitly set, try falling back to GOPROXY=direct for the next attempt. + if "proxy.golang.org" in out.lower(): + if base_env is None: + # Caller didn't pass env: copy process env so we can safely override. + next_env = dict(os.environ) + else: + next_env = dict(base_env) + if "GOPROXY" not in next_env: + next_env["GOPROXY"] = "direct" + cur_env = next_env + time.sleep(backoff_s) + backoff_s *= 2.0 + continue + + hint = _go_network_hint(out) + if hint: + out = out.rstrip("\n") + hint + "\n" + raise BuildError(f"command failed: {' '.join(cmd)}\n{out}") + + # Defensive: loop always returns/raises, but keep a fallback. + hint = _go_network_hint(last_out) + if hint: + last_out = last_out.rstrip("\n") + hint + "\n" + raise BuildError(f"command failed: {' '.join(cmd)}\n{last_out}") def _read_module_path(module_dir: Path) -> str: @@ -44,8 +136,8 @@ def _read_module_path(module_dir: Path) -> str: raise BuildError("failed to parse module path from go.mod") -def _list_packages(module_dir: Path) -> list[str]: - out = _run(["go", "list", "./..."], cwd=module_dir) +def _list_packages(module_dir: Path, *, env: dict[str, str] | None = None) -> list[str]: + out = _run(["go", "list", "./..."], cwd=module_dir, env=env) pkgs = [ln.strip() for ln in out.splitlines() if ln.strip()] # Exclude internal packages (Go import rules). return [p for p in pkgs if "/internal/" not in p and not p.endswith("/internal")] @@ -64,6 +156,8 @@ def _is_supported_type(t: str, *, struct_types: set[str] | None = None) -> bool: "float32", "float64", } + if t == "any": + return True if t in scalars: return True if t == "time.Time": @@ -77,6 +171,8 @@ def _is_supported_type(t: str, *, struct_types: set[str] | None = None) -> bool: if t.startswith("[]"): # []byte is handled above as a special scalar. return _is_supported_type(t[2:], struct_types=struct_types) + if t.startswith("..."): + return _is_supported_type("[]" + t[3:], struct_types=struct_types) if t.startswith("map[string]"): vt = t[len("map[string]") :] return _is_supported_type(vt, struct_types=struct_types) @@ -97,6 +193,152 @@ def _is_supported_sig(fn: ExportedFunc, *, struct_types: set[str] | None = None) return True +def _is_supported_method_sig(m: ExportedMethod, *, struct_types: set[str] | None = None) -> bool: + if any(not _is_supported_type(t, struct_types=struct_types) for t in m.params): + return False + if any(not _is_supported_type(t, struct_types=struct_types) and t != "error" for t in m.results): + return False + if len(m.results) > 2: + return False + if len(m.results) == 2 and m.results[1] != "error": + return False + return True + + +def _substitute_type(t: str, mapping: dict[str, str]) -> str: + t = t.strip() + if t.startswith("*"): + return "*" + _substitute_type(t[1:], mapping) + if t.startswith("[]"): + return "[]" + _substitute_type(t[2:], mapping) + if t.startswith("map[string]"): + return "map[string]" + _substitute_type(t[len("map[string]") :], mapping) + return mapping.get(t, t) + + +def _mangle_type_for_symbol(t: str) -> str: + t = t.strip() + # Stable-ish mangling for type argument tokens. + t = t.replace("map[string]", "mapstr_") + t = t.replace("[]", "slice_") + t = t.replace("*", "ptr_") + for ch in ["/", ".", "[", "]", " ", "{", "}", ",", ":", ";", "(", ")", "<", ">", "|", "~", "&", "!"]: + t = t.replace(ch, "_") + while "__" in t: + t = t.replace("__", "_") + return t.strip("_") or "T" + + +def _default_generic_symbol(name: str, type_args: list[str]) -> str: + suffix = "_".join(_mangle_type_for_symbol(t) for t in type_args) + return f"{name}__{suffix}" if suffix else f"{name}__inst" + + +def _load_generic_instantiations( + *, + generics: Path, + defs: list[GenericFuncDef], + struct_types_by_pkg: dict[str, set[str]], +) -> list[GenericInstantiation]: + generics = Path(generics) + if not generics.exists(): + raise BuildError(f"generics config not found: {generics}") + try: + obj = json.loads(generics.read_text(encoding="utf-8")) + except Exception as e: # noqa: BLE001 + raise BuildError(f"failed to parse generics config JSON: {e}") from e + insts = obj.get("instantiations") if isinstance(obj, dict) else None + if not isinstance(insts, list): + raise BuildError("generics config must be a JSON object with an 'instantiations' list") + + defs_by_pkg_name: dict[tuple[str, str], GenericFuncDef] = {} + for d in defs: + defs_by_pkg_name[(d.pkg, d.name)] = d + + out: list[GenericInstantiation] = [] + seen_symbols: set[tuple[str, str]] = set() + ident_re = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + for item in insts: + if not isinstance(item, dict): + continue + pkg = item.get("pkg") + name = item.get("name") + type_args = item.get("type_args") + symbol = item.get("symbol") + if not isinstance(pkg, str) or not isinstance(name, str): + raise BuildError("generics instantiation entry must include 'pkg' and 'name' strings") + if not isinstance(type_args, list) or not all(isinstance(x, str) for x in type_args): + raise BuildError("generics instantiation entry must include 'type_args' list[str]") + if symbol is None: + symbol = _default_generic_symbol(name, list(type_args)) + if not isinstance(symbol, str) or not symbol: + raise BuildError("generics instantiation 'symbol' must be a non-empty string when provided") + if not ident_re.fullmatch(symbol): + raise BuildError( + f"generics instantiation 'symbol' must be a valid identifier (got {symbol!r})" + ) + + d = defs_by_pkg_name.get((pkg, name)) + if d is None: + avail = sorted({f"{dp}.{dn}" for (dp, dn) in defs_by_pkg_name.keys()}) + raise BuildError( + f"generic function not found: {pkg}.{name}. " + f"available: {', '.join(avail) if avail else '(none)'}" + ) + if len(type_args) != len(d.type_params): + raise BuildError( + f"generic instantiation arity mismatch for {pkg}.{name}: " + f"expected {len(d.type_params)} type args, got {len(type_args)}" + ) + + # Validate type args: must be supported scalar/adapter/container or a named struct in the same pkg. + struct_types = struct_types_by_pkg.get(pkg, set()) + for ta in type_args: + base = ta.strip() + while True: + if base.startswith("*"): + base = base[1:].strip() + continue + if base.startswith("[]"): + base = base[2:].strip() + continue + if base.startswith("map[string]"): + base = base[len("map[string]") :].strip() + continue + break + if base in struct_types: + continue + if not _is_supported_type(ta, struct_types=struct_types): + raise BuildError(f"unsupported generic type arg: {ta} (for {pkg}.{name})") + + mapping = {tp: ta for tp, ta in zip(d.type_params, type_args, strict=True)} + inst_params = [_substitute_type(t, mapping) for t in d.params] + inst_results = [_substitute_type(t, mapping) for t in d.results] + + gi = GenericInstantiation( + pkg=pkg, + generic_name=name, + type_args=list(type_args), + symbol=symbol, + params=inst_params, + results=inst_results, + doc=d.doc, + ) + if not _is_supported_sig( + ExportedFunc(pkg=pkg, name=symbol, params=inst_params, results=inst_results), + struct_types=struct_types_by_pkg.get(pkg), + ): + raise BuildError(f"generic instantiation produces unsupported signature: {pkg}:{symbol}") + + key = (pkg, symbol) + if key in seen_symbols: + raise BuildError(f"duplicate generic symbol: {pkg}:{symbol}") + seen_symbols.add(key) + out.append(gi) + + return out + + def _bridge_go_mod( *, bridge_dir: Path, @@ -160,22 +402,69 @@ def build_artifact( out_dir: Path, version: str | None = None, force: bool = False, + generics: Path | None = None, + gomodcache_dir: Path | None = None, + clean_gomodcache: bool = False, ) -> Path: - resolved = resolve_module_target(target=str(module), version=version) + env_base = dict(os.environ) + if gomodcache_dir is not None: + gomodcache_dir = Path(gomodcache_dir).resolve() + if clean_gomodcache and gomodcache_dir.exists(): + shutil.rmtree(gomodcache_dir, ignore_errors=True) + gomodcache_dir.mkdir(parents=True, exist_ok=True) + env_base["GOMODCACHE"] = str(gomodcache_dir) + + resolved = resolve_module_target(target=str(module), version=version, env=env_base) module_dir = resolved.module_dir out_root = Path(out_dir).resolve() out_root.mkdir(parents=True, exist_ok=True) module_path = resolved.module_path - packages = _list_packages(module_dir) - scan = scan_module(module_dir=module_dir) + packages = _list_packages(module_dir, env=env_base) + scan = scan_module(module_dir=module_dir, env=env_base) exported = scan.funcs + methods = scan.methods + generic_defs = scan.generic_funcs + vars_ = scan.vars exported = [ fn for fn in exported if _is_supported_sig(fn, struct_types=scan.struct_types_by_pkg.get(fn.pkg)) ] + methods = [ + m + for m in methods + if _is_supported_method_sig(m, struct_types=scan.struct_types_by_pkg.get(m.pkg)) + ] + + # Vars: only expose exported vars whose type is a named struct type in the same package. + # This supports "namespace" patterns like `isr.DL.Of(...)`. + usable_vars: list[ExportedVar] = [] + for v in vars_: + base = v.type.strip() + if base.startswith("*"): + base = base[1:].strip() + if not base: + continue + if "." in base or "/" in base: + continue + if base not in scan.struct_types_by_pkg.get(v.pkg, set()): + continue + usable_vars.append(v) + + # Methods for bridge wrappers require exported receiver type names (the bridge package + # cannot reference unexported identifiers from imported packages). Unexported receiver + # methods remain in the manifest schema and are invoked via reflective dispatch. + methods_for_bridge = [m for m in methods if m.recv[:1].isupper()] + + generic_insts: list[GenericInstantiation] = [] + if generics is not None: + generic_insts = _load_generic_instantiations( + generics=Path(generics), + defs=generic_defs, + struct_types_by_pkg=scan.struct_types_by_pkg, + ) with tempfile.TemporaryDirectory(prefix="usegolib-bridge-") as td: bridge_dir = Path(td) @@ -198,6 +487,20 @@ def build_artifact( adapter_types.add("time.Duration") if "uuid.UUID" in fn.params or "uuid.UUID" in fn.results: adapter_types.add("uuid.UUID") + for m in methods: + if "time.Time" in m.params or "time.Time" in m.results: + adapter_types.add("time.Time") + if "time.Duration" in m.params or "time.Duration" in m.results: + adapter_types.add("time.Duration") + if "uuid.UUID" in m.params or "uuid.UUID" in m.results: + adapter_types.add("uuid.UUID") + for gi in generic_insts: + if "time.Time" in gi.params or "time.Time" in gi.results: + adapter_types.add("time.Time") + if "time.Duration" in gi.params or "time.Duration" in gi.results: + adapter_types.add("time.Duration") + if "uuid.UUID" in gi.params or "uuid.UUID" in gi.results: + adapter_types.add("uuid.UUID") for pkg, by_name in scan.structs_by_pkg.items(): for _name, fields in by_name.items(): for f in fields: @@ -208,11 +511,29 @@ def build_artifact( if "uuid.UUID" in f.type: adapter_types.add("uuid.UUID") + # Struct types discovered in the package, but with no exported fields in the + # schema. For pointers to these types, we treat values as opaque object + # handles (uint64 ids), so the returned value remains callable in Python. + opaque_struct_types_by_pkg: dict[str, set[str]] = {} + all_pkgs = set(scan.struct_types_by_pkg.keys()) | set(scan.structs_by_pkg.keys()) + for pkg in all_pkgs: + all_structs = set(scan.struct_types_by_pkg.get(pkg, set())) + with_fields = set(scan.structs_by_pkg.get(pkg, {}).keys()) + opaque = all_structs - with_fields + # Unexported structs are always treated as opaque object handles. + opaque |= {n for n in all_structs if not n[:1].isupper()} + if opaque: + opaque_struct_types_by_pkg[pkg] = opaque + write_bridge( bridge_dir=bridge_dir, module_path=module_path, functions=exported, + methods=methods_for_bridge, + generic_instantiations=generic_insts, + vars=usable_vars, struct_types_by_pkg=scan.struct_types_by_pkg, + opaque_struct_types_by_pkg=opaque_struct_types_by_pkg, adapter_types=adapter_types, ) @@ -242,7 +563,7 @@ def build_artifact( lib_path = out_leaf / lib_name zig = ensure_zig() - env = dict(os.environ) + env = dict(env_base) env["CGO_ENABLED"] = "1" env["CC"] = f"{zig} cc" @@ -260,6 +581,63 @@ def build_artifact( go_version = _run(["go", "version"], cwd=module_dir).strip() zig_version = _run([str(zig), "version"], cwd=module_dir).strip() + all_symbol_entries: list[dict[str, Any]] = [ + { + "pkg": fn.pkg, + "name": fn.name, + "params": fn.params, + "results": fn.results, + "doc": fn.doc, + } + for fn in exported + ] + all_symbol_entries.extend( + [ + { + "pkg": gi.pkg, + "name": gi.symbol, + "params": gi.params, + "results": gi.results, + "doc": gi.doc, + "generic": {"name": gi.generic_name, "type_args": gi.type_args}, + } + for gi in generic_insts + ] + ) + + # Ensure schema includes struct entries for all discovered struct types, + # even when a struct has zero exported fields (empty schema). This matters + # for methods that return fluent receivers like `*DataList`. + struct_schema: dict[str, dict[str, list[dict[str, Any]]]] = {} + all_pkgs = set(scan.struct_types_by_pkg.keys()) | set(scan.structs_by_pkg.keys()) + for pkg in sorted(all_pkgs): + names = set(scan.struct_types_by_pkg.get(pkg, set())) | set( + scan.structs_by_pkg.get(pkg, {}).keys() + ) + by_name: dict[str, list[dict[str, Any]]] = {} + for name in sorted(names): + # Treat unexported structs as opaque by default: even if they have + # exported fields (e.g. embedded pointers), passing them as record + # dicts breaks fluent/method APIs. They are represented as object handles. + if not name[:1].isupper(): + fields = [] + else: + fields = scan.structs_by_pkg.get(pkg, {}).get(name, []) + by_name[name] = [ + { + "name": f.name, + "type": f.type, + "key": f.key, + "aliases": f.aliases, + "omitempty": f.omitempty, + "embedded": f.embedded, + "required": ((not f.type.strip().startswith("*")) and (not f.omitempty)), + } + for f in fields + ] + if by_name: + struct_schema[pkg] = by_name + manifest = { "manifest_version": 1, "abi_version": 0, @@ -270,45 +648,38 @@ def build_artifact( "go_version": go_version, "zig_version": zig_version, "packages": packages, - "symbols": [ - { - "pkg": fn.pkg, - "name": fn.name, - "params": fn.params, - "results": fn.results, - } - for fn in exported - ], + "symbols": list(all_symbol_entries), "schema": { - "structs": { - pkg: { - name: [ - { - "name": f.name, - "type": f.type, - "key": f.key, - "aliases": f.aliases, - "omitempty": f.omitempty, - "embedded": f.embedded, - "required": ( - (not f.type.strip().startswith("*")) - and (not f.omitempty) - ), - } - for f in fields - ] - for name, fields in scan.structs_by_pkg.get(pkg, {}).items() + "structs": struct_schema, + "symbols": list(all_symbol_entries), + "methods": [ + { + "pkg": m.pkg, + "recv": m.recv, + "name": m.name, + "params": m.params, + "results": m.results, + "doc": m.doc, } - for pkg in sorted(scan.structs_by_pkg.keys()) - }, - "symbols": [ + for m in methods + ], + "generics": [ + { + "pkg": gi.pkg, + "name": gi.generic_name, + "type_args": gi.type_args, + "symbol": gi.symbol, + } + for gi in generic_insts + ], + "vars": [ { - "pkg": fn.pkg, - "name": fn.name, - "params": fn.params, - "results": fn.results, + "pkg": v.pkg, + "name": v.name, + "type": v.type, + "doc": v.doc, } - for fn in exported + for v in usable_vars ], }, "library": {"path": lib_name, "sha256": sha}, diff --git a/src/usegolib/builder/gobridge.py b/src/usegolib/builder/gobridge.py index 28a3394..d07ddcf 100644 --- a/src/usegolib/builder/gobridge.py +++ b/src/usegolib/builder/gobridge.py @@ -2,7 +2,7 @@ from pathlib import Path -from .symbols import ExportedFunc +from .symbols import ExportedFunc, ExportedMethod, ExportedVar, GenericInstantiation def write_bridge( @@ -10,22 +10,43 @@ def write_bridge( bridge_dir: Path, module_path: str, functions: list[ExportedFunc], + methods: list[ExportedMethod] | None = None, + generic_instantiations: list[GenericInstantiation] | None = None, + vars: list[ExportedVar] | None = None, struct_types_by_pkg: dict[str, set[str]] | None = None, + opaque_struct_types_by_pkg: dict[str, set[str]] | None = None, adapter_types: set[str] | None = None, ) -> None: # Generate a single `main` package for `-buildmode=c-shared`. struct_types_by_pkg = struct_types_by_pkg or {} + opaque_struct_types_by_pkg = opaque_struct_types_by_pkg or {} + methods = methods or [] + generic_instantiations = generic_instantiations or [] + vars = vars or [] adapter_types = adapter_types or set() imports: dict[str, str] = {} for fn in functions: if fn.pkg not in imports: alias = f"p{len(imports)}" imports[fn.pkg] = alias + for m in methods: + if m.pkg not in imports: + alias = f"p{len(imports)}" + imports[m.pkg] = alias + for gi in generic_instantiations: + if gi.pkg not in imports: + alias = f"p{len(imports)}" + imports[gi.pkg] = alias + for v in vars: + if v.pkg not in imports: + alias = f"p{len(imports)}" + imports[v.pkg] = alias - reg_lines: list[str] = [] + func_reg_lines: list[str] = [] + method_reg_lines: list[str] = [] wrap_lines: list[str] = [] - needs_reflect = False + needs_reflect = True # Object handles require reflection helpers. allowed_struct_type_keys: set[str] = set() for pkg, names in struct_types_by_pkg.items(): for n in names: @@ -36,8 +57,9 @@ def write_bridge( key = f"{fn.pkg}:{fn.name}" wrap_name = f"wrap_{alias}_{fn.name}" - reg_lines.append(f' "{key}": {wrap_name},') + func_reg_lines.append(f' "{key}": {wrap_name},') struct_types = struct_types_by_pkg.get(fn.pkg, set()) + opaque_struct_types = opaque_struct_types_by_pkg.get(fn.pkg, set()) if any(_base_type(t) in struct_types for t in fn.params) or any( _base_type(t) in struct_types for t in fn.results ): @@ -52,10 +74,76 @@ def write_bridge( alias=alias, fn=fn, struct_types=struct_types, + opaque_struct_types=opaque_struct_types, ) ) wrap_lines.append("") + for gi in generic_instantiations: + alias = imports[gi.pkg] + key = f"{gi.pkg}:{gi.symbol}" + wrap_name = f"wrapg_{alias}_{gi.symbol}" + + func_reg_lines.append(f' "{key}": {wrap_name},') + struct_types = struct_types_by_pkg.get(gi.pkg, set()) + opaque_struct_types = opaque_struct_types_by_pkg.get(gi.pkg, set()) + wrap_lines.extend( + _write_generic_wrapper( + wrap_name=wrap_name, + alias=alias, + gi=gi, + struct_types=struct_types, + opaque_struct_types=opaque_struct_types, + ) + ) + wrap_lines.append("") + + for v in vars: + alias = imports[v.pkg] + key = f"{v.pkg}:__usegolib_getvar_{v.name}" + wrap_name = f"wrapv_{alias}_{v.name}" + func_reg_lines.append(f' "{key}": {wrap_name},') + wrap_lines.extend(_write_var_get_wrapper(wrap_name=wrap_name, alias=alias, v=v)) + wrap_lines.append("") + + obj_type_keys: set[str] = set() + for m in methods: + alias = imports[m.pkg] + type_key = f"{m.pkg}.{m.recv}" + obj_type_keys.add(type_key) + method_key = f"{type_key}:{m.name}" + wrap_name = f"wrapm_{alias}_{m.recv}_{m.name}" + method_reg_lines.append(f' "{method_key}": {wrap_name},') + + struct_types = struct_types_by_pkg.get(m.pkg, set()) + opaque_struct_types = opaque_struct_types_by_pkg.get(m.pkg, set()) + if any(_base_type(t) in struct_types for t in m.params) or any( + _base_type(t) in struct_types for t in m.results + ): + needs_reflect = True + if any(_base_type(t) in adapter_types for t in m.params) or any( + _base_type(t) in adapter_types for t in m.results + ): + needs_reflect = True + wrap_lines.extend( + _write_method_wrapper( + wrap_name=wrap_name, + alias=alias, + m=m, + struct_types=struct_types, + opaque_struct_types=opaque_struct_types, + ) + ) + wrap_lines.append("") + type_lines: list[str] = [] + for m in methods: + type_key = f"{m.pkg}.{m.recv}" + # De-dupe by key. + if any(ln.startswith(f' "{type_key}":') for ln in type_lines): + continue + alias = imports[m.pkg] + type_lines.append(f' "{type_key}": reflect.TypeOf({alias}.{m.recv}{{}}),') + src = "\n".join( [ "package main", @@ -77,6 +165,10 @@ def write_bridge( ' Op string `msgpack:"op"`', ' Pkg string `msgpack:"pkg"`', ' Fn string `msgpack:"fn"`', + ' Type string `msgpack:"type,omitempty"`', + ' ID uint64 `msgpack:"id,omitempty"`', + ' Method string `msgpack:"method,omitempty"`', + ' Init any `msgpack:"init,omitempty"`', ' Args []any `msgpack:"args"`', "}", "", @@ -93,12 +185,194 @@ def write_bridge( "}", "", "type Handler func(args []any) (any, *ErrorObj)", + "type MethodHandler func(obj any, args []any) (any, *ErrorObj)", "", "var dispatch = map[string]Handler{}", + "var methodDispatch = map[string]MethodHandler{}", + "var typeByKey = map[string]reflect.Type{}", + "", + "type ObjEntry struct {", + " Key string", + " Obj any", + "}", + "", + "var objNext uint64", + "var objMu sync.RWMutex", + "var objByID = map[uint64]ObjEntry{}", + "", + "func storeObj(typeKey string, obj any) uint64 {", + " id := atomic.AddUint64(&objNext, 1)", + " objMu.Lock()", + " objByID[id] = ObjEntry{Key: typeKey, Obj: obj}", + " objMu.Unlock()", + " return id", + "}", + "", + "func isExportedIdent(name string) bool {", + " if name == \"\" {", + " return false", + " }", + " r := rune(name[0])", + " return r >= 'A' && r <= 'Z'", + "}", + "", + "func reflectCallMethod(pkg string, typeName string, obj any, method string, args []any) (any, *ErrorObj) {", + " rv := reflect.ValueOf(obj)", + " if !rv.IsValid() {", + ' return nil, &ErrorObj{Type: "ObjectNotFound", Message: "invalid object"}', + " }", + " mv := rv.MethodByName(method)", + " if !mv.IsValid() {", + ' return nil, &ErrorObj{Type: "MethodNotFound", Message: "method not found"}', + " }", + " mt := mv.Type()", + " nin := mt.NumIn()", + " isVar := mt.IsVariadic()", + " if !isVar {", + " if len(args) != nin {", + ' return nil, &ErrorObj{Type: "ABIError", Message: "wrong arity"}', + " }", + " } else {", + " if len(args) < nin-1 {", + ' return nil, &ErrorObj{Type: "ABIError", Message: "wrong arity"}', + " }", + " }", + "", + " // Variadic calls are represented over the ABI as a single final argument: a slice.", + " // When len(args) == nin, the last arg is already packed and must be expanded with CallSlice.", + " if isVar && len(args) == nin {", + " callArgs := make([]reflect.Value, 0, nin)", + " for i := 0; i < nin-1; i++ {", + " pt := mt.In(i)", + " cv, ok := convertToType(args[i], pt)", + " if !ok {", + ' return nil, &ErrorObj{Type: "UnsupportedTypeError", Message: "unsupported arg type"}', + " }", + " callArgs = append(callArgs, cv)", + " }", + " pt := mt.In(nin - 1)", + " cv, ok := convertToType(args[nin-1], pt)", + " if !ok {", + ' return nil, &ErrorObj{Type: "UnsupportedTypeError", Message: "unsupported arg type"}', + " }", + " callArgs = append(callArgs, cv)", + " outs := mv.CallSlice(callArgs)", + " if len(outs) == 0 {", + " return nil, nil", + " }", + " // Treat trailing error specially (0/1/2 return conventions).", + " if len(outs) == 1 {", + " o0 := outs[0]", + " if o0.IsValid() && o0.Type().Implements(reflect.TypeOf((*error)(nil)).Elem()) {", + " if o0.IsNil() {", + " return nil, nil", + " }", + " return nil, &ErrorObj{Type: \"GoError\", Message: o0.Interface().(error).Error()}", + " }", + " return exportResultAsAny(pkg, typeName, o0)", + " }", + " if len(outs) == 2 {", + " o0 := outs[0]", + " o1 := outs[1]", + " if o1.IsValid() && o1.Type().Implements(reflect.TypeOf((*error)(nil)).Elem()) {", + " if !o1.IsNil() {", + " return nil, &ErrorObj{Type: \"GoError\", Message: o1.Interface().(error).Error()}", + " }", + " return exportResultAsAny(pkg, typeName, o0)", + " }", + " }", + ' return nil, &ErrorObj{Type: "UnsupportedSignatureError", Message: "unsupported method signature"}', + " }", + "", + " callArgs := make([]reflect.Value, 0, len(args))", + " for i := 0; i < nin; i++ {", + " pt := mt.In(i)", + " if isVar && i == nin-1 {", + " // Expand remaining args into the variadic element type.", + " et := pt.Elem()", + " for j := i; j < len(args); j++ {", + " cv, ok := convertToType(args[j], et)", + " if !ok {", + ' return nil, &ErrorObj{Type: "UnsupportedTypeError", Message: "unsupported arg type"}', + " }", + " callArgs = append(callArgs, cv)", + " }", + " break", + " }", + " cv, ok := convertToType(args[i], pt)", + " if !ok {", + ' return nil, &ErrorObj{Type: "UnsupportedTypeError", Message: "unsupported arg type"}', + " }", + " callArgs = append(callArgs, cv)", + " }", + "", + " outs := mv.Call(callArgs)", + " if len(outs) == 0 {", + " return nil, nil", + " }", + " // Treat trailing error specially (0/1/2 return conventions).", + " if len(outs) == 1 {", + " o0 := outs[0]", + " if o0.IsValid() && o0.Type().Implements(reflect.TypeOf((*error)(nil)).Elem()) {", + " if o0.IsNil() {", + " return nil, nil", + " }", + " return nil, &ErrorObj{Type: \"GoError\", Message: o0.Interface().(error).Error()}", + " }", + " return exportResultAsAny(pkg, typeName, o0)", + " }", + " if len(outs) == 2 {", + " o0 := outs[0]", + " o1 := outs[1]", + " if o1.IsValid() && o1.Type().Implements(reflect.TypeOf((*error)(nil)).Elem()) {", + " if !o1.IsNil() {", + " return nil, &ErrorObj{Type: \"GoError\", Message: o1.Interface().(error).Error()}", + " }", + " return exportResultAsAny(pkg, typeName, o0)", + " }", + " }", + ' return nil, &ErrorObj{Type: "UnsupportedSignatureError", Message: "unsupported method signature"}', + "}", + "", + "func exportResultAsAny(pkg string, typeName string, v reflect.Value) (any, *ErrorObj) {", + " if !v.IsValid() {", + " return nil, nil", + " }", + " if v.Kind() == reflect.Interface {", + " if v.IsNil() {", + " return nil, nil", + " }", + " v = v.Elem()", + " }", + " if v.Kind() == reflect.Ptr {", + " if v.IsNil() {", + " return nil, nil", + " }", + " if v.Elem().Kind() == reflect.Struct {", + " // Only support returning object handles for structs defined in the same package as the request.", + " rt := v.Elem().Type()", + " if rt.PkgPath() == pkg {", + " id := storeObj(pkg+\".\"+rt.Name(), v.Interface())", + " return id, nil", + " }", + " }", + " }", + " av, ok := exportAny(v)", + " if !ok {", + ' return nil, &ErrorObj{Type: "UnsupportedTypeError", Message: "unsupported return type"}', + " }", + " return av, nil", + "}", "", "func init() {", " dispatch = map[string]Handler{", - *reg_lines, + *func_reg_lines, + " }", + " methodDispatch = map[string]MethodHandler{", + *method_reg_lines, + " }", + " typeByKey = map[string]reflect.Type{", + *type_lines, " }", "}", "", @@ -117,35 +391,125 @@ def write_bridge( ' writeError(respPtr, respLen, "UnsupportedABIVersion", "unsupported abi version", map[string]any{"abi": req.ABI})', " return 0", " }", - ' if req.Op != "call" {', - ' writeError(respPtr, respLen, "UnsupportedOperation", "unsupported op", map[string]any{"op": req.Op})', - " return 0", - " }", "", - ' key := req.Pkg + ":" + req.Fn', - " h := dispatch[key]", - " if h == nil {", - ' writeError(respPtr, respLen, "SymbolNotFound", "symbol not found", map[string]any{"pkg": req.Pkg, "fn": req.Fn})', + " switch req.Op {", + ' case "call":', + ' key := req.Pkg + ":" + req.Fn', + " h := dispatch[key]", + " if h == nil {", + ' writeError(respPtr, respLen, "SymbolNotFound", "symbol not found", map[string]any{"pkg": req.Pkg, "fn": req.Fn})', + " return 0", + " }", + "", + " var result any", + " var errObj *ErrorObj", + " func() {", + " defer func() {", + " if r := recover(); r != nil {", + ' errObj = &ErrorObj{Type: "GoPanicError", Message: "panic"}', + " }", + " }()", + " result, errObj = h(req.Args)", + " }()", + "", + " if errObj != nil {", + " writeResp(respPtr, respLen, &Response{Ok: false, Error: errObj})", + " return 0", + " }", + " writeResp(respPtr, respLen, &Response{Ok: true, Result: result})", " return 0", - " }", + ' case "obj_new":', + " typeKey := req.Pkg + \".\" + req.Type", + " rt, ok := typeByKey[typeKey]", + " if !ok {", + ' writeError(respPtr, respLen, "TypeNotFound", "type not found", map[string]any{"type": typeKey})', + " return 0", + " }", + " if rt.Kind() != reflect.Struct {", + ' writeError(respPtr, respLen, "ABIError", "type is not a struct", map[string]any{"type": typeKey})', + " return 0", + " }", + " sv := reflect.New(rt).Elem()", + " if req.Init != nil {", + " cv, ok := convertToType(req.Init, rt)", + " if !ok {", + ' writeError(respPtr, respLen, "UnsupportedTypeError", "invalid init", map[string]any{"type": typeKey})', + " return 0", + " }", + " sv = cv", + " }", + " pv := reflect.New(rt)", + " pv.Elem().Set(sv)", + " obj := pv.Interface()", "", - " var result any", - " var errObj *ErrorObj", - " func() {", - " defer func() {", - " if r := recover(); r != nil {", - ' errObj = &ErrorObj{Type: "GoPanicError", Message: "panic"}', + " id := storeObj(typeKey, obj)", + " writeResp(respPtr, respLen, &Response{Ok: true, Result: id})", + " return 0", + ' case "obj_call":', + " typeKey := req.Pkg + \".\" + req.Type", + " objMu.RLock()", + " ent, ok := objByID[req.ID]", + " objMu.RUnlock()", + " if !ok {", + ' writeError(respPtr, respLen, "ObjectNotFound", "object not found", map[string]any{"id": req.ID})', + " return 0", + " }", + " if ent.Key != typeKey {", + ' writeError(respPtr, respLen, "ABIError", "object type mismatch", map[string]any{"id": req.ID, "type": typeKey})', + " return 0", + " }", + " mk := typeKey + \":\" + req.Method", + " mh := methodDispatch[mk]", + " if mh == nil {", + " // Unexported receiver types cannot be referenced from the bridge package, so we", + " // fall back to reflection-based method invocation for them.", + " if isExportedIdent(req.Type) {", + ' writeError(respPtr, respLen, "MethodNotFound", "method not found", map[string]any{\"type\": typeKey, \"method\": req.Method})', + " return 0", " }", + " var result any", + " var errObj *ErrorObj", + " func() {", + " defer func() {", + " if r := recover(); r != nil {", + ' errObj = &ErrorObj{Type: "GoPanicError", Message: "panic"}', + " }", + " }()", + " result, errObj = reflectCallMethod(req.Pkg, req.Type, ent.Obj, req.Method, req.Args)", + " }()", + " if errObj != nil {", + " writeResp(respPtr, respLen, &Response{Ok: false, Error: errObj})", + " return 0", + " }", + " writeResp(respPtr, respLen, &Response{Ok: true, Result: result})", + " return 0", + " }", + " var result any", + " var errObj *ErrorObj", + " func() {", + " defer func() {", + " if r := recover(); r != nil {", + ' errObj = &ErrorObj{Type: "GoPanicError", Message: "panic"}', + " }", + " }()", + " result, errObj = mh(ent.Obj, req.Args)", " }()", - " result, errObj = h(req.Args)", - " }()", - "", - " if errObj != nil {", - " writeResp(respPtr, respLen, &Response{Ok: false, Error: errObj})", + " if errObj != nil {", + " writeResp(respPtr, respLen, &Response{Ok: false, Error: errObj})", + " return 0", + " }", + " writeResp(respPtr, respLen, &Response{Ok: true, Result: result})", + " return 0", + ' case "obj_free":', + " objMu.Lock()", + " delete(objByID, req.ID)", + " objMu.Unlock()", + " writeResp(respPtr, respLen, &Response{Ok: true, Result: nil})", + " return 0", + " default:", + ' writeError(respPtr, respLen, "UnsupportedOperation", "unsupported op", map[string]any{"op": req.Op})', " return 0", " }", - " writeResp(respPtr, respLen, &Response{Ok: true, Result: result})", - " return 0", "}", "", "func writeError(respPtr **C.uchar, respLen *C.size_t, typ string, msg string, detail map[string]any) {", @@ -186,11 +550,11 @@ def write_bridge( "", ' "github.com/vmihailenco/msgpack/v5"', ] - if needs_reflect: - import_block.append(' "reflect"') - import_block.append(' "strings"') - if needs_reflect: - import_block.append(' "time"') + import_block.append(' "sync"') + import_block.append(' "sync/atomic"') + import_block.append(' "reflect"') + import_block.append(' "strings"') + import_block.append(' "time"') if "uuid.UUID" in adapter_types: import_block.append(' uuid "github.com/google/uuid"') for pkg, alias in imports.items(): @@ -225,6 +589,7 @@ def _write_wrapper( alias: str, fn: ExportedFunc, struct_types: set[str], + opaque_struct_types: set[str], ) -> list[str]: # Only support a small set of v0 types. lines: list[str] = [] @@ -249,13 +614,37 @@ def _write_wrapper( ) ) - call = f"{alias}.{fn.name}({', '.join(arg_names)})" + call_args = list(arg_names) + if fn.params and fn.params[-1].strip().startswith("..."): + call_args[-1] = call_args[-1] + "..." + call = f"{alias}.{fn.name}({', '.join(call_args)})" if len(fn.results) == 0: lines.append(f" {call}") lines.append(" return nil, nil") elif len(fn.results) == 1: + t0 = fn.results[0].strip() + if t0 == "error": + lines.append(f" err := {call}") + lines.append(" if err != nil {") + lines.append(' return nil, &ErrorObj{Type: "GoError", Message: err.Error()}') + lines.append(" }") + lines.append(" return nil, nil") + lines.append("}") + return lines + lines.append(f" r0 := {call}") - if _base_type(fn.results[0]) in struct_types or _base_type(fn.results[0]) in { + opaque_ptr = None + if t0.startswith("*"): + inner = t0[1:].strip() + if inner and not inner.startswith(("[]", "map[string]", "...", "*")) and inner in opaque_struct_types: + opaque_ptr = inner + if opaque_ptr is not None: + lines.append(" if r0 == nil {") + lines.append(" return nil, nil") + lines.append(" }") + lines.append(f' id := storeObj("{fn.pkg}.{opaque_ptr}", r0)') + lines.append(" return id, nil") + elif _base_type(t0) == "any" or _base_type(t0) in struct_types or _base_type(t0) in { # noqa: PLR1714 "time.Time", "time.Duration", "uuid.UUID", @@ -275,7 +664,243 @@ def _write_wrapper( lines.append(" if err != nil {") lines.append(' return nil, &ErrorObj{Type: "GoError", Message: err.Error()}') lines.append(" }") - if _base_type(fn.results[0]) in struct_types or _base_type(fn.results[0]) in { + t0 = fn.results[0].strip() + opaque_ptr = None + if t0.startswith("*"): + inner = t0[1:].strip() + if inner and not inner.startswith(("[]", "map[string]", "...", "*")) and inner in opaque_struct_types: + opaque_ptr = inner + if opaque_ptr is not None: + lines.append(" if r0 == nil {") + lines.append(" return nil, nil") + lines.append(" }") + lines.append(f' id := storeObj("{fn.pkg}.{opaque_ptr}", r0)') + lines.append(" return id, nil") + elif _base_type(t0) in struct_types or _base_type(t0) in { + "time.Time", + "time.Duration", + "uuid.UUID", + }: + lines.append(" v0, ok := exportAny(reflect.ValueOf(r0))") + lines.append(" if !ok {") + lines.append( + ' return nil, &ErrorObj{Type: "UnsupportedTypeError", Message: "unsupported return type"}' + ) + lines.append(" }") + lines.append(" return v0, nil") + else: + lines.append(" return r0, nil") + + lines.append("}") + return lines + + +def _write_method_wrapper( + *, + wrap_name: str, + alias: str, + m: ExportedMethod, + struct_types: set[str], + opaque_struct_types: set[str], +) -> list[str]: + lines: list[str] = [] + recv_go = f"*{alias}.{m.recv}" + lines.append(f"func {wrap_name}(obj any, args []any) (any, *ErrorObj) {{") + lines.append(f" recv, ok := obj.({recv_go})") + lines.append(" if !ok {") + lines.append(' return nil, &ErrorObj{Type: "ABIError", Message: "wrong receiver type"}') + lines.append(" }") + lines.append(f" if len(args) != {len(m.params)} {{") + lines.append(' return nil, &ErrorObj{Type: "ABIError", Message: "wrong arity"}') + lines.append(" }") + + arg_names: list[str] = [] + for i, t in enumerate(m.params): + vn = f"a{i}" + arg_names.append(vn) + lines.extend( + _write_arg_convert( + var_name=vn, + go_type=t, + value_expr=f"args[{i}]", + pkg_alias=alias, + struct_types=struct_types, + ) + ) + + call_args = list(arg_names) + if m.params and m.params[-1].strip().startswith("..."): + call_args[-1] = call_args[-1] + "..." + call = f"recv.{m.name}({', '.join(call_args)})" + if len(m.results) == 0: + lines.append(f" {call}") + lines.append(" return nil, nil") + elif len(m.results) == 1: + t0 = m.results[0].strip() + if t0 == "error": + lines.append(f" err := {call}") + lines.append(" if err != nil {") + lines.append(' return nil, &ErrorObj{Type: "GoError", Message: err.Error()}') + lines.append(" }") + lines.append(" return nil, nil") + lines.append("}") + return lines + + lines.append(f" r0 := {call}") + opaque_ptr = None + if t0.startswith("*"): + inner = t0[1:].strip() + if inner and not inner.startswith(("[]", "map[string]", "...", "*")) and inner in opaque_struct_types: + opaque_ptr = inner + if opaque_ptr is not None: + lines.append(" if r0 == nil {") + lines.append(" return nil, nil") + lines.append(" }") + lines.append(f' id := storeObj("{m.pkg}.{opaque_ptr}", r0)') + lines.append(" return id, nil") + elif _base_type(t0) == "any" or _base_type(t0) in struct_types or _base_type(t0) in { # noqa: PLR1714 + "time.Time", + "time.Duration", + "uuid.UUID", + }: + lines.append(" v0, ok := exportAny(reflect.ValueOf(r0))") + lines.append(" if !ok {") + lines.append( + ' return nil, &ErrorObj{Type: "UnsupportedTypeError", Message: "unsupported return type"}' + ) + lines.append(" }") + lines.append(" return v0, nil") + else: + lines.append(" return r0, nil") + else: + lines.append(f" r0, err := {call}") + lines.append(" if err != nil {") + lines.append(' return nil, &ErrorObj{Type: "GoError", Message: err.Error()}') + lines.append(" }") + t0 = m.results[0].strip() + opaque_ptr = None + if t0.startswith("*"): + inner = t0[1:].strip() + if inner and not inner.startswith(("[]", "map[string]", "...", "*")) and inner in opaque_struct_types: + opaque_ptr = inner + if opaque_ptr is not None: + lines.append(" if r0 == nil {") + lines.append(" return nil, nil") + lines.append(" }") + lines.append(f' id := storeObj("{m.pkg}.{opaque_ptr}", r0)') + lines.append(" return id, nil") + elif _base_type(t0) in struct_types or _base_type(t0) in { + "time.Time", + "time.Duration", + "uuid.UUID", + }: + lines.append(" v0, ok := exportAny(reflect.ValueOf(r0))") + lines.append(" if !ok {") + lines.append( + ' return nil, &ErrorObj{Type: "UnsupportedTypeError", Message: "unsupported return type"}' + ) + lines.append(" }") + lines.append(" return v0, nil") + else: + lines.append(" return r0, nil") + + lines.append("}") + return lines + + +def _write_generic_wrapper( + *, + wrap_name: str, + alias: str, + gi: GenericInstantiation, + struct_types: set[str], + opaque_struct_types: set[str], +) -> list[str]: + lines: list[str] = [] + lines.append(f"func {wrap_name}(args []any) (any, *ErrorObj) {{") + lines.append(f" if len(args) != {len(gi.params)} {{") + lines.append(' return nil, &ErrorObj{Type: "ABIError", Message: "wrong arity"}') + lines.append(" }") + + arg_names: list[str] = [] + for i, t in enumerate(gi.params): + vn = f"a{i}" + arg_names.append(vn) + lines.extend( + _write_arg_convert( + var_name=vn, + go_type=t, + value_expr=f"args[{i}]", + pkg_alias=alias, + struct_types=struct_types, + ) + ) + + type_args_exprs = [ + _qualify_type(t, pkg_alias=alias, struct_types=struct_types) for t in gi.type_args + ] + call_args = list(arg_names) + if gi.params and gi.params[-1].strip().startswith("..."): + call_args[-1] = call_args[-1] + "..." + call = f"{alias}.{gi.generic_name}[{', '.join(type_args_exprs)}]({', '.join(call_args)})" + if len(gi.results) == 0: + lines.append(f" {call}") + lines.append(" return nil, nil") + elif len(gi.results) == 1: + t0 = gi.results[0].strip() + if t0 == "error": + lines.append(f" err := {call}") + lines.append(" if err != nil {") + lines.append(' return nil, &ErrorObj{Type: "GoError", Message: err.Error()}') + lines.append(" }") + lines.append(" return nil, nil") + lines.append("}") + return lines + + lines.append(f" r0 := {call}") + opaque_ptr = None + if t0.startswith("*"): + inner = t0[1:].strip() + if inner and not inner.startswith(("[]", "map[string]", "...", "*")) and inner in opaque_struct_types: + opaque_ptr = inner + if opaque_ptr is not None: + lines.append(" if r0 == nil {") + lines.append(" return nil, nil") + lines.append(" }") + lines.append(f' id := storeObj("{gi.pkg}.{opaque_ptr}", r0)') + lines.append(" return id, nil") + elif _base_type(t0) == "any" or _base_type(t0) in struct_types or _base_type(t0) in { # noqa: PLR1714 + "time.Time", + "time.Duration", + "uuid.UUID", + }: + lines.append(" v0, ok := exportAny(reflect.ValueOf(r0))") + lines.append(" if !ok {") + lines.append( + ' return nil, &ErrorObj{Type: "UnsupportedTypeError", Message: "unsupported return type"}' + ) + lines.append(" }") + lines.append(" return v0, nil") + else: + lines.append(" return r0, nil") + else: + lines.append(f" r0, err := {call}") + lines.append(" if err != nil {") + lines.append(' return nil, &ErrorObj{Type: "GoError", Message: err.Error()}') + lines.append(" }") + t0 = gi.results[0].strip() + opaque_ptr = None + if t0.startswith("*"): + inner = t0[1:].strip() + if inner and not inner.startswith(("[]", "map[string]", "...", "*")) and inner in opaque_struct_types: + opaque_ptr = inner + if opaque_ptr is not None: + lines.append(" if r0 == nil {") + lines.append(" return nil, nil") + lines.append(" }") + lines.append(f' id := storeObj("{gi.pkg}.{opaque_ptr}", r0)') + lines.append(" return id, nil") + elif _base_type(t0) in struct_types or _base_type(t0) in { "time.Time", "time.Duration", "uuid.UUID", @@ -289,7 +914,26 @@ def _write_wrapper( lines.append(" return v0, nil") else: lines.append(" return r0, nil") + lines.append("}") + return lines + +def _write_var_get_wrapper(*, wrap_name: str, alias: str, v: ExportedVar) -> list[str]: + base = v.type.strip() + if base.startswith("*"): + base = base[1:].strip() + lines: list[str] = [] + lines.append(f"func {wrap_name}(args []any) (any, *ErrorObj) {{") + lines.append(" if len(args) != 0 {") + lines.append(' return nil, &ErrorObj{Type: "ABIError", Message: "wrong arity"}') + lines.append(" }") + # Store a pointer to the var when possible so pointer-receiver methods work. + if v.type.strip().startswith("*"): + lines.append(f" obj := {alias}.{v.name}") + else: + lines.append(f" obj := &{alias}.{v.name}") + lines.append(f' id := storeObj("{v.pkg}.{base}", obj)') + lines.append(" return id, nil") lines.append("}") return lines @@ -301,6 +945,10 @@ def _conv_expr(go_type: str, v: str) -> str: def _base_type(go_type: str) -> str: t = go_type.strip() while True: + # Variadic `...T` behaves like a slice `[]T` inside the wrapper. + if t.startswith("..."): + t = t[3:].strip() + continue if t.startswith("*"): t = t[1:].strip() continue @@ -317,6 +965,9 @@ def _qualify_type(go_type: str, *, pkg_alias: str, struct_types: set[str]) -> st t = go_type.strip() if t.startswith("*"): return "*" + _qualify_type(t[1:], pkg_alias=pkg_alias, struct_types=struct_types) + if t.startswith("..."): + # Variadic `...T` is a slice type `[]T` when referenced as a value type. + return "[]" + _qualify_type(t[3:], pkg_alias=pkg_alias, struct_types=struct_types) if t.startswith("[]"): return "[]" + _qualify_type(t[2:], pkg_alias=pkg_alias, struct_types=struct_types) if t.startswith("map[string]"): @@ -345,6 +996,14 @@ def unsupported() -> None: ' return nil, &ErrorObj{Type: "UnsupportedTypeError", Message: "unsupported arg type"}' ) + # Normalize variadics: `...T` is received over ABI as a single slice arg. + if go_type.strip().startswith("..."): + go_type = "[]" + go_type.strip()[3:].strip() + + if go_type == "any": + lines.append(f" {var_name} := {value_expr}") + return lines + if _base_type(go_type) in struct_types: typ = _qualify_type(go_type, pkg_alias=pkg_alias, struct_types=struct_types) lines.append(f" {var_name}, ok := toGoValue[{typ}]({value_expr})") @@ -381,6 +1040,13 @@ def unsupported() -> None: vt = go_type[len("map[string]") :] tmp = f"t_{var_name}" + if vt == "any": + lines.append(f" {var_name}, ok := toAnyMap({value_expr})") + lines.append(" if !ok {") + unsupported() + lines.append(" }") + return lines + if vt == "int64": lines.append(f" {var_name}, ok := toStringInt64Map({value_expr})") lines.append(" if !ok {") @@ -533,6 +1199,31 @@ def unsupported() -> None: lines.append(" }") return lines + if go_type == "[]any": + lines.append(f" {var_name}, ok := toAnySlice({value_expr})") + lines.append(" if !ok {") + unsupported() + lines.append(" }") + return lines + + if go_type == "[]map[string]any": + tmp = f"t_{var_name}" + lines.append(f" {tmp}, ok := toAnySlice({value_expr})") + lines.append(" if !ok {") + unsupported() + lines.append(" }") + lines.append(f" {var_name} := make([]map[string]any, 0, len({tmp}))") + lines.append(f" for _, item := range {tmp} {{") + lines.append(" m, ok := toAnyMap(item)") + lines.append(" if !ok {") + lines.append( + ' return nil, &ErrorObj{Type: "UnsupportedTypeError", Message: "unsupported arg type"}' + ) + lines.append(" }") + lines.append(f" {var_name} = append({var_name}, m)") + lines.append(" }") + return lines + if go_type == "[]float64": lines.append(f" {var_name}, ok := toFloat64Slice({value_expr})") lines.append(" if !ok {") @@ -1080,6 +1771,22 @@ def _write_helpers( " return out, true", " }", " switch t.Kind() {", + " case reflect.Interface:", + " if v == nil {", + " return reflect.Zero(t), true", + " }", + " rv := reflect.ValueOf(v)", + " if rv.Type().AssignableTo(t) {", + " return rv, true", + " }", + " if rv.Type().ConvertibleTo(t) {", + " return rv.Convert(t), true", + " }", + " // Empty interface: accept any value as-is.", + " if t.NumMethod() == 0 {", + " return rv, true", + " }", + " return reflect.Value{}, false", " case reflect.Bool:", " b, ok := toBool(v)", " if !ok {", @@ -1249,6 +1956,12 @@ def _write_helpers( " }", " v = v.Elem()", " }", + " if v.Kind() == reflect.Interface {", + " if v.IsNil() {", + " return nil, true", + " }", + " v = v.Elem()", + " }", " // Adapter: github.com/google/uuid.UUID encoded as string.", " if v.Type().PkgPath() == \"github.com/google/uuid\" && v.Type().Name() == \"UUID\" {", " id := v.Interface().(uuid.UUID)", diff --git a/src/usegolib/builder/resolve.py b/src/usegolib/builder/resolve.py index 91561cc..4f36743 100644 --- a/src/usegolib/builder/resolve.py +++ b/src/usegolib/builder/resolve.py @@ -1,8 +1,11 @@ from __future__ import annotations import json +import os +import re import subprocess import tempfile +import time from dataclasses import dataclass from pathlib import Path @@ -16,22 +19,34 @@ class ResolvedModule: module_dir: Path -def resolve_module_target(*, target: str, version: str | None) -> ResolvedModule: - """Resolve a builder target to a local module directory and concrete version. +_GO_TRANSIENT_NET_RE = re.compile( + r"(" + r"proxy\.golang\.org" + r"|sum\.golang\.org" + r"|wsarecv" + r"|connection (?:attempt failed|reset)" + r"|i/o timeout" + r"|tls handshake timeout" + r"|unexpected eof" + r"|temporary failure" + r"|no such host" + r"|502 bad gateway" + r"|503 service unavailable" + r"|504 gateway timeout" + r")", + re.IGNORECASE, +) - - If `target` is a local directory, parse its go.mod and return version "local". - - Otherwise treat `target` as a Go import path (module or package path) and use - `go mod download -json` to resolve module root and version (defaults to @latest). - """ - p = Path(target) - if p.exists() and p.is_dir(): - module_dir = p.resolve() - module_path = _read_module_path(module_dir) - return ResolvedModule(module_path=module_path, version="local", module_dir=module_dir) - wanted = version or "latest" - mod_path, mod_version, mod_dir = _resolve_remote_module(import_path=target, wanted=wanted) - return ResolvedModule(module_path=mod_path, version=mod_version, module_dir=mod_dir) +def _go_network_hint(out: str) -> str | None: + if not _GO_TRANSIENT_NET_RE.search(out): + return None + return ( + "\n\nHint: Go module download failed due to a network/proxy error. " + "Try re-running the command. If `proxy.golang.org` is blocked/unreliable " + "in your environment, try setting `GOPROXY=direct` (or another reachable proxy) " + "and retry." + ) def _read_module_path(module_dir: Path) -> str: @@ -45,39 +60,160 @@ def _read_module_path(module_dir: Path) -> str: raise BuildError("failed to parse module path from go.mod") -def _resolve_remote_module(*, import_path: str, wanted: str) -> tuple[str, str, Path]: +def _find_module_root(start: Path) -> Path: + p = start + while True: + if (p / "go.mod").exists(): + return p + if p.parent == p: + break + p = p.parent + raise BuildError(f"go.mod not found in {start} or any parent directory") + + +def resolve_module_target( + *, target: str, version: str | None, env: dict[str, str] | None = None +) -> ResolvedModule: + """Resolve a builder target to a local module directory and concrete version. + + - If `target` is a local directory, locate the nearest parent containing go.mod + (allows passing a subdirectory of a module) and return version "local". + - Otherwise treat `target` as a Go import path (module or package path) and use + `go mod download -json` to resolve module root and version (defaults to @latest). + + `env` is passed through to `go` subprocesses (for example to set `GOMODCACHE`). + """ + p = Path(target) + if p.exists() and p.is_dir(): + module_dir = _find_module_root(p.resolve()) + module_path = _read_module_path(module_dir) + return ResolvedModule(module_path=module_path, version="local", module_dir=module_dir) + + # Support `go get` style syntax: `module@version`. + # We only parse this for remote targets (local directory paths are handled above). + if "@" in target: + base, inline = target.rsplit("@", 1) + if base and inline: + if version is not None: + raise BuildError("conflicting version: use either target@version or version=, not both") + target = base + version = inline + + wanted = version + if wanted is None or wanted == "latest": + wanted = "@latest" + mod_path, mod_version, mod_dir = _resolve_remote_module( + import_path=target, wanted=wanted, env=env + ) + return ResolvedModule(module_path=mod_path, version=mod_version, module_dir=mod_dir) + + +def _resolve_remote_module( + *, import_path: str, wanted: str, env: dict[str, str] | None +) -> tuple[str, str, Path]: # For non-module package paths, trim segments until download succeeds. candidate = import_path while True: try: - info = _go_mod_download_json(f"{candidate}@{wanted}") + # `go mod download` uses special queries like `@latest`. + # Allow callers to pass `wanted` with or without the leading "@". + arg = f"{candidate}{wanted}" if wanted.startswith("@") else f"{candidate}@{wanted}" + info = _go_mod_download_json(arg, env=env) mod_path = str(info["Path"]) mod_version = str(info["Version"]) mod_dir = Path(str(info["Dir"])).resolve() return mod_path, mod_version, mod_dir - except BuildError: + except BuildError as e: + # If the failure looks like a transient network/proxy problem, do not + # "walk up" the import path: it hides the real error and can produce + # misleading messages like `github.com@vX.Y.Z`. + if _GO_TRANSIENT_NET_RE.search(str(e)): + raise # Trim one path segment and try again. if "/" not in candidate: raise candidate = candidate.rsplit("/", 1)[0] -def _go_mod_download_json(arg: str) -> dict: +def _go_mod_download_json(arg: str, *, env: dict[str, str] | None) -> dict: # `go mod download` does not require being inside a module, but to be robust # across environments, run in a temp directory. with tempfile.TemporaryDirectory(prefix="usegolib-moddl-") as td: - proc = subprocess.run( - ["go", "mod", "download", "-json", arg], - cwd=td, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - check=False, - ) + max_attempts = 3 + backoff_s = 0.5 + base_env = env + cur_env = env + last_out = "" + + for attempt in range(max_attempts): + try: + proc = subprocess.run( + ["go", "mod", "download", "-json", arg], + cwd=td, + env=cur_env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=False, + check=False, + ) + except FileNotFoundError as e: + raise BuildError( + "Go toolchain not found (`go` is missing from PATH). " + "Install Go and ensure `go` is available on PATH. " + "If you do not want auto-build on import, pass `build_if_missing=False` " + "(and use prebuilt artifacts/wheels)." + ) from e + + out = (proc.stdout or b"").decode("utf-8", errors="replace") + last_out = out + if proc.returncode == 0: + break + + if attempt < max_attempts - 1 and _GO_TRANSIENT_NET_RE.search(out): + if "proxy.golang.org" in out.lower(): + if base_env is None: + next_env = dict(os.environ) + else: + next_env = dict(base_env) + if "GOPROXY" not in next_env: + next_env["GOPROXY"] = "direct" + cur_env = next_env + time.sleep(backoff_s) + backoff_s *= 2.0 + continue + + hint = _go_network_hint(out) + if hint: + out = out.rstrip("\n") + hint + "\n" + raise BuildError(f"go mod download failed for {arg}\n{out}") + if proc.returncode != 0: - raise BuildError(f"go mod download failed for {arg}\n{proc.stdout}") + hint = _go_network_hint(last_out) + if hint: + last_out = last_out.rstrip("\n") + hint + "\n" + raise BuildError(f"go mod download failed for {arg}\n{last_out}") try: - return json.loads(proc.stdout) - except Exception as e: # noqa: BLE001 - raise BuildError(f"failed to parse go mod download output for {arg}: {e}") from e - + return json.loads(last_out) + except Exception: + # Go may print non-JSON lines (e.g. toolchain switching messages) to stderr, + # which we merge into stdout for portability. Extract the first JSON object. + out = last_out + start = out.find("{") + if start == -1: + raise BuildError(f"failed to parse go mod download output for {arg}") + depth = 0 + end = -1 + for i, ch in enumerate(out[start:], start=start): + if ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + end = i + 1 + break + if end == -1: + raise BuildError(f"failed to parse go mod download output for {arg}") + try: + return json.loads(out[start:end]) + except Exception as e: # noqa: BLE001 + raise BuildError(f"failed to parse go mod download output for {arg}: {e}") from e diff --git a/src/usegolib/builder/scan.py b/src/usegolib/builder/scan.py index 4e9ee7e..ab8c7e3 100644 --- a/src/usegolib/builder/scan.py +++ b/src/usegolib/builder/scan.py @@ -1,15 +1,48 @@ from __future__ import annotations import json +import os +import re import subprocess import tempfile +import time from pathlib import Path from ..errors import BuildError -from .symbols import ExportedFunc, ModuleScan, StructField +from .symbols import ExportedFunc, ExportedMethod, ExportedVar, GenericFuncDef, ModuleScan, StructField + + +_GO_TRANSIENT_NET_RE = re.compile( + r"(" + r"proxy\.golang\.org" + r"|sum\.golang\.org" + r"|wsarecv" + r"|connection (?:attempt failed|reset)" + r"|i/o timeout" + r"|tls handshake timeout" + r"|unexpected eof" + r"|temporary failure" + r"|no such host" + r"|502 bad gateway" + r"|503 service unavailable" + r"|504 gateway timeout" + r")", + re.IGNORECASE, +) + + +def _go_network_hint(out: str) -> str | None: + if not _GO_TRANSIENT_NET_RE.search(out): + return None + return ( + "\n\nHint: Go module download failed due to a network/proxy error. " + "Try re-running the command. If `proxy.golang.org` is blocked/unreliable " + "in your environment, try setting `GOPROXY=direct` (or another reachable proxy) " + "and retry." + ) -def scan_module(*, module_dir: Path) -> ModuleScan: +def scan_module(*, module_dir: Path, env: dict[str, str] | None = None) -> ModuleScan: """Scan exported top-level functions by parsing Go source (not `go doc` text). This is used by the builder to decide which functions are callable through the @@ -32,21 +65,78 @@ def scan_module(*, module_dir: Path) -> ModuleScan: ) (scan_dir / "main.go").write_text(_scanner_go_source(), encoding="utf-8") - proc = subprocess.run( - ["go", "run", ".", "--module-dir", str(module_dir)], - cwd=str(scan_dir), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - check=False, - ) - if proc.returncode != 0: - raise BuildError(f"go scan failed\n{proc.stdout}") + proc = None + out = "" + max_attempts = 3 + backoff_s = 0.5 + base_env = env + cur_env = env + try: + for attempt in range(max_attempts): + proc = subprocess.run( + ["go", "run", ".", "--module-dir", str(module_dir)], + cwd=str(scan_dir), + env=cur_env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=False, + check=False, + ) + out = (proc.stdout or b"").decode("utf-8", errors="replace") + if proc.returncode == 0: + break + + if attempt < max_attempts - 1 and _GO_TRANSIENT_NET_RE.search(out): + if "proxy.golang.org" in out.lower(): + if base_env is None: + next_env = dict(os.environ) + else: + next_env = dict(base_env) + if "GOPROXY" not in next_env: + next_env["GOPROXY"] = "direct" + cur_env = next_env + time.sleep(backoff_s) + backoff_s *= 2.0 + continue + break + except FileNotFoundError as e: + raise BuildError( + "Go toolchain not found (`go` is missing from PATH). " + "Install Go and ensure `go` is available on PATH. " + "If you do not want auto-build on import, pass `build_if_missing=False` " + "(and use prebuilt artifacts/wheels)." + ) from e + if proc is None or proc.returncode != 0: + hint = _go_network_hint(out) + if hint: + out = out.rstrip("\n") + hint + "\n" + raise BuildError(f"go scan failed\n{out}") try: - obj = json.loads(proc.stdout) + try: + obj = json.loads(out) + except Exception: + # Best-effort: some Go toolchains print non-JSON lines (e.g. toolchain + # switching messages) that get merged into stdout. Extract the first + # JSON object and retry. + start = out.find("{") + if start == -1: + raise + depth = 0 + end = -1 + for i, ch in enumerate(out[start:], start=start): + if ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + end = i + 1 + break + if end == -1: + raise + obj = json.loads(out[start:end]) except Exception as e: # noqa: BLE001 - raise BuildError(f"failed to parse go scan output: {e}\n{proc.stdout}") from e + raise BuildError(f"failed to parse go scan output: {e}\n{out}") from e funcs: list[ExportedFunc] = [] for item in obj.get("funcs", []): @@ -56,6 +146,7 @@ def scan_module(*, module_dir: Path) -> ModuleScan: name = item.get("name") params = item.get("params") results = item.get("results") + doc = item.get("doc") if not isinstance(pkg, str) or not isinstance(name, str): continue if not isinstance(params, list) or not isinstance(results, list): @@ -64,7 +155,99 @@ def scan_module(*, module_dir: Path) -> ModuleScan: continue if not all(isinstance(t, str) for t in results): continue - funcs.append(ExportedFunc(pkg=pkg, name=name, params=list(params), results=list(results))) + doc_str: str | None = None + if isinstance(doc, str): + doc_str = doc.strip() or None + funcs.append( + ExportedFunc( + pkg=pkg, + name=name, + params=list(params), + results=list(results), + doc=doc_str, + ) + ) + + methods: list[ExportedMethod] = [] + for item in obj.get("methods", []): + if not isinstance(item, dict): + continue + pkg = item.get("pkg") + recv = item.get("recv") + name = item.get("name") + params = item.get("params") + results = item.get("results") + doc = item.get("doc") + if not (isinstance(pkg, str) and isinstance(recv, str) and isinstance(name, str)): + continue + if not isinstance(params, list) or not isinstance(results, list): + continue + if not all(isinstance(t, str) for t in params): + continue + if not all(isinstance(t, str) for t in results): + continue + methods.append( + ExportedMethod( + pkg=pkg, + recv=recv, + name=name, + params=list(params), + results=list(results), + doc=(doc.strip() or None) if isinstance(doc, str) else None, + ) + ) + + vars_: list[ExportedVar] = [] + for item in obj.get("vars", []): + if not isinstance(item, dict): + continue + pkg = item.get("pkg") + name = item.get("name") + typ = item.get("type") + doc = item.get("doc") + if not (isinstance(pkg, str) and isinstance(name, str) and isinstance(typ, str)): + continue + if not pkg or not name or not typ: + continue + vars_.append( + ExportedVar( + pkg=pkg, + name=name, + type=typ, + doc=(doc.strip() or None) if isinstance(doc, str) else None, + ) + ) + + generic_funcs: list[GenericFuncDef] = [] + for item in obj.get("generic_funcs", []): + if not isinstance(item, dict): + continue + pkg = item.get("pkg") + name = item.get("name") + type_params = item.get("type_params") + params = item.get("params") + results = item.get("results") + doc = item.get("doc") + if not (isinstance(pkg, str) and isinstance(name, str)): + continue + if not isinstance(type_params, list) or not all(isinstance(x, str) for x in type_params): + continue + if not isinstance(params, list) or not isinstance(results, list): + continue + if not all(isinstance(t, str) for t in params): + continue + if not all(isinstance(t, str) for t in results): + continue + generic_funcs.append( + GenericFuncDef( + pkg=pkg, + name=name, + type_params=list(type_params), + params=list(params), + results=list(results), + doc=(doc.strip() or None) if isinstance(doc, str) else None, + ) + ) struct_types_by_pkg: dict[str, set[str]] = {} st = obj.get("struct_types") @@ -123,13 +306,16 @@ def scan_module(*, module_dir: Path) -> ModuleScan: return ModuleScan( funcs=funcs, + methods=methods, + generic_funcs=generic_funcs, + vars=vars_, struct_types_by_pkg=struct_types_by_pkg, structs_by_pkg=structs_by_pkg, ) -def scan_exported_funcs(*, module_dir: Path) -> list[ExportedFunc]: - return scan_module(module_dir=module_dir).funcs +def scan_exported_funcs(*, module_dir: Path, env: dict[str, str] | None = None) -> list[ExportedFunc]: + return scan_module(module_dir=module_dir, env=env).funcs def _scanner_go_source() -> str: @@ -167,13 +353,42 @@ def _scanner_go_source() -> str: Name string `json:"name"` Params []string `json:"params"` Results []string `json:"results"` + Doc string `json:"doc"` } -type outObj struct { - Funcs []outFunc `json:"funcs"` - StructTypes map[string][]string `json:"struct_types"` - Structs map[string][]outStruct `json:"structs"` -} + type outMethod struct { + Pkg string `json:"pkg"` + Recv string `json:"recv"` + Name string `json:"name"` + Params []string `json:"params"` + Results []string `json:"results"` + Doc string `json:"doc"` + } + + type outVar struct { + Pkg string `json:"pkg"` + Name string `json:"name"` + Type string `json:"type"` + Doc string `json:"doc"` + } + + type outGenericFunc struct { + Pkg string `json:"pkg"` + Name string `json:"name"` + TypeParams []string `json:"type_params"` + Params []string `json:"params"` + Results []string `json:"results"` + Doc string `json:"doc"` + } + + type outObj struct { + Funcs []outFunc `json:"funcs"` + Methods []outMethod `json:"methods"` + GenericFuncs []outGenericFunc `json:"generic_funcs"` + Vars []outVar `json:"vars"` + StructTypes map[string][]string `json:"struct_types"` + Structs map[string][]outStruct `json:"structs"` + } type outStructField struct { Name string `json:"name"` @@ -210,12 +425,15 @@ def _scanner_go_source() -> str: os.Exit(1) } - out := outObj{ - Funcs: make([]outFunc, 0, 128), - StructTypes: map[string][]string{}, - Structs: map[string][]outStruct{}, - } - for _, p := range pkgs { + out := outObj{ + Funcs: make([]outFunc, 0, 128), + Methods: make([]outMethod, 0, 128), + GenericFuncs: make([]outGenericFunc, 0, 128), + Vars: make([]outVar, 0, 128), + StructTypes: map[string][]string{}, + Structs: map[string][]outStruct{}, + } + for _, p := range pkgs { if isInternalPkg(p.ImportPath) { continue } @@ -233,39 +451,149 @@ def _scanner_go_source() -> str: structNames := map[string]bool{} structFields := map[string][]outStructField{} for _, file := range files { - af, err := parser.ParseFile(fs, file, nil, 0) + af, err := parser.ParseFile(fs, file, nil, parser.ParseComments) if err != nil { continue } im := fileImports(af) collectStructTypes(af, im, structNames, structFields) - for _, decl := range af.Decls { - fd, ok := decl.(*ast.FuncDecl) - if !ok { + for _, decl := range af.Decls { + // Exported package-level vars (namespace-style singletons). + gd, ok := decl.(*ast.GenDecl) + if ok && gd.Tok == token.VAR { + for _, spec := range gd.Specs { + vs, ok := spec.(*ast.ValueSpec) + if !ok || vs == nil || len(vs.Names) == 0 { + continue + } + typ := "" + if vs.Type != nil { + typ = renderType(vs.Type, im) + } else if len(vs.Values) == 1 && vs.Values[0] != nil { + switch vv := vs.Values[0].(type) { + case *ast.CompositeLit: + typ = renderType(vv.Type, im) + case *ast.UnaryExpr: + if vv.Op == token.AND { + if cl, ok := vv.X.(*ast.CompositeLit); ok { + inner := renderType(cl.Type, im) + if inner != "" { + typ = "*" + inner + } + } + } + } + } + if typ == "" { + continue + } + doc := "" + if vs.Doc != nil { + doc = docText(vs.Doc) + } else if gd.Doc != nil { + doc = docText(gd.Doc) + } + for _, nm := range vs.Names { + if nm == nil || !nm.IsExported() { + continue + } + out.Vars = append(out.Vars, outVar{ + Pkg: p.ImportPath, + Name: nm.Name, + Type: typ, + Doc: doc, + }) + } + } + continue + } + + fd, ok := decl.(*ast.FuncDecl) + if !ok { + continue + } + if fd.Name == nil || !fd.Name.IsExported() { continue } - if fd.Recv != nil { + // Exported generic functions are reported separately. + if fd.Recv == nil && fd.Type != nil && fd.Type.TypeParams != nil && len(fd.Type.TypeParams.List) > 0 { + typeParams := []string{} + for _, tp := range fd.Type.TypeParams.List { + if tp == nil || len(tp.Names) == 0 { + continue + } + for _, nm := range tp.Names { + if nm == nil || nm.Name == "" { + continue + } + typeParams = append(typeParams, nm.Name) + } + } + if len(typeParams) == 0 { + continue + } + params := fieldListTypes(fd.Type.Params, im) + results := fieldListTypes(fd.Type.Results, im) + if params == nil || results == nil { + continue + } + out.GenericFuncs = append(out.GenericFuncs, outGenericFunc{ + Pkg: p.ImportPath, + Name: fd.Name.Name, + TypeParams: typeParams, + Params: params, + Results: results, + Doc: docText(fd.Doc), + }) continue } - if fd.Name == nil || !fd.Name.IsExported() { + + // Top-level function. + if fd.Recv == nil { + params := fieldListTypes(fd.Type.Params, im) + results := fieldListTypes(fd.Type.Results, im) + if params == nil || results == nil { continue } - // Skip generic functions for v0. - if fd.Type != nil && fd.Type.TypeParams != nil && len(fd.Type.TypeParams.List) > 0 { + + out.Funcs = append(out.Funcs, outFunc{ + Pkg: p.ImportPath, + Name: fd.Name.Name, + Params: params, + Results: results, + Doc: docText(fd.Doc), + }) continue } + // Exported method on exported struct receiver type. + if fd.Recv.List == nil || len(fd.Recv.List) == 0 || fd.Recv.List[0] == nil || fd.Recv.List[0].Type == nil { + continue + } + rt := renderType(fd.Recv.List[0].Type, im) + if rt == "" { + continue + } + recv := strings.TrimPrefix(rt, "*") + if recv == "" { + continue + } + if !structNames[recv] { + continue + } + params := fieldListTypes(fd.Type.Params, im) results := fieldListTypes(fd.Type.Results, im) if params == nil || results == nil { continue } - - out.Funcs = append(out.Funcs, outFunc{ + out.Methods = append(out.Methods, outMethod{ Pkg: p.ImportPath, + Recv: recv, Name: fd.Name.Name, Params: params, Results: results, + Doc: docText(fd.Doc), }) } } @@ -342,6 +670,13 @@ def _scanner_go_source() -> str: return out } +func docText(cg *ast.CommentGroup) string { + if cg == nil { + return "" + } + return strings.TrimSpace(cg.Text()) +} + func collectStructTypes(af *ast.File, im map[string]string, out map[string]bool, fieldsOut map[string][]outStructField) { for _, decl := range af.Decls { gd, ok := decl.(*ast.GenDecl) @@ -512,6 +847,12 @@ def _scanner_go_source() -> str: switch t := e.(type) { case *ast.Ident: return t.Name + case *ast.Ellipsis: + inner := renderType(t.Elt, im) + if inner == "" { + return "" + } + return "..." + inner case *ast.ArrayType: if t.Len != nil { return "" diff --git a/src/usegolib/builder/symbols.py b/src/usegolib/builder/symbols.py index 5d4c217..2d7874d 100644 --- a/src/usegolib/builder/symbols.py +++ b/src/usegolib/builder/symbols.py @@ -9,6 +9,25 @@ class ExportedFunc: name: str params: list[str] results: list[str] + doc: str | None = None + + +@dataclass(frozen=True) +class ExportedMethod: + pkg: str + recv: str # receiver struct type name (no leading '*') + name: str + params: list[str] + results: list[str] + doc: str | None = None + + +@dataclass(frozen=True) +class ExportedVar: + pkg: str + name: str + type: str + doc: str | None = None @dataclass(frozen=True) @@ -24,5 +43,29 @@ class StructField: @dataclass(frozen=True) class ModuleScan: funcs: list[ExportedFunc] + methods: list[ExportedMethod] + generic_funcs: list["GenericFuncDef"] + vars: list[ExportedVar] struct_types_by_pkg: dict[str, set[str]] structs_by_pkg: dict[str, dict[str, list[StructField]]] + + +@dataclass(frozen=True) +class GenericFuncDef: + pkg: str + name: str + type_params: list[str] + params: list[str] + results: list[str] + doc: str | None = None + + +@dataclass(frozen=True) +class GenericInstantiation: + pkg: str + generic_name: str + type_args: list[str] + symbol: str + params: list[str] + results: list[str] + doc: str | None = None diff --git a/src/usegolib/cli.py b/src/usegolib/cli.py index 8fa59b8..c59aa37 100644 --- a/src/usegolib/cli.py +++ b/src/usegolib/cli.py @@ -1,10 +1,28 @@ from __future__ import annotations import argparse +import hashlib import importlib.metadata from pathlib import Path +def _split_target_and_version(value: str) -> tuple[str, str | None]: + """Parse `module@version` syntax for remote targets. + + For local module directories, callers should pass the raw path string and + let the builder resolve it as a directory. + """ + p = Path(value) + if p.exists() and p.is_dir(): + return value, None + if "@" not in value: + return value, None + base, ver = value.rsplit("@", 1) + if not base or not ver: + return value, None + return base, ver + + def main() -> None: parser = argparse.ArgumentParser(prog="usegolib") sub = parser.add_subparsers(dest="cmd", required=True) @@ -12,28 +30,99 @@ def main() -> None: sub.add_parser("version", help="Print usegolib version.") p_build = sub.add_parser("build", help="Build a Go module into an artifact directory.") - p_build.add_argument("--module", required=True, help="Go module directory path (v0).") + p_build.add_argument( + "--module", + required=True, + help="Go module directory path OR Go module/package import path (v0).", + ) p_build.add_argument("--out", required=True, help="Output artifact directory.") - p_build.add_argument("--version", default=None, help="Go module version (remote only; default: @latest).") p_build.add_argument("--force", action="store_true", help="Force rebuild even if artifact exists.") + p_build.add_argument( + "--gomodcache", + default=None, + help="Optional Go module cache root (sets GOMODCACHE for the build).", + ) + p_build.add_argument( + "--redownload", + action="store_true", + help="Re-download Go modules before building (implies --force).", + ) + p_build.add_argument( + "--generics", + default=None, + help="Path to generics instantiation config JSON (optional).", + ) p_pkg = sub.add_parser( "package", help="Generate a Python package project embedding the built artifact (v0).", ) - p_pkg.add_argument("--module", required=True, help="Go module directory path (v0).") + p_pkg.add_argument( + "--module", + required=True, + help="Go module directory path OR Go module/package import path (v0).", + ) p_pkg.add_argument("--python-package-name", required=True, help="Python package name to generate.") p_pkg.add_argument("--out", required=True, help="Output directory for the generated project.") - p_pkg.add_argument("--version", default=None, help="Go module version (remote only; default: @latest).") + + p_art = sub.add_parser("artifact", help="Manage the local artifact cache.") + art = p_art.add_subparsers(dest="artifact_cmd", required=True) + + p_art_rm = art.add_parser("rm", help="Delete cached artifacts for a module/package.") + p_art_rm.add_argument( + "--module", + required=True, + help="Same value you pass to usegolib.import_: local module dir OR import path (supports @version).", + ) + p_art_rm.add_argument( + "--artifact-dir", + default=None, + help="Artifact root directory (default: USEGOLIB_ARTIFACT_DIR or OS cache).", + ) + p_art_rm.add_argument( + "--all-versions", + action="store_true", + help="Delete all versions for this module/package on the current platform.", + ) + p_art_rm.add_argument( + "--yes", + action="store_true", + help="Actually perform deletion. Without --yes, prints the matched directories.", + ) + + p_art_rebuild = art.add_parser("rebuild", help="Rebuild artifacts into an artifact root.") + p_art_rebuild.add_argument( + "--module", + required=True, + help="Local module dir OR import path (module or package; supports @version).", + ) + p_art_rebuild.add_argument( + "--artifact-dir", + default=None, + help="Artifact root directory (default: USEGOLIB_ARTIFACT_DIR or OS cache).", + ) + p_art_rebuild.add_argument( + "--redownload", + action="store_true", + help="Re-download Go modules before rebuilding (uses an isolated GOMODCACHE under the artifact root).", + ) + p_art_rebuild.add_argument( + "--clean", + action="store_true", + help="Delete any existing matching artifacts before rebuilding.", + ) p_gen = sub.add_parser( "gen", help="Generate a static Python bindings module from an artifact manifest schema.", ) p_gen.add_argument("--artifact-dir", required=True, help="Artifact root directory containing manifest.json.") - p_gen.add_argument("--package", required=True, help="Go package import path to generate bindings for.") + p_gen.add_argument( + "--package", + required=True, + help="Go package import path to generate bindings for (supports @version).", + ) p_gen.add_argument("--out", required=True, help="Output .py file path.") - p_gen.add_argument("--version", default=None, help="Artifact version to resolve (optional).") args = parser.parse_args() if args.cmd == "version": @@ -47,11 +136,26 @@ def main() -> None: if args.cmd == "build": from .builder.build import build_artifact + module, version = _split_target_and_version(args.module) + gomodcache = Path(args.gomodcache) if args.gomodcache else None + clean_gomodcache = False + force = bool(args.force) + if args.redownload: + force = True + clean_gomodcache = True + if gomodcache is None: + # Use a deterministic isolated cache directory under the output root. + h = hashlib.sha256(f"{module}@{version or '@latest'}".encode("utf-8")).hexdigest() + gomodcache = Path(args.out) / ".usegolib-gomodcache" / h + build_artifact( - module=args.module, + module=module, out_dir=Path(args.out), - version=args.version, - force=bool(args.force), + version=version, + force=force, + generics=Path(args.generics) if args.generics else None, + gomodcache_dir=gomodcache, + clean_gomodcache=clean_gomodcache, ) return @@ -60,7 +164,7 @@ def main() -> None: from .artifact import read_manifest from .packager.generate import generate_project - module_dir = args.module + module_dir, version = _split_target_and_version(args.module) out_dir = Path(args.out) # Build artifacts into a temporary root, then embed them into the project. @@ -68,7 +172,7 @@ def main() -> None: if tmp_root.exists(): raise SystemExit(f"temporary artifact dir already exists: {tmp_root}") try: - manifest_path = build_artifact(module=module_dir, out_dir=tmp_root, version=args.version) + manifest_path = build_artifact(module=module_dir, out_dir=tmp_root, version=version) manifest = read_manifest(manifest_path.parent) generate_project( python_package_name=args.python_package_name, @@ -84,21 +188,101 @@ def main() -> None: shutil.rmtree(tmp_root, ignore_errors=True) return + if args.cmd == "artifact": + from .artifact import delete_artifacts, find_manifest_dirs + from .paths import default_artifact_root + + artifact_root = Path(args.artifact_dir) if args.artifact_dir else default_artifact_root() + artifact_root.mkdir(parents=True, exist_ok=True) + + # Match import_ behavior for local paths: map a module subdir to a subpackage import path. + module, version = _split_target_and_version(args.module) + runtime_pkg = module + module_path_arg = Path(module) + if module_path_arg.exists() and module_path_arg.is_dir(): + from .builder.resolve import resolve_module_target + + resolved = resolve_module_target(target=module, version=version, env=None) + rel = module_path_arg.resolve().relative_to(resolved.module_dir) + if str(rel) == ".": + runtime_pkg = resolved.module_path + else: + runtime_pkg = f"{resolved.module_path}/{'/'.join(rel.parts)}" + + if args.artifact_cmd == "rm": + if version in {"latest", "@latest"}: + raise SystemExit("artifact rm does not support @latest; use --all-versions or a concrete @vX.Y.Z") + if version is None and not args.all_versions: + raise SystemExit("artifact rm requires module@version unless --all-versions is set") + dirs = find_manifest_dirs( + artifact_root, + package=runtime_pkg, + version=version, + all_versions=bool(args.all_versions), + ) + if not dirs: + print("no matching artifacts found") + return + if not args.yes: + print("matched artifact directories (use --yes to delete):") + for d in dirs: + print(str(d)) + raise SystemExit(2) + deleted = delete_artifacts( + artifact_root, + package=runtime_pkg, + version=version, + all_versions=bool(args.all_versions), + ) + for d in deleted: + print(f"deleted: {d}") + return + + if args.artifact_cmd == "rebuild": + if getattr(args, "clean", False): + _ = delete_artifacts( + artifact_root, + package=runtime_pkg, + version=version, + all_versions=(version is None), + ) + + gomodcache = None + clean_gomodcache = False + if getattr(args, "redownload", False): + clean_gomodcache = True + h = hashlib.sha256(f"{module}@{version or '@latest'}".encode("utf-8")).hexdigest() + gomodcache = artifact_root / ".usegolib-gomodcache" / h + + from .builder.build import build_artifact + + manifest_path = build_artifact( + module=module, + out_dir=artifact_root, + version=version, + force=True, + gomodcache_dir=gomodcache, + clean_gomodcache=clean_gomodcache, + ) + print(str(manifest_path)) + return + if args.cmd == "gen": from .artifact import resolve_manifest from .bindgen import BindgenOptions, generate_python_bindings from .schema import Schema artifact_root = Path(args.artifact_dir) - manifest = resolve_manifest(artifact_root, package=args.package, version=args.version) + pkg, version = _split_target_and_version(args.package) + manifest = resolve_manifest(artifact_root, package=pkg, version=version) schema = Schema.from_manifest(manifest.schema) if schema is None: raise SystemExit("manifest schema is missing; rebuild artifact with schema exchange enabled") generate_python_bindings( schema=schema, - pkg=args.package, + pkg=pkg, out_file=Path(args.out), - opts=BindgenOptions(package=args.package), + opts=BindgenOptions(package=pkg), ) return diff --git a/src/usegolib/handle.py b/src/usegolib/handle.py index 0c3c9b7..eed0095 100644 --- a/src/usegolib/handle.py +++ b/src/usegolib/handle.py @@ -21,7 +21,14 @@ VersionConflictError, ) from .runtime.cbridge import SharedLibClient -from .schema import Schema, validate_call_args, validate_call_result +from .schema import ( + Schema, + validate_call_args, + validate_call_result, + validate_method_args, + validate_method_result, + validate_struct_value, +) from .runtime.platform import host_goarch, host_goos @@ -36,6 +43,102 @@ class _Runtime: _LOADED_RUNTIMES: dict[str, _Runtime] = {} +def _loaded_version_for_package(pkg: str) -> str | None: + """Return the already-loaded module version for `pkg` (module or subpackage). + + This allows `import_(..., version=None)` for subpackages to follow the already + loaded module version in the current process, avoiding ambiguity when multiple + artifact versions exist on disk. + """ + best_key = None + for mod in _LOADED_RUNTIMES.keys(): + if pkg == mod or pkg.startswith(mod + "/"): + if best_key is None or len(mod) > len(best_key): + best_key = mod + if best_key is None: + return None + return _LOADED_RUNTIMES[best_key].version + + +def _pack_variadic_args(*, params: list[str], args: list[Any]) -> list[Any]: + """Pack Python varargs for Go variadic parameters. + + Go variadics are represented in the ABI as a single final argument: a list of + elements. This enables `f(1, 2, 3)` in Python to map to `f(...T)` in Go. + """ + if not params: + return args + if not params[-1].strip().startswith("..."): + return args + + fixed = len(params) - 1 + if len(args) < fixed: + # Missing required fixed args; let schema validation raise a good error. + return args + + if len(args) == fixed: + # No variadic args provided. + return [*args, []] + + # If the caller already provided the packed variadic list, don't repack. + if len(args) == len(params) and isinstance(args[-1], (list, tuple)): + return args + + tail = list(args[fixed:]) + return [*args[:fixed], tail] + + +def _format_go_sig(*, pkg: str, name: str, params: list[str], results: list[str], recv: str | None = None) -> str: + if recv is None: + head = f"{pkg}.{name}" + else: + head = f"(*{recv}).{name}" + p = ", ".join(params) + if not results: + r = "" + elif len(results) == 1: + r = results[0] + else: + r = ", ".join(results) + r = f"({r})" + return f"Go: {head}({p}){(' ' + r) if r else ''}" + + +def _attach_doc(*, fn: Callable[..., Any], doc: str | None, sig: str | None) -> None: + """Best-effort attach __doc__ to a dynamic callable.""" + doc = (doc or "").strip() + sig = (sig or "").strip() + if doc and sig: + fn.__doc__ = f"{doc}\n\n{sig}" + return + if doc: + fn.__doc__ = doc + return + if sig: + fn.__doc__ = sig + + +def _opaque_ptr_target(*, schema: Schema, pkg: str, go_type: str) -> str | None: + """If go_type is `*T` for an opaque struct T, return `T` else None. + + Opaque structs are represented in the manifest schema as struct entries with + zero exported fields. For pointers to those structs, the runtime uses object + handles (uint64 ids) instead of record dicts. + """ + t = go_type.strip() + if not t.startswith("*"): + return None + inner = t[1:].strip() + if not inner or inner.startswith(("[]", "map[string]", "...", "*")): + return None + st = schema.structs_by_pkg.get(pkg, {}).get(inner) + if st is None: + return None + if st.fields_by_name: + return None + return inner + + @dataclass class PackageHandle: module: str @@ -44,6 +147,7 @@ class PackageHandle: package: str _client: SharedLibClient _schema: Schema | None = None + _var_cache: dict[str, "GoObject"] = field(default_factory=dict, repr=False) @classmethod def from_manifest(cls, manifest: ArtifactManifest, *, package: str) -> "PackageHandle": @@ -75,10 +179,51 @@ def from_manifest(cls, manifest: ArtifactManifest, *, package: str) -> "PackageH ) def __getattr__(self, name: str) -> Callable[..., Any]: + # Exported package variables can be used as namespace singletons (e.g. isr.DL.Of()). + # When schema declares a var, resolve it to an object handle so methods can be called. + if self._schema is not None: + vt = self._schema.vars_by_pkg.get(self.package, {}).get(name) + if vt is not None: + existing = self._var_cache.get(name) + if existing is not None: + return existing + + fn = f"__usegolib_getvar_{name}" + try: + req = abi.encode_call_request(pkg=self.package, fn=fn, args=[]) + except Exception as e: # noqa: BLE001 - encode boundary + raise ABIEncodeError(str(e)) from e + + resp_bytes = self._client.call(req) + resp = abi.decode_response(resp_bytes) + if resp.ok: + if not isinstance(resp.result, int) or isinstance(resp.result, bool): + raise ABIDecodeError("getvar: expected integer object id") + obj = GoObject(_pkg=self, _type=vt, _id=resp.result) + self._var_cache[name] = obj + return obj + + err = resp.error + if err is None: + raise ABIDecodeError("missing error object in failed response") + if err.type == "GoError": + raise GoError(err.message) + if err.type == "GoPanicError": + raise GoPanicError(err.message) + raise UseGoLibError(f"{err.type}: {err.message}") + # Treat any missing attribute as a Go function call. def _call(*args: Any) -> Any: args_list = list(args) + result_go_type: str | None = None if self._schema is not None: + sig = self._schema.symbols_by_pkg.get(self.package, {}).get(name) + if sig is not None: + params, _results = sig + args_list = _pack_variadic_args(params=params, args=args_list) + if _results: + result_go_type = _results[0] + # Allow passing generated dataclasses; encode them to record-struct dicts first. from .typed import encode_value @@ -101,6 +246,16 @@ def _call(*args: Any) -> Any: fn=name, result=resp.result, ) + if result_go_type is not None: + type_name = _opaque_ptr_target( + schema=self._schema, pkg=self.package, go_type=result_go_type + ) + if type_name is not None: + if not isinstance(resp.result, int) or isinstance(resp.result, bool): + raise ABIDecodeError( + f"expected integer object id for opaque pointer result {result_go_type}" + ) + return GoObject(_pkg=self, _type=type_name, _id=resp.result) return resp.result err = resp.error @@ -118,6 +273,15 @@ def _call(*args: Any) -> Any: raise UseGoLibError(f"{err.type}: {err.message}") + if self._schema is not None: + doc = self._schema.symbol_docs_by_pkg.get(self.package, {}).get(name) + sig = self._schema.symbols_by_pkg.get(self.package, {}).get(name) + sig_txt = None + if sig is not None: + params, results = sig + sig_txt = _format_go_sig(pkg=self.package, name=name, params=params, results=results) + _attach_doc(fn=_call, doc=doc, sig=sig_txt) + return _call @property @@ -127,6 +291,49 @@ def schema(self) -> Schema | None: def typed(self) -> "TypedPackageHandle": return TypedPackageHandle(self) + def generic(self, name: str, type_args: list[str]) -> Callable[..., Any]: + if self._schema is None: + raise UseGoLibError("generic() requires manifest schema") + sym = ( + self._schema.generics_by_pkg.get(self.package, {}) + .get(name, {}) + .get(tuple(type_args)) + ) + if sym is None: + raise UseGoLibError(f"generic instantiation not found: {self.package}.{name}{type_args!r}") + return getattr(self, sym) + + def object(self, type_name: str, init: Any | None = None) -> "GoObject": + if self._schema is not None and init is not None: + from .typed import encode_value + + init = encode_value(schema=self._schema, pkg=self.package, v=init) + validate_struct_value(schema=self._schema, pkg=self.package, struct=type_name, value=init) + try: + req = abi.encode_obj_new_request(pkg=self.package, type_name=type_name, init=init) + except Exception as e: # noqa: BLE001 - encode boundary + raise ABIEncodeError(str(e)) from e + + resp_bytes = self._client.call(req) + resp = abi.decode_response(resp_bytes) + if resp.ok: + if not isinstance(resp.result, int) or isinstance(resp.result, bool): + raise ABIDecodeError("obj_new: expected integer object id") + return GoObject(_pkg=self, _type=type_name, _id=resp.result) + + err = resp.error + if err is None: + raise ABIDecodeError("missing error object in failed response") + if err.type == "GoError": + raise GoError(err.message) + if err.type == "GoPanicError": + raise GoPanicError(err.message) + if err.type == "UnsupportedTypeError": + raise UnsupportedTypeError(err.message) + if err.type == "UnsupportedSignatureError": + raise UnsupportedSignatureError(err.message) + raise UseGoLibError(f"{err.type}: {err.message}") + _SHA256_RE = re.compile(r"^[0-9a-f]{64}$") @@ -202,4 +409,218 @@ def _call(*args: Any) -> Any: return decode_value(types=self._types, go_type=results[0], v=result) + # Preserve docstrings from the base callable (GoDoc/signature). + _call.__doc__ = getattr(fn, "__doc__", None) + return _call + + def object(self, type_name: str, init: Any | None = None) -> "TypedGoObject": + obj = self._base.object(type_name, init=init) + schema = self._base._schema # noqa: SLF001 - internal linkage + assert schema is not None + return TypedGoObject(_base=obj, _types=self._types, _schema=schema, _pkg=self._base.package) + + def generic(self, name: str, type_args: list[str]) -> Callable[..., Any]: + schema = self._base._schema # noqa: SLF001 - internal linkage + assert schema is not None + sym = schema.generics_by_pkg.get(self._base.package, {}).get(name, {}).get(tuple(type_args)) + if sym is None: + raise UseGoLibError(f"generic instantiation not found: {self._base.package}.{name}{type_args!r}") + fn = getattr(self._base, sym) + + def _call(*args: Any) -> Any: + result = fn(*args) + sig = schema.symbols_by_pkg.get(self._base.package, {}).get(sym) + if sig is None: + return result + _params, results = sig + if not results: + return result + from .typed import decode_value + + return decode_value(types=self._types, go_type=results[0], v=result) + + _call.__doc__ = getattr(fn, "__doc__", None) + return _call + + +@dataclass +class GoObject: + _pkg: PackageHandle + _type: str + _id: int + _closed: bool = False + + @property + def id(self) -> int: + return self._id + + @property + def type_name(self) -> str: + return self._type + + def close(self) -> None: + if self._closed: + return + self._closed = True + try: + req = abi.encode_obj_free_request(obj_id=self._id) + except Exception: + return + try: + _ = self._pkg._client.call(req) # noqa: SLF001 - internal linkage + except Exception: + return + + def __enter__(self) -> "GoObject": + return self + + def __exit__(self, exc_type, exc, tb) -> None: # noqa: ANN001 + self.close() + + def __del__(self) -> None: + # Best-effort cleanup; ignore errors at interpreter shutdown. + try: + self.close() + except Exception: + return + + def __getattr__(self, name: str) -> Callable[..., Any]: + def _call(*args: Any) -> Any: + if self._closed: + raise UseGoLibError("object is closed") + args_list = list(args) + schema = self._pkg._schema # noqa: SLF001 - internal linkage + result_go_type: str | None = None + if schema is not None: + sig = ( + schema.methods_by_pkg.get(self._pkg.package, {}) + .get(self._type, {}) + .get(name) + ) + if sig is not None: + params, _results = sig + args_list = _pack_variadic_args(params=params, args=args_list) + if _results: + result_go_type = _results[0] + + from .typed import encode_value + + args_list = [encode_value(schema=schema, pkg=self._pkg.package, v=a) for a in args_list] + validate_method_args( + schema=schema, + pkg=self._pkg.package, + recv=self._type, + method=name, + args=args_list, + ) + try: + req = abi.encode_obj_call_request( + pkg=self._pkg.package, + type_name=self._type, + obj_id=self._id, + method=name, + args=args_list, + ) + except Exception as e: # noqa: BLE001 - encode boundary + raise ABIEncodeError(str(e)) from e + + resp_bytes = self._pkg._client.call(req) # noqa: SLF001 - internal linkage + resp = abi.decode_response(resp_bytes) + if resp.ok: + if schema is not None: + validate_method_result( + schema=schema, + pkg=self._pkg.package, + recv=self._type, + method=name, + result=resp.result, + ) + if result_go_type is not None: + type_name = _opaque_ptr_target( + schema=schema, pkg=self._pkg.package, go_type=result_go_type + ) + if type_name is not None: + if not isinstance(resp.result, int) or isinstance(resp.result, bool): + raise ABIDecodeError( + f"expected integer object id for opaque pointer result {result_go_type}" + ) + return GoObject(_pkg=self._pkg, _type=type_name, _id=resp.result) + return resp.result + + err = resp.error + if err is None: + raise ABIDecodeError("missing error object in failed response") + if err.type == "GoError": + raise GoError(err.message) + if err.type == "GoPanicError": + raise GoPanicError(err.message) + if err.type == "UnsupportedTypeError": + raise UnsupportedTypeError(err.message) + if err.type == "UnsupportedSignatureError": + raise UnsupportedSignatureError(err.message) + raise UseGoLibError(f"{err.type}: {err.message}") + + schema = self._pkg._schema # noqa: SLF001 - internal linkage + if schema is not None: + doc = ( + schema.method_docs_by_pkg.get(self._pkg.package, {}) + .get(self._type, {}) + .get(name) + ) + sig = ( + schema.methods_by_pkg.get(self._pkg.package, {}) + .get(self._type, {}) + .get(name) + ) + sig_txt = None + if sig is not None: + params, results = sig + sig_txt = _format_go_sig( + pkg=self._pkg.package, recv=self._type, name=name, params=params, results=results + ) + _attach_doc(fn=_call, doc=doc, sig=sig_txt) + + return _call + + +@dataclass(frozen=True) +class TypedGoObject: + _base: GoObject + _types: Any + _schema: Schema + _pkg: str + + @property + def id(self) -> int: + return self._base.id + + @property + def type_name(self) -> str: + return self._base.type_name + + def close(self) -> None: + self._base.close() + + def __enter__(self) -> "TypedGoObject": + return self + + def __exit__(self, exc_type, exc, tb) -> None: # noqa: ANN001 + self.close() + + def __getattr__(self, name: str) -> Callable[..., Any]: + fn = getattr(self._base, name) + + def _call(*args: Any) -> Any: + result = fn(*args) + sig = self._schema.methods_by_pkg.get(self._pkg, {}).get(self._base.type_name, {}).get(name) + if sig is None: + return result + _params, results = sig + if not results: + return result + from .typed import decode_value + + return decode_value(types=self._types, go_type=results[0], v=result) + + _call.__doc__ = getattr(fn, "__doc__", None) return _call diff --git a/src/usegolib/importer.py b/src/usegolib/importer.py index 227d6a3..e72ac8b 100644 --- a/src/usegolib/importer.py +++ b/src/usegolib/importer.py @@ -12,31 +12,58 @@ def import_( version: str | None = None, *, artifact_dir: str | Path | None = None, - build_if_missing: bool = False, + build_if_missing: bool | None = None, ): """Import a Go module by loading a prebuilt artifact. - By default, this function requires a prebuilt artifact to exist on disk. - In development workflows, `build_if_missing=True` can be used to build into - `artifact_dir` and then import the newly built artifact. + If `artifact_dir` is omitted, a default artifact root is used. + + `build_if_missing` is tri-state: + - True: build missing artifacts into the selected artifact root. + - False: never build; missing artifacts raise ArtifactNotFoundError. + - None (auto): build only when `artifact_dir` is omitted. """ from .artifact import resolve_manifest from .handle import PackageHandle + from .paths import default_artifact_root + + auto_root = artifact_dir is None + artifact_root = Path(artifact_dir) if artifact_dir is not None else default_artifact_root() + artifact_root.mkdir(parents=True, exist_ok=True) - if artifact_dir is None: - raise ArtifactNotFoundError("artifact_dir is required in v0 runtime mode") + if build_if_missing is None: + build_if_missing = auto_root # Convenience for dev: allow passing a local module directory path. - # When `module` is a directory, resolve its module path and import that. + # When `module` is a directory, resolve its module path and import that, + # including mapping subdirectories to subpackage import paths. runtime_pkg = module build_target = module - if Path(module).is_dir(): + module_path_arg = Path(module) + if module_path_arg.exists() and module_path_arg.is_dir(): from .builder.resolve import resolve_module_target resolved = resolve_module_target(target=module, version=version) - runtime_pkg = resolved.module_path + build_target = str(resolved.module_dir) + # If `module` points to a subdirectory of the module, treat it as a + # subpackage import path under the module. + rel = module_path_arg.resolve().relative_to(resolved.module_dir) + if str(rel) == ".": + runtime_pkg = resolved.module_path + else: + suffix = "/".join(rel.parts) + runtime_pkg = f"{resolved.module_path}/{suffix}" + + # If the module is already loaded in this process, and the caller did not + # specify a version, follow the loaded version. This avoids raising + # AmbiguousArtifactError when multiple artifact versions exist on disk. + if version is None: + from .handle import _loaded_version_for_package + + loaded = _loaded_version_for_package(runtime_pkg) + if loaded is not None: + version = loaded - artifact_root = Path(artifact_dir) try: manifest = resolve_manifest(artifact_root, package=runtime_pkg, version=version) except ArtifactNotFoundError: diff --git a/src/usegolib/paths.py b/src/usegolib/paths.py new file mode 100644 index 0000000..984702a --- /dev/null +++ b/src/usegolib/paths.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import os +from pathlib import Path + + +def default_artifact_root() -> Path: + """Return the default artifact root directory. + + Override with `USEGOLIB_ARTIFACT_DIR`. + """ + override = os.environ.get("USEGOLIB_ARTIFACT_DIR") + if override: + return Path(override) + + if os.name == "nt": + base = os.environ.get("LOCALAPPDATA") or os.path.expanduser("~") + return Path(base) / "usegolib" / "artifacts" + return Path(os.path.expanduser("~/.cache/usegolib/artifacts")) + diff --git a/src/usegolib/schema.py b/src/usegolib/schema.py index dcd86e0..5387e32 100644 --- a/src/usegolib/schema.py +++ b/src/usegolib/schema.py @@ -20,6 +20,8 @@ def _split_prefix(t: str) -> tuple[str, str]: t = t.strip() if t.startswith("*"): return "*", t[1:].strip() + if t.startswith("..."): + return "...", t[3:].strip() if t.startswith("[]"): return "[]", t[2:].strip() if t.startswith("map[string]"): @@ -60,6 +62,15 @@ class Schema: # pkg -> structName -> schema structs_by_pkg: dict[str, dict[str, StructSchema]] symbols_by_pkg: dict[str, dict[str, tuple[list[str], list[str]]]] + symbol_docs_by_pkg: dict[str, dict[str, str]] + # pkg -> recvType -> methodName -> (params, results) + methods_by_pkg: dict[str, dict[str, dict[str, tuple[list[str], list[str]]]]] + method_docs_by_pkg: dict[str, dict[str, dict[str, str]]] + # pkg -> genericName -> typeArgsTuple -> concreteSymbolName + generics_by_pkg: dict[str, dict[str, dict[tuple[str, ...], str]]] + generic_docs_by_pkg: dict[str, dict[str, str]] + vars_by_pkg: dict[str, dict[str, str]] + var_docs_by_pkg: dict[str, dict[str, str]] @classmethod def from_manifest(cls, manifest_schema: dict[str, Any] | None) -> "Schema | None": @@ -77,6 +88,12 @@ def from_manifest(cls, manifest_schema: dict[str, Any] | None) -> "Schema | None continue key_to_name: dict[str, str] = {} fields_by_name: dict[str, tuple[str, bool]] = {} + if not fields: + # Allow empty struct schemas to represent "opaque" structs that + # have no exported fields. This prevents runtime validation from + # failing with "unknown type" for fluent APIs that return `*T`. + pkg_out[name] = StructSchema(key_to_name={}, fields_by_name={}) + continue for f in fields: if not isinstance(f, dict): continue @@ -118,13 +135,12 @@ def from_manifest(cls, manifest_schema: dict[str, Any] | None) -> "Schema | None key_to_name[k] = fn if key_to_name and fields_by_name: - pkg_out[name] = StructSchema( - key_to_name=key_to_name, fields_by_name=fields_by_name - ) + pkg_out[name] = StructSchema(key_to_name=key_to_name, fields_by_name=fields_by_name) if pkg_out: structs[pkg] = pkg_out symbols_by_pkg: dict[str, dict[str, tuple[list[str], list[str]]]] = {} + symbol_docs_by_pkg: dict[str, dict[str, str]] = {} raw_symbols = manifest_schema.get("symbols") if isinstance(raw_symbols, list): for s in raw_symbols: @@ -134,6 +150,7 @@ def from_manifest(cls, manifest_schema: dict[str, Any] | None) -> "Schema | None name = s.get("name") params = s.get("params") results = s.get("results") + doc = s.get("doc") if not isinstance(pkg, str) or not isinstance(name, str): continue if not isinstance(params, list) or not isinstance(results, list): @@ -143,8 +160,98 @@ def from_manifest(cls, manifest_schema: dict[str, Any] | None) -> "Schema | None ): continue symbols_by_pkg.setdefault(pkg, {})[name] = (list(params), list(results)) - - return cls(structs_by_pkg=structs, symbols_by_pkg=symbols_by_pkg) + if isinstance(doc, str) and doc.strip(): + symbol_docs_by_pkg.setdefault(pkg, {})[name] = doc.strip() + + methods_by_pkg: dict[str, dict[str, dict[str, tuple[list[str], list[str]]]]] = {} + method_docs_by_pkg: dict[str, dict[str, dict[str, str]]] = {} + raw_methods = manifest_schema.get("methods") + if isinstance(raw_methods, list): + for m in raw_methods: + if not isinstance(m, dict): + continue + pkg = m.get("pkg") + recv = m.get("recv") + name = m.get("name") + params = m.get("params") + results = m.get("results") + doc = m.get("doc") + if not (isinstance(pkg, str) and isinstance(recv, str) and isinstance(name, str)): + continue + if not isinstance(params, list) or not isinstance(results, list): + continue + if not all(isinstance(x, str) for x in params) or not all( + isinstance(x, str) for x in results + ): + continue + methods_by_pkg.setdefault(pkg, {}).setdefault(recv, {})[name] = ( + list(params), + list(results), + ) + if isinstance(doc, str) and doc.strip(): + method_docs_by_pkg.setdefault(pkg, {}).setdefault(recv, {})[name] = doc.strip() + + generics_by_pkg: dict[str, dict[str, dict[tuple[str, ...], str]]] = {} + generic_docs_by_pkg: dict[str, dict[str, str]] = {} + raw_generics = manifest_schema.get("generics") + if isinstance(raw_generics, list): + for g in raw_generics: + if not isinstance(g, dict): + continue + pkg = g.get("pkg") + name = g.get("name") + type_args = g.get("type_args") + symbol = g.get("symbol") + doc = g.get("doc") + if not (isinstance(pkg, str) and isinstance(name, str) and isinstance(symbol, str)): + continue + if not isinstance(type_args, list) or not all(isinstance(x, str) for x in type_args): + continue + generics_by_pkg.setdefault(pkg, {}).setdefault(name, {})[tuple(type_args)] = symbol + if isinstance(doc, str) and doc.strip(): + generic_docs_by_pkg.setdefault(pkg, {})[name] = doc.strip() + + vars_by_pkg: dict[str, dict[str, str]] = {} + var_docs_by_pkg: dict[str, dict[str, str]] = {} + raw_vars = manifest_schema.get("vars") + if isinstance(raw_vars, list): + for v in raw_vars: + if not isinstance(v, dict): + continue + pkg = v.get("pkg") + name = v.get("name") + typ = v.get("type") + doc = v.get("doc") + if not (isinstance(pkg, str) and isinstance(name, str) and isinstance(typ, str)): + continue + if not pkg or not name or not typ: + continue + base = typ.strip() + if base.startswith("*"): + base = base[1:].strip() + if base: + vars_by_pkg.setdefault(pkg, {})[name] = base + if isinstance(doc, str) and doc.strip(): + var_docs_by_pkg.setdefault(pkg, {})[name] = doc.strip() + + return cls( + structs_by_pkg=structs, + symbols_by_pkg=symbols_by_pkg, + symbol_docs_by_pkg=symbol_docs_by_pkg, + methods_by_pkg=methods_by_pkg, + method_docs_by_pkg=method_docs_by_pkg, + generics_by_pkg=generics_by_pkg, + generic_docs_by_pkg=generic_docs_by_pkg, + vars_by_pkg=vars_by_pkg, + var_docs_by_pkg=var_docs_by_pkg, + ) + + +def validate_struct_value(*, schema: Schema, pkg: str, struct: str, value: Any) -> None: + try: + _validate_value(schema=schema, pkg=pkg, t=struct, v=value) + except UnsupportedTypeError as e: + raise UnsupportedTypeError(f"schema: {struct}: {e}") from None def validate_call_args(*, schema: Schema, pkg: str, fn: str, args: list[Any]) -> None: @@ -178,9 +285,52 @@ def validate_call_result(*, schema: Schema, pkg: str, fn: str, result: Any) -> N raise UnsupportedTypeError(f"schema: result ({t0}): {e}") from None +def validate_method_args( + *, schema: Schema, pkg: str, recv: str, method: str, args: list[Any] +) -> None: + sig = schema.methods_by_pkg.get(pkg, {}).get(recv, {}).get(method) + if sig is None: + return + params, _results = sig + if len(args) != len(params): + raise UnsupportedTypeError(f"schema: wrong arity (expected {len(params)}, got {len(args)})") + for i, (t, v) in enumerate(zip(params, args, strict=True)): + try: + _validate_value(schema=schema, pkg=pkg, t=t, v=v) + except UnsupportedTypeError as e: + raise UnsupportedTypeError(f"schema: arg{i} ({t}): {e}") from None + + +def validate_method_result( + *, schema: Schema, pkg: str, recv: str, method: str, result: Any +) -> None: + sig = schema.methods_by_pkg.get(pkg, {}).get(recv, {}).get(method) + if sig is None: + return + _params, results = sig + if not results: + if result is not None: + raise UnsupportedTypeError("schema: expected nil result") + return + t0 = results[0] + try: + _validate_value(schema=schema, pkg=pkg, t=t0, v=result) + except UnsupportedTypeError as e: + raise UnsupportedTypeError(f"schema: result ({t0}): {e}") from None + + def _validate_value(*, schema: Schema, pkg: str, t: str, v: Any) -> None: t = t.strip() + if t == "any": + return + if t == "error": + # For v0 we represent successful `error` results as nil. Non-nil errors + # are reported via the ABI error envelope instead of as values. + if v is not None: + raise UnsupportedTypeError("expected nil") + return + # Special-case `[]byte`: represented as bytes, not list[int]. if t == "[]byte": if not isinstance(v, (bytes, bytearray)): @@ -205,7 +355,22 @@ def _validate_value(*, schema: Schema, pkg: str, t: str, v: Any) -> None: if t.startswith("*"): if v is None: return - _validate_value(schema=schema, pkg=pkg, t=t[1:].strip(), v=v) + inner = t[1:].strip() + # Opaque pointer handles: if the pointed-to struct exists in schema but has + # no exported fields, allow passing/returning a uint64-ish object id (int). + st = schema.structs_by_pkg.get(pkg, {}).get(inner) + if st is not None and not st.fields_by_name: + if isinstance(v, int) and not isinstance(v, bool): + return + _validate_value(schema=schema, pkg=pkg, t=inner, v=v) + return + + if t.startswith("..."): + if not isinstance(v, (list, tuple)): + raise UnsupportedTypeError("expected list") + inner = t[3:].strip() + for item in v: + _validate_value(schema=schema, pkg=pkg, t=inner, v=item) return if t.startswith("[]"): diff --git a/src/usegolib/typed.py b/src/usegolib/typed.py index bd9f4c0..d875ef1 100644 --- a/src/usegolib/typed.py +++ b/src/usegolib/typed.py @@ -38,7 +38,7 @@ def _py_type_for_go(schema: Schema, pkg: str, go_type: str) -> Any: for op in reversed(ops): if op == "*": ty = Optional[ty] - elif op == "[]": + elif op in {"[]", "..."}: ty = list[ty] # type: ignore[valid-type] elif op == "map[string]": ty = dict[str, ty] # type: ignore[valid-type] @@ -153,8 +153,8 @@ def decode_value(*, types: PackageTypes, go_type: str, v: Any) -> Any: if v is None: return None return decode_value(types=types, go_type=go_type[1:].strip(), v=v) - if ops and ops[0] == "[]": - inner = go_type[2:].strip() + if ops and ops[0] in {"[]", "..."}: + inner = go_type[2:].strip() if ops[0] == "[]" else go_type[3:].strip() if not isinstance(v, list): return v return [decode_value(types=types, go_type=inner, v=item) for item in v] diff --git a/tests/test_artifact_delete.py b/tests/test_artifact_delete.py new file mode 100644 index 0000000..9f27259 --- /dev/null +++ b/tests/test_artifact_delete.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from usegolib.artifact import delete_artifacts, find_manifest_dirs +from usegolib.runtime.platform import host_goarch, host_goos + + +def _write_manifest(dir_path: Path, *, module: str, version: str, packages: list[str]) -> None: + dir_path.mkdir(parents=True, exist_ok=True) + obj = { + "manifest_version": 1, + "abi_version": 0, + "module": module, + "version": version, + "goos": host_goos(), + "goarch": host_goarch(), + "packages": packages, + "symbols": [], + "schema": None, + "library": {"path": "libusegolib.so", "sha256": "0" * 64}, + } + (dir_path / "manifest.json").write_text(json.dumps(obj), encoding="utf-8") + + +def test_delete_artifacts_by_version(tmp_path: Path) -> None: + root = tmp_path / "artifacts" + leaf1 = root / "example.com" / "p@v1.0.0" / f"{host_goos()}-{host_goarch()}" + leaf2 = root / "example.com" / "p@v2.0.0" / f"{host_goos()}-{host_goarch()}" + _write_manifest(leaf1, module="example.com/p", version="v1.0.0", packages=["example.com/p"]) + _write_manifest(leaf2, module="example.com/p", version="v2.0.0", packages=["example.com/p"]) + + dirs = find_manifest_dirs(root, package="example.com/p", version="v1.0.0") + assert leaf1 in dirs + assert leaf2 not in dirs + + deleted = delete_artifacts(root, package="example.com/p", version="v1.0.0") + assert leaf1 in deleted + assert not leaf1.exists() + assert leaf2.exists() + + +def test_delete_artifacts_all_versions(tmp_path: Path) -> None: + root = tmp_path / "artifacts" + leaf1 = root / "example.com" / "p@v1.0.0" / f"{host_goos()}-{host_goarch()}" + leaf2 = root / "example.com" / "p@v2.0.0" / f"{host_goos()}-{host_goarch()}" + _write_manifest(leaf1, module="example.com/p", version="v1.0.0", packages=["example.com/p"]) + _write_manifest(leaf2, module="example.com/p", version="v2.0.0", packages=["example.com/p"]) + + deleted = delete_artifacts(root, package="example.com/p", version=None, all_versions=True) + assert leaf1 in deleted + assert leaf2 in deleted + assert not leaf1.exists() + assert not leaf2.exists() + diff --git a/tests/test_bindgen_docstrings.py b/tests/test_bindgen_docstrings.py new file mode 100644 index 0000000..34ad738 --- /dev/null +++ b/tests/test_bindgen_docstrings.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from pathlib import Path + + +def test_bindgen_emits_docstrings(tmp_path: Path): + from usegolib.bindgen import BindgenOptions, generate_python_bindings + from usegolib.schema import Schema + + manifest_schema = { + "structs": {}, + "symbols": [ + { + "pkg": "example.com/p", + "name": "Add", + "params": ["int64", "int64"], + "results": ["int64"], + "doc": "Add returns a+b.\n\nThis is a test doc.", + } + ], + "methods": [], + "generics": [], + } + schema = Schema.from_manifest(manifest_schema) + assert schema is not None + assert schema.symbol_docs_by_pkg["example.com/p"]["Add"].startswith("Add returns") + + out = tmp_path / "bindings.py" + generate_python_bindings( + schema=schema, + pkg="example.com/p", + out_file=out, + opts=BindgenOptions(package="example.com/p"), + ) + + txt = out.read_text(encoding="utf-8") + assert 'def Add(' in txt + assert '"""' in txt + assert "Add returns a+b." in txt + assert "This is a test doc." in txt + diff --git a/tests/test_builder_missing_go_hint.py b/tests/test_builder_missing_go_hint.py new file mode 100644 index 0000000..7e181ba --- /dev/null +++ b/tests/test_builder_missing_go_hint.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import subprocess + +import pytest + + +def test_builder_run_missing_go_raises_build_error(monkeypatch, tmp_path): + from usegolib.builder import build + from usegolib.errors import BuildError + + def fake_run(*args, **kwargs): # noqa: ANN001 + raise FileNotFoundError("go") + + monkeypatch.setattr(subprocess, "run", fake_run) + + with pytest.raises(BuildError, match=r"Go toolchain not found"): + build._run(["go", "version"], cwd=tmp_path) # noqa: SLF001 + + +def test_resolve_missing_go_raises_build_error(monkeypatch): + from usegolib.builder import resolve + from usegolib.errors import BuildError + + def fake_run(*args, **kwargs): # noqa: ANN001 + raise FileNotFoundError("go") + + monkeypatch.setattr(subprocess, "run", fake_run) + + with pytest.raises(BuildError, match=r"Go toolchain not found"): + resolve._go_mod_download_json("example.com/mod@v1.2.3", env=None) # noqa: SLF001 + + +def test_scan_missing_go_raises_build_error(monkeypatch, tmp_path): + from usegolib.builder.scan import scan_module + from usegolib.errors import BuildError + + def fake_run(*args, **kwargs): # noqa: ANN001 + raise FileNotFoundError("go") + + monkeypatch.setattr(subprocess, "run", fake_run) + + with pytest.raises(BuildError, match=r"Go toolchain not found"): + scan_module(module_dir=tmp_path) diff --git a/tests/test_builder_network_resilience.py b/tests/test_builder_network_resilience.py new file mode 100644 index 0000000..4e4cd1b --- /dev/null +++ b/tests/test_builder_network_resilience.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +def test_go_command_retries_and_falls_back_to_goproxy_direct(monkeypatch: pytest.MonkeyPatch): + # BuildError logs in the wild show transient failures fetching from proxy.golang.org. + # We retry and (when GOPROXY isn't explicitly set) fall back to GOPROXY=direct. + import usegolib.builder.build as b + + calls: list[dict[str, str] | None] = [] + + def fake_run(*args, **kwargs): # noqa: ANN001 + calls.append(kwargs.get("env")) + if len(calls) == 1: + return SimpleNamespace( + returncode=1, + stdout=b"", + stderr=( + b'read "https://proxy.golang.org/x/y/@v/v0.0.0.zip": wsarecv: ' + b"A connection attempt failed because the connected party did not properly respond" + ), + ) + return SimpleNamespace(returncode=0, stdout=b"ok\n", stderr=b"") + + monkeypatch.setattr(b.subprocess, "run", fake_run) + monkeypatch.setattr(b.time, "sleep", lambda _s: None) + + out = b._run(["go", "list", "./..."], cwd=Path("."), env={}) + assert out == "ok\n" + assert len(calls) == 2 + assert calls[1] is not None + assert calls[1].get("GOPROXY") == "direct" + + +def test_go_command_failure_includes_actionable_hint(monkeypatch: pytest.MonkeyPatch): + import usegolib.builder.build as b + from usegolib.errors import BuildError + + def fake_run(*args, **kwargs): # noqa: ANN001 + return SimpleNamespace( + returncode=1, + stdout=b"", + stderr=b'read "https://proxy.golang.org/x/y/@v/v0.0.0.zip": i/o timeout', + ) + + monkeypatch.setattr(b.subprocess, "run", fake_run) + monkeypatch.setattr(b.time, "sleep", lambda _s: None) + + with pytest.raises(BuildError) as ei: + b._run(["go", "list", "./..."], cwd=Path("."), env={}) + + msg = str(ei.value) + assert "GOPROXY=direct" in msg + diff --git a/tests/test_builder_resolve.py b/tests/test_builder_resolve.py index c8874ba..378ec96 100644 --- a/tests/test_builder_resolve.py +++ b/tests/test_builder_resolve.py @@ -14,3 +14,105 @@ def test_resolve_local_module_dir(tmp_path: Path): assert r.version == "local" assert r.module_dir == mod_dir.resolve() + +def test_resolve_local_module_subdir(tmp_path: Path): + from usegolib.builder.resolve import resolve_module_target + + mod_dir = tmp_path / "gomod" + sub_dir = mod_dir / "subpkg" + sub_dir.mkdir(parents=True) + (mod_dir / "go.mod").write_text("module example.com/localmod\n\ngo 1.22\n", encoding="utf-8") + (mod_dir / "local.go").write_text("package localmod\n", encoding="utf-8") + (sub_dir / "sub.go").write_text("package subpkg\n", encoding="utf-8") + + r = resolve_module_target(target=str(sub_dir), version=None) + assert r.module_path == "example.com/localmod" + assert r.version == "local" + assert r.module_dir == mod_dir.resolve() + + +def test_resolve_remote_defaults_to_at_latest(monkeypatch): + from usegolib.builder import resolve as rmod + from usegolib.builder.resolve import resolve_module_target + + seen: list[str] = [] + + def fake_download(arg: str, *, env=None) -> dict: # noqa: ANN001 + seen.append(arg) + return {"Path": "example.com/remote", "Version": "v1.2.3", "Dir": "/tmp/x"} + + monkeypatch.setattr(rmod, "_go_mod_download_json", fake_download) + r = resolve_module_target(target="example.com/remote", version=None) + assert r.module_path == "example.com/remote" + assert r.version == "v1.2.3" + assert seen == ["example.com/remote@latest"] + + +def test_resolve_remote_subpackage_trims_segments(monkeypatch): + from usegolib.builder import resolve as rmod + from usegolib.builder.resolve import resolve_module_target + from usegolib.errors import BuildError + + calls: list[str] = [] + + def fake_download(arg: str, *, env=None) -> dict: # noqa: ANN001 + calls.append(arg) + if arg.startswith("example.com/mod/subpkg@"): + raise BuildError("fail") + return {"Path": "example.com/mod", "Version": "v9.9.9", "Dir": "/tmp/m"} + + monkeypatch.setattr(rmod, "_go_mod_download_json", fake_download) + r = resolve_module_target(target="example.com/mod/subpkg", version=None) + assert r.module_path == "example.com/mod" + assert r.version == "v9.9.9" + assert calls[0].startswith("example.com/mod/subpkg@") + assert calls[1].startswith("example.com/mod@") + + +def test_resolve_remote_does_not_trim_on_transient_network_errors(monkeypatch): + import pytest + + from usegolib.builder import resolve as rmod + from usegolib.builder.resolve import resolve_module_target + from usegolib.errors import BuildError + + calls: list[str] = [] + + def fake_download(arg: str, *, env=None) -> dict: # noqa: ANN001 + calls.append(arg) + raise BuildError('read "https://proxy.golang.org/x/y/@v/v0.0.0.zip": i/o timeout') + + monkeypatch.setattr(rmod, "_go_mod_download_json", fake_download) + + with pytest.raises(BuildError): + resolve_module_target(target="example.com/mod/subpkg", version="v1.2.3") + + # Should not try parent segments (would hide the real error). + assert calls == ["example.com/mod/subpkg@v1.2.3"] + + +def test_resolve_remote_parses_inline_at_version(monkeypatch): + from usegolib.builder import resolve as rmod + from usegolib.builder.resolve import resolve_module_target + + seen: list[str] = [] + + def fake_download(arg: str, *, env=None) -> dict: # noqa: ANN001 + seen.append(arg) + return {"Path": "example.com/remote", "Version": "v1.2.3", "Dir": "/tmp/x"} + + monkeypatch.setattr(rmod, "_go_mod_download_json", fake_download) + r = resolve_module_target(target="example.com/remote@v1.2.3", version=None) + assert r.module_path == "example.com/remote" + assert r.version == "v1.2.3" + assert seen == ["example.com/remote@v1.2.3"] + + +def test_resolve_remote_inline_version_conflict(monkeypatch): + import pytest + + from usegolib.builder.resolve import resolve_module_target + from usegolib.errors import BuildError + + with pytest.raises(BuildError, match=r"conflicting version"): + resolve_module_target(target="example.com/remote@v1.2.3", version="v9.9.9") diff --git a/tests/test_handle_package_vars.py b/tests/test_handle_package_vars.py new file mode 100644 index 0000000..63f0d36 --- /dev/null +++ b/tests/test_handle_package_vars.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import msgpack + + +class _FakeClient: + def __init__(self) -> None: + self.calls: list[bytes] = [] + + def call(self, req: bytes) -> bytes: # noqa: ANN001 + self.calls.append(req) + # Return object id 7. + return msgpack.packb({"ok": True, "result": 7}, use_bin_type=True) + + +def test_package_handle_var_returns_goobject_and_caches(): + from usegolib.handle import PackageHandle + from usegolib.schema import Schema + + schema = Schema.from_manifest( + { + "structs": {"example.com/p": {"dl": []}}, + "methods": [ + {"pkg": "example.com/p", "recv": "dl", "name": "Of", "params": ["...any"], "results": ["*dl"]} + ], + "vars": [{"pkg": "example.com/p", "name": "DL", "type": "dl"}], + } + ) + assert schema is not None + + h = PackageHandle( + module="example.com/p", + version="v1.0.0", + abi_version=0, + package="example.com/p", + _client=_FakeClient(), # type: ignore[arg-type] + _schema=schema, + ) + + dl1 = h.DL # type: ignore[attr-defined] + dl2 = h.DL # type: ignore[attr-defined] + assert dl1 is dl2 + assert dl1.type_name == "dl" + assert dl1.id == 7 + diff --git a/tests/test_import_resolution.py b/tests/test_import_resolution.py index 6f9e0f0..c1be58d 100644 --- a/tests/test_import_resolution.py +++ b/tests/test_import_resolution.py @@ -59,6 +59,36 @@ def test_import_raises_ambiguous_when_multiple_versions(tmp_path: Path): usegolib.import_("example.com/mod", artifact_dir=tmp_path) +def test_import_omitted_version_follows_loaded_module_version(tmp_path: Path): + import usegolib + + # Two artifacts for the same module, same platform, different versions. + _write_manifest( + tmp_path / "a", + module="example.com/mod", + version="v1.0.0", + packages=["example.com/mod", "example.com/mod/subpkg"], + ) + _write_manifest( + tmp_path / "b", + module="example.com/mod", + version="v2.0.0", + packages=["example.com/mod", "example.com/mod/subpkg"], + ) + + # Load a specific version first. + h = usegolib.import_("example.com/mod", version="v2.0.0", artifact_dir=tmp_path) + assert h.version == "v2.0.0" + + # Then omit version: should follow the already-loaded version (no ambiguity). + h2 = usegolib.import_("example.com/mod", artifact_dir=tmp_path) + assert h2.version == "v2.0.0" + + # Subpackages should also follow the loaded module version. + hs = usegolib.import_("example.com/mod/subpkg", artifact_dir=tmp_path) + assert hs.version == "v2.0.0" + + def test_import_selects_requested_version(tmp_path: Path): import usegolib diff --git a/tests/test_integration_build_and_call.py b/tests/test_integration_build_and_call.py index 3615962..ee69a36 100644 --- a/tests/test_integration_build_and_call.py +++ b/tests/test_integration_build_and_call.py @@ -161,6 +161,13 @@ def _write_go_test_module(mod_dir: Path) -> None: " return \"\", errors.New(msg)", "}", "", + "func FailOnly(msg string) error {", + " if msg == \"\" {", + " return nil", + " }", + " return errors.New(msg)", + "}", + "", ] ), encoding="utf-8", @@ -280,6 +287,11 @@ def test_build_and_call(tmp_path: Path): with pytest.raises(usegolib.errors.GoError): h.Fail("boom") + # `error`-only results return nil on success and GoError on failure. + assert h.FailOnly("") is None + with pytest.raises(usegolib.errors.GoError): + h.FailOnly("boom") + # Typed wrapper: decode record-struct results into dataclasses and accept dataclasses as inputs. ht = h.typed() p = ht.MakePerson("bob", 20) diff --git a/tests/test_integration_generics.py b/tests/test_integration_generics.py new file mode 100644 index 0000000..a9992f0 --- /dev/null +++ b/tests/test_integration_generics.py @@ -0,0 +1,108 @@ +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + + +def _write_go_test_module(mod_dir: Path) -> None: + (mod_dir / "go.mod").write_text( + "\n".join( + [ + "module example.com/genericmod", + "", + "go 1.22", + "", + ] + ), + encoding="utf-8", + ) + (mod_dir / "genericmod.go").write_text( + "\n".join( + [ + "package genericmod", + "", + "type Person struct {", + " Name string `json:\"name\"`", + "}", + "", + "func Id[T any](x T) T {", + " return x", + "}", + "", + "func Wrap[T any](x T) map[string]T {", + " return map[string]T{\"x\": x}", + "}", + "", + "func Pair[T any](a, b T) []T {", + " return []T{a, b}", + "}", + "", + ] + ), + encoding="utf-8", + ) + + +@pytest.mark.skipif( + os.environ.get("USEGOLIB_INTEGRATION") != "1", + reason="set USEGOLIB_INTEGRATION=1 to run integration tests", +) +def test_build_and_call_generics(tmp_path: Path): + import usegolib + + mod_dir = tmp_path / "gomod" + mod_dir.mkdir() + _write_go_test_module(mod_dir) + + generics_cfg = tmp_path / "generics.json" + generics_cfg.write_text( + json.dumps( + { + "instantiations": [ + {"pkg": "example.com/genericmod", "name": "Id", "type_args": ["int64"]}, + {"pkg": "example.com/genericmod", "name": "Id", "type_args": ["string"]}, + {"pkg": "example.com/genericmod", "name": "Id", "type_args": ["Person"]}, + {"pkg": "example.com/genericmod", "name": "Wrap", "type_args": ["int64"]}, + {"pkg": "example.com/genericmod", "name": "Pair", "type_args": ["int64"]}, + ] + }, + indent=2, + ), + encoding="utf-8", + ) + + out_dir = tmp_path / "artifact" + subprocess.check_call( + [ + sys.executable, + "-m", + "usegolib", + "build", + "--module", + str(mod_dir), + "--out", + str(out_dir), + "--generics", + str(generics_cfg), + ] + ) + + h = usegolib.import_("example.com/genericmod", artifact_dir=out_dir) + + assert h.Id__int64(1) == 1 + assert h.Id__string("x") == "x" + assert h.Id__Person({"name": "bob"}) == {"name": "bob"} + assert h.Wrap__int64(5) == {"x": 5} + assert h.Pair__int64(1, 2) == [1, 2] + + assert h.generic("Id", ["int64"])(2) == 2 + + th = h.typed() + types = th.types + + p = types.Person(Name="alice") + assert th.generic("Id", ["Person"])(p) == p + diff --git a/tests/test_integration_import_auto_build.py b/tests/test_integration_import_auto_build.py new file mode 100644 index 0000000..8603c1c --- /dev/null +++ b/tests/test_integration_import_auto_build.py @@ -0,0 +1,89 @@ +import os +from pathlib import Path + +import pytest + + +def _write_go_module(mod_dir: Path, *, module_path: str) -> None: + (mod_dir / "go.mod").write_text( + "\n".join( + [ + f"module {module_path}", + "", + "go 1.22", + "", + ] + ), + encoding="utf-8", + ) + + +@pytest.mark.skipif( + os.environ.get("USEGOLIB_INTEGRATION") != "1", + reason="set USEGOLIB_INTEGRATION=1 to run integration tests", +) +def test_import_auto_build_default_artifact_root(tmp_path: Path, monkeypatch): + import usegolib + + mod_dir = tmp_path / "gomod" + mod_dir.mkdir() + _write_go_module(mod_dir, module_path="example.com/testauto") + (mod_dir / "m.go").write_text( + "\n".join( + [ + "package testauto", + "", + "func AddInt(a int64, b int64) int64 {", + " return a + b", + "}", + "", + ] + ), + encoding="utf-8", + ) + + artifact_root = tmp_path / "artifacts" + monkeypatch.setenv("USEGOLIB_ARTIFACT_DIR", str(artifact_root)) + + h = usegolib.import_(str(mod_dir)) + assert h.package == "example.com/testauto" + assert h.AddInt(1, 2) == 3 + + +@pytest.mark.skipif( + os.environ.get("USEGOLIB_INTEGRATION") != "1", + reason="set USEGOLIB_INTEGRATION=1 to run integration tests", +) +def test_import_local_subpackage_dir_maps_package_path(tmp_path: Path, monkeypatch): + import usegolib + + mod_dir = tmp_path / "gomodsub" + sub_dir = mod_dir / "subpkg" + sub_dir.mkdir(parents=True) + + _write_go_module(mod_dir, module_path="example.com/testautosub") + (sub_dir / "s.go").write_text( + "\n".join( + [ + "package subpkg", + "", + "func AddInt(a int64, b int64) int64 {", + " return a + b", + "}", + "", + ] + ), + encoding="utf-8", + ) + + artifact_root = tmp_path / "artifacts" + monkeypatch.setenv("USEGOLIB_ARTIFACT_DIR", str(artifact_root)) + + h = usegolib.import_(str(sub_dir)) + assert h.package == "example.com/testautosub/subpkg" + assert h.AddInt(10, 20) == 30 + + # After building, importing by Go package path should work against the artifact root. + h2 = usegolib.import_("example.com/testautosub/subpkg", artifact_dir=artifact_root) + assert h2.AddInt(5, 6) == 11 + diff --git a/tests/test_integration_object_handles.py b/tests/test_integration_object_handles.py new file mode 100644 index 0000000..294e343 --- /dev/null +++ b/tests/test_integration_object_handles.py @@ -0,0 +1,144 @@ +import os +import subprocess +import sys +from pathlib import Path + +import pytest + + +def _write_go_test_module(mod_dir: Path) -> None: + (mod_dir / "go.mod").write_text( + "\n".join( + [ + "module example.com/objmod", + "", + "go 1.22", + "", + ] + ), + encoding="utf-8", + ) + (mod_dir / "objmod.go").write_text( + "\n".join( + [ + "package objmod", + "", + 'import (', + ' "errors"', + ')', + "", + "type AddReq struct {", + " Delta int64 `json:\"delta\"`", + "}", + "", + "type Snapshot struct {", + " N int64 `json:\"n\"`", + "}", + "", + "type Counter struct {", + " N int64 `json:\"n\"`", + "}", + "", + "func (c *Counter) Inc(delta int64) int64 {", + " c.N += delta", + " return c.N", + "}", + "", + "func (c *Counter) AddReq(req AddReq) int64 {", + " c.N += req.Delta", + " return c.N", + "}", + "", + "func (c *Counter) Snapshot() Snapshot {", + " return Snapshot{N: c.N}", + "}", + "", + "func (c *Counter) AddAndGet(delta int64) (int64, error) {", + " if delta < 0 {", + " return 0, errors.New(\"negative\")", + " }", + " c.N += delta", + " return c.N, nil", + "}", + "", + "// Opaque struct: no exported fields. Returned pointers should become object handles.", + "type Opaque struct {", + " n int64", + "}", + "", + "func NewOpaque() *Opaque {", + " return &Opaque{n: 1}", + "}", + "", + "func (o *Opaque) Inc(delta int64) int64 {", + " o.n += delta", + " return o.n", + "}", + "", + "func (o *Opaque) Self() *Opaque {", + " return o", + "}", + "", + ] + ), + encoding="utf-8", + ) + + +@pytest.mark.skipif( + os.environ.get("USEGOLIB_INTEGRATION") != "1", + reason="set USEGOLIB_INTEGRATION=1 to run integration tests", +) +def test_object_handles(tmp_path: Path): + import usegolib + from usegolib.errors import UseGoLibError + + mod_dir = tmp_path / "gomod" + mod_dir.mkdir() + _write_go_test_module(mod_dir) + + out_dir = tmp_path / "artifact" + subprocess.check_call( + [ + sys.executable, + "-m", + "usegolib", + "build", + "--module", + str(mod_dir), + "--out", + str(out_dir), + ] + ) + + h = usegolib.import_("example.com/objmod", artifact_dir=out_dir) + + with h.object("Counter", {"n": 1}) as c: + assert c.Inc(2) == 3 + assert c.AddReq({"delta": 5}) == 8 + assert c.Snapshot() == {"n": 8} + assert c.AddAndGet(1) == 9 + with pytest.raises(usegolib.errors.GoError): + c.AddAndGet(-1) + + with pytest.raises(UseGoLibError) as ei: + c.Inc(1) # type: ignore[misc] # closed object + assert "closed" in str(ei.value) + + th = h.typed() + types = th.types + with th.object("Counter", types.Counter(N=10)) as tc: + assert tc.Inc(1) == 11 + assert tc.AddReq(types.AddReq(Delta=2)) == 13 + snap = tc.Snapshot() + assert snap == types.Snapshot(N=13) + + # Opaque pointer returns should be callable objects, not empty dicts. + o = h.NewOpaque() + assert o.type_name == "Opaque" + assert o.Inc(2) == 3 + o2 = o.Self() + assert o2.type_name == "Opaque" + assert o2.Inc(1) == 4 + o.close() + o2.close() diff --git a/tests/test_integration_package_vars.py b/tests/test_integration_package_vars.py new file mode 100644 index 0000000..e282eec --- /dev/null +++ b/tests/test_integration_package_vars.py @@ -0,0 +1,102 @@ +import os +import subprocess +import sys +from pathlib import Path + +import pytest + + +def _write_go_test_module(mod_dir: Path) -> None: + (mod_dir / "go.mod").write_text( + "\n".join( + [ + "module example.com/nsmod", + "", + "go 1.22", + "", + ] + ), + encoding="utf-8", + ) + pkg = mod_dir / "isr" + pkg.mkdir() + (pkg / "isr.go").write_text( + "\n".join( + [ + "package isr", + "", + "// Use `DL.Of` to create a new list.", + "var DL = dl{}", + "", + "type dl struct {", + " items []any", + "}", + "", + "func (d dl) Of(items ...any) *dl {", + " x := dl{items: items}", + " return &x", + "}", + "", + "func (l *dl) Push(items ...any) *dl {", + " l.items = append(l.items, items...)", + " return l", + "}", + "", + "func (l *dl) Len() int64 {", + " return int64(len(l.items))", + "}", + "", + "func (l *dl) At(i int64) any {", + " return l.items[i]", + "}", + "", + ] + ), + encoding="utf-8", + ) + + +@pytest.mark.skipif( + os.environ.get("USEGOLIB_INTEGRATION") != "1", + reason="set USEGOLIB_INTEGRATION=1 to run integration tests", +) +def test_exported_package_var_namespace_methods(tmp_path: Path): + import usegolib + + mod_dir = tmp_path / "gomod" + mod_dir.mkdir() + _write_go_test_module(mod_dir) + + out_dir = tmp_path / "artifact" + subprocess.check_call( + [ + sys.executable, + "-m", + "usegolib", + "build", + "--module", + str(mod_dir), + "--out", + str(out_dir), + ] + ) + + h = usegolib.import_("example.com/nsmod/isr", artifact_dir=out_dir) + + # Exported var resolves to an object handle. + ns = h.DL + assert ns.type_name == "dl" + + dl = h.DL.Of(1, 2, 3) + assert dl.type_name == "dl" + assert dl.Len() == 3 + assert dl.At(0) == 1 + + dl2 = dl.Push(4, 5) + assert dl2.type_name == "dl" + assert dl2.Len() == 5 + + ns.close() + dl.close() + dl2.close() + diff --git a/tests/test_runtime_docstrings.py b/tests/test_runtime_docstrings.py new file mode 100644 index 0000000..0549253 --- /dev/null +++ b/tests/test_runtime_docstrings.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import pytest + +from usegolib.handle import GoObject, PackageHandle, TypedGoObject, TypedPackageHandle +from usegolib.schema import Schema + + +class _DummyClient: + def call(self, req: bytes) -> bytes: # noqa: ARG002 + raise RuntimeError("not used") + + +def test_runtime_attaches_godoc_and_signature_to_function_callables(): + schema = Schema.from_manifest( + { + "structs": {"example.com/p": {}}, + "symbols": [ + { + "pkg": "example.com/p", + "name": "Add", + "params": ["int64", "int64"], + "results": ["int64"], + "doc": "Add returns a+b.", + }, + {"pkg": "example.com/p", "name": "NoDoc", "params": [], "results": ["int64"]}, + ], + "methods": [], + "generics": [], + } + ) + assert schema is not None + h = PackageHandle( + module="example.com/p", + version="v0.0.0", + abi_version=0, + package="example.com/p", + _client=_DummyClient(), # type: ignore[arg-type] + _schema=schema, + ) + + f = h.Add + assert f.__doc__ is not None + assert "Add returns a+b." in f.__doc__ + assert "Go:" in f.__doc__ + + g = h.NoDoc + assert g.__doc__ is not None + assert g.__doc__.startswith("Go:") + + +def test_runtime_attaches_godoc_and_signature_to_method_callables(): + schema = Schema.from_manifest( + { + "structs": {"example.com/p": {"T": []}}, + "symbols": [], + "methods": [ + { + "pkg": "example.com/p", + "recv": "T", + "name": "M", + "params": ["int64"], + "results": [], + "doc": "M does a thing.", + }, + {"pkg": "example.com/p", "recv": "T", "name": "NoDoc", "params": [], "results": ["int64"]}, + ], + "generics": [], + } + ) + assert schema is not None + h = PackageHandle( + module="example.com/p", + version="v0.0.0", + abi_version=0, + package="example.com/p", + _client=_DummyClient(), # type: ignore[arg-type] + _schema=schema, + ) + + obj = GoObject(_pkg=h, _type="T", _id=1) + m = obj.M + assert m.__doc__ is not None + assert "M does a thing." in m.__doc__ + assert "Go:" in m.__doc__ + + nd = obj.NoDoc + assert nd.__doc__ is not None + assert nd.__doc__.startswith("Go:") + + +def test_typed_wrappers_preserve_docstrings(): + schema = Schema.from_manifest( + { + "structs": {"example.com/p": {"T": []}}, + "symbols": [{"pkg": "example.com/p", "name": "NoDoc", "params": [], "results": ["int64"]}], + "methods": [{"pkg": "example.com/p", "recv": "T", "name": "NoDoc", "params": [], "results": ["int64"]}], + "generics": [], + } + ) + assert schema is not None + h = PackageHandle( + module="example.com/p", + version="v0.0.0", + abi_version=0, + package="example.com/p", + _client=_DummyClient(), # type: ignore[arg-type] + _schema=schema, + ) + + th = TypedPackageHandle(h) + assert isinstance(th.NoDoc.__doc__, str) + assert th.NoDoc.__doc__.startswith("Go:") + + obj = GoObject(_pkg=h, _type="T", _id=1) + tobj = TypedGoObject(_base=obj, _types=th.types, _schema=schema, _pkg="example.com/p") + assert isinstance(tobj.NoDoc.__doc__, str) + assert tobj.NoDoc.__doc__.startswith("Go:") + diff --git a/tests/test_schema_generics_mapping.py b/tests/test_schema_generics_mapping.py new file mode 100644 index 0000000..f13e657 --- /dev/null +++ b/tests/test_schema_generics_mapping.py @@ -0,0 +1,30 @@ +from __future__ import annotations + + +def test_schema_parses_generics_mapping(): + from usegolib.schema import Schema + + manifest_schema = { + "structs": {}, + "symbols": [ + { + "pkg": "example.com/mod", + "name": "Id__int64", + "params": ["int64"], + "results": ["int64"], + } + ], + "generics": [ + { + "pkg": "example.com/mod", + "name": "Id", + "type_args": ["int64"], + "symbol": "Id__int64", + } + ], + } + + schema = Schema.from_manifest(manifest_schema) + assert schema is not None + assert schema.generics_by_pkg["example.com/mod"]["Id"][("int64",)] == "Id__int64" + diff --git a/tests/test_schema_methods_validation.py b/tests/test_schema_methods_validation.py new file mode 100644 index 0000000..2e06446 --- /dev/null +++ b/tests/test_schema_methods_validation.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import pytest + + +def test_schema_methods_roundtrip_and_validation(): + from usegolib.schema import ( + Schema, + UnsupportedTypeError, + validate_method_args, + validate_method_result, + validate_struct_value, + ) + + manifest_schema = { + "structs": { + "example.com/mod": { + "Counter": [ + { + "name": "N", + "type": "int64", + "key": "n", + "aliases": ["N", "n"], + "omitempty": False, + "embedded": False, + "required": True, + } + ] + } + }, + "symbols": [], + "methods": [ + { + "pkg": "example.com/mod", + "recv": "Counter", + "name": "Inc", + "params": ["int64"], + "results": ["int64"], + } + ], + } + + schema = Schema.from_manifest(manifest_schema) + assert schema is not None + + validate_struct_value(schema=schema, pkg="example.com/mod", struct="Counter", value={"n": 1}) + + validate_method_args(schema=schema, pkg="example.com/mod", recv="Counter", method="Inc", args=[1]) + validate_method_result(schema=schema, pkg="example.com/mod", recv="Counter", method="Inc", result=2) + + with pytest.raises(UnsupportedTypeError): + validate_method_args(schema=schema, pkg="example.com/mod", recv="Counter", method="Inc", args=["x"]) + diff --git a/tests/test_schema_validation_unit.py b/tests/test_schema_validation_unit.py index 1364915..12c49f7 100644 --- a/tests/test_schema_validation_unit.py +++ b/tests/test_schema_validation_unit.py @@ -3,7 +3,12 @@ import pytest from usegolib.errors import UnsupportedTypeError -from usegolib.schema import Schema, validate_call_args +from usegolib.schema import ( + Schema, + validate_call_args, + validate_call_result, + validate_method_result, +) def test_schema_validation_rejects_missing_required_fields(): @@ -32,3 +37,26 @@ def test_schema_validation_rejects_missing_required_fields(): # Omitempty fields are optional. validate_call_args(schema=schema, pkg="example.com/p", fn="IsAdult", args=[{"Name": "x", "Age": 18}]) + +def test_schema_allows_opaque_pointer_handles_as_int_ids(): + schema = Schema.from_manifest( + { + "structs": {"example.com/p": {"Opaque": []}}, + "symbols": [{"pkg": "example.com/p", "name": "NewOpaque", "params": [], "results": ["*Opaque"]}], + "methods": [{"pkg": "example.com/p", "recv": "Opaque", "name": "Self", "params": [], "results": ["*Opaque"]}], + } + ) + + validate_call_result(schema=schema, pkg="example.com/p", fn="NewOpaque", result=1) + validate_method_result(schema=schema, pkg="example.com/p", recv="Opaque", method="Self", result=2) + + +def test_schema_error_only_result_is_nil(): + schema = Schema.from_manifest( + { + "structs": {"example.com/p": {}}, + "symbols": [{"pkg": "example.com/p", "name": "Do", "params": [], "results": ["error"]}], + } + ) + + validate_call_result(schema=schema, pkg="example.com/p", fn="Do", result=None) diff --git a/tests/test_schema_vars.py b/tests/test_schema_vars.py new file mode 100644 index 0000000..c9920b6 --- /dev/null +++ b/tests/test_schema_vars.py @@ -0,0 +1,19 @@ +from __future__ import annotations + + +def test_schema_parses_vars_base_type(): + from usegolib.schema import Schema + + s = Schema.from_manifest( + { + "vars": [ + {"pkg": "example.com/p", "name": "DL", "type": "dl", "doc": "doc"}, + {"pkg": "example.com/p", "name": "PDL", "type": "*dl", "doc": ""}, + ] + } + ) + assert s is not None + assert s.vars_by_pkg["example.com/p"]["DL"] == "dl" + assert s.vars_by_pkg["example.com/p"]["PDL"] == "dl" + assert s.var_docs_by_pkg["example.com/p"]["DL"] == "doc" + diff --git a/tests/test_variadic_any_support.py b/tests/test_variadic_any_support.py new file mode 100644 index 0000000..b87ee02 --- /dev/null +++ b/tests/test_variadic_any_support.py @@ -0,0 +1,43 @@ +from __future__ import annotations + + +def test_pack_variadic_args(): + from usegolib.handle import _pack_variadic_args + + assert _pack_variadic_args(params=["...any"], args=[]) == [[]] + assert _pack_variadic_args(params=["...any"], args=[1, 2]) == [[1, 2]] + assert _pack_variadic_args(params=["int64", "...any"], args=[1]) == [1, []] + assert _pack_variadic_args(params=["int64", "...any"], args=[1, 2, 3]) == [1, [2, 3]] + assert _pack_variadic_args(params=["int64", "...any"], args=[1, [2, 3]]) == [1, [2, 3]] + + +def test_schema_accepts_any_and_variadic(): + from usegolib.schema import Schema, validate_call_args + + schema = Schema.from_manifest( + { + "structs": {}, + "symbols": [ + {"pkg": "example.com/p", "name": "VarAny", "params": ["...any"], "results": []}, + {"pkg": "example.com/p", "name": "EchoAny", "params": ["any"], "results": ["any"]}, + {"pkg": "example.com/p", "name": "EchoAnySlice", "params": ["[]any"], "results": ["[]any"]}, + { + "pkg": "example.com/p", + "name": "EchoAnyMap", + "params": ["map[string]any"], + "results": ["map[string]any"], + }, + ], + "methods": [], + "generics": [], + } + ) + assert schema is not None + + validate_call_args(schema=schema, pkg="example.com/p", fn="VarAny", args=[[]]) + validate_call_args(schema=schema, pkg="example.com/p", fn="VarAny", args=[[1, "x", {"k": 1}]]) + + validate_call_args(schema=schema, pkg="example.com/p", fn="EchoAny", args=[{"k": 1}]) + validate_call_args(schema=schema, pkg="example.com/p", fn="EchoAnySlice", args=[[1, {"k": 1}]]) + validate_call_args(schema=schema, pkg="example.com/p", fn="EchoAnyMap", args=[{"k": 1, "x": [1, 2]}]) + diff --git a/tests/test_windows_subprocess_decoding.py b/tests/test_windows_subprocess_decoding.py new file mode 100644 index 0000000..e93e7d2 --- /dev/null +++ b/tests/test_windows_subprocess_decoding.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + + +def test_go_mod_download_json_tolerates_non_utf8_prefix(monkeypatch): + from usegolib.builder import resolve as rmod + + payload = b"\x88\x00" + json.dumps( + {"Path": "example.com/mod", "Version": "v1.2.3", "Dir": "C:/tmp/x"} + ).encode("utf-8") + + def fake_run(*args, **kwargs): # noqa: ANN001 + return subprocess.CompletedProcess(args=args[0], returncode=0, stdout=payload) + + monkeypatch.setattr(subprocess, "run", fake_run) + info = rmod._go_mod_download_json("example.com/mod@v1.2.3", env=None) # noqa: SLF001 + assert info["Path"] == "example.com/mod" + assert info["Version"] == "v1.2.3" + + +def test_scan_module_tolerates_non_utf8_prefix(monkeypatch, tmp_path: Path): + from usegolib.builder.scan import scan_module + + obj = { + "funcs": [], + "methods": [], + "generic_funcs": [], + "struct_types": {}, + "structs": {}, + } + payload = b"\x88\x00" + json.dumps(obj).encode("utf-8") + + def fake_run(*args, **kwargs): # noqa: ANN001 + return subprocess.CompletedProcess(args=args[0], returncode=0, stdout=payload) + + monkeypatch.setattr(subprocess, "run", fake_run) + + # module_dir only needs to exist; scan_module calls subprocess.run which we patched. + mod_dir = tmp_path / "gomod" + mod_dir.mkdir() + + scan = scan_module(module_dir=mod_dir, env=None) + assert scan.funcs == [] + assert scan.methods == [] +