From d76c1f79431c65274f17dee356b07f0a39f84e2a Mon Sep 17 00:00:00 2001 From: guangtao Date: Tue, 21 Apr 2026 14:45:11 -0700 Subject: [PATCH 1/8] Align bootstrap env with WendaoArrow main --- .github/workflows/ci.yml | 7 +- .github/workflows/nightly.yml | 7 +- Project.toml | 8 +- scripts/prepare_wendao_code_parser_env.jl | 109 ++++++++++++++++++++++ scripts/test_wendao_code_parser.sh | 16 ++++ 5 files changed, 140 insertions(+), 7 deletions(-) create mode 100644 scripts/prepare_wendao_code_parser_env.jl create mode 100755 scripts/test_wendao_code_parser.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a0d91c..428f493 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,11 +73,14 @@ jobs: - name: Resolve and build package env: JULIA_PKG_PRECOMPILE_AUTO: "0" + WENDAO_CODE_PARSER_BOOTSTRAP_ENV: ${{ runner.temp }}/wendaocodeparser-env run: | - julia --project=. -e 'using Pkg; Pkg.resolve(); Pkg.instantiate(); Pkg.build()' + julia ./scripts/prepare_wendao_code_parser_env.jl - name: Run package tests env: JULIA_NUM_THREADS: ${{ matrix.nthreads }} JULIA_PKG_PRECOMPILE_AUTO: "0" + WENDAO_CODE_PARSER_BOOTSTRAP_ENV: ${{ runner.temp }}/wendaocodeparser-env run: | - julia --project=. -e 'using Pkg; Pkg.test("WendaoCodeParser"; coverage=false)' + chmod +x ./scripts/test_wendao_code_parser.sh + ./scripts/test_wendao_code_parser.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 155665b..20a9f5d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -65,11 +65,14 @@ jobs: - name: Resolve and build package env: JULIA_PKG_PRECOMPILE_AUTO: "0" + WENDAO_CODE_PARSER_BOOTSTRAP_ENV: ${{ runner.temp }}/wendaocodeparser-env run: | - julia --project=. -e 'using Pkg; Pkg.resolve(); Pkg.instantiate(); Pkg.build()' + julia ./scripts/prepare_wendao_code_parser_env.jl - name: Run package tests env: JULIA_NUM_THREADS: ${{ matrix.nthreads }} JULIA_PKG_PRECOMPILE_AUTO: "0" + WENDAO_CODE_PARSER_BOOTSTRAP_ENV: ${{ runner.temp }}/wendaocodeparser-env run: | - julia --project=. -e 'using Pkg; Pkg.test("WendaoCodeParser"; coverage=false)' + chmod +x ./scripts/test_wendao_code_parser.sh + ./scripts/test_wendao_code_parser.sh diff --git a/Project.toml b/Project.toml index 52ed288..26830d2 100644 --- a/Project.toml +++ b/Project.toml @@ -19,14 +19,15 @@ WendaoArrow = "561c8d8d-4bcf-4807-873b-a6b7d1e55843" [sources] Absyn = {rev = "master", url = "https://github.com/OpenModelica/Absyn.jl"} -Arrow = {rev = "2ec9e65ef648c5da4060f0ef8b8e78e7ed00226d", url = "https://github.com/JuliaCN/arrow-julia.git"} -ArrowTypes = {rev = "2ec9e65ef648c5da4060f0ef8b8e78e7ed00226d", subdir = "src/ArrowTypes", url = "https://github.com/JuliaCN/arrow-julia.git"} +Arrow = {rev = "82562ef144a5d8b7d488710cea609fb7488d4efe", url = "https://github.com/JuliaCN/arrow-julia.git"} +ArrowTypes = {rev = "82562ef144a5d8b7d488710cea609fb7488d4efe", subdir = "src/ArrowTypes", url = "https://github.com/JuliaCN/arrow-julia.git"} +gRPCServer = {rev = "261cd70ac0b76c060cef0507245c400da0b5def9", url = "https://github.com/tao3k/gRPCServer.jl"} ImmutableList = {rev = "master", url = "https://github.com/OpenModelica/ImmutableList.jl"} JuliaSyntax = {rev = "main", url = "https://github.com/JuliaLang/JuliaSyntax.jl"} MetaModelica = {rev = "master", url = "https://github.com/OpenModelica/MetaModelica.jl"} OMParser = {rev = "d59051069e43fb2624aa13fe8935532ca15aecec", url = "https://github.com/tao3k/OMParser.jl"} PureHTTP2 = {rev = "03d8853e9556a4355d7b650853cda62b1906d88e", url = "https://github.com/s-celles/PureHTTP2.jl"} -WendaoArrow = {rev = "ab02bc080e90a0683978051cc4165b7173c2992b", url = "https://github.com/tao3k/WendaoArrow.jl.git"} +WendaoArrow = {rev = "e992839d84dc92ffc4972e10d160ee4ce53ce126", url = "https://github.com/tao3k/WendaoArrow.jl.git"} [compat] Absyn = "1.3" @@ -42,6 +43,7 @@ WendaoArrow = "0.1" julia = "1.12" [extras] +gRPCServer = "608c6337-0d7d-447f-bb69-0f5674ee3959" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] diff --git a/scripts/prepare_wendao_code_parser_env.jl b/scripts/prepare_wendao_code_parser_env.jl new file mode 100644 index 0000000..9862d3f --- /dev/null +++ b/scripts/prepare_wendao_code_parser_env.jl @@ -0,0 +1,109 @@ +import Pkg +using TOML + +const SCRIPT_ROOT = @__DIR__ +const WENDAO_ROOT = normpath(joinpath(SCRIPT_ROOT, "..")) +const PROJECT_TOML = joinpath(WENDAO_ROOT, "Project.toml") +const BOOTSTRAP_ENV = "WENDAO_CODE_PARSER_BOOTSTRAP_ENV" +const LOCAL_ARROW_ENV = "WENDAO_CODE_PARSER_LOCAL_ARROW_PATH" +const LOCAL_WENDAOARROW_ENV = "WENDAO_CODE_PARSER_LOCAL_WENDAO_ARROW_PATH" + +function valid_arrow_checkout(path::AbstractString) + return isfile(joinpath(path, "Project.toml")) && + isfile(joinpath(path, "src", "ArrowTypes", "Project.toml")) +end + +function valid_wendaoarrow_checkout(path::AbstractString) + return isfile(joinpath(path, "Project.toml")) && + isfile(joinpath(path, "src", "WendaoArrow.jl")) +end + +function candidate_arrow_checkouts() + candidates = String[] + + if haskey(ENV, LOCAL_ARROW_ENV) + push!(candidates, abspath(ENV[LOCAL_ARROW_ENV])) + end + + push!(candidates, normpath(joinpath(dirname(WENDAO_ROOT), "arrow-julia"))) + + if haskey(ENV, "PRJ_ROOT") + push!(candidates, normpath(joinpath(ENV["PRJ_ROOT"], ".data", "arrow-julia"))) + end + + return unique(candidates) +end + +function candidate_wendaoarrow_checkouts() + candidates = String[] + + if haskey(ENV, LOCAL_WENDAOARROW_ENV) + push!(candidates, abspath(ENV[LOCAL_WENDAOARROW_ENV])) + end + + push!(candidates, normpath(joinpath(dirname(WENDAO_ROOT), "WendaoArrow.jl"))) + + if haskey(ENV, "PRJ_ROOT") + push!(candidates, normpath(joinpath(ENV["PRJ_ROOT"], ".data", "WendaoArrow.jl"))) + end + + return unique(candidates) +end + +function maybe_local_checkout(candidates::Vector{String}, validator::Function) + for candidate in candidates + validator(candidate) && return candidate + end + return nothing +end + +function remote_source_spec(name::String, entry::Dict{String,Any}) + kwargs = Dict{Symbol,Any}(:name => name) + haskey(entry, "url") && (kwargs[:url] = entry["url"]) + haskey(entry, "rev") && (kwargs[:rev] = entry["rev"]) + haskey(entry, "subdir") && (kwargs[:subdir] = entry["subdir"]) + haskey(entry, "path") && (kwargs[:path] = abspath(joinpath(WENDAO_ROOT, entry["path"]))) + return Pkg.PackageSpec(; kwargs...) +end + +project = TOML.parsefile(PROJECT_TOML) +sources = get(project, "sources", Dict{String,Any}()) + +env_path = get(ENV, BOOTSTRAP_ENV, mktempdir()) +Pkg.activate(env_path) + +arrow_checkout = maybe_local_checkout(candidate_arrow_checkouts(), valid_arrow_checkout) +wendaoarrow_checkout = + maybe_local_checkout(candidate_wendaoarrow_checkouts(), valid_wendaoarrow_checkout) + +add_specs = Pkg.PackageSpec[] +develop_specs = Pkg.PackageSpec[] + +if isnothing(arrow_checkout) + push!(add_specs, remote_source_spec("Arrow", sources["Arrow"])) + push!(add_specs, remote_source_spec("ArrowTypes", sources["ArrowTypes"])) +else + push!(develop_specs, Pkg.PackageSpec(path = arrow_checkout)) + push!(develop_specs, Pkg.PackageSpec(path = joinpath(arrow_checkout, "src", "ArrowTypes"))) +end + +if isnothing(wendaoarrow_checkout) + push!(add_specs, remote_source_spec("WendaoArrow", sources["WendaoArrow"])) +else + push!(develop_specs, Pkg.PackageSpec(path = wendaoarrow_checkout)) +end + +for (name, entry) in sources + entry isa Dict{String,Any} || continue + name in ("Arrow", "ArrowTypes", "WendaoArrow") && continue + push!(add_specs, remote_source_spec(name, entry)) +end + +isempty(add_specs) || Pkg.add(add_specs; preserve = Pkg.PRESERVE_DIRECT) +isempty(develop_specs) || Pkg.develop(develop_specs; preserve = Pkg.PRESERVE_DIRECT) +Pkg.develop([Pkg.PackageSpec(path = WENDAO_ROOT)]; preserve = Pkg.PRESERVE_DIRECT) +Pkg.add([Pkg.PackageSpec(name = "Tables")]; preserve = Pkg.PRESERVE_DIRECT) + +Pkg.resolve() +Pkg.instantiate() +Pkg.build("WendaoCodeParser") diff --git a/scripts/test_wendao_code_parser.sh b/scripts/test_wendao_code_parser.sh new file mode 100755 index 0000000..a000329 --- /dev/null +++ b/scripts/test_wendao_code_parser.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +ENV_PATH="${WENDAO_CODE_PARSER_BOOTSTRAP_ENV:-$(mktemp -d)}" +if [[ -z "${WENDAO_CODE_PARSER_BOOTSTRAP_ENV:-}" ]]; then + trap 'rm -rf "${ENV_PATH}"' EXIT +fi +export WENDAO_CODE_PARSER_BOOTSTRAP_ENV="${ENV_PATH}" + +if [[ ! -f "${ENV_PATH}/Project.toml" ]]; then + "${JULIA:-julia}" "${ROOT}/scripts/prepare_wendao_code_parser_env.jl" +fi + +exec "${JULIA:-julia}" --project="${ENV_PATH}" "${ROOT}/test/runtests.jl" From f84f6b5013d2b44649008f2bb575160ec460da70 Mon Sep 17 00:00:00 2001 From: guangtao Date: Mon, 27 Apr 2026 16:21:26 -0700 Subject: [PATCH 2/8] fix: align compat with parser source pins --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 26830d2..512c8a4 100644 --- a/Project.toml +++ b/Project.toml @@ -33,9 +33,9 @@ WendaoArrow = {rev = "e992839d84dc92ffc4972e10d160ee4ce53ce126", url = "https:// Absyn = "1.3" Arrow = "2.8.1" ArrowTypes = "2.3.0" -ImmutableList = "0.1" +ImmutableList = "0.1, 0.3" JuliaSyntax = "2" -MetaModelica = "0.0.5" +MetaModelica = "0.0.5, 0.1" OMParser = "0.0.3" PureHTTP2 = "0.5.0" Tables = "1" From c8d4aaad6a6b3a290e2e3df083b91308ce5690f6 Mon Sep 17 00:00:00 2001 From: guangtao Date: Fri, 1 May 2026 00:23:18 -0700 Subject: [PATCH 3/8] feat: add parser service entrypoint --- Project.toml | 8 +- README.md | 39 +- config/live/parser_summary.toml | 10 + .../wendaocodeparser_parser_summary.toml | 21 + scripts/run_service.jl | 152 +++++++ src/WendaoCodeParser.jl | 7 + src/service/runtime.jl | 397 ++++++++++++++++++ test/cases/flight_services.jl | 53 +++ 8 files changed, 672 insertions(+), 15 deletions(-) create mode 100644 config/live/parser_summary.toml create mode 100644 contracts/wendaocodeparser_parser_summary.toml create mode 100644 scripts/run_service.jl diff --git a/Project.toml b/Project.toml index 512c8a4..94c1e04 100644 --- a/Project.toml +++ b/Project.toml @@ -15,13 +15,13 @@ MetaModelica = "9d7f2a79-07b5-5542-8b19-c0100dda6b06" OMParser = "11f87224-cae7-4e99-a924-e50d12f62c59" PureHTTP2 = "7d1e1b98-28e7-4969-8df9-5a308937986a" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" WendaoArrow = "561c8d8d-4bcf-4807-873b-a6b7d1e55843" [sources] Absyn = {rev = "master", url = "https://github.com/OpenModelica/Absyn.jl"} -Arrow = {rev = "82562ef144a5d8b7d488710cea609fb7488d4efe", url = "https://github.com/JuliaCN/arrow-julia.git"} -ArrowTypes = {rev = "82562ef144a5d8b7d488710cea609fb7488d4efe", subdir = "src/ArrowTypes", url = "https://github.com/JuliaCN/arrow-julia.git"} -gRPCServer = {rev = "261cd70ac0b76c060cef0507245c400da0b5def9", url = "https://github.com/tao3k/gRPCServer.jl"} +Arrow = {rev = "91944a6d968bbe5acd2f864346269ef0c350bd2a", url = "https://github.com/JuliaCN/arrow-julia.git"} +ArrowTypes = {rev = "91944a6d968bbe5acd2f864346269ef0c350bd2a", subdir = "src/ArrowTypes", url = "https://github.com/JuliaCN/arrow-julia.git"} ImmutableList = {rev = "master", url = "https://github.com/OpenModelica/ImmutableList.jl"} JuliaSyntax = {rev = "main", url = "https://github.com/JuliaLang/JuliaSyntax.jl"} MetaModelica = {rev = "master", url = "https://github.com/OpenModelica/MetaModelica.jl"} @@ -39,11 +39,11 @@ MetaModelica = "0.0.5, 0.1" OMParser = "0.0.3" PureHTTP2 = "0.5.0" Tables = "1" +TOML = "1" WendaoArrow = "0.1" julia = "1.12" [extras] -gRPCServer = "608c6337-0d7d-447f-bb69-0f5674ee3959" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] diff --git a/README.md b/README.md index 2a9034d..6d4e726 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,29 @@ Package boundary: The initial slice keeps the Rust client cutover out of scope and proves the provider contract first. -`WendaoSearch.jl` can also mount these parser routes into its existing live gRPC -service with `--code-parser-route-names`, so the same Arrow Flight process can -serve both graph-search routes and AST-query routes during local loopback tests. +Service runtime: + +1. `scripts/run_service.jl` starts the parser-summary and AST-query Flight + service directly from this package +2. `config/live/parser_summary.toml` is the package-local live-service + descriptor for the default Julia and Modelica parser routes +3. `contracts/wendaocodeparser_parser_summary.toml` is the package-local + route and transport contract consumed by Rust integration tests + +Start the default service: + +```bash +julia --project=. scripts/run_service.jl --config config/live/parser_summary.toml +``` + +Override listener fields without changing the package-owned route contract: + +```bash +julia --project=. scripts/run_service.jl \ + --config config/live/parser_summary.toml \ + --host 127.0.0.1 \ + --port 41081 +``` Package docs now also live under `docs/`: @@ -57,8 +77,7 @@ Native bridge note: `ImmutableList`, and `MetaModelica` from `Main` during parser initialization 2. `WendaoCodeParser.jl` therefore aliases those already-loaded modules into - `Main` before the first Modelica parse, especially for mounted live-child - startup under `WendaoSearch.jl` + `Main` before the first Modelica parse 3. This runtime requirement is separate from the upstream `OMParser.jl` build/bootstrap lane: the upstream PR still matters for `Pkg.build(...)`, release assets, and CI coverage, but it does not by itself close the live @@ -278,12 +297,10 @@ Contract note: distinct AST nodes instead of being collapsed globally 28. package tests are now split under `test/support/` and `test/cases/`, so `test/runtests.jl` stays as a small runner instead of a monolithic file -29. parser-specific Flight round-trip coverage is now isolated in - `test/cases/flight_native_columns.jl`, and mounted shared-service parser - regressions are isolated under `WendaoSearch.jl/test/integration/`, - including `live_code_parser.jl`, `live_dependency_semantics.jl`, - `live_relative_dependencies.jl`, `live_modelica_import_forms.jl`, and - `live_julia_type_headers.jl` +29. parser-specific Flight round-trip coverage is isolated in + `test/cases/flight_native_columns.jl`, while parser-service route parsing, + listener config, and multiplexed live-service behavior are covered in + `test/cases/flight_services.jl` 30. AST match rows now also promote parser-owned stable columns such as `match_target_name`, `match_root_module_name`, `match_top_level`, `match_reexported`, `match_visibility`, `match_type_name`, diff --git a/config/live/parser_summary.toml b/config/live/parser_summary.toml new file mode 100644 index 0000000..4587ccd --- /dev/null +++ b/config/live/parser_summary.toml @@ -0,0 +1,10 @@ +code_parser_route_names = [ + "julia_file_summary", + "julia_root_summary", + "modelica_file_summary", + "modelica_ast_query", +] + +[interface] +host = "127.0.0.1" +port = 41081 diff --git a/contracts/wendaocodeparser_parser_summary.toml b/contracts/wendaocodeparser_parser_summary.toml new file mode 100644 index 0000000..358c0e0 --- /dev/null +++ b/contracts/wendaocodeparser_parser_summary.toml @@ -0,0 +1,21 @@ +contract_version = 1 + +[service] +script = "scripts/run_service.jl" +config = "config/live/parser_summary.toml" +host = "127.0.0.1" +port = 41081 +default_code_parser_route_names = [ + "julia_file_summary", + "julia_root_summary", + "modelica_file_summary", + "modelica_ast_query", +] + +[modelica_transport] +schema_version = "v3" +file_summary_route_name = "modelica_file_summary" +ast_query_route_name = "modelica_ast_query" +file_summary_path = "/wendao/code-parser/modelica/file-summary" +ast_query_path = "/wendao/code-parser/modelica/ast-query" +readiness_route_names = ["modelica_file_summary", "modelica_ast_query"] diff --git a/scripts/run_service.jl b/scripts/run_service.jl new file mode 100644 index 0000000..a53d732 --- /dev/null +++ b/scripts/run_service.jl @@ -0,0 +1,152 @@ +if "@" ∉ Base.LOAD_PATH + pushfirst!(Base.LOAD_PATH, "@") +end + +if "@stdlib" ∉ Base.LOAD_PATH + push!(Base.LOAD_PATH, "@stdlib") +end + +using Logging +using TOML +using WendaoArrow +using WendaoCodeParser + +const SCRIPT_ROOT = @__DIR__ +const WENDAOCODEPARSER_ROOT = normpath(joinpath(SCRIPT_ROOT, "..")) +const DEFAULT_CONFIG_PATH = + joinpath(WENDAOCODEPARSER_ROOT, "config", "live", "parser_summary.toml") + +function _usage() + return """ + usage: julia --project=. scripts/run_service.jl [--config PATH] [--host HOST] [--port PORT] [--code-parser-route-names ROUTES] + + Starts the WendaoCodeParser parser-summary Flight service. + """ +end + +function _help_requested(args::Vector{String}) + return any(argument -> argument == "--help" || argument == "-h", args) +end + +function _has_config_arg(args::Vector{String}) + return any( + argument == "--config" || startswith(argument, "--config=") for argument in args + ) +end + +function _config_arg_path(args::Vector{String}) + index = 1 + while index <= length(args) + argument = args[index] + if startswith(argument, "--config=") + return abspath(split(argument, "=", limit = 2)[2]) + elseif argument == "--config" + index += 1 + index > length(args) && + error("WendaoCodeParser service requires one value after --config") + return abspath(args[index]) + end + index += 1 + end + return nothing +end + +function _has_code_parser_route_arg(args::Vector{String}) + return any( + startswith(argument, "--code-parser-route-name=") || + startswith(argument, "--code-parser-route-names=") || + startswith(argument, "--code-parser-routes=") || + argument == "--code-parser-route-name" || + argument == "--code-parser-route-names" || + argument == "--code-parser-routes" for argument in args + ) +end + +function _effective_config_path(args::Vector{String}) + config_path = _config_arg_path(args) + !isnothing(config_path) && return config_path + if haskey(ENV, "WENDAOCODEPARSER_CONFIG") + return abspath(ENV["WENDAOCODEPARSER_CONFIG"]) + end + return DEFAULT_CONFIG_PATH +end + +function _code_parser_route_args_from_config(config_path::AbstractString) + resolved_path = abspath(String(config_path)) + isfile(resolved_path) || + error("WendaoCodeParser service config does not exist: $(resolved_path)") + config = TOML.parsefile(resolved_path) + route_name = get(config, "code_parser_route_name", nothing) + route_names = get(config, "code_parser_route_names", nothing) + (!isnothing(route_name) && !isnothing(route_names)) && error( + "WendaoCodeParser service config must not set both code_parser_route_name and code_parser_route_names", + ) + if !isnothing(route_name) + return String["--code-parser-route-name", String(route_name)] + end + if !isnothing(route_names) + route_names isa AbstractVector || error( + "WendaoCodeParser service config code_parser_route_names must be an array of strings", + ) + return String[ + "--code-parser-route-names", + join(String[String(value) for value in route_names], ","), + ] + end + return String[] +end + +function service_entry_args(args::Vector{String}) + entry_args = String[] + config_path = _effective_config_path(args) + !_has_config_arg(args) && append!(entry_args, ["--config", config_path]) + if !_has_code_parser_route_arg(args) + route_args = _code_parser_route_args_from_config(config_path) + isempty(route_args) && error( + "WendaoCodeParser service requires code_parser_route_name(s) via --config or explicit CLI args", + ) + append!(entry_args, route_args) + end + append!(entry_args, args) + return entry_args +end + +function main(args::Vector{String}) + if _help_requested(args) + print(_usage()) + return nothing + end + + entry_args = service_entry_args(args) + route_names = WendaoCodeParser.parser_service_route_names(entry_args) + isempty(route_names) && + error("WendaoCodeParser service requires at least one parser route") + listener = WendaoCodeParser.parser_service_listener_config(entry_args) + config = WendaoArrow.config_from_args( + WendaoCodeParser.parser_service_interface_args(entry_args), + ) + + @info( + "WendaoCodeParser service startup", + host = String(config.host), + port = Int(config.port), + route_names = String[String(route_name) for route_name in route_names], + listener_max_active_requests = listener.max_active_requests, + listener_request_capacity = listener.request_capacity, + listener_response_capacity = listener.response_capacity, + schema_version = WendaoCodeParser.WENDAOCODEPARSER_SCHEMA_VERSION, + ) + live_service = WendaoCodeParser.build_parser_live_flight_service(route_names) + WendaoCodeParser.warm_parser_live_flight_service(live_service, route_names) + server = WendaoArrow.flight_server( + live_service; + host = String(config.host), + port = Int(config.port), + WendaoCodeParser.parser_service_flight_server_kwargs(listener)..., + ) + WendaoArrow._wait_for_flight_server(server; block = true) +end + +if abspath(PROGRAM_FILE) == @__FILE__ + main(copy(ARGS)) +end diff --git a/src/WendaoCodeParser.jl b/src/WendaoCodeParser.jl index 55a0b8c..f278b65 100644 --- a/src/WendaoCodeParser.jl +++ b/src/WendaoCodeParser.jl @@ -48,8 +48,15 @@ export parse_julia_root_summary export parse_modelica_file_summary export parser_route_descriptor export parser_route_request_headers +export parser_service_flight_server_kwargs +export parser_service_interface_args +export parser_service_listener_config +export parser_service_route_names +export ParserServiceListenerConfig export supported_parser_route_names +export build_parser_live_flight_service export search_julia_ast export search_modelica_ast +export warm_parser_live_flight_service end diff --git a/src/service/runtime.jl b/src/service/runtime.jl index ca773d8..1567a8d 100644 --- a/src/service/runtime.jl +++ b/src/service/runtime.jl @@ -37,6 +37,403 @@ _optional_request_text(value) = ismissing(value) ? nothing : String(value) _optional_request_int(value) = ismissing(value) ? nothing : Int(value) supported_parser_route_names() = collect(PARSER_ROUTE_NAMES) +const CODE_PARSER_WARMUP_JULIA_SOURCE = """ +module WarmupCodeParser +foo(x)=x +export foo +end +""" + +const CODE_PARSER_WARMUP_MODELICA_SOURCE = """ +model WarmupCodeParser +end WarmupCodeParser; +""" + +const CODE_PARSER_WARMUP_MODELICA_AST_SOURCE = """ +within Modelica; +package WarmupCodeParser + import SI = Modelica.Units.SI; + extends Icons.Package; + + block Controller + parameter Real k = 1; + input Real u; + output Real y; + equation + y = k * u; + end Controller; + + model Plant + SI.Time t(start = 0); + Controller c; + end Plant; + + package Types + type Gain = Real(unit = "1"); + end Types; +end WarmupCodeParser; +""" + +struct ParserServiceListenerConfig + max_active_requests::Int + request_capacity::Int + response_capacity::Int +end + +function ParserServiceListenerConfig(; + max_active_requests::Integer = max(Threads.nthreads() * 8, 32), + request_capacity::Integer = 16, + response_capacity::Integer = 16, +) + max_active_requests > 0 || + error("WendaoCodeParser listener max_active_requests must be greater than zero") + request_capacity > 0 || + error("WendaoCodeParser listener request_capacity must be greater than zero") + response_capacity > 0 || + error("WendaoCodeParser listener response_capacity must be greater than zero") + return ParserServiceListenerConfig( + Int(max_active_requests), + Int(request_capacity), + Int(response_capacity), + ) +end + +function parser_service_route_names(args::Vector{String}) + route_name = nothing + route_names = nothing + index = 1 + + while index <= length(args) + argument = args[index] + if startswith(argument, "--code-parser-route-name=") + route_name = split(argument, "=", limit = 2)[2] + elseif argument == "--code-parser-route-name" + index += 1 + index > length(args) && error( + "WendaoCodeParser service requires one value after --code-parser-route-name", + ) + route_name = args[index] + elseif startswith(argument, "--code-parser-route-names=") || + startswith(argument, "--code-parser-routes=") + route_names = split(argument, "=", limit = 2)[2] + elseif argument == "--code-parser-route-names" || argument == "--code-parser-routes" + index += 1 + index > length(args) && error( + "WendaoCodeParser service requires one value after --code-parser-route-names", + ) + route_names = args[index] + end + index += 1 + end + + (!isnothing(route_name) && !isnothing(route_names)) && error( + "WendaoCodeParser service must not set both code_parser route_name and route_names", + ) + isnothing(route_name) && isnothing(route_names) && return Symbol[] + return _resolved_parser_service_route_names(something(route_name, route_names)) +end + +function parser_service_listener_config(args::Vector{String}) + defaults = ParserServiceListenerConfig() + max_active_requests = defaults.max_active_requests + request_capacity = defaults.request_capacity + response_capacity = defaults.response_capacity + index = 1 + + while index <= length(args) + argument = args[index] + if startswith(argument, "--max-active-requests=") + max_active_requests = Base.parse(Int, split(argument, "=", limit = 2)[2]) + elseif argument == "--max-active-requests" + index += 1 + index > length(args) && error( + "WendaoCodeParser service requires one value after --max-active-requests", + ) + max_active_requests = Base.parse(Int, args[index]) + elseif startswith(argument, "--request-capacity=") + request_capacity = Base.parse(Int, split(argument, "=", limit = 2)[2]) + elseif argument == "--request-capacity" + index += 1 + index > length(args) && error( + "WendaoCodeParser service requires one value after --request-capacity", + ) + request_capacity = Base.parse(Int, args[index]) + elseif startswith(argument, "--response-capacity=") + response_capacity = Base.parse(Int, split(argument, "=", limit = 2)[2]) + elseif argument == "--response-capacity" + index += 1 + index > length(args) && error( + "WendaoCodeParser service requires one value after --response-capacity", + ) + response_capacity = Base.parse(Int, args[index]) + end + index += 1 + end + + return ParserServiceListenerConfig( + max_active_requests = max_active_requests, + request_capacity = request_capacity, + response_capacity = response_capacity, + ) +end + +function parser_service_flight_server_kwargs(listener::ParserServiceListenerConfig) + return ( + max_active_requests = listener.max_active_requests, + request_capacity = listener.request_capacity, + response_capacity = listener.response_capacity, + ) +end + +function parser_service_interface_args(args::Vector{String}) + filtered = String[] + index = 1 + while index <= length(args) + argument = args[index] + if startswith(argument, "--code-parser-route-name=") || + startswith(argument, "--code-parser-route-names=") || + startswith(argument, "--code-parser-routes=") + nothing + elseif argument == "--code-parser-route-name" || + argument == "--code-parser-route-names" || + argument == "--code-parser-routes" || + argument == "--max-active-requests" || + argument == "--request-capacity" || + argument == "--response-capacity" + index += 1 + index > length(args) && + error("WendaoCodeParser service requires one value after $(argument)") + elseif startswith(argument, "--max-active-requests=") || + startswith(argument, "--request-capacity=") || + startswith(argument, "--response-capacity=") + nothing + else + push!(filtered, argument) + end + index += 1 + end + return filtered +end + +function build_parser_live_flight_service(route_names = Symbol[]) + parser_routes = _resolved_parser_service_route_names(route_names) + isempty(parser_routes) && + error("WendaoCodeParser live service requires at least one parser route") + _prewarm_modelica_backend_if_needed(parser_routes) + length(parser_routes) == 1 && return build_parser_flight_service(only(parser_routes)) + + route_entries = Dict{Tuple,NamedTuple}() + for route_name in parser_routes + route_entries[_descriptor_path_key(parser_route_descriptor(route_name))] = ( + processor = build_parser_table_processor(route_name), + expected_schema_version = WENDAOCODEPARSER_SCHEMA_VERSION, + subject = "WendaoCodeParser parser-summary exchange request", + ) + end + + return _build_routed_parser_live_flight_service( + route_entries; + missing_descriptor_message = "WendaoCodeParser live service requires one Flight descriptor", + unsupported_descriptor_prefix = "unsupported WendaoCodeParser descriptor path", + ) +end + +function warm_parser_live_flight_service( + service::WendaoArrow.Arrow.Flight.Service, + route_names = Symbol[], +) + parser_routes = _resolved_parser_service_route_names(route_names) + isempty(parser_routes) && + error("WendaoCodeParser live warmup requires at least one parser route") + for route_name in parser_routes + _warm_parser_flight_service(service, route_name) + end + return nothing +end + +function _build_routed_parser_live_flight_service( + route_entries; + missing_descriptor_message::AbstractString, + unsupported_descriptor_prefix::AbstractString, +) + return WendaoArrow.Arrow.Flight.exchangeservice( + function (incoming_messages, request_descriptor, _) + descriptor_path = + isnothing(request_descriptor) ? nothing : request_descriptor.path + table_like = try + WendaoArrow.Arrow.Flight.table(incoming_messages; convert = true) + catch error + @error "WendaoCodeParser routed Flight service failed to decode request" exception = + (error, catch_backtrace()) descriptor_path = descriptor_path + rethrow() + end + + isnothing(request_descriptor) && error(missing_descriptor_message) + requested_path = _descriptor_path_key(request_descriptor) + route_entry = get(route_entries, requested_path, nothing) + isnothing(route_entry) && + error("$(unsupported_descriptor_prefix): $(join(requested_path, "/"))") + + try + WendaoArrow.require_schema_version( + table_like; + subject = route_entry.subject, + expected = route_entry.expected_schema_version, + ) + return ( + output_table = route_entry.processor(table_like), + schema_version = route_entry.expected_schema_version, + subject = route_entry.subject, + ) + catch error + @error "WendaoCodeParser routed Flight processor failed" exception = + (error, catch_backtrace()) descriptor_path = request_descriptor.path subject = + route_entry.subject + rethrow() + end + end; + writer = function (response, routed_output, request_descriptor, _) + descriptor_path = + isnothing(request_descriptor) ? nothing : request_descriptor.path + try + return WendaoArrow.Arrow.Flight.putflightdata!( + response, + routed_output.output_table; + metadata = WendaoArrow.merge_schema_metadata( + WendaoArrow.schema_metadata(routed_output.output_table); + schema_version = routed_output.schema_version, + ), + ) + catch error + @error "WendaoCodeParser routed Flight service failed to encode response" exception = + (error, catch_backtrace()) descriptor_path = descriptor_path subject = + routed_output.subject + rethrow() + end + end, + ) +end + +function _resolved_parser_service_route_names(route_names) + isnothing(route_names) && return Symbol[] + raw_values = + route_names isa AbstractVector ? collect(route_names) : + split(String(route_names), ',') + isempty(raw_values) && return Symbol[] + + normalized = Symbol[] + for raw_value in raw_values + route_text = lowercase(strip(String(raw_value))) + isempty(route_text) && + error("WendaoCodeParser route_names must not contain blanks") + if route_text in ("all", "multi", "full") + append!(normalized, supported_parser_route_names()) + else + push!(normalized, _normalized_parser_service_route_name(route_text)) + end + end + return unique(normalized) +end + +function _normalized_parser_service_route_name(route_name) + route_text = lowercase(strip(String(route_name))) + isempty(route_text) && error("WendaoCodeParser route_name must not be blank") + if route_text in + ("julia_file_summary", "julia-file-summary", "julia-file", "julia_summary") + return JULIA_FILE_SUMMARY_ROUTE + elseif route_text in ("julia_root_summary", "julia-root-summary", "julia-root") + return JULIA_ROOT_SUMMARY_ROUTE + elseif route_text in ( + "modelica_file_summary", + "modelica-file-summary", + "modelica-file", + "modelica_summary", + ) + return MODELICA_FILE_SUMMARY_ROUTE + elseif route_text in ("julia_ast_query", "julia-ast-query", "julia-query") + return JULIA_AST_QUERY_ROUTE + elseif route_text in ("modelica_ast_query", "modelica-ast-query", "modelica-query") + return MODELICA_AST_QUERY_ROUTE + end + error("unsupported WendaoCodeParser route_name: $(route_name)") +end + +function _warm_parser_flight_service( + service::WendaoArrow.Arrow.Flight.Service, + route_name::Symbol, +) + request = _warm_parser_exchange_request(route_name) + WendaoArrow.flight_exchange_table( + service, + WendaoArrow.Arrow.Flight.ServerCallContext(), + request, + ) + return nothing +end + +function _prewarm_modelica_backend_if_needed(route_names::AbstractVector{Symbol}) + any(_is_modelica_parser_route, route_names) || return nothing + prewarm_modelica_backend!() + return nothing +end + +_is_modelica_parser_route(route_name::Symbol) = + route_name == MODELICA_FILE_SUMMARY_ROUTE || route_name == MODELICA_AST_QUERY_ROUTE + +function _warm_parser_exchange_request(route_name::Symbol) + if route_name == JULIA_FILE_SUMMARY_ROUTE + requests = [ + ParserRequest( + "warmup-julia-file-summary", + "WarmupCodeParser.jl", + CODE_PARSER_WARMUP_JULIA_SOURCE, + ), + ] + elseif route_name == JULIA_ROOT_SUMMARY_ROUTE + requests = [ + ParserRequest( + "warmup-julia-root-summary", + "WarmupCodeParser.jl", + CODE_PARSER_WARMUP_JULIA_SOURCE, + ), + ] + elseif route_name == MODELICA_FILE_SUMMARY_ROUTE + requests = [ + ParserRequest( + "warmup-modelica-file-summary", + "WarmupCodeParser.mo", + CODE_PARSER_WARMUP_MODELICA_SOURCE, + ), + ] + elseif route_name == JULIA_AST_QUERY_ROUTE + requests = [ + ParserRequest( + "warmup-julia-ast-query", + "WarmupCodeParser.jl", + CODE_PARSER_WARMUP_JULIA_SOURCE; + node_kind = "function", + limit = 1, + ), + ] + elseif route_name == MODELICA_AST_QUERY_ROUTE + requests = [ + ParserRequest( + "warmup-modelica-ast-query", + "WarmupCodeParser/package.mo", + CODE_PARSER_WARMUP_MODELICA_AST_SOURCE; + limit = 64, + ), + ] + else + error("unsupported WendaoCodeParser route_name: $(route_name)") + end + return parser_exchange_request(route_name, requests) +end + +function _descriptor_path_key(descriptor) + return Tuple(String(segment) for segment in descriptor.path) +end + function _optional_request_text(columns, column_name::Symbol, index::Int) column_name in propertynames(columns) || return nothing return _optional_request_text(getproperty(columns, column_name)[index]) diff --git a/test/cases/flight_services.jl b/test/cases/flight_services.jl index ee611f0..0c0a307 100644 --- a/test/cases/flight_services.jl +++ b/test/cases/flight_services.jl @@ -88,6 +88,59 @@ @test hasproperty(docstring_query_columns, :match_attribute_value) end +@testset "Parser service route parsing and routed live service" begin + @test parser_service_route_names([ + "--code-parser-route-names", + "julia_file_summary,julia-ast-query", + ]) == [JULIA_FILE_SUMMARY_ROUTE, JULIA_AST_QUERY_ROUTE] + @test parser_service_route_names(["--code-parser-routes", "all"]) == + supported_parser_route_names() + + listener = parser_service_listener_config([ + "--max-active-requests", + "4", + "--request-capacity=3", + "--response-capacity", + "2", + ]) + @test listener.max_active_requests == 4 + @test listener.request_capacity == 3 + @test listener.response_capacity == 2 + @test parser_service_interface_args([ + "--config", + "config/live/parser_summary.toml", + "--code-parser-route-names", + "julia_ast_query", + "--max-active-requests", + "4", + "--host", + "127.0.0.1", + "--port=41081", + ]) == [ + "--config", + "config/live/parser_summary.toml", + "--host", + "127.0.0.1", + "--port=41081", + ] + + live_service = + build_parser_live_flight_service([JULIA_FILE_SUMMARY_ROUTE, JULIA_AST_QUERY_ROUTE]) + request = parser_exchange_request( + JULIA_AST_QUERY_ROUTE, + [ParserRequest("live-query", "Demo.jl", JULIA_SOURCE; node_kind = "function")], + ) + table = WendaoCodeParser.WendaoArrow.flight_exchange_table( + live_service, + WendaoCodeParser.WendaoArrow.Arrow.Flight.ServerCallContext(), + request, + ) + columns = Tables.columntable(table) + @test columns.request_id == ["live-query"] + @test columns.success == [true] + @test columns.match_name == ["foo"] +end + @testset "Modelica Flight services round-trip summary response" begin summary_service = build_parser_flight_service(MODELICA_FILE_SUMMARY_ROUTE) summary_request = parser_exchange_request( From 742c4fc56f900262700e4b08e3e86e7c5f333ba4 Mon Sep 17 00:00:00 2001 From: guangtao Date: Fri, 1 May 2026 00:47:34 -0700 Subject: [PATCH 4/8] fix: repair ci bootstrap sources --- Project.toml | 2 +- scripts/prepare_wendao_code_parser_env.jl | 67 +++++++++++++++++++++-- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/Project.toml b/Project.toml index 94c1e04..da7e33b 100644 --- a/Project.toml +++ b/Project.toml @@ -35,7 +35,7 @@ Arrow = "2.8.1" ArrowTypes = "2.3.0" ImmutableList = "0.1, 0.3" JuliaSyntax = "2" -MetaModelica = "0.0.5, 0.1" +MetaModelica = "0.0.5, 0.1, 0.2" OMParser = "0.0.3" PureHTTP2 = "0.5.0" Tables = "1" diff --git a/scripts/prepare_wendao_code_parser_env.jl b/scripts/prepare_wendao_code_parser_env.jl index 9862d3f..f048f35 100644 --- a/scripts/prepare_wendao_code_parser_env.jl +++ b/scripts/prepare_wendao_code_parser_env.jl @@ -7,6 +7,13 @@ const PROJECT_TOML = joinpath(WENDAO_ROOT, "Project.toml") const BOOTSTRAP_ENV = "WENDAO_CODE_PARSER_BOOTSTRAP_ENV" const LOCAL_ARROW_ENV = "WENDAO_CODE_PARSER_LOCAL_ARROW_PATH" const LOCAL_WENDAOARROW_ENV = "WENDAO_CODE_PARSER_LOCAL_WENDAO_ARROW_PATH" +const WENDAOARROW_REQUIRED_SOURCES = ("gRPCServer",) +const WENDAOARROW_SOURCE_FALLBACKS = Dict{String,Dict{String,Any}}( + "gRPCServer" => Dict{String,Any}( + "url" => "https://github.com/tao3k/gRPCServer.jl", + "rev" => "261cd70ac0b76c060cef0507245c400da0b5def9", + ), +) function valid_arrow_checkout(path::AbstractString) return isfile(joinpath(path, "Project.toml")) && @@ -57,15 +64,50 @@ function maybe_local_checkout(candidates::Vector{String}, validator::Function) return nothing end -function remote_source_spec(name::String, entry::Dict{String,Any}) +function remote_source_spec(name::String, entry::Dict{String,Any}; root::AbstractString = WENDAO_ROOT) kwargs = Dict{Symbol,Any}(:name => name) haskey(entry, "url") && (kwargs[:url] = entry["url"]) haskey(entry, "rev") && (kwargs[:rev] = entry["rev"]) haskey(entry, "subdir") && (kwargs[:subdir] = entry["subdir"]) - haskey(entry, "path") && (kwargs[:path] = abspath(joinpath(WENDAO_ROOT, entry["path"]))) + haskey(entry, "path") && (kwargs[:path] = abspath(joinpath(root, entry["path"]))) return Pkg.PackageSpec(; kwargs...) end +function push_remote_source_spec!( + specs::Vector{Pkg.PackageSpec}, + seen_sources::Set{String}, + name::String, + entry, + ; + root::AbstractString = WENDAO_ROOT, +) + entry isa Dict{String,Any} || return + name in seen_sources && return + push!(specs, remote_source_spec(name, entry; root = root)) + push!(seen_sources, name) + return +end + +function wendaoarrow_source_entry( + name::String, + sources::Dict{String,Any}, + wendaoarrow_checkout::Union{Nothing,String}, +) + entry = get(sources, name, nothing) + entry isa Dict{String,Any} && return (; entry, root = WENDAO_ROOT) + + if !isnothing(wendaoarrow_checkout) + wendaoarrow_project = TOML.parsefile(joinpath(wendaoarrow_checkout, "Project.toml")) + wendaoarrow_sources = get(wendaoarrow_project, "sources", Dict{String,Any}()) + entry = get(wendaoarrow_sources, name, nothing) + entry isa Dict{String,Any} && return (; entry, root = wendaoarrow_checkout) + end + + entry = get(WENDAOARROW_SOURCE_FALLBACKS, name, nothing) + entry isa Dict{String,Any} && return (; entry, root = WENDAO_ROOT) + return nothing +end + project = TOML.parsefile(PROJECT_TOML) sources = get(project, "sources", Dict{String,Any}()) @@ -78,25 +120,38 @@ wendaoarrow_checkout = add_specs = Pkg.PackageSpec[] develop_specs = Pkg.PackageSpec[] +seen_sources = Set{String}() if isnothing(arrow_checkout) - push!(add_specs, remote_source_spec("Arrow", sources["Arrow"])) - push!(add_specs, remote_source_spec("ArrowTypes", sources["ArrowTypes"])) + push_remote_source_spec!(add_specs, seen_sources, "Arrow", sources["Arrow"]) + push_remote_source_spec!(add_specs, seen_sources, "ArrowTypes", sources["ArrowTypes"]) else push!(develop_specs, Pkg.PackageSpec(path = arrow_checkout)) push!(develop_specs, Pkg.PackageSpec(path = joinpath(arrow_checkout, "src", "ArrowTypes"))) end if isnothing(wendaoarrow_checkout) - push!(add_specs, remote_source_spec("WendaoArrow", sources["WendaoArrow"])) + push_remote_source_spec!(add_specs, seen_sources, "WendaoArrow", sources["WendaoArrow"]) else push!(develop_specs, Pkg.PackageSpec(path = wendaoarrow_checkout)) end +for name in WENDAOARROW_REQUIRED_SOURCES + source = wendaoarrow_source_entry(name, sources, wendaoarrow_checkout) + isnothing(source) && continue + push_remote_source_spec!( + add_specs, + seen_sources, + name, + source.entry; + root = source.root, + ) +end + for (name, entry) in sources entry isa Dict{String,Any} || continue name in ("Arrow", "ArrowTypes", "WendaoArrow") && continue - push!(add_specs, remote_source_spec(name, entry)) + push_remote_source_spec!(add_specs, seen_sources, name, entry) end isempty(add_specs) || Pkg.add(add_specs; preserve = Pkg.PRESERVE_DIRECT) From c34847a3e4f2019f393dab67360f7c46923cd63b Mon Sep 17 00:00:00 2001 From: guangtao Date: Fri, 1 May 2026 01:00:11 -0700 Subject: [PATCH 5/8] fix: derive ci source pins from WendaoArrow --- .github/workflows/ci.yml | 4 +- .github/workflows/nightly.yml | 4 +- scripts/prepare_wendao_code_parser_env.jl | 102 ++++++++++++++-------- scripts/test_wendao_code_parser.sh | 6 +- 4 files changed, 75 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 428f493..6c01370 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,14 +73,14 @@ jobs: - name: Resolve and build package env: JULIA_PKG_PRECOMPILE_AUTO: "0" - WENDAO_CODE_PARSER_BOOTSTRAP_ENV: ${{ runner.temp }}/wendaocodeparser-env + WENDAO_CODE_PARSER_TEST_ENV: ${{ runner.temp }}/wendaocodeparser-env run: | julia ./scripts/prepare_wendao_code_parser_env.jl - name: Run package tests env: JULIA_NUM_THREADS: ${{ matrix.nthreads }} JULIA_PKG_PRECOMPILE_AUTO: "0" - WENDAO_CODE_PARSER_BOOTSTRAP_ENV: ${{ runner.temp }}/wendaocodeparser-env + WENDAO_CODE_PARSER_TEST_ENV: ${{ runner.temp }}/wendaocodeparser-env run: | chmod +x ./scripts/test_wendao_code_parser.sh ./scripts/test_wendao_code_parser.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 20a9f5d..5c70848 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -65,14 +65,14 @@ jobs: - name: Resolve and build package env: JULIA_PKG_PRECOMPILE_AUTO: "0" - WENDAO_CODE_PARSER_BOOTSTRAP_ENV: ${{ runner.temp }}/wendaocodeparser-env + WENDAO_CODE_PARSER_TEST_ENV: ${{ runner.temp }}/wendaocodeparser-env run: | julia ./scripts/prepare_wendao_code_parser_env.jl - name: Run package tests env: JULIA_NUM_THREADS: ${{ matrix.nthreads }} JULIA_PKG_PRECOMPILE_AUTO: "0" - WENDAO_CODE_PARSER_BOOTSTRAP_ENV: ${{ runner.temp }}/wendaocodeparser-env + WENDAO_CODE_PARSER_TEST_ENV: ${{ runner.temp }}/wendaocodeparser-env run: | chmod +x ./scripts/test_wendao_code_parser.sh ./scripts/test_wendao_code_parser.sh diff --git a/scripts/prepare_wendao_code_parser_env.jl b/scripts/prepare_wendao_code_parser_env.jl index f048f35..ca16b8c 100644 --- a/scripts/prepare_wendao_code_parser_env.jl +++ b/scripts/prepare_wendao_code_parser_env.jl @@ -1,19 +1,13 @@ +import Downloads import Pkg using TOML const SCRIPT_ROOT = @__DIR__ const WENDAO_ROOT = normpath(joinpath(SCRIPT_ROOT, "..")) const PROJECT_TOML = joinpath(WENDAO_ROOT, "Project.toml") -const BOOTSTRAP_ENV = "WENDAO_CODE_PARSER_BOOTSTRAP_ENV" +const TEST_ENV = "WENDAO_CODE_PARSER_TEST_ENV" const LOCAL_ARROW_ENV = "WENDAO_CODE_PARSER_LOCAL_ARROW_PATH" const LOCAL_WENDAOARROW_ENV = "WENDAO_CODE_PARSER_LOCAL_WENDAO_ARROW_PATH" -const WENDAOARROW_REQUIRED_SOURCES = ("gRPCServer",) -const WENDAOARROW_SOURCE_FALLBACKS = Dict{String,Dict{String,Any}}( - "gRPCServer" => Dict{String,Any}( - "url" => "https://github.com/tao3k/gRPCServer.jl", - "rev" => "261cd70ac0b76c060cef0507245c400da0b5def9", - ), -) function valid_arrow_checkout(path::AbstractString) return isfile(joinpath(path, "Project.toml")) && @@ -88,30 +82,66 @@ function push_remote_source_spec!( return end -function wendaoarrow_source_entry( +function github_project_toml_url(entry::Dict{String,Any}) + url = get(entry, "url", nothing) + rev = get(entry, "rev", nothing) + url isa AbstractString || return nothing + rev isa AbstractString || return nothing + + match_result = match(r"^https://github\.com/([^/]+)/([^/]+?)(?:\.git)?/?$", url) + isnothing(match_result) && return nothing + + owner, repo = match_result.captures + subdir = get(entry, "subdir", "") + project_path = isempty(subdir) ? "Project.toml" : "$(subdir)/Project.toml" + return "https://raw.githubusercontent.com/$(owner)/$(repo)/$(rev)/$(project_path)" +end + +function source_project_from_entry( name::String, - sources::Dict{String,Any}, - wendaoarrow_checkout::Union{Nothing,String}, + entry::Dict{String,Any}; + root::AbstractString = WENDAO_ROOT, ) - entry = get(sources, name, nothing) - entry isa Dict{String,Any} && return (; entry, root = WENDAO_ROOT) - - if !isnothing(wendaoarrow_checkout) - wendaoarrow_project = TOML.parsefile(joinpath(wendaoarrow_checkout, "Project.toml")) - wendaoarrow_sources = get(wendaoarrow_project, "sources", Dict{String,Any}()) - entry = get(wendaoarrow_sources, name, nothing) - entry isa Dict{String,Any} && return (; entry, root = wendaoarrow_checkout) + if haskey(entry, "path") + return TOML.parsefile(joinpath(root, entry["path"], "Project.toml")) end - entry = get(WENDAOARROW_SOURCE_FALLBACKS, name, nothing) - entry isa Dict{String,Any} && return (; entry, root = WENDAO_ROOT) + project_url = github_project_toml_url(entry) + isnothing(project_url) && return nothing + + project_toml = Downloads.download(project_url) + try + return TOML.parsefile(project_toml) + catch err + error("Could not read source Project.toml for $(name) from $(project_url): $(err)") + finally + rm(project_toml; force = true) + end +end + +function push_dependency_source_specs!( + specs::Vector{Pkg.PackageSpec}, + seen_sources::Set{String}, + source_project::Union{Nothing,Dict{String,Any}}, + ; + root::AbstractString = WENDAO_ROOT, +) + isnothing(source_project) && return + + dependencies = get(source_project, "deps", Dict{String,Any}()) + dependency_sources = get(source_project, "sources", Dict{String,Any}()) + + for name in sort(collect(keys(dependencies))) + entry = get(dependency_sources, name, nothing) + push_remote_source_spec!(specs, seen_sources, name, entry; root = root) + end return nothing end project = TOML.parsefile(PROJECT_TOML) sources = get(project, "sources", Dict{String,Any}()) -env_path = get(ENV, BOOTSTRAP_ENV, mktempdir()) +env_path = get(ENV, TEST_ENV, mktempdir()) Pkg.activate(env_path) arrow_checkout = maybe_local_checkout(candidate_arrow_checkouts(), valid_arrow_checkout) @@ -128,25 +158,29 @@ if isnothing(arrow_checkout) else push!(develop_specs, Pkg.PackageSpec(path = arrow_checkout)) push!(develop_specs, Pkg.PackageSpec(path = joinpath(arrow_checkout, "src", "ArrowTypes"))) + push!(seen_sources, "Arrow") + push!(seen_sources, "ArrowTypes") end +wendaoarrow_project = nothing +wendaoarrow_project_root = WENDAO_ROOT if isnothing(wendaoarrow_checkout) - push_remote_source_spec!(add_specs, seen_sources, "WendaoArrow", sources["WendaoArrow"]) + wendaoarrow_source = sources["WendaoArrow"] + push_remote_source_spec!(add_specs, seen_sources, "WendaoArrow", wendaoarrow_source) + wendaoarrow_project = source_project_from_entry("WendaoArrow", wendaoarrow_source) else push!(develop_specs, Pkg.PackageSpec(path = wendaoarrow_checkout)) + push!(seen_sources, "WendaoArrow") + wendaoarrow_project = TOML.parsefile(joinpath(wendaoarrow_checkout, "Project.toml")) + wendaoarrow_project_root = wendaoarrow_checkout end -for name in WENDAOARROW_REQUIRED_SOURCES - source = wendaoarrow_source_entry(name, sources, wendaoarrow_checkout) - isnothing(source) && continue - push_remote_source_spec!( - add_specs, - seen_sources, - name, - source.entry; - root = source.root, - ) -end +push_dependency_source_specs!( + add_specs, + seen_sources, + wendaoarrow_project; + root = wendaoarrow_project_root, +) for (name, entry) in sources entry isa Dict{String,Any} || continue diff --git a/scripts/test_wendao_code_parser.sh b/scripts/test_wendao_code_parser.sh index a000329..c29291d 100755 --- a/scripts/test_wendao_code_parser.sh +++ b/scripts/test_wendao_code_parser.sh @@ -3,11 +3,11 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -ENV_PATH="${WENDAO_CODE_PARSER_BOOTSTRAP_ENV:-$(mktemp -d)}" -if [[ -z "${WENDAO_CODE_PARSER_BOOTSTRAP_ENV:-}" ]]; then +ENV_PATH="${WENDAO_CODE_PARSER_TEST_ENV:-$(mktemp -d)}" +if [[ -z "${WENDAO_CODE_PARSER_TEST_ENV:-}" ]]; then trap 'rm -rf "${ENV_PATH}"' EXIT fi -export WENDAO_CODE_PARSER_BOOTSTRAP_ENV="${ENV_PATH}" +export WENDAO_CODE_PARSER_TEST_ENV="${ENV_PATH}" if [[ ! -f "${ENV_PATH}/Project.toml" ]]; then "${JULIA:-julia}" "${ROOT}/scripts/prepare_wendao_code_parser_env.jl" From ed4644adc6c0936ad619a89d8bc8ac9093fc2f50 Mon Sep 17 00:00:00 2001 From: guangtao Date: Fri, 1 May 2026 01:12:03 -0700 Subject: [PATCH 6/8] docs: clarify ci test environment --- .github/workflows/ci.yml | 2 +- .github/workflows/nightly.yml | 2 +- README.md | 19 +++++++++---------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c01370..78aa5e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,7 @@ jobs: if: runner.os == 'macOS' run: | brew install autoconf automake libtool pkg-config - - name: Bootstrap registries + - name: Prepare registries shell: julia --project=. {0} run: | using Pkg diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5c70848..53019de 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -42,7 +42,7 @@ jobs: if: runner.os == 'macOS' run: | brew install autoconf automake libtool pkg-config - - name: Bootstrap registries + - name: Prepare registries shell: julia --project=. {0} run: | using Pkg diff --git a/README.md b/README.md index 6d4e726..bafdc23 100644 --- a/README.md +++ b/README.md @@ -61,15 +61,15 @@ Current backend status: `lib/parser -> autoconf -> ./configure -> make` 4. The current workspace lock pins `OMParser.jl` to `https://github.com/tao3k/OMParser.jl` at - `cebc0696407385e52496608fcc13e95a556da3b5` until the bootstrap fixes are - consumed upstream + `d59051069e43fb2624aa13fe8935532ca15aecec` until the upstream source-build + fixes are consumed 5. The current workspace lock pins `WendaoArrow.jl` to `https://github.com/tao3k/WendaoArrow.jl.git` at - `3325a646785e022a3286d08f28b19dafb4e7c8dd` + `e992839d84dc92ffc4972e10d160ee4ce53ce126` 6. The package also pins the inherited `Arrow.jl`, `ArrowTypes`, and `PureHTTP2.jl` transport sources directly in `Project.toml`, so clean package resolution and GitHub Actions do not rely on a workflow-local - inherited-source bootstrap + inherited-source shim Native bridge note: @@ -79,7 +79,7 @@ Native bridge note: 2. `WendaoCodeParser.jl` therefore aliases those already-loaded modules into `Main` before the first Modelica parse 3. This runtime requirement is separate from the upstream `OMParser.jl` - build/bootstrap lane: the upstream PR still matters for `Pkg.build(...)`, + source-build lane: the upstream PR still matters for `Pkg.build(...)`, release assets, and CI coverage, but it does not by itself close the live child startup contract @@ -111,7 +111,7 @@ Parser layout note: 10. `src/parsers/julia/collect.jl` owns SyntaxNode traversal and Julia summary or AST state collection 11. `src/parsers/modelica/backend.jl` now only owns the `OMParser.jl` native - bridge and shared-library/runtime bootstrap + bridge and shared-library/runtime initialization 12. `src/parsers/modelica/nodes.jl` owns generic Modelica AST node materialization 13. `src/parsers/modelica/dependencies.jl` owns Modelica `import` / `extends` @@ -323,7 +323,6 @@ GitHub Actions note: 1. package-local CI now runs `Pkg.build()` plus `Pkg.test()` on `ubuntu-latest` and `macos-latest` for Julia `1.12` and `pre` 2. a separate nightly workflow runs weekly on `ubuntu-latest` -3. both workflows bootstrap `General` plus `OpenModelicaRegistry` before - running `Pkg.resolve()`, `Pkg.instantiate()`, `Pkg.build()`, and package - tests, so remote runners resolve the same source-locked transport stack as - local runs +3. both workflows prepare `General` plus `OpenModelicaRegistry` before running + `Pkg.resolve()`, `Pkg.instantiate()`, `Pkg.build()`, and package tests, so + remote runners resolve the same source-locked transport stack as local runs From f93afd2635254b8130da5044fdba817fe8e5c87e Mon Sep 17 00:00:00 2001 From: guangtao Date: Fri, 1 May 2026 09:42:59 -0700 Subject: [PATCH 7/8] ci: simplify package project setup --- .github/workflows/ci.yml | 26 +-- .github/workflows/nightly.yml | 26 +-- Project.toml | 3 + README.md | 12 +- scripts/prepare_wendao_code_parser_env.jl | 198 ---------------------- scripts/run_service.jl | 23 ++- scripts/test_wendao_code_parser.sh | 22 ++- test/cases/mod.jl | 1 + test/cases/service_entrypoint.jl | 21 +++ 9 files changed, 69 insertions(+), 263 deletions(-) delete mode 100644 scripts/prepare_wendao_code_parser_env.jl create mode 100644 test/cases/service_entrypoint.jl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78aa5e0..4ad7d1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,37 +50,21 @@ jobs: if: runner.os == 'macOS' run: | brew install autoconf automake libtool pkg-config - - name: Prepare registries + - name: Ensure General registry shell: julia --project=. {0} run: | using Pkg - - function ensure_registry(name::String; spec=nothing) - registries = Pkg.Registry.reachable_registries() - any(registry -> registry.name == name, registries) && return - if isnothing(spec) - Pkg.Registry.add(name) - else - Pkg.Registry.add(spec) - end - end - - ensure_registry("General") - ensure_registry( - "OpenModelica"; - spec=Pkg.RegistrySpec(url="https://github.com/JKRT/OpenModelicaRegistry.git"), - ) - - name: Resolve and build package + registries = Pkg.Registry.reachable_registries() + any(registry -> registry.name == "General", registries) || Pkg.Registry.add("General") + - name: Instantiate package env: JULIA_PKG_PRECOMPILE_AUTO: "0" - WENDAO_CODE_PARSER_TEST_ENV: ${{ runner.temp }}/wendaocodeparser-env run: | - julia ./scripts/prepare_wendao_code_parser_env.jl + julia --project="$(mktemp -d)" -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.build("WendaoCodeParser")' - name: Run package tests env: JULIA_NUM_THREADS: ${{ matrix.nthreads }} JULIA_PKG_PRECOMPILE_AUTO: "0" - WENDAO_CODE_PARSER_TEST_ENV: ${{ runner.temp }}/wendaocodeparser-env run: | chmod +x ./scripts/test_wendao_code_parser.sh ./scripts/test_wendao_code_parser.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 53019de..2b2c38f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -42,37 +42,21 @@ jobs: if: runner.os == 'macOS' run: | brew install autoconf automake libtool pkg-config - - name: Prepare registries + - name: Ensure General registry shell: julia --project=. {0} run: | using Pkg - - function ensure_registry(name::String; spec=nothing) - registries = Pkg.Registry.reachable_registries() - any(registry -> registry.name == name, registries) && return - if isnothing(spec) - Pkg.Registry.add(name) - else - Pkg.Registry.add(spec) - end - end - - ensure_registry("General") - ensure_registry( - "OpenModelica"; - spec=Pkg.RegistrySpec(url="https://github.com/JKRT/OpenModelicaRegistry.git"), - ) - - name: Resolve and build package + registries = Pkg.Registry.reachable_registries() + any(registry -> registry.name == "General", registries) || Pkg.Registry.add("General") + - name: Instantiate package env: JULIA_PKG_PRECOMPILE_AUTO: "0" - WENDAO_CODE_PARSER_TEST_ENV: ${{ runner.temp }}/wendaocodeparser-env run: | - julia ./scripts/prepare_wendao_code_parser_env.jl + julia --project="$(mktemp -d)" -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.build("WendaoCodeParser")' - name: Run package tests env: JULIA_NUM_THREADS: ${{ matrix.nthreads }} JULIA_PKG_PRECOMPILE_AUTO: "0" - WENDAO_CODE_PARSER_TEST_ENV: ${{ runner.temp }}/wendaocodeparser-env run: | chmod +x ./scripts/test_wendao_code_parser.sh ./scripts/test_wendao_code_parser.sh diff --git a/Project.toml b/Project.toml index e5e3247..a711214 100644 --- a/Project.toml +++ b/Project.toml @@ -7,6 +7,7 @@ authors = ["CyberXiuXian Workshop"] Absyn = "ce2f92e2-a952-11e9-0543-8b443f216f1d" Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45" ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" +gRPCServer = "608c6337-0d7d-447f-bb69-0f5674ee3959" ImmutableList = "4a558cac-c1ed-11e9-20da-3584bcd8709a" JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -22,6 +23,7 @@ WendaoArrow = "561c8d8d-4bcf-4807-873b-a6b7d1e55843" Absyn = {rev = "master", url = "https://github.com/OpenModelica/Absyn.jl"} Arrow = {rev = "82562ef144a5d8b7d488710cea609fb7488d4efe", url = "https://github.com/JuliaCN/arrow-julia.git"} ArrowTypes = {rev = "82562ef144a5d8b7d488710cea609fb7488d4efe", subdir = "src/ArrowTypes", url = "https://github.com/JuliaCN/arrow-julia.git"} +gRPCServer = {rev = "261cd70ac0b76c060cef0507245c400da0b5def9", url = "https://github.com/tao3k/gRPCServer.jl"} ImmutableList = {rev = "master", url = "https://github.com/OpenModelica/ImmutableList.jl"} JuliaSyntax = {rev = "main", url = "https://github.com/JuliaLang/JuliaSyntax.jl"} MetaModelica = {rev = "master", url = "https://github.com/OpenModelica/MetaModelica.jl"} @@ -33,6 +35,7 @@ WendaoArrow = {rev = "e992839d84dc92ffc4972e10d160ee4ce53ce126", url = "https:// Absyn = "1.3" Arrow = "2.8.1" ArrowTypes = "2.3.0" +gRPCServer = "0.1" ImmutableList = "0.1, 0.3" JuliaSyntax = "2" MetaModelica = "0.0.5, 0.1, 0.2" diff --git a/README.md b/README.md index bafdc23..6a67c20 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,9 @@ Current backend status: 5. The current workspace lock pins `WendaoArrow.jl` to `https://github.com/tao3k/WendaoArrow.jl.git` at `e992839d84dc92ffc4972e10d160ee4ce53ce126` -6. The package also pins the inherited `Arrow.jl`, `ArrowTypes`, and - `PureHTTP2.jl` transport sources directly in `Project.toml`, so clean +6. The package also pins the inherited `Arrow.jl`, `ArrowTypes`, + `gRPCServer.jl`, and `PureHTTP2.jl` transport sources directly in + `Project.toml`, so clean package resolution and GitHub Actions do not rely on a workflow-local inherited-source shim @@ -323,6 +324,7 @@ GitHub Actions note: 1. package-local CI now runs `Pkg.build()` plus `Pkg.test()` on `ubuntu-latest` and `macos-latest` for Julia `1.12` and `pre` 2. a separate nightly workflow runs weekly on `ubuntu-latest` -3. both workflows prepare `General` plus `OpenModelicaRegistry` before running - `Pkg.resolve()`, `Pkg.instantiate()`, `Pkg.build()`, and package tests, so - remote runners resolve the same source-locked transport stack as local runs +3. both workflows ensure `General`, then develop the package in a temporary + Julia project before `Pkg.instantiate()`, `Pkg.build()`, and package tests, + so remote runners resolve from the package-owned source contract instead of + workflow-local dependency shims diff --git a/scripts/prepare_wendao_code_parser_env.jl b/scripts/prepare_wendao_code_parser_env.jl deleted file mode 100644 index ca16b8c..0000000 --- a/scripts/prepare_wendao_code_parser_env.jl +++ /dev/null @@ -1,198 +0,0 @@ -import Downloads -import Pkg -using TOML - -const SCRIPT_ROOT = @__DIR__ -const WENDAO_ROOT = normpath(joinpath(SCRIPT_ROOT, "..")) -const PROJECT_TOML = joinpath(WENDAO_ROOT, "Project.toml") -const TEST_ENV = "WENDAO_CODE_PARSER_TEST_ENV" -const LOCAL_ARROW_ENV = "WENDAO_CODE_PARSER_LOCAL_ARROW_PATH" -const LOCAL_WENDAOARROW_ENV = "WENDAO_CODE_PARSER_LOCAL_WENDAO_ARROW_PATH" - -function valid_arrow_checkout(path::AbstractString) - return isfile(joinpath(path, "Project.toml")) && - isfile(joinpath(path, "src", "ArrowTypes", "Project.toml")) -end - -function valid_wendaoarrow_checkout(path::AbstractString) - return isfile(joinpath(path, "Project.toml")) && - isfile(joinpath(path, "src", "WendaoArrow.jl")) -end - -function candidate_arrow_checkouts() - candidates = String[] - - if haskey(ENV, LOCAL_ARROW_ENV) - push!(candidates, abspath(ENV[LOCAL_ARROW_ENV])) - end - - push!(candidates, normpath(joinpath(dirname(WENDAO_ROOT), "arrow-julia"))) - - if haskey(ENV, "PRJ_ROOT") - push!(candidates, normpath(joinpath(ENV["PRJ_ROOT"], ".data", "arrow-julia"))) - end - - return unique(candidates) -end - -function candidate_wendaoarrow_checkouts() - candidates = String[] - - if haskey(ENV, LOCAL_WENDAOARROW_ENV) - push!(candidates, abspath(ENV[LOCAL_WENDAOARROW_ENV])) - end - - push!(candidates, normpath(joinpath(dirname(WENDAO_ROOT), "WendaoArrow.jl"))) - - if haskey(ENV, "PRJ_ROOT") - push!(candidates, normpath(joinpath(ENV["PRJ_ROOT"], ".data", "WendaoArrow.jl"))) - end - - return unique(candidates) -end - -function maybe_local_checkout(candidates::Vector{String}, validator::Function) - for candidate in candidates - validator(candidate) && return candidate - end - return nothing -end - -function remote_source_spec(name::String, entry::Dict{String,Any}; root::AbstractString = WENDAO_ROOT) - kwargs = Dict{Symbol,Any}(:name => name) - haskey(entry, "url") && (kwargs[:url] = entry["url"]) - haskey(entry, "rev") && (kwargs[:rev] = entry["rev"]) - haskey(entry, "subdir") && (kwargs[:subdir] = entry["subdir"]) - haskey(entry, "path") && (kwargs[:path] = abspath(joinpath(root, entry["path"]))) - return Pkg.PackageSpec(; kwargs...) -end - -function push_remote_source_spec!( - specs::Vector{Pkg.PackageSpec}, - seen_sources::Set{String}, - name::String, - entry, - ; - root::AbstractString = WENDAO_ROOT, -) - entry isa Dict{String,Any} || return - name in seen_sources && return - push!(specs, remote_source_spec(name, entry; root = root)) - push!(seen_sources, name) - return -end - -function github_project_toml_url(entry::Dict{String,Any}) - url = get(entry, "url", nothing) - rev = get(entry, "rev", nothing) - url isa AbstractString || return nothing - rev isa AbstractString || return nothing - - match_result = match(r"^https://github\.com/([^/]+)/([^/]+?)(?:\.git)?/?$", url) - isnothing(match_result) && return nothing - - owner, repo = match_result.captures - subdir = get(entry, "subdir", "") - project_path = isempty(subdir) ? "Project.toml" : "$(subdir)/Project.toml" - return "https://raw.githubusercontent.com/$(owner)/$(repo)/$(rev)/$(project_path)" -end - -function source_project_from_entry( - name::String, - entry::Dict{String,Any}; - root::AbstractString = WENDAO_ROOT, -) - if haskey(entry, "path") - return TOML.parsefile(joinpath(root, entry["path"], "Project.toml")) - end - - project_url = github_project_toml_url(entry) - isnothing(project_url) && return nothing - - project_toml = Downloads.download(project_url) - try - return TOML.parsefile(project_toml) - catch err - error("Could not read source Project.toml for $(name) from $(project_url): $(err)") - finally - rm(project_toml; force = true) - end -end - -function push_dependency_source_specs!( - specs::Vector{Pkg.PackageSpec}, - seen_sources::Set{String}, - source_project::Union{Nothing,Dict{String,Any}}, - ; - root::AbstractString = WENDAO_ROOT, -) - isnothing(source_project) && return - - dependencies = get(source_project, "deps", Dict{String,Any}()) - dependency_sources = get(source_project, "sources", Dict{String,Any}()) - - for name in sort(collect(keys(dependencies))) - entry = get(dependency_sources, name, nothing) - push_remote_source_spec!(specs, seen_sources, name, entry; root = root) - end - return nothing -end - -project = TOML.parsefile(PROJECT_TOML) -sources = get(project, "sources", Dict{String,Any}()) - -env_path = get(ENV, TEST_ENV, mktempdir()) -Pkg.activate(env_path) - -arrow_checkout = maybe_local_checkout(candidate_arrow_checkouts(), valid_arrow_checkout) -wendaoarrow_checkout = - maybe_local_checkout(candidate_wendaoarrow_checkouts(), valid_wendaoarrow_checkout) - -add_specs = Pkg.PackageSpec[] -develop_specs = Pkg.PackageSpec[] -seen_sources = Set{String}() - -if isnothing(arrow_checkout) - push_remote_source_spec!(add_specs, seen_sources, "Arrow", sources["Arrow"]) - push_remote_source_spec!(add_specs, seen_sources, "ArrowTypes", sources["ArrowTypes"]) -else - push!(develop_specs, Pkg.PackageSpec(path = arrow_checkout)) - push!(develop_specs, Pkg.PackageSpec(path = joinpath(arrow_checkout, "src", "ArrowTypes"))) - push!(seen_sources, "Arrow") - push!(seen_sources, "ArrowTypes") -end - -wendaoarrow_project = nothing -wendaoarrow_project_root = WENDAO_ROOT -if isnothing(wendaoarrow_checkout) - wendaoarrow_source = sources["WendaoArrow"] - push_remote_source_spec!(add_specs, seen_sources, "WendaoArrow", wendaoarrow_source) - wendaoarrow_project = source_project_from_entry("WendaoArrow", wendaoarrow_source) -else - push!(develop_specs, Pkg.PackageSpec(path = wendaoarrow_checkout)) - push!(seen_sources, "WendaoArrow") - wendaoarrow_project = TOML.parsefile(joinpath(wendaoarrow_checkout, "Project.toml")) - wendaoarrow_project_root = wendaoarrow_checkout -end - -push_dependency_source_specs!( - add_specs, - seen_sources, - wendaoarrow_project; - root = wendaoarrow_project_root, -) - -for (name, entry) in sources - entry isa Dict{String,Any} || continue - name in ("Arrow", "ArrowTypes", "WendaoArrow") && continue - push_remote_source_spec!(add_specs, seen_sources, name, entry) -end - -isempty(add_specs) || Pkg.add(add_specs; preserve = Pkg.PRESERVE_DIRECT) -isempty(develop_specs) || Pkg.develop(develop_specs; preserve = Pkg.PRESERVE_DIRECT) -Pkg.develop([Pkg.PackageSpec(path = WENDAO_ROOT)]; preserve = Pkg.PRESERVE_DIRECT) -Pkg.add([Pkg.PackageSpec(name = "Tables")]; preserve = Pkg.PRESERVE_DIRECT) - -Pkg.resolve() -Pkg.instantiate() -Pkg.build("WendaoCodeParser") diff --git a/scripts/run_service.jl b/scripts/run_service.jl index a53d732..8626499 100644 --- a/scripts/run_service.jl +++ b/scripts/run_service.jl @@ -6,13 +6,24 @@ if "@stdlib" ∉ Base.LOAD_PATH push!(Base.LOAD_PATH, "@stdlib") end +using Pkg + +const SCRIPT_ROOT = @__DIR__ +const WENDAOCODEPARSER_ROOT = normpath(joinpath(SCRIPT_ROOT, "..")) + +function activate_code_parser_project() + Pkg.activate(; temp = true, io = devnull) + Pkg.develop(Pkg.PackageSpec(path = WENDAOCODEPARSER_ROOT); io = devnull) + Pkg.instantiate(; io = devnull) + return WENDAOCODEPARSER_ROOT +end + +activate_code_parser_project() + using Logging using TOML -using WendaoArrow using WendaoCodeParser -const SCRIPT_ROOT = @__DIR__ -const WENDAOCODEPARSER_ROOT = normpath(joinpath(SCRIPT_ROOT, "..")) const DEFAULT_CONFIG_PATH = joinpath(WENDAOCODEPARSER_ROOT, "config", "live", "parser_summary.toml") @@ -122,7 +133,7 @@ function main(args::Vector{String}) isempty(route_names) && error("WendaoCodeParser service requires at least one parser route") listener = WendaoCodeParser.parser_service_listener_config(entry_args) - config = WendaoArrow.config_from_args( + config = WendaoCodeParser.WendaoArrow.config_from_args( WendaoCodeParser.parser_service_interface_args(entry_args), ) @@ -138,13 +149,13 @@ function main(args::Vector{String}) ) live_service = WendaoCodeParser.build_parser_live_flight_service(route_names) WendaoCodeParser.warm_parser_live_flight_service(live_service, route_names) - server = WendaoArrow.flight_server( + server = WendaoCodeParser.WendaoArrow.flight_server( live_service; host = String(config.host), port = Int(config.port), WendaoCodeParser.parser_service_flight_server_kwargs(listener)..., ) - WendaoArrow._wait_for_flight_server(server; block = true) + WendaoCodeParser.WendaoArrow._wait_for_flight_server(server; block = true) end if abspath(PROGRAM_FILE) == @__FILE__ diff --git a/scripts/test_wendao_code_parser.sh b/scripts/test_wendao_code_parser.sh index c29291d..7755be4 100755 --- a/scripts/test_wendao_code_parser.sh +++ b/scripts/test_wendao_code_parser.sh @@ -2,15 +2,13 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - -ENV_PATH="${WENDAO_CODE_PARSER_TEST_ENV:-$(mktemp -d)}" -if [[ -z "${WENDAO_CODE_PARSER_TEST_ENV:-}" ]]; then - trap 'rm -rf "${ENV_PATH}"' EXIT -fi -export WENDAO_CODE_PARSER_TEST_ENV="${ENV_PATH}" - -if [[ ! -f "${ENV_PATH}/Project.toml" ]]; then - "${JULIA:-julia}" "${ROOT}/scripts/prepare_wendao_code_parser_env.jl" -fi - -exec "${JULIA:-julia}" --project="${ENV_PATH}" "${ROOT}/test/runtests.jl" +ENV_PATH="$(mktemp -d)" +trap 'rm -rf "${ENV_PATH}"' EXIT + +"${JULIA:-julia}" --project="${ENV_PATH}" -e ' +using Pkg +package_path = popfirst!(ARGS) +Pkg.develop(PackageSpec(path = package_path)) +Pkg.instantiate() +Pkg.test("WendaoCodeParser"; coverage = false, test_args = ARGS) +' "${ROOT}" "$@" diff --git a/test/cases/mod.jl b/test/cases/mod.jl index 87f1365..cb1d714 100644 --- a/test/cases/mod.jl +++ b/test/cases/mod.jl @@ -32,3 +32,4 @@ include("flight_julia_parameter_owner_signatures.jl") include("flight_search_attribute_lists.jl") include("flight_search_attribute_scalars.jl") include("flight_scope_target_columns.jl") +include("service_entrypoint.jl") diff --git a/test/cases/service_entrypoint.jl b/test/cases/service_entrypoint.jl new file mode 100644 index 0000000..8ebc03b --- /dev/null +++ b/test/cases/service_entrypoint.jl @@ -0,0 +1,21 @@ +module ServiceEntrypointHarness +include(joinpath(@__DIR__, "..", "..", "scripts", "run_service.jl")) +end + +@testset "Service entrypoint uses package project" begin + @test ServiceEntrypointHarness.activate_code_parser_project() == + ServiceEntrypointHarness.WENDAOCODEPARSER_ROOT + @test isfile(Base.active_project()) + @test Base.active_project() != + joinpath(ServiceEntrypointHarness.WENDAOCODEPARSER_ROOT, "Project.toml") +end + +@testset "Service entrypoint default config args" begin + entry_args = ServiceEntrypointHarness.service_entry_args(String[]) + @test entry_args[1:2] == [ + "--config", + ServiceEntrypointHarness.DEFAULT_CONFIG_PATH, + ] + @test "--code-parser-route-name" in entry_args || + "--code-parser-route-names" in entry_args +end From 0b2de07f9bf2491570419af0c0b2a7bf6f79b12d Mon Sep 17 00:00:00 2001 From: guangtao Date: Fri, 1 May 2026 09:45:30 -0700 Subject: [PATCH 8/8] ci: use Julia package actions --- .github/workflows/ci.yml | 15 ++------------- .github/workflows/nightly.yml | 15 ++------------- README.md | 7 +++---- 3 files changed, 7 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ad7d1a..a907b3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,21 +50,10 @@ jobs: if: runner.os == 'macOS' run: | brew install autoconf automake libtool pkg-config - - name: Ensure General registry - shell: julia --project=. {0} - run: | - using Pkg - registries = Pkg.Registry.reachable_registries() - any(registry -> registry.name == "General", registries) || Pkg.Registry.add("General") - - name: Instantiate package + - uses: julia-actions/julia-buildpkg@v1 env: JULIA_PKG_PRECOMPILE_AUTO: "0" - run: | - julia --project="$(mktemp -d)" -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.build("WendaoCodeParser")' - - name: Run package tests + - uses: julia-actions/julia-runtest@v1 env: JULIA_NUM_THREADS: ${{ matrix.nthreads }} JULIA_PKG_PRECOMPILE_AUTO: "0" - run: | - chmod +x ./scripts/test_wendao_code_parser.sh - ./scripts/test_wendao_code_parser.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2b2c38f..482f699 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -42,21 +42,10 @@ jobs: if: runner.os == 'macOS' run: | brew install autoconf automake libtool pkg-config - - name: Ensure General registry - shell: julia --project=. {0} - run: | - using Pkg - registries = Pkg.Registry.reachable_registries() - any(registry -> registry.name == "General", registries) || Pkg.Registry.add("General") - - name: Instantiate package + - uses: julia-actions/julia-buildpkg@v1 env: JULIA_PKG_PRECOMPILE_AUTO: "0" - run: | - julia --project="$(mktemp -d)" -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.build("WendaoCodeParser")' - - name: Run package tests + - uses: julia-actions/julia-runtest@v1 env: JULIA_NUM_THREADS: ${{ matrix.nthreads }} JULIA_PKG_PRECOMPILE_AUTO: "0" - run: | - chmod +x ./scripts/test_wendao_code_parser.sh - ./scripts/test_wendao_code_parser.sh diff --git a/README.md b/README.md index 6a67c20..3e84181 100644 --- a/README.md +++ b/README.md @@ -324,7 +324,6 @@ GitHub Actions note: 1. package-local CI now runs `Pkg.build()` plus `Pkg.test()` on `ubuntu-latest` and `macos-latest` for Julia `1.12` and `pre` 2. a separate nightly workflow runs weekly on `ubuntu-latest` -3. both workflows ensure `General`, then develop the package in a temporary - Julia project before `Pkg.instantiate()`, `Pkg.build()`, and package tests, - so remote runners resolve from the package-owned source contract instead of - workflow-local dependency shims +3. both workflows use `julia-actions/julia-buildpkg` and + `julia-actions/julia-runtest`, so remote runners resolve from the + package-owned source contract instead of workflow-local dependency shims