diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 428f493..a907b3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,37 +50,10 @@ jobs: if: runner.os == 'macOS' run: | brew install autoconf automake libtool pkg-config - - name: Bootstrap registries - shell: julia --project=. {0} - run: | - using Pkg - - function ensure_registry(name::String; spec=nothing) - registries = Pkg.Registry.reachable_registries() - any(registry -> registry.name == name, registries) && return - if isnothing(spec) - Pkg.Registry.add(name) - else - Pkg.Registry.add(spec) - end - end - - ensure_registry("General") - ensure_registry( - "OpenModelica"; - spec=Pkg.RegistrySpec(url="https://github.com/JKRT/OpenModelicaRegistry.git"), - ) - - name: Resolve and build package + - uses: julia-actions/julia-buildpkg@v1 env: JULIA_PKG_PRECOMPILE_AUTO: "0" - WENDAO_CODE_PARSER_BOOTSTRAP_ENV: ${{ runner.temp }}/wendaocodeparser-env - run: | - julia ./scripts/prepare_wendao_code_parser_env.jl - - name: Run package tests + - uses: julia-actions/julia-runtest@v1 env: JULIA_NUM_THREADS: ${{ matrix.nthreads }} JULIA_PKG_PRECOMPILE_AUTO: "0" - WENDAO_CODE_PARSER_BOOTSTRAP_ENV: ${{ runner.temp }}/wendaocodeparser-env - run: | - chmod +x ./scripts/test_wendao_code_parser.sh - ./scripts/test_wendao_code_parser.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 20a9f5d..482f699 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -42,37 +42,10 @@ jobs: if: runner.os == 'macOS' run: | brew install autoconf automake libtool pkg-config - - name: Bootstrap registries - shell: julia --project=. {0} - run: | - using Pkg - - function ensure_registry(name::String; spec=nothing) - registries = Pkg.Registry.reachable_registries() - any(registry -> registry.name == name, registries) && return - if isnothing(spec) - Pkg.Registry.add(name) - else - Pkg.Registry.add(spec) - end - end - - ensure_registry("General") - ensure_registry( - "OpenModelica"; - spec=Pkg.RegistrySpec(url="https://github.com/JKRT/OpenModelicaRegistry.git"), - ) - - name: Resolve and build package + - uses: julia-actions/julia-buildpkg@v1 env: JULIA_PKG_PRECOMPILE_AUTO: "0" - WENDAO_CODE_PARSER_BOOTSTRAP_ENV: ${{ runner.temp }}/wendaocodeparser-env - run: | - julia ./scripts/prepare_wendao_code_parser_env.jl - - name: Run package tests + - uses: julia-actions/julia-runtest@v1 env: JULIA_NUM_THREADS: ${{ matrix.nthreads }} JULIA_PKG_PRECOMPILE_AUTO: "0" - WENDAO_CODE_PARSER_BOOTSTRAP_ENV: ${{ runner.temp }}/wendaocodeparser-env - run: | - chmod +x ./scripts/test_wendao_code_parser.sh - ./scripts/test_wendao_code_parser.sh diff --git a/Project.toml b/Project.toml index 512c8a4..a711214 100644 --- a/Project.toml +++ b/Project.toml @@ -7,6 +7,7 @@ authors = ["CyberXiuXian Workshop"] Absyn = "ce2f92e2-a952-11e9-0543-8b443f216f1d" Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45" ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" +gRPCServer = "608c6337-0d7d-447f-bb69-0f5674ee3959" ImmutableList = "4a558cac-c1ed-11e9-20da-3584bcd8709a" JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -15,6 +16,7 @@ MetaModelica = "9d7f2a79-07b5-5542-8b19-c0100dda6b06" OMParser = "11f87224-cae7-4e99-a924-e50d12f62c59" PureHTTP2 = "7d1e1b98-28e7-4969-8df9-5a308937986a" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" WendaoArrow = "561c8d8d-4bcf-4807-873b-a6b7d1e55843" [sources] @@ -33,17 +35,18 @@ WendaoArrow = {rev = "e992839d84dc92ffc4972e10d160ee4ce53ce126", url = "https:// Absyn = "1.3" Arrow = "2.8.1" ArrowTypes = "2.3.0" +gRPCServer = "0.1" ImmutableList = "0.1, 0.3" JuliaSyntax = "2" -MetaModelica = "0.0.5, 0.1" +MetaModelica = "0.0.5, 0.1, 0.2" OMParser = "0.0.3" PureHTTP2 = "0.5.0" Tables = "1" +TOML = "1" WendaoArrow = "0.1" julia = "1.12" [extras] -gRPCServer = "608c6337-0d7d-447f-bb69-0f5674ee3959" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] diff --git a/README.md b/README.md index 2a9034d..3e84181 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,29 @@ Package boundary: The initial slice keeps the Rust client cutover out of scope and proves the provider contract first. -`WendaoSearch.jl` can also mount these parser routes into its existing live gRPC -service with `--code-parser-route-names`, so the same Arrow Flight process can -serve both graph-search routes and AST-query routes during local loopback tests. +Service runtime: + +1. `scripts/run_service.jl` starts the parser-summary and AST-query Flight + service directly from this package +2. `config/live/parser_summary.toml` is the package-local live-service + descriptor for the default Julia and Modelica parser routes +3. `contracts/wendaocodeparser_parser_summary.toml` is the package-local + route and transport contract consumed by Rust integration tests + +Start the default service: + +```bash +julia --project=. scripts/run_service.jl --config config/live/parser_summary.toml +``` + +Override listener fields without changing the package-owned route contract: + +```bash +julia --project=. scripts/run_service.jl \ + --config config/live/parser_summary.toml \ + --host 127.0.0.1 \ + --port 41081 +``` Package docs now also live under `docs/`: @@ -41,15 +61,16 @@ Current backend status: `lib/parser -> autoconf -> ./configure -> make` 4. The current workspace lock pins `OMParser.jl` to `https://github.com/tao3k/OMParser.jl` at - `cebc0696407385e52496608fcc13e95a556da3b5` until the bootstrap fixes are - consumed upstream + `d59051069e43fb2624aa13fe8935532ca15aecec` until the upstream source-build + fixes are consumed 5. The current workspace lock pins `WendaoArrow.jl` to `https://github.com/tao3k/WendaoArrow.jl.git` at - `3325a646785e022a3286d08f28b19dafb4e7c8dd` -6. The package also pins the inherited `Arrow.jl`, `ArrowTypes`, and - `PureHTTP2.jl` transport sources directly in `Project.toml`, so clean + `e992839d84dc92ffc4972e10d160ee4ce53ce126` +6. The package also pins the inherited `Arrow.jl`, `ArrowTypes`, + `gRPCServer.jl`, and `PureHTTP2.jl` transport sources directly in + `Project.toml`, so clean package resolution and GitHub Actions do not rely on a workflow-local - inherited-source bootstrap + inherited-source shim Native bridge note: @@ -57,10 +78,9 @@ Native bridge note: `ImmutableList`, and `MetaModelica` from `Main` during parser initialization 2. `WendaoCodeParser.jl` therefore aliases those already-loaded modules into - `Main` before the first Modelica parse, especially for mounted live-child - startup under `WendaoSearch.jl` + `Main` before the first Modelica parse 3. This runtime requirement is separate from the upstream `OMParser.jl` - build/bootstrap lane: the upstream PR still matters for `Pkg.build(...)`, + source-build lane: the upstream PR still matters for `Pkg.build(...)`, release assets, and CI coverage, but it does not by itself close the live child startup contract @@ -92,7 +112,7 @@ Parser layout note: 10. `src/parsers/julia/collect.jl` owns SyntaxNode traversal and Julia summary or AST state collection 11. `src/parsers/modelica/backend.jl` now only owns the `OMParser.jl` native - bridge and shared-library/runtime bootstrap + bridge and shared-library/runtime initialization 12. `src/parsers/modelica/nodes.jl` owns generic Modelica AST node materialization 13. `src/parsers/modelica/dependencies.jl` owns Modelica `import` / `extends` @@ -278,12 +298,10 @@ Contract note: distinct AST nodes instead of being collapsed globally 28. package tests are now split under `test/support/` and `test/cases/`, so `test/runtests.jl` stays as a small runner instead of a monolithic file -29. parser-specific Flight round-trip coverage is now isolated in - `test/cases/flight_native_columns.jl`, and mounted shared-service parser - regressions are isolated under `WendaoSearch.jl/test/integration/`, - including `live_code_parser.jl`, `live_dependency_semantics.jl`, - `live_relative_dependencies.jl`, `live_modelica_import_forms.jl`, and - `live_julia_type_headers.jl` +29. parser-specific Flight round-trip coverage is isolated in + `test/cases/flight_native_columns.jl`, while parser-service route parsing, + listener config, and multiplexed live-service behavior are covered in + `test/cases/flight_services.jl` 30. AST match rows now also promote parser-owned stable columns such as `match_target_name`, `match_root_module_name`, `match_top_level`, `match_reexported`, `match_visibility`, `match_type_name`, @@ -306,7 +324,6 @@ GitHub Actions note: 1. package-local CI now runs `Pkg.build()` plus `Pkg.test()` on `ubuntu-latest` and `macos-latest` for Julia `1.12` and `pre` 2. a separate nightly workflow runs weekly on `ubuntu-latest` -3. both workflows bootstrap `General` plus `OpenModelicaRegistry` before - running `Pkg.resolve()`, `Pkg.instantiate()`, `Pkg.build()`, and package - tests, so remote runners resolve the same source-locked transport stack as - local runs +3. both workflows use `julia-actions/julia-buildpkg` and + `julia-actions/julia-runtest`, so remote runners resolve from the + package-owned source contract instead of workflow-local dependency shims diff --git a/config/live/parser_summary.toml b/config/live/parser_summary.toml new file mode 100644 index 0000000..4587ccd --- /dev/null +++ b/config/live/parser_summary.toml @@ -0,0 +1,10 @@ +code_parser_route_names = [ + "julia_file_summary", + "julia_root_summary", + "modelica_file_summary", + "modelica_ast_query", +] + +[interface] +host = "127.0.0.1" +port = 41081 diff --git a/contracts/wendaocodeparser_parser_summary.toml b/contracts/wendaocodeparser_parser_summary.toml new file mode 100644 index 0000000..358c0e0 --- /dev/null +++ b/contracts/wendaocodeparser_parser_summary.toml @@ -0,0 +1,21 @@ +contract_version = 1 + +[service] +script = "scripts/run_service.jl" +config = "config/live/parser_summary.toml" +host = "127.0.0.1" +port = 41081 +default_code_parser_route_names = [ + "julia_file_summary", + "julia_root_summary", + "modelica_file_summary", + "modelica_ast_query", +] + +[modelica_transport] +schema_version = "v3" +file_summary_route_name = "modelica_file_summary" +ast_query_route_name = "modelica_ast_query" +file_summary_path = "/wendao/code-parser/modelica/file-summary" +ast_query_path = "/wendao/code-parser/modelica/ast-query" +readiness_route_names = ["modelica_file_summary", "modelica_ast_query"] diff --git a/scripts/prepare_wendao_code_parser_env.jl b/scripts/prepare_wendao_code_parser_env.jl deleted file mode 100644 index 9862d3f..0000000 --- a/scripts/prepare_wendao_code_parser_env.jl +++ /dev/null @@ -1,109 +0,0 @@ -import Pkg -using TOML - -const SCRIPT_ROOT = @__DIR__ -const WENDAO_ROOT = normpath(joinpath(SCRIPT_ROOT, "..")) -const PROJECT_TOML = joinpath(WENDAO_ROOT, "Project.toml") -const BOOTSTRAP_ENV = "WENDAO_CODE_PARSER_BOOTSTRAP_ENV" -const LOCAL_ARROW_ENV = "WENDAO_CODE_PARSER_LOCAL_ARROW_PATH" -const LOCAL_WENDAOARROW_ENV = "WENDAO_CODE_PARSER_LOCAL_WENDAO_ARROW_PATH" - -function valid_arrow_checkout(path::AbstractString) - return isfile(joinpath(path, "Project.toml")) && - isfile(joinpath(path, "src", "ArrowTypes", "Project.toml")) -end - -function valid_wendaoarrow_checkout(path::AbstractString) - return isfile(joinpath(path, "Project.toml")) && - isfile(joinpath(path, "src", "WendaoArrow.jl")) -end - -function candidate_arrow_checkouts() - candidates = String[] - - if haskey(ENV, LOCAL_ARROW_ENV) - push!(candidates, abspath(ENV[LOCAL_ARROW_ENV])) - end - - push!(candidates, normpath(joinpath(dirname(WENDAO_ROOT), "arrow-julia"))) - - if haskey(ENV, "PRJ_ROOT") - push!(candidates, normpath(joinpath(ENV["PRJ_ROOT"], ".data", "arrow-julia"))) - end - - return unique(candidates) -end - -function candidate_wendaoarrow_checkouts() - candidates = String[] - - if haskey(ENV, LOCAL_WENDAOARROW_ENV) - push!(candidates, abspath(ENV[LOCAL_WENDAOARROW_ENV])) - end - - push!(candidates, normpath(joinpath(dirname(WENDAO_ROOT), "WendaoArrow.jl"))) - - if haskey(ENV, "PRJ_ROOT") - push!(candidates, normpath(joinpath(ENV["PRJ_ROOT"], ".data", "WendaoArrow.jl"))) - end - - return unique(candidates) -end - -function maybe_local_checkout(candidates::Vector{String}, validator::Function) - for candidate in candidates - validator(candidate) && return candidate - end - return nothing -end - -function remote_source_spec(name::String, entry::Dict{String,Any}) - kwargs = Dict{Symbol,Any}(:name => name) - haskey(entry, "url") && (kwargs[:url] = entry["url"]) - haskey(entry, "rev") && (kwargs[:rev] = entry["rev"]) - haskey(entry, "subdir") && (kwargs[:subdir] = entry["subdir"]) - haskey(entry, "path") && (kwargs[:path] = abspath(joinpath(WENDAO_ROOT, entry["path"]))) - return Pkg.PackageSpec(; kwargs...) -end - -project = TOML.parsefile(PROJECT_TOML) -sources = get(project, "sources", Dict{String,Any}()) - -env_path = get(ENV, BOOTSTRAP_ENV, mktempdir()) -Pkg.activate(env_path) - -arrow_checkout = maybe_local_checkout(candidate_arrow_checkouts(), valid_arrow_checkout) -wendaoarrow_checkout = - maybe_local_checkout(candidate_wendaoarrow_checkouts(), valid_wendaoarrow_checkout) - -add_specs = Pkg.PackageSpec[] -develop_specs = Pkg.PackageSpec[] - -if isnothing(arrow_checkout) - push!(add_specs, remote_source_spec("Arrow", sources["Arrow"])) - push!(add_specs, remote_source_spec("ArrowTypes", sources["ArrowTypes"])) -else - push!(develop_specs, Pkg.PackageSpec(path = arrow_checkout)) - push!(develop_specs, Pkg.PackageSpec(path = joinpath(arrow_checkout, "src", "ArrowTypes"))) -end - -if isnothing(wendaoarrow_checkout) - push!(add_specs, remote_source_spec("WendaoArrow", sources["WendaoArrow"])) -else - push!(develop_specs, Pkg.PackageSpec(path = wendaoarrow_checkout)) -end - -for (name, entry) in sources - entry isa Dict{String,Any} || continue - name in ("Arrow", "ArrowTypes", "WendaoArrow") && continue - push!(add_specs, remote_source_spec(name, entry)) -end - -isempty(add_specs) || Pkg.add(add_specs; preserve = Pkg.PRESERVE_DIRECT) -isempty(develop_specs) || Pkg.develop(develop_specs; preserve = Pkg.PRESERVE_DIRECT) -Pkg.develop([Pkg.PackageSpec(path = WENDAO_ROOT)]; preserve = Pkg.PRESERVE_DIRECT) -Pkg.add([Pkg.PackageSpec(name = "Tables")]; preserve = Pkg.PRESERVE_DIRECT) - -Pkg.resolve() -Pkg.instantiate() -Pkg.build("WendaoCodeParser") diff --git a/scripts/run_service.jl b/scripts/run_service.jl new file mode 100644 index 0000000..8626499 --- /dev/null +++ b/scripts/run_service.jl @@ -0,0 +1,163 @@ +if "@" ∉ Base.LOAD_PATH + pushfirst!(Base.LOAD_PATH, "@") +end + +if "@stdlib" ∉ Base.LOAD_PATH + push!(Base.LOAD_PATH, "@stdlib") +end + +using Pkg + +const SCRIPT_ROOT = @__DIR__ +const WENDAOCODEPARSER_ROOT = normpath(joinpath(SCRIPT_ROOT, "..")) + +function activate_code_parser_project() + Pkg.activate(; temp = true, io = devnull) + Pkg.develop(Pkg.PackageSpec(path = WENDAOCODEPARSER_ROOT); io = devnull) + Pkg.instantiate(; io = devnull) + return WENDAOCODEPARSER_ROOT +end + +activate_code_parser_project() + +using Logging +using TOML +using WendaoCodeParser + +const DEFAULT_CONFIG_PATH = + joinpath(WENDAOCODEPARSER_ROOT, "config", "live", "parser_summary.toml") + +function _usage() + return """ + usage: julia --project=. scripts/run_service.jl [--config PATH] [--host HOST] [--port PORT] [--code-parser-route-names ROUTES] + + Starts the WendaoCodeParser parser-summary Flight service. + """ +end + +function _help_requested(args::Vector{String}) + return any(argument -> argument == "--help" || argument == "-h", args) +end + +function _has_config_arg(args::Vector{String}) + return any( + argument == "--config" || startswith(argument, "--config=") for argument in args + ) +end + +function _config_arg_path(args::Vector{String}) + index = 1 + while index <= length(args) + argument = args[index] + if startswith(argument, "--config=") + return abspath(split(argument, "=", limit = 2)[2]) + elseif argument == "--config" + index += 1 + index > length(args) && + error("WendaoCodeParser service requires one value after --config") + return abspath(args[index]) + end + index += 1 + end + return nothing +end + +function _has_code_parser_route_arg(args::Vector{String}) + return any( + startswith(argument, "--code-parser-route-name=") || + startswith(argument, "--code-parser-route-names=") || + startswith(argument, "--code-parser-routes=") || + argument == "--code-parser-route-name" || + argument == "--code-parser-route-names" || + argument == "--code-parser-routes" for argument in args + ) +end + +function _effective_config_path(args::Vector{String}) + config_path = _config_arg_path(args) + !isnothing(config_path) && return config_path + if haskey(ENV, "WENDAOCODEPARSER_CONFIG") + return abspath(ENV["WENDAOCODEPARSER_CONFIG"]) + end + return DEFAULT_CONFIG_PATH +end + +function _code_parser_route_args_from_config(config_path::AbstractString) + resolved_path = abspath(String(config_path)) + isfile(resolved_path) || + error("WendaoCodeParser service config does not exist: $(resolved_path)") + config = TOML.parsefile(resolved_path) + route_name = get(config, "code_parser_route_name", nothing) + route_names = get(config, "code_parser_route_names", nothing) + (!isnothing(route_name) && !isnothing(route_names)) && error( + "WendaoCodeParser service config must not set both code_parser_route_name and code_parser_route_names", + ) + if !isnothing(route_name) + return String["--code-parser-route-name", String(route_name)] + end + if !isnothing(route_names) + route_names isa AbstractVector || error( + "WendaoCodeParser service config code_parser_route_names must be an array of strings", + ) + return String[ + "--code-parser-route-names", + join(String[String(value) for value in route_names], ","), + ] + end + return String[] +end + +function service_entry_args(args::Vector{String}) + entry_args = String[] + config_path = _effective_config_path(args) + !_has_config_arg(args) && append!(entry_args, ["--config", config_path]) + if !_has_code_parser_route_arg(args) + route_args = _code_parser_route_args_from_config(config_path) + isempty(route_args) && error( + "WendaoCodeParser service requires code_parser_route_name(s) via --config or explicit CLI args", + ) + append!(entry_args, route_args) + end + append!(entry_args, args) + return entry_args +end + +function main(args::Vector{String}) + if _help_requested(args) + print(_usage()) + return nothing + end + + entry_args = service_entry_args(args) + route_names = WendaoCodeParser.parser_service_route_names(entry_args) + isempty(route_names) && + error("WendaoCodeParser service requires at least one parser route") + listener = WendaoCodeParser.parser_service_listener_config(entry_args) + config = WendaoCodeParser.WendaoArrow.config_from_args( + WendaoCodeParser.parser_service_interface_args(entry_args), + ) + + @info( + "WendaoCodeParser service startup", + host = String(config.host), + port = Int(config.port), + route_names = String[String(route_name) for route_name in route_names], + listener_max_active_requests = listener.max_active_requests, + listener_request_capacity = listener.request_capacity, + listener_response_capacity = listener.response_capacity, + schema_version = WendaoCodeParser.WENDAOCODEPARSER_SCHEMA_VERSION, + ) + live_service = WendaoCodeParser.build_parser_live_flight_service(route_names) + WendaoCodeParser.warm_parser_live_flight_service(live_service, route_names) + server = WendaoCodeParser.WendaoArrow.flight_server( + live_service; + host = String(config.host), + port = Int(config.port), + WendaoCodeParser.parser_service_flight_server_kwargs(listener)..., + ) + WendaoCodeParser.WendaoArrow._wait_for_flight_server(server; block = true) +end + +if abspath(PROGRAM_FILE) == @__FILE__ + main(copy(ARGS)) +end diff --git a/scripts/test_wendao_code_parser.sh b/scripts/test_wendao_code_parser.sh index a000329..7755be4 100755 --- a/scripts/test_wendao_code_parser.sh +++ b/scripts/test_wendao_code_parser.sh @@ -2,15 +2,13 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - -ENV_PATH="${WENDAO_CODE_PARSER_BOOTSTRAP_ENV:-$(mktemp -d)}" -if [[ -z "${WENDAO_CODE_PARSER_BOOTSTRAP_ENV:-}" ]]; then - trap 'rm -rf "${ENV_PATH}"' EXIT -fi -export WENDAO_CODE_PARSER_BOOTSTRAP_ENV="${ENV_PATH}" - -if [[ ! -f "${ENV_PATH}/Project.toml" ]]; then - "${JULIA:-julia}" "${ROOT}/scripts/prepare_wendao_code_parser_env.jl" -fi - -exec "${JULIA:-julia}" --project="${ENV_PATH}" "${ROOT}/test/runtests.jl" +ENV_PATH="$(mktemp -d)" +trap 'rm -rf "${ENV_PATH}"' EXIT + +"${JULIA:-julia}" --project="${ENV_PATH}" -e ' +using Pkg +package_path = popfirst!(ARGS) +Pkg.develop(PackageSpec(path = package_path)) +Pkg.instantiate() +Pkg.test("WendaoCodeParser"; coverage = false, test_args = ARGS) +' "${ROOT}" "$@" diff --git a/src/WendaoCodeParser.jl b/src/WendaoCodeParser.jl index 55a0b8c..f278b65 100644 --- a/src/WendaoCodeParser.jl +++ b/src/WendaoCodeParser.jl @@ -48,8 +48,15 @@ export parse_julia_root_summary export parse_modelica_file_summary export parser_route_descriptor export parser_route_request_headers +export parser_service_flight_server_kwargs +export parser_service_interface_args +export parser_service_listener_config +export parser_service_route_names +export ParserServiceListenerConfig export supported_parser_route_names +export build_parser_live_flight_service export search_julia_ast export search_modelica_ast +export warm_parser_live_flight_service end diff --git a/src/service/runtime.jl b/src/service/runtime.jl index ca773d8..1567a8d 100644 --- a/src/service/runtime.jl +++ b/src/service/runtime.jl @@ -37,6 +37,403 @@ _optional_request_text(value) = ismissing(value) ? nothing : String(value) _optional_request_int(value) = ismissing(value) ? nothing : Int(value) supported_parser_route_names() = collect(PARSER_ROUTE_NAMES) +const CODE_PARSER_WARMUP_JULIA_SOURCE = """ +module WarmupCodeParser +foo(x)=x +export foo +end +""" + +const CODE_PARSER_WARMUP_MODELICA_SOURCE = """ +model WarmupCodeParser +end WarmupCodeParser; +""" + +const CODE_PARSER_WARMUP_MODELICA_AST_SOURCE = """ +within Modelica; +package WarmupCodeParser + import SI = Modelica.Units.SI; + extends Icons.Package; + + block Controller + parameter Real k = 1; + input Real u; + output Real y; + equation + y = k * u; + end Controller; + + model Plant + SI.Time t(start = 0); + Controller c; + end Plant; + + package Types + type Gain = Real(unit = "1"); + end Types; +end WarmupCodeParser; +""" + +struct ParserServiceListenerConfig + max_active_requests::Int + request_capacity::Int + response_capacity::Int +end + +function ParserServiceListenerConfig(; + max_active_requests::Integer = max(Threads.nthreads() * 8, 32), + request_capacity::Integer = 16, + response_capacity::Integer = 16, +) + max_active_requests > 0 || + error("WendaoCodeParser listener max_active_requests must be greater than zero") + request_capacity > 0 || + error("WendaoCodeParser listener request_capacity must be greater than zero") + response_capacity > 0 || + error("WendaoCodeParser listener response_capacity must be greater than zero") + return ParserServiceListenerConfig( + Int(max_active_requests), + Int(request_capacity), + Int(response_capacity), + ) +end + +function parser_service_route_names(args::Vector{String}) + route_name = nothing + route_names = nothing + index = 1 + + while index <= length(args) + argument = args[index] + if startswith(argument, "--code-parser-route-name=") + route_name = split(argument, "=", limit = 2)[2] + elseif argument == "--code-parser-route-name" + index += 1 + index > length(args) && error( + "WendaoCodeParser service requires one value after --code-parser-route-name", + ) + route_name = args[index] + elseif startswith(argument, "--code-parser-route-names=") || + startswith(argument, "--code-parser-routes=") + route_names = split(argument, "=", limit = 2)[2] + elseif argument == "--code-parser-route-names" || argument == "--code-parser-routes" + index += 1 + index > length(args) && error( + "WendaoCodeParser service requires one value after --code-parser-route-names", + ) + route_names = args[index] + end + index += 1 + end + + (!isnothing(route_name) && !isnothing(route_names)) && error( + "WendaoCodeParser service must not set both code_parser route_name and route_names", + ) + isnothing(route_name) && isnothing(route_names) && return Symbol[] + return _resolved_parser_service_route_names(something(route_name, route_names)) +end + +function parser_service_listener_config(args::Vector{String}) + defaults = ParserServiceListenerConfig() + max_active_requests = defaults.max_active_requests + request_capacity = defaults.request_capacity + response_capacity = defaults.response_capacity + index = 1 + + while index <= length(args) + argument = args[index] + if startswith(argument, "--max-active-requests=") + max_active_requests = Base.parse(Int, split(argument, "=", limit = 2)[2]) + elseif argument == "--max-active-requests" + index += 1 + index > length(args) && error( + "WendaoCodeParser service requires one value after --max-active-requests", + ) + max_active_requests = Base.parse(Int, args[index]) + elseif startswith(argument, "--request-capacity=") + request_capacity = Base.parse(Int, split(argument, "=", limit = 2)[2]) + elseif argument == "--request-capacity" + index += 1 + index > length(args) && error( + "WendaoCodeParser service requires one value after --request-capacity", + ) + request_capacity = Base.parse(Int, args[index]) + elseif startswith(argument, "--response-capacity=") + response_capacity = Base.parse(Int, split(argument, "=", limit = 2)[2]) + elseif argument == "--response-capacity" + index += 1 + index > length(args) && error( + "WendaoCodeParser service requires one value after --response-capacity", + ) + response_capacity = Base.parse(Int, args[index]) + end + index += 1 + end + + return ParserServiceListenerConfig( + max_active_requests = max_active_requests, + request_capacity = request_capacity, + response_capacity = response_capacity, + ) +end + +function parser_service_flight_server_kwargs(listener::ParserServiceListenerConfig) + return ( + max_active_requests = listener.max_active_requests, + request_capacity = listener.request_capacity, + response_capacity = listener.response_capacity, + ) +end + +function parser_service_interface_args(args::Vector{String}) + filtered = String[] + index = 1 + while index <= length(args) + argument = args[index] + if startswith(argument, "--code-parser-route-name=") || + startswith(argument, "--code-parser-route-names=") || + startswith(argument, "--code-parser-routes=") + nothing + elseif argument == "--code-parser-route-name" || + argument == "--code-parser-route-names" || + argument == "--code-parser-routes" || + argument == "--max-active-requests" || + argument == "--request-capacity" || + argument == "--response-capacity" + index += 1 + index > length(args) && + error("WendaoCodeParser service requires one value after $(argument)") + elseif startswith(argument, "--max-active-requests=") || + startswith(argument, "--request-capacity=") || + startswith(argument, "--response-capacity=") + nothing + else + push!(filtered, argument) + end + index += 1 + end + return filtered +end + +function build_parser_live_flight_service(route_names = Symbol[]) + parser_routes = _resolved_parser_service_route_names(route_names) + isempty(parser_routes) && + error("WendaoCodeParser live service requires at least one parser route") + _prewarm_modelica_backend_if_needed(parser_routes) + length(parser_routes) == 1 && return build_parser_flight_service(only(parser_routes)) + + route_entries = Dict{Tuple,NamedTuple}() + for route_name in parser_routes + route_entries[_descriptor_path_key(parser_route_descriptor(route_name))] = ( + processor = build_parser_table_processor(route_name), + expected_schema_version = WENDAOCODEPARSER_SCHEMA_VERSION, + subject = "WendaoCodeParser parser-summary exchange request", + ) + end + + return _build_routed_parser_live_flight_service( + route_entries; + missing_descriptor_message = "WendaoCodeParser live service requires one Flight descriptor", + unsupported_descriptor_prefix = "unsupported WendaoCodeParser descriptor path", + ) +end + +function warm_parser_live_flight_service( + service::WendaoArrow.Arrow.Flight.Service, + route_names = Symbol[], +) + parser_routes = _resolved_parser_service_route_names(route_names) + isempty(parser_routes) && + error("WendaoCodeParser live warmup requires at least one parser route") + for route_name in parser_routes + _warm_parser_flight_service(service, route_name) + end + return nothing +end + +function _build_routed_parser_live_flight_service( + route_entries; + missing_descriptor_message::AbstractString, + unsupported_descriptor_prefix::AbstractString, +) + return WendaoArrow.Arrow.Flight.exchangeservice( + function (incoming_messages, request_descriptor, _) + descriptor_path = + isnothing(request_descriptor) ? nothing : request_descriptor.path + table_like = try + WendaoArrow.Arrow.Flight.table(incoming_messages; convert = true) + catch error + @error "WendaoCodeParser routed Flight service failed to decode request" exception = + (error, catch_backtrace()) descriptor_path = descriptor_path + rethrow() + end + + isnothing(request_descriptor) && error(missing_descriptor_message) + requested_path = _descriptor_path_key(request_descriptor) + route_entry = get(route_entries, requested_path, nothing) + isnothing(route_entry) && + error("$(unsupported_descriptor_prefix): $(join(requested_path, "/"))") + + try + WendaoArrow.require_schema_version( + table_like; + subject = route_entry.subject, + expected = route_entry.expected_schema_version, + ) + return ( + output_table = route_entry.processor(table_like), + schema_version = route_entry.expected_schema_version, + subject = route_entry.subject, + ) + catch error + @error "WendaoCodeParser routed Flight processor failed" exception = + (error, catch_backtrace()) descriptor_path = request_descriptor.path subject = + route_entry.subject + rethrow() + end + end; + writer = function (response, routed_output, request_descriptor, _) + descriptor_path = + isnothing(request_descriptor) ? nothing : request_descriptor.path + try + return WendaoArrow.Arrow.Flight.putflightdata!( + response, + routed_output.output_table; + metadata = WendaoArrow.merge_schema_metadata( + WendaoArrow.schema_metadata(routed_output.output_table); + schema_version = routed_output.schema_version, + ), + ) + catch error + @error "WendaoCodeParser routed Flight service failed to encode response" exception = + (error, catch_backtrace()) descriptor_path = descriptor_path subject = + routed_output.subject + rethrow() + end + end, + ) +end + +function _resolved_parser_service_route_names(route_names) + isnothing(route_names) && return Symbol[] + raw_values = + route_names isa AbstractVector ? collect(route_names) : + split(String(route_names), ',') + isempty(raw_values) && return Symbol[] + + normalized = Symbol[] + for raw_value in raw_values + route_text = lowercase(strip(String(raw_value))) + isempty(route_text) && + error("WendaoCodeParser route_names must not contain blanks") + if route_text in ("all", "multi", "full") + append!(normalized, supported_parser_route_names()) + else + push!(normalized, _normalized_parser_service_route_name(route_text)) + end + end + return unique(normalized) +end + +function _normalized_parser_service_route_name(route_name) + route_text = lowercase(strip(String(route_name))) + isempty(route_text) && error("WendaoCodeParser route_name must not be blank") + if route_text in + ("julia_file_summary", "julia-file-summary", "julia-file", "julia_summary") + return JULIA_FILE_SUMMARY_ROUTE + elseif route_text in ("julia_root_summary", "julia-root-summary", "julia-root") + return JULIA_ROOT_SUMMARY_ROUTE + elseif route_text in ( + "modelica_file_summary", + "modelica-file-summary", + "modelica-file", + "modelica_summary", + ) + return MODELICA_FILE_SUMMARY_ROUTE + elseif route_text in ("julia_ast_query", "julia-ast-query", "julia-query") + return JULIA_AST_QUERY_ROUTE + elseif route_text in ("modelica_ast_query", "modelica-ast-query", "modelica-query") + return MODELICA_AST_QUERY_ROUTE + end + error("unsupported WendaoCodeParser route_name: $(route_name)") +end + +function _warm_parser_flight_service( + service::WendaoArrow.Arrow.Flight.Service, + route_name::Symbol, +) + request = _warm_parser_exchange_request(route_name) + WendaoArrow.flight_exchange_table( + service, + WendaoArrow.Arrow.Flight.ServerCallContext(), + request, + ) + return nothing +end + +function _prewarm_modelica_backend_if_needed(route_names::AbstractVector{Symbol}) + any(_is_modelica_parser_route, route_names) || return nothing + prewarm_modelica_backend!() + return nothing +end + +_is_modelica_parser_route(route_name::Symbol) = + route_name == MODELICA_FILE_SUMMARY_ROUTE || route_name == MODELICA_AST_QUERY_ROUTE + +function _warm_parser_exchange_request(route_name::Symbol) + if route_name == JULIA_FILE_SUMMARY_ROUTE + requests = [ + ParserRequest( + "warmup-julia-file-summary", + "WarmupCodeParser.jl", + CODE_PARSER_WARMUP_JULIA_SOURCE, + ), + ] + elseif route_name == JULIA_ROOT_SUMMARY_ROUTE + requests = [ + ParserRequest( + "warmup-julia-root-summary", + "WarmupCodeParser.jl", + CODE_PARSER_WARMUP_JULIA_SOURCE, + ), + ] + elseif route_name == MODELICA_FILE_SUMMARY_ROUTE + requests = [ + ParserRequest( + "warmup-modelica-file-summary", + "WarmupCodeParser.mo", + CODE_PARSER_WARMUP_MODELICA_SOURCE, + ), + ] + elseif route_name == JULIA_AST_QUERY_ROUTE + requests = [ + ParserRequest( + "warmup-julia-ast-query", + "WarmupCodeParser.jl", + CODE_PARSER_WARMUP_JULIA_SOURCE; + node_kind = "function", + limit = 1, + ), + ] + elseif route_name == MODELICA_AST_QUERY_ROUTE + requests = [ + ParserRequest( + "warmup-modelica-ast-query", + "WarmupCodeParser/package.mo", + CODE_PARSER_WARMUP_MODELICA_AST_SOURCE; + limit = 64, + ), + ] + else + error("unsupported WendaoCodeParser route_name: $(route_name)") + end + return parser_exchange_request(route_name, requests) +end + +function _descriptor_path_key(descriptor) + return Tuple(String(segment) for segment in descriptor.path) +end + function _optional_request_text(columns, column_name::Symbol, index::Int) column_name in propertynames(columns) || return nothing return _optional_request_text(getproperty(columns, column_name)[index]) diff --git a/test/cases/flight_services.jl b/test/cases/flight_services.jl index ee611f0..0c0a307 100644 --- a/test/cases/flight_services.jl +++ b/test/cases/flight_services.jl @@ -88,6 +88,59 @@ @test hasproperty(docstring_query_columns, :match_attribute_value) end +@testset "Parser service route parsing and routed live service" begin + @test parser_service_route_names([ + "--code-parser-route-names", + "julia_file_summary,julia-ast-query", + ]) == [JULIA_FILE_SUMMARY_ROUTE, JULIA_AST_QUERY_ROUTE] + @test parser_service_route_names(["--code-parser-routes", "all"]) == + supported_parser_route_names() + + listener = parser_service_listener_config([ + "--max-active-requests", + "4", + "--request-capacity=3", + "--response-capacity", + "2", + ]) + @test listener.max_active_requests == 4 + @test listener.request_capacity == 3 + @test listener.response_capacity == 2 + @test parser_service_interface_args([ + "--config", + "config/live/parser_summary.toml", + "--code-parser-route-names", + "julia_ast_query", + "--max-active-requests", + "4", + "--host", + "127.0.0.1", + "--port=41081", + ]) == [ + "--config", + "config/live/parser_summary.toml", + "--host", + "127.0.0.1", + "--port=41081", + ] + + live_service = + build_parser_live_flight_service([JULIA_FILE_SUMMARY_ROUTE, JULIA_AST_QUERY_ROUTE]) + request = parser_exchange_request( + JULIA_AST_QUERY_ROUTE, + [ParserRequest("live-query", "Demo.jl", JULIA_SOURCE; node_kind = "function")], + ) + table = WendaoCodeParser.WendaoArrow.flight_exchange_table( + live_service, + WendaoCodeParser.WendaoArrow.Arrow.Flight.ServerCallContext(), + request, + ) + columns = Tables.columntable(table) + @test columns.request_id == ["live-query"] + @test columns.success == [true] + @test columns.match_name == ["foo"] +end + @testset "Modelica Flight services round-trip summary response" begin summary_service = build_parser_flight_service(MODELICA_FILE_SUMMARY_ROUTE) summary_request = parser_exchange_request( diff --git a/test/cases/mod.jl b/test/cases/mod.jl index 87f1365..cb1d714 100644 --- a/test/cases/mod.jl +++ b/test/cases/mod.jl @@ -32,3 +32,4 @@ include("flight_julia_parameter_owner_signatures.jl") include("flight_search_attribute_lists.jl") include("flight_search_attribute_scalars.jl") include("flight_scope_target_columns.jl") +include("service_entrypoint.jl") diff --git a/test/cases/service_entrypoint.jl b/test/cases/service_entrypoint.jl new file mode 100644 index 0000000..8ebc03b --- /dev/null +++ b/test/cases/service_entrypoint.jl @@ -0,0 +1,21 @@ +module ServiceEntrypointHarness +include(joinpath(@__DIR__, "..", "..", "scripts", "run_service.jl")) +end + +@testset "Service entrypoint uses package project" begin + @test ServiceEntrypointHarness.activate_code_parser_project() == + ServiceEntrypointHarness.WENDAOCODEPARSER_ROOT + @test isfile(Base.active_project()) + @test Base.active_project() != + joinpath(ServiceEntrypointHarness.WENDAOCODEPARSER_ROOT, "Project.toml") +end + +@testset "Service entrypoint default config args" begin + entry_args = ServiceEntrypointHarness.service_entry_args(String[]) + @test entry_args[1:2] == [ + "--config", + ServiceEntrypointHarness.DEFAULT_CONFIG_PATH, + ] + @test "--code-parser-route-name" in entry_args || + "--code-parser-route-names" in entry_args +end