diff --git a/news/changelog-1.7.md b/news/changelog-1.7.md index c4941e75be2..27efc6443b0 100644 --- a/news/changelog-1.7.md +++ b/news/changelog-1.7.md @@ -88,6 +88,7 @@ All changes included in 1.7: - ([#12121](https://github.com/quarto-dev/quarto-cli/pull/12121)): Update QuartoNotebookRunner to 0.14.0. Support for evaluating Python cells via [PythonCall.jl](https://github.com/JuliaPy/PythonCall.jl) added. Support for notebook caching via `execute.cache` added. - ([#12151](https://github.com/quarto-dev/quarto-cli/pull/12151)): Basic YAML validation is now active for document using Julia engine. +- ([#11803](https://github.com/quarto-dev/quarto-cli/pull/11803)): Added subcommands `status`, `kill`, `close [--force]` and `log` under the new CLI command `quarto call engine julia`. ### `jupyter` @@ -105,3 +106,4 @@ All changes included in 1.7: - ([#11951](https://github.com/quarto-dev/quarto-cli/issues/11951)): Raw LaTeX table without `tbl-` prefix label for using Quarto crossref are now correctly passed through unmodified. - ([#12117](https://github.com/quarto-dev/quarto-cli/issues/12117)): Color output to stdout and stderr is now correctly rendered for `html` format in the Jupyter and Julia engines. - ([#12264](https://github.com/quarto-dev/quarto-cli/issues/12264)): Upgrade `dart-sass` to 1.85.1. +- ([#11803](https://github.com/quarto-dev/quarto-cli/pull/11803)): Added a new CLI command `quarto call`. First users of this interface are the new `quarto call engine julia ...` subcommands. diff --git a/src/command/call/cmd.ts b/src/command/call/cmd.ts new file mode 100644 index 00000000000..1fd10c50b6a --- /dev/null +++ b/src/command/call/cmd.ts @@ -0,0 +1,10 @@ +import { Command } from "cliffy/command/mod.ts"; +import { engineCommand } from "../../execute/engine.ts"; + +export const callCommand = new Command() + .name("call") + .description("Access functions of Quarto subsystems such as its rendering engines.") + .action(() => { + callCommand.showHelp(); + Deno.exit(1); + }).command("engine", engineCommand); diff --git a/src/command/command.ts b/src/command/command.ts index 478b63dccd4..ab4b69f0117 100644 --- a/src/command/command.ts +++ b/src/command/command.ts @@ -29,6 +29,7 @@ import { addCommand } from "./add/cmd.ts"; import { uninstallCommand } from "./uninstall/cmd.ts"; import { createCommand } from "./create/cmd.ts"; import { editorSupportCommand } from "./editor-support/cmd.ts"; +import { callCommand } from "./call/cmd.ts"; // deno-lint-ignore no-explicit-any export function commands(): Command[] { @@ -57,5 +58,6 @@ export function commands(): Command[] { checkCommand, buildJsCommand, editorSupportCommand, + callCommand, ]; } diff --git a/src/execute/engine.ts b/src/execute/engine.ts index 5a1f7e886bf..9386b542e53 100644 --- a/src/execute/engine.ts +++ b/src/execute/engine.ts @@ -31,6 +31,7 @@ import { pandocBuiltInFormats } from "../core/pandoc/pandoc-formats.ts"; import { gitignoreEntries } from "../project/project-gitignore.ts"; import { juliaEngine } from "./julia.ts"; import { ensureFileInformationCache } from "../project/project-shared.ts"; +import { Command } from "cliffy/command/mod.ts"; const kEngines: Map = new Map(); @@ -276,3 +277,30 @@ export function projectIgnoreGlobs(dir: string) { gitignoreEntries(dir).map((ignore) => `**/${ignore}**`), ); } + +export const engineCommand = new Command() + .name("engine") + .description( + `Access functionality specific to quarto's different rendering engines.`, + ) + .action(() => { + engineCommand.showHelp(); + Deno.exit(1); + }); + +kEngines.forEach((engine, name) => { + if (engine.populateCommand) { + const engineSubcommand = new Command(); + // fill in some default behavior for each engine command + engineSubcommand + .description( + `Access functionality specific to the ${name} rendering engine.`, + ) + .action(() => { + engineSubcommand.showHelp(); + Deno.exit(1); + }); + engine.populateCommand(engineSubcommand); + engineCommand.command(name, engineSubcommand); + } +}); diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 8d8a676eccf..4ed8e7fffcd 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -47,15 +47,14 @@ import { executeResultIncludes, } from "./jupyter/jupyter.ts"; import { isWindows } from "../deno_ral/platform.ts"; +import { Command } from "cliffy/command/mod.ts"; import { isJupyterPercentScript, markdownFromJupyterPercentScript, } from "./jupyter/percent.ts"; export interface JuliaExecuteOptions extends ExecuteOptions { - julia_cmd: string; oneShot: boolean; // if true, the file's worker process is closed before and after running - supervisor_pid?: number; } function isJuliaPercentScript(file: string) { @@ -146,9 +145,7 @@ export const juliaEngine: ExecutionEngine = { }; const juliaExecOptions: JuliaExecuteOptions = { - julia_cmd: Deno.env.get("QUARTO_JULIA") ?? "julia", oneShot: !executeDaemon, - supervisor_pid: options.previewServer ? Deno.pid : undefined, ...execOptions, }; @@ -261,8 +258,14 @@ export const juliaEngine: ExecutionEngine = { }; return Promise.resolve(target); }, + + populateCommand: populateJuliaEngineCommand, }; +function juliaCmd() { + return Deno.env.get("QUARTO_JULIA") ?? "julia"; +} + function powershell_argument_list_to_string(...args: string[]): string { // formats as '"arg 1" "arg 2" "arg 3"' const inner = args.map((arg) => `"${arg}"`).join(" "); @@ -291,7 +294,7 @@ async function startOrReuseJuliaServer( options, `Custom julia project set via QUARTO_JULIA_PROJECT="${juliaProject}". Checking if QuartoNotebookRunner can be loaded.`, ); - const qnrTestCommand = new Deno.Command(options.julia_cmd, { + const qnrTestCommand = new Deno.Command(juliaCmd(), { args: [ "--startup-file=no", `--project=${juliaProject}`, @@ -330,13 +333,14 @@ async function startOrReuseJuliaServer( args: [ "-Command", "Start-Process", - options.julia_cmd, + juliaCmd(), "-ArgumentList", powershell_argument_list_to_string( "--startup-file=no", `--project=${juliaProject}`, resourcePath("julia/quartonotebookrunner.jl"), transportFile, + juliaServerLogFile(), ), "-WindowStyle", "Hidden", @@ -355,14 +359,15 @@ async function startOrReuseJuliaServer( throw new Error(new TextDecoder().decode(result.stderr)); } } else { - const command = new Deno.Command(options.julia_cmd, { + const command = new Deno.Command(juliaCmd(), { args: [ "--startup-file=no", resourcePath("julia/start_quartonotebookrunner_detached.jl"), - options.julia_cmd, + juliaCmd(), juliaProject, resourcePath("julia/quartonotebookrunner.jl"), transportFile, + juliaServerLogFile(), ], env: { "JULIA_LOAD_PATH": "@:@stdlib", // ignore the main env @@ -393,7 +398,7 @@ async function ensureQuartoNotebookRunnerEnvironment( const projectTomlTemplate = juliaResourcePath("Project.toml"); const projectToml = join(juliaRuntimeDir(), "Project.toml"); Deno.copyFileSync(projectTomlTemplate, projectToml); - const command = new Deno.Command(options.julia_cmd, { + const command = new Deno.Command(juliaCmd(), { args: [ "--startup-file=no", `--project=${juliaRuntimeDir()}`, @@ -416,6 +421,9 @@ interface JuliaTransportFile { port: number; pid: number; key: string; + juliaVersion: string; + environment: string; + runnerVersion: string; } async function pollTransportFile( @@ -423,11 +431,11 @@ async function pollTransportFile( ): Promise { const transportFile = juliaTransportFile(); - for (let i = 0; i < 20; i++) { + for (let i = 0; i < 15; i++) { if (existsSync(transportFile)) { - const content = Deno.readTextFileSync(transportFile); + const transportOptions = readTransportFile(transportFile); trace(options, "Transport file read successfully."); - return JSON.parse(content) as JuliaTransportFile; + return transportOptions; } trace(options, "Transport file did not exist, yet."); await sleep(i * 100); @@ -435,11 +443,75 @@ async function pollTransportFile( return Promise.reject(); } +function readTransportFile(transportFile: string): JuliaTransportFile { + // As we know the json file ends with \n but we might accidentally read + // it too quickly once it exists, for example when not the whole string + // has been written to it, yet, we just repeat reading until the string + // ends in a newline. The overhead doesn't matter as the file is so small. + let content = Deno.readTextFileSync(transportFile); + let i = 0; + while (i < 20 && !content.endsWith("\n")) { + sleep(100); + content = Deno.readTextFileSync(transportFile); + i += 1; + } + if (!content.endsWith("\n")) { + throw ("Read invalid transport file that did not end with a newline"); + } + return JSON.parse(content) as JuliaTransportFile; +} + +async function getReadyServerConnection( + transportOptions: JuliaTransportFile, + executeOptions: JuliaExecuteOptions, +) { + const conn = await Deno.connect({ + port: transportOptions.port, + }); + const isready = writeJuliaCommand( + conn, + { type: "isready", content: {} }, + transportOptions.key, + executeOptions, + ); + const timeoutMilliseconds = 10000; + const timeout = new Promise((accept, _) => + setTimeout(() => { + accept( + `Timed out after getting no response for ${timeoutMilliseconds} milliseconds.`, + ); + }, timeoutMilliseconds) + ); + const result = await Promise.race([isready, timeout]); + if (typeof result === "string") { + return result; + } else if (result !== true) { + conn.close(); + return `Expected isready command to return true, returned ${isready} instead. Closing connection.`; + } else { + return conn; + } +} + async function getJuliaServerConnection( options: JuliaExecuteOptions, ): Promise { const { reused } = await startOrReuseJuliaServer(options); - const transportOptions = await pollTransportFile(options); + + let transportOptions: JuliaTransportFile; + try { + transportOptions = await pollTransportFile(options); + } catch (err) { + if (!reused) { + info( + "No transport file was found after the timeout. This is the log from the server process:", + ); + info("#### BEGIN LOG ####"); + printJuliaServerLog(); + info("#### END LOG ####"); + } + throw err; + } if (!reused) { info("Julia server process started."); @@ -451,35 +523,13 @@ async function getJuliaServerConnection( ); try { - const conn = await Deno.connect({ - port: transportOptions.port, - }); - const isready = writeJuliaCommand( - conn, - "isready", - transportOptions.key, - options, - ) as Promise; - const timeoutMilliseconds = 10000; - const timeout = new Promise((accept, _) => - setTimeout(() => { - accept( - `Timed out after getting no response for ${timeoutMilliseconds} milliseconds.`, - ); - }, timeoutMilliseconds) - ); - const result = await Promise.race([isready, timeout]); - if (typeof result === "string") { - // timed out - throw new Error(result); - } else if (result !== true) { - error( - `Expected isready command to return true, returned ${isready} instead. Closing connection.`, - ); - conn.close(); - return Promise.reject(); + const conn = await getReadyServerConnection(transportOptions, options); + if (typeof conn === "string") { + // timed out or otherwise not ready + throw new Error(conn); + } else { + return conn; } - return conn; } catch (e) { if (reused) { trace( @@ -533,20 +583,26 @@ async function executeJulia( ): Promise { const conn = await getJuliaServerConnection(options); const transportOptions = await pollTransportFile(options); + const file = options.target.input; if (options.oneShot || options.format.execute[kExecuteDaemonRestart]) { const isopen = await writeJuliaCommand( conn, - "isopen", + { type: "isopen", content: { file } }, transportOptions.key, options, - ) as boolean; + ); if (isopen) { - await writeJuliaCommand(conn, "close", transportOptions.key, options); + await writeJuliaCommand( + conn, + { type: "close", content: { file } }, + transportOptions.key, + options, + ); } } const response = await writeJuliaCommand( conn, - "run", + { type: "run", content: { file, options } }, transportOptions.key, options, (update: ProgressUpdate) => { @@ -565,14 +621,15 @@ async function executeJulia( ); if (options.oneShot) { - await writeJuliaCommand(conn, "close", transportOptions.key, options); - } - - if (response.error !== undefined) { - throw new Error("Running notebook failed:\n" + response.juliaError); + await writeJuliaCommand( + conn, + { type: "close", content: { file } }, + transportOptions.key, + options, + ); } - return response.notebook as JupyterNotebook; + return response.notebook; } interface ProgressUpdate { @@ -583,26 +640,51 @@ interface ProgressUpdate { line: number; } -async function writeJuliaCommand( +type empty = Record; + +type ServerCommand = + | { type: "run"; content: { file: string; options: JuliaExecuteOptions } } + | { type: "close"; content: { file: string } } + | { type: "forceclose"; content: { file: string } } + | { type: "isopen"; content: { file: string } } + | { type: "stop"; content: empty } + | { type: "isready"; content: empty } + | { type: "status"; content: empty }; + +type ServerCommandResponseMap = { + run: { notebook: JupyterNotebook }; + close: { status: true }; + forceclose: { status: true }; + stop: { message: "Server stopped." }; + isopen: boolean; + isready: true; + status: string; +}; + +type ServerCommandError = { + error: string; + juliaError?: string; +}; + +type ServerCommandResponse = + ServerCommandResponseMap[T]; + +function isProgressUpdate(data: any): data is ProgressUpdate { + return data && data.type === "progress_update"; +} + +function isServerCommandError(data: any): data is ServerCommandError { + return data && typeof (data.error) === "string"; +} + +async function writeJuliaCommand( conn: Deno.Conn, - command: "run" | "close" | "stop" | "isready" | "isopen", + command: Extract, secret: string, options: JuliaExecuteOptions, onProgressUpdate?: (update: ProgressUpdate) => void, -) { - // send the options along with the "run" command - const content = command === "run" - ? { file: options.target.input, options } - : command === "stop" || command === "isready" - ? {} - : options.target.input; - - const commandData = { - type: command, - content, - }; - - const payload = JSON.stringify(commandData); +): Promise> { + const payload = JSON.stringify(command); const key = await crypto.subtle.importKey( "raw", new TextEncoder().encode(secret), @@ -610,9 +692,7 @@ async function writeJuliaCommand( true, ["sign"], ); - const canonicalRequestBytes = new TextEncoder().encode( - JSON.stringify(commandData), - ); + const canonicalRequestBytes = new TextEncoder().encode(payload); const signatureArrayBuffer = await crypto.subtle.sign( "HMAC", key, @@ -625,7 +705,7 @@ async function writeJuliaCommand( const messageBytes = new TextEncoder().encode(message); - trace(options, `write command "${command}" to socket server`); + trace(options, `write command "${command.type}" to socket server`); const bytesWritten = await conn.write(messageBytes); if (bytesWritten !== messageBytes.length) { throw new Error("Internal Error"); @@ -695,28 +775,40 @@ async function writeJuliaCommand( // one command should be sent, ended by a newline, currently just throwing away anything else because we don't // expect multiple commmands at once const json = response.split("\n")[0]; - const data = JSON.parse(json); + const responseData = JSON.parse(json); - if (data.type === "progress_update") { - trace( - options, - "received progress update response, listening for further responses", - ); - if (onProgressUpdate !== undefined) { - onProgressUpdate(data as ProgressUpdate); + if (isServerCommandError(responseData)) { + const data = responseData; + let errorMessage = + `Julia server returned error after receiving "${command.type}" command:\n\n${data.error}`; + + if (data.juliaError) { + errorMessage += + `\n\nThe underlying Julia error was:\n\n${data.juliaError}`; } - continue; // wait for the next message + + throw new Error(errorMessage); } - const err = data.error; - if (err !== undefined) { - const juliaError = data.juliaError ?? "No julia error message available."; - error( - `Julia server returned error after receiving "${command}" command:\n` + - err, - ); - error(juliaError); - throw new Error("Internal julia server error"); + let data: ServerCommandResponse; + if (command.type === "run") { + const data_or_update: ServerCommandResponse | ProgressUpdate = + responseData; + if (isProgressUpdate(data_or_update)) { + const update = data_or_update; + trace( + options, + "received progress update response, listening for further responses", + ); + if (onProgressUpdate !== undefined) { + onProgressUpdate(update); + } + continue; // wait for the next message + } else { + data = data_or_update; + } + } else { + data = responseData; } return data; @@ -741,12 +833,155 @@ function juliaRuntimeDir(): string { } } -function juliaTransportFile() { +export function juliaTransportFile() { return join(juliaRuntimeDir(), "julia_transport.txt"); } +export function juliaServerLogFile() { + return join(juliaRuntimeDir(), "julia_server_log.txt"); +} + function trace(options: ExecuteOptions, msg: string) { - if (options.format.execute[kExecuteDebug]) { + if (options.format?.execute[kExecuteDebug] === true) { info("- " + msg, { bold: true }); } } + +function populateJuliaEngineCommand(command: Command) { + command + .command("status", "Status") + .description( + "Get status information on the currently running Julia server process.", + ).action(logStatus) + .command("kill", "Kill server") + .description("Kill the control server if it is currently running. This will also kill all notebook worker processes.") + .action(killJuliaServer) + .command("log", "Print julia server log") + .description( + "Print the content of the julia server log file if it exists which can be used to diagnose problems.", + ) + .action(printJuliaServerLog) + .command( + "close", + "Close the worker for a given notebook. If it is currently running, it will not be interrupted.", + ) + .arguments("") + .option( + "-f, --force", + "Force closing. This will terminate the worker if it is running.", + { default: false }, + ) + .action(async (options, file) => { + await closeWorker(file, options.force); + }) + .command("stop", "Stop the server") + .description( + "Send a message to the server that it should close all notebooks and exit. This will fail if any notebooks are not idle.", + ) + .action(stopServer); + return; +} + +async function logStatus() { + const transportFile = juliaTransportFile(); + if (!existsSync(transportFile)) { + info("Julia control server is not running."); + return; + } + const transportOptions = readTransportFile(transportFile); + + const conn = await getReadyServerConnection( + transportOptions, + {} as JuliaExecuteOptions, + ); + const successfullyConnected = typeof conn !== "string"; + + if (successfullyConnected) { + const status: string = await writeJuliaCommand( + conn, + { type: "status", content: {} }, + transportOptions.key, + {} as JuliaExecuteOptions, + ); + + Deno.stdout.writeSync((new TextEncoder()).encode(status)); + + conn.close(); + } else { + info(`Found transport file but can't connect to control server.`); + } +} + +function killJuliaServer() { + const transportFile = juliaTransportFile(); + if (!existsSync(transportFile)) { + info("Julia control server is not running."); + return; + } + const transportOptions = readTransportFile(transportFile); + Deno.kill(transportOptions.pid, "SIGTERM"); + info("Sent SIGTERM to server process"); +} + +function printJuliaServerLog() { + if (existsSync(juliaServerLogFile())) { + Deno.stdout.writeSync(Deno.readFileSync(juliaServerLogFile())); + } else { + info("Server log file doesn't exist"); + } + return; +} + +// todo: this could use a refactor with the other functions that make +// server connections or execute commands, this one is just supposed to +// simplify the pattern where a running server is expected (there will be an error if there is none) +// and we want to get the API response out quickly +async function connectAndWriteJuliaCommandToRunningServer< + T extends ServerCommand["type"], +>( + command: Extract, +): Promise> { + const transportFile = juliaTransportFile(); + if (!existsSync(transportFile)) { + throw new Error("Julia control server is not running."); + } + const transportOptions = readTransportFile(transportFile); + + const conn = await getReadyServerConnection( + transportOptions, + {} as JuliaExecuteOptions, + ); + const successfullyConnected = typeof conn !== "string"; + + if (successfullyConnected) { + const result = await writeJuliaCommand( + conn, + command, + transportOptions.key, + {} as JuliaExecuteOptions, + ); + conn.close(); + return result; + } else { + throw new Error( + `Found transport file but can't connect to control server.`, + ); + } +} + +async function closeWorker(file: string, force: boolean) { + const absfile = normalizePath(file); + await connectAndWriteJuliaCommandToRunningServer({ + type: force ? "forceclose" : "close", + content: { file: absfile }, + }); + info(`Worker ${force ? "force-" : ""}closed successfully.`); +} + +async function stopServer() { + const result = await connectAndWriteJuliaCommandToRunningServer({ + type: "stop", + content: {}, + }); + info(result.message); +} diff --git a/src/execute/types.ts b/src/execute/types.ts index bb85ca46a70..1f57a02c15e 100644 --- a/src/execute/types.ts +++ b/src/execute/types.ts @@ -15,6 +15,7 @@ import { RenderOptions, RenderResultFile } from "../command/render/types.ts"; import { MappedString } from "../core/lib/text-types.ts"; import { HandlerContextResults } from "../core/handlers/types.ts"; import { ProjectContext } from "../project/types.ts"; +import { Command } from "cliffy/command/mod.ts"; export const kQmdExtensions = [".qmd"]; @@ -61,6 +62,7 @@ export interface ExecutionEngine { file: RenderResultFile, project?: ProjectContext, ) => Promise; + populateCommand?: (command: Command) => void; } // execution target (filename and context 'cookie') diff --git a/src/quarto.ts b/src/quarto.ts index a0c61b831c1..35698339837 100644 --- a/src/quarto.ts +++ b/src/quarto.ts @@ -51,6 +51,7 @@ import "./format/imports.ts"; import { kCliffyImplicitCwd } from "./config/constants.ts"; import { mainRunner } from "./core/main.ts"; +import { engineCommand } from "./execute/engine.ts"; const checkVersionRequirement = () => { const versionReq = Deno.env.get("QUARTO_VERSION_REQUIREMENT"); diff --git a/src/resources/julia/Project.toml b/src/resources/julia/Project.toml index 65d2fd8a72b..6b96f044ad1 100644 --- a/src/resources/julia/Project.toml +++ b/src/resources/julia/Project.toml @@ -2,4 +2,4 @@ QuartoNotebookRunner = "4c0109c6-14e9-4c88-93f0-2b974d3468f4" [compat] -QuartoNotebookRunner = "=0.14.0" +QuartoNotebookRunner = "=0.15.0" diff --git a/src/resources/julia/quartonotebookrunner.jl b/src/resources/julia/quartonotebookrunner.jl index ee36de4138e..d2a394da981 100644 --- a/src/resources/julia/quartonotebookrunner.jl +++ b/src/resources/julia/quartonotebookrunner.jl @@ -1,7 +1,43 @@ +logfile = ARGS[2] +filehandle = open(logfile, "w") + +# We cannot start Julia in a way such that it uses line buffering or no buffering (see https://github.com/JuliaLang/julia/issues/13050) +# which means that if we just redirect all its output into a logfile (for example with `pipeline(julia_cmd, stdout = logfile)`) +# it will not actually write to the logfile until the buffer is filled or it's explicitly flushed, which can take a while. +# So when we check the logfile, we don't actually see anything if the buffer is not filled first, which +# may never be if there's little output. So instead we redirect stdout and stderr to a pipe which we manually check +# for available data in shorter intervals. We then write the data to the logfile and flush. + +pipe = Pipe() +pipe_with_color = IOContext(pipe, :color => Base.get_have_color()) +redirect_stdout(pipe_with_color) +redirect_stderr(pipe_with_color) + +function update_logfile() + data = readavailable(pipe) + if !isempty(data) + write(filehandle, data) + flush(filehandle) + end + return +end + +@async while true + update_logfile() + sleep(1) +end + +# we might lose printout from crashes if we don't do another update at the end +atexit() do + update_logfile() +end + +using Dates: now +@info "Log started at $(now())" + using QuartoNotebookRunner using Sockets - transport_file = ARGS[1] transport_dir = dirname(transport_file) @@ -16,4 +52,6 @@ open(transport_file, "w") do io println(io, """{"port": $port, "pid": $(Base.Libc.getpid()), "key": "$(server.key)"}""") end +@info "Starting server at $(now())" wait(server) +@info "Server stopped at $(now())" diff --git a/src/resources/julia/start_quartonotebookrunner_detached.jl b/src/resources/julia/start_quartonotebookrunner_detached.jl index dd5be56781d..3771ca52322 100644 --- a/src/resources/julia/start_quartonotebookrunner_detached.jl +++ b/src/resources/julia/start_quartonotebookrunner_detached.jl @@ -4,12 +4,14 @@ julia_bin = ARGS[1] project = ARGS[2] julia_file = ARGS[3] transport_file = ARGS[4] +logfile = ARGS[5] -if length(ARGS) > 4 +if length(ARGS) > 5 error("Too many arguments") end env = copy(ENV) env["JULIA_LOAD_PATH"] = "@:@stdlib" # ignore the main env -cmd = `$julia_bin --startup-file=no --project=$project $julia_file $transport_file` -run(detach(setenv(cmd, env)), wait = false) +cmd = `$julia_bin --startup-file=no --project=$project $julia_file $transport_file $logfile` +cmd = setenv(cmd, env) +run(detach(cmd), wait = false) diff --git a/tests/docs/call/engine/julia/sleep.qmd b/tests/docs/call/engine/julia/sleep.qmd new file mode 100644 index 00000000000..6e8011d326b --- /dev/null +++ b/tests/docs/call/engine/julia/sleep.qmd @@ -0,0 +1,9 @@ +--- +engine: julia +params: + sleep_duration: 0 +--- + +```{julia} +sleep(sleep_duration) +``` \ No newline at end of file diff --git a/tests/smoke/call/engine/julia/julia.test.ts b/tests/smoke/call/engine/julia/julia.test.ts new file mode 100644 index 00000000000..330b5f9e23b --- /dev/null +++ b/tests/smoke/call/engine/julia/julia.test.ts @@ -0,0 +1,166 @@ +import { assert, assertStringIncludes } from "testing/asserts"; +import { docs, quartoDevCmd } from "../../../../utils.ts"; +import { existsSync } from "fs/exists"; +import { juliaServerLogFile, juliaTransportFile } from "../../../../../src/execute/julia.ts"; +import { sleep } from "../../../../../src/core/wait.ts"; + +const sleepQmd = docs("call/engine/julia/sleep.qmd"); +assert(existsSync(sleepQmd)); + +function assertSuccess(output: Deno.CommandOutput) { + if (!output.success) { + console.error("Command failed:"); + console.error("stdout:\n" + new TextDecoder().decode(output.stdout)); + console.error("stderr:\n" + new TextDecoder().decode(output.stderr)); + throw new Error("Command execution was not successful"); + } +} + +function assertStdoutIncludes(output: Deno.CommandOutput, str: string) { + assertStringIncludes(new TextDecoder().decode(output.stdout), str); +} +function assertStderrIncludes(output: Deno.CommandOutput, str: string) { + assertStringIncludes(new TextDecoder().decode(output.stderr), str); +} + +// make sure we don't have a server process running by sending a kill command +// and then also try to remove the transport file in case one still exists +const killcmd = new Deno.Command( + quartoDevCmd(), + {args: ["call", "engine", "julia", "kill"]} +).outputSync(); +assertSuccess(killcmd); +try { + await Deno.remove(juliaTransportFile()); +} catch { +} + +Deno.test("kill without server running", () => { + const output = new Deno.Command( + quartoDevCmd(), + {args: ["call", "engine", "julia", "kill"]} + ).outputSync(); + assertSuccess(output); + assertStderrIncludes(output, "Julia control server is not running."); +}); + +Deno.test("status without server running", () => { + const output = new Deno.Command( + quartoDevCmd(), + {args: ["call", "engine", "julia", "status"]} + ).outputSync(); + assertSuccess(output); + assertStderrIncludes(output, "Julia control server is not running."); +}); + +try { + await Deno.remove(juliaServerLogFile()); +} catch { +} + +Deno.test("log file doesn't exist", () => { + const log_output = new Deno.Command( + quartoDevCmd(), + {args: ["call", "engine", "julia", "log"]} + ).outputSync(); + assertSuccess(log_output); + assertStderrIncludes(log_output, "Server log file doesn't exist"); +}); + +Deno.test("status with server and worker running", () => { + const render_output = new Deno.Command( + quartoDevCmd(), + {args: ["render", sleepQmd, "-P", "sleep_duration:0", "--execute-daemon", "60"]} + ).outputSync(); + assertSuccess(render_output); + + const status_output = new Deno.Command( + quartoDevCmd(), + {args: ["call", "engine", "julia", "status"]} + ).outputSync(); + assertSuccess(status_output); + assertStdoutIncludes(status_output, "workers active: 1"); +}); + +Deno.test("closing an idling worker", () => { + const close_output = new Deno.Command( + quartoDevCmd(), + {args: ["call", "engine", "julia", "close", sleepQmd]} + ).outputSync(); + assertSuccess(close_output); + assertStderrIncludes(close_output, "Worker closed successfully"); + + const status_output = new Deno.Command( + quartoDevCmd(), + {args: ["call", "engine", "julia", "status"]} + ).outputSync(); + assertSuccess(status_output); + assertStdoutIncludes(status_output, "workers active: 0"); +}); + +Deno.test("force-closing a running worker", async () => { + // spawn a long-running command + const render_cmd = new Deno.Command( + quartoDevCmd(), + {args: ["render", sleepQmd, "-P", "sleep_duration:30"]} + ).output(); + + await sleep(3000); + + const close_output = new Deno.Command( + quartoDevCmd(), + {args: ["call", "engine", "julia", "close", sleepQmd]} + ).outputSync(); + assertStderrIncludes(close_output, "worker is busy"); + + const status_output = new Deno.Command( + quartoDevCmd(), + {args: ["call", "engine", "julia", "status"]} + ).outputSync(); + assertSuccess(status_output); + assertStdoutIncludes(status_output, "workers active: 1"); + + const force_close_output = new Deno.Command( + quartoDevCmd(), + {args: ["call", "engine", "julia", "close", "--force", sleepQmd]} + ).outputSync(); + assertSuccess(force_close_output); + assertStderrIncludes(force_close_output, "Worker force-closed successfully"); + + const status_output_2 = new Deno.Command( + quartoDevCmd(), + {args: ["call", "engine", "julia", "status"]} + ).outputSync(); + assertSuccess(status_output_2); + assertStdoutIncludes(status_output_2, "workers active: 0"); + + const render_output = await render_cmd; + assertStderrIncludes(render_output, "File was force-closed during run") +}); + +Deno.test("log exists", () => { + const log_output = new Deno.Command( + quartoDevCmd(), + {args: ["call", "engine", "julia", "log"]} + ).outputSync(); + assertSuccess(log_output); + assertStdoutIncludes(log_output, "Log started at"); +}); + +Deno.test("stop the idling server", async () => { + const stop_output = new Deno.Command( + quartoDevCmd(), + {args: ["call", "engine", "julia", "stop"]} + ).outputSync(); + assertSuccess(stop_output); + assertStderrIncludes(stop_output, "Server stopped"); + + await sleep(2000); // allow a little bit of time for the server to stop and the log message to be written + + const log_output = new Deno.Command( + quartoDevCmd(), + {args: ["call", "engine", "julia", "log"]} + ).outputSync(); + assertSuccess(log_output); + assertStdoutIncludes(log_output, "Server stopped"); +});