-
Notifications
You must be signed in to change notification settings - Fork 2.7k
[stdlib] Adds functionality to spawn and manage processes from exec. file #3998
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
7ece193
446d8a9
ae307a9
6c1c302
2bf983b
63d5dc0
08acf1c
2d96bb9
d78ccb6
b4d0c02
1610ce3
48f65d7
bb0b684
2491143
cf9015e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -44,3 +44,4 @@ from .os import ( | |
| unlink, | ||
| ) | ||
| from .pathlike import PathLike | ||
| from .process import Process, Pipe | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,290 @@ | ||
| # ===----------------------------------------------------------------------=== # | ||
| # Copyright (c) 2025, Modular Inc. All rights reserved. | ||
| # | ||
| # Licensed under the Apache License v2.0 with LLVM Exceptions: | ||
| # https://llvm.org/LICENSE.txt | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # ===----------------------------------------------------------------------=== # | ||
| """Implements os methods for dealing with processes. | ||
|
|
||
| Example: | ||
|
|
||
| ```mojo | ||
| from os import Process | ||
| ``` | ||
| """ | ||
| from collections import List, Optional | ||
| from collections.string import StringSlice | ||
|
|
||
| from sys import CompilationTarget | ||
| from sys._libc import ( | ||
| vfork, | ||
| execvp, | ||
| exit, | ||
| kill, | ||
| SignalCodes, | ||
| pipe, | ||
| fcntl, | ||
| FcntlCommands, | ||
| FcntlFDFlags, | ||
| close, | ||
| ) | ||
| from sys.ffi import c_char, c_int | ||
| from sys.os import sep | ||
|
|
||
|
|
||
| # ===----------------------------------------------------------------------=== # | ||
| # Process comm. | ||
| # ===----------------------------------------------------------------------=== # | ||
| struct Pipe: | ||
| """Create a pipe for interprocess communication. | ||
|
|
||
| Example usage: | ||
| ``` | ||
| pipe().write_bytes("TEST".as_bytes()) | ||
| ``` | ||
| """ | ||
|
|
||
| var fd_in: Optional[FileDescriptor] | ||
| """File descriptor for pipe input.""" | ||
| var fd_out: Optional[FileDescriptor] | ||
| """File descriptor for pipe output.""" | ||
|
|
||
| fn __init__( | ||
| out self, | ||
| in_close_on_exec: Bool = False, | ||
| out_close_on_exec: Bool = False, | ||
| ) raises: | ||
| """Struct to manage interprocess pipe comms. | ||
|
|
||
| Args: | ||
| in_close_on_exec: Close the read side of pipe if `exec` sys. call is issued in process. | ||
| out_close_on_exec: Close the write side of pipe if `exec` sys. call is issued in process. | ||
| """ | ||
| var pipe_fds = UnsafePointer[c_int].alloc(2) | ||
| if pipe(pipe_fds) < 0: | ||
| pipe_fds.free() | ||
| raise Error("Failed to create pipe") | ||
|
|
||
| if in_close_on_exec: | ||
| if not self._set_close_on_exec(pipe_fds[0]): | ||
| pipe_fds.free() | ||
| raise Error("Failed to configure input pipe close on exec") | ||
|
|
||
| if out_close_on_exec: | ||
| if not self._set_close_on_exec(pipe_fds[1]): | ||
| pipe_fds.free() | ||
| raise Error("Failed to configure output pipe close on exec") | ||
|
|
||
| self.fd_in = FileDescriptor(Int(pipe_fds[0])) | ||
| self.fd_out = FileDescriptor(Int(pipe_fds[1])) | ||
| pipe_fds.free() | ||
|
|
||
| fn __del__(deinit self): | ||
| """Ensures pipes input and output file descriptors are closed, when the object is destroyed. | ||
| """ | ||
| self.set_input_only() | ||
| self.set_output_only() | ||
|
|
||
| @staticmethod | ||
| fn _set_close_on_exec(fd: c_int) -> Bool: | ||
| return ( | ||
| fcntl( | ||
| fd, | ||
| FcntlCommands.F_SETFD, | ||
| fcntl(fd, FcntlCommands.F_GETFD, 0) | FcntlFDFlags.FD_CLOEXEC, | ||
| ) | ||
| == 0 | ||
| ) | ||
|
|
||
| @always_inline | ||
| fn set_input_only(mut self): | ||
| """Close the output descriptor/ channel for this side of the pipe.""" | ||
| if self.fd_out: | ||
| _ = close(rebind[Int](self.fd_out.value())) | ||
| self.fd_out = None | ||
|
|
||
| @always_inline | ||
| fn set_output_only(mut self): | ||
| """Close the input descriptor/ channel for this side of the pipe.""" | ||
| if self.fd_in: | ||
| _ = close(rebind[Int](self.fd_in.value())) | ||
| self.fd_in = None | ||
|
|
||
| @always_inline | ||
| fn write_bytes(mut self, bytes: Span[Byte, _]) raises: | ||
| """ | ||
| Write a span of bytes to the pipe. | ||
|
|
||
| Args: | ||
| bytes: The byte span to write to this pipe. | ||
|
|
||
| """ | ||
| if self.fd_out: | ||
| self.fd_out.value().write_bytes(bytes) | ||
| else: | ||
| raise Error("Can not write from read only side of pipe") | ||
|
|
||
| @always_inline | ||
| fn read_bytes(mut self, mut buffer: Span[mut=True, Byte]) raises -> UInt: | ||
| """ | ||
| Read a number of bytes from this pipe. | ||
|
|
||
| Args: | ||
| buffer: Span[Byte] of length n where to store read bytes. n = number of bytes to read. | ||
|
|
||
| Returns: | ||
| Actual number of bytes read. | ||
| """ | ||
| if self.fd_in: | ||
| return self.fd_in.value().read_bytes(buffer) | ||
|
|
||
| raise Error("Can not read from write only side of pipe") | ||
|
|
||
|
|
||
| # ===----------------------------------------------------------------------=== # | ||
| # Process execution | ||
| # ===----------------------------------------------------------------------=== # | ||
|
|
||
| alias ERR_STR_LEN = 8 | ||
|
|
||
|
|
||
| struct Process: | ||
| """Create and manage child processes from file executables. | ||
|
|
||
| Example usage: | ||
| ``` | ||
| child_process = Process.run("ls", List[String]("-lha")) | ||
| if child_process.interrupt(): | ||
| print("Successfully interrupted.") | ||
| ``` | ||
| """ | ||
|
|
||
| var child_pid: c_int | ||
| """Child process id.""" | ||
|
|
||
| fn __init__(out self, child_pid: c_int): | ||
| """Struct to manage metadata about child process. | ||
| Use the `run` static method to create new process. | ||
|
|
||
| Args: | ||
| child_pid: The pid of child processed returned by `vfork` that the struct will manage. | ||
| """ | ||
|
|
||
| self.child_pid = child_pid | ||
|
|
||
| fn _kill(self, signal: Int) -> Bool: | ||
| # `kill` returns 0 on success and -1 on failure | ||
| return kill(self.child_pid, signal) > -1 | ||
|
|
||
| fn hangup(self) -> Bool: | ||
| """Send the Hang up signal to the managed child process. | ||
|
|
||
| Returns: | ||
| Upon successful completion, True is returned else False. | ||
| """ | ||
| return self._kill(SignalCodes.HUP) | ||
|
|
||
| fn interrupt(self) -> Bool: | ||
| """Send the Interrupt signal to the managed child process. | ||
|
|
||
| Returns: | ||
| Upon successful completion, True is returned else False. | ||
| """ | ||
| return self._kill(SignalCodes.INT) | ||
|
|
||
| fn kill(self) -> Bool: | ||
| """Send the Kill signal to the managed child process. | ||
|
|
||
| Returns: | ||
| Upon successful completion, True is returned else False. | ||
| """ | ||
| return self._kill(SignalCodes.KILL) | ||
|
|
||
| @staticmethod | ||
| fn run(var path: String, argv: List[String]) raises -> Process: | ||
| """Spawn new process from file executable. | ||
|
|
||
| Args: | ||
| path: The path to the file. | ||
| argv: A list of string arguments to be passed to executable. | ||
|
|
||
| Returns: | ||
| An instance of `Process` struct. | ||
| """ | ||
|
|
||
| @parameter | ||
| if CompilationTarget.is_linux() or CompilationTarget.is_macos(): | ||
| var file_name = String(path.split(sep)[-1]) | ||
| var pipe = Pipe(out_close_on_exec=True) | ||
| var exec_err_code = StaticString("EXEC_ERR") | ||
|
|
||
| var pid = vfork() | ||
|
|
||
| if pid == 0: | ||
| # Child process. | ||
| pipe.set_output_only() | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nb: this is only OK because pipe is on the stack. Would prefer we just close the fd and not write onto the stack. |
||
|
|
||
| var arg_count = len(argv) | ||
| var argv_array_ptr_cstr_ptr = UnsafePointer[ | ||
| UnsafePointer[c_char, mut=False] | ||
| ].alloc(arg_count + 2) | ||
| var offset = 0 | ||
| # Arg 0 in `argv` ptr array should be the file name | ||
| argv_array_ptr_cstr_ptr[offset] = file_name.unsafe_cstr_ptr() | ||
| offset += 1 | ||
|
|
||
| for var arg in argv: | ||
izo0x90 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| argv_array_ptr_cstr_ptr[offset] = arg.unsafe_cstr_ptr() | ||
| offset += 1 | ||
|
|
||
| # `argv` ptr array terminates with NULL PTR | ||
| argv_array_ptr_cstr_ptr[offset] = UnsafePointer[c_char]() | ||
|
|
||
| var path_cptr = path.unsafe_cstr_ptr() | ||
|
|
||
| _ = execvp(path_cptr, argv_array_ptr_cstr_ptr) | ||
|
|
||
| # This will only get reached if exec call fails to replace currently executing code | ||
| argv_array_ptr_cstr_ptr.free() | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yes, and because of that I'm pretty sure the allocation leaks in the parent process. vfork doesn't change the page-table mapping, so the allocation is in BOTH processes and it's freed ONLY in the child by virtue of the exec wiping everything out. |
||
|
|
||
| # Canonical fork/ exec error handling pattern of using a pipe that closes on exec is | ||
| # used to signal error to parent process `https://cr.yp.to/docs/selfpipe.html` | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not sure about macOS but on linux you can use
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. and this is another reason to just call |
||
| pipe.write_bytes(exec_err_code.as_bytes()) | ||
|
|
||
| exit(1) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think this will call the parent's atexit handlers and flush the parents io buffers |
||
|
|
||
| elif pid < 0: | ||
| raise Error("Unable to fork parent") | ||
|
|
||
| var err: Optional[StringSlice[MutableAnyOrigin]] = None | ||
| var err_buff_data = InlineArray[Byte, ERR_STR_LEN](fill=0) | ||
|
|
||
| try: | ||
| pipe.set_input_only() | ||
| var buf = Span[Byte, MutableAnyOrigin]( | ||
| ptr=err_buff_data.unsafe_ptr(), length=ERR_STR_LEN | ||
| ) | ||
| var bytes_read = pipe.read_bytes(buf) | ||
| err = StringSlice(unsafe_from_utf8=buf) | ||
| except e: | ||
| raise Error( | ||
| "Failed to read child process response from pipe, exception" | ||
| " was: " | ||
| + String(e) | ||
| ) | ||
|
|
||
| if err and len(err.value()) > 0 and err.value() == exec_err_code: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. does the child need to be reaped in this case? |
||
| raise Error("Failed to execute " + path) | ||
|
|
||
| return Process(child_pid=pid) | ||
| else: | ||
| constrained[ | ||
| False, "Unknown platform process execution not implemented" | ||
| ]() | ||
| return abort[Process]() | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| # ===----------------------------------------------------------------------=== # | ||
| # Copyright (c) 2025, Modular Inc. All rights reserved. | ||
| # | ||
| # Licensed under the Apache License v2.0 with LLVM Exceptions: | ||
| # https://llvm.org/LICENSE.txt | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # ===----------------------------------------------------------------------=== # | ||
| # RUN: %mojo %s | FileCheck %s | ||
|
|
||
| from collections import List | ||
| from os.path import exists | ||
| from os import Process | ||
|
|
||
| from testing import assert_false, assert_raises | ||
|
|
||
|
|
||
| def test_process_run(): | ||
| var command = "echo" | ||
| _ = Process.run(command, List[String]("== TEST_ECHO")) | ||
|
|
||
|
|
||
| def test_process_run_missing(): | ||
| missing_executable_file = "ThIsFiLeCoUlDNoTPoSsIbLlYExIsT.NoTAnExTeNsIoN" | ||
|
|
||
| # verify that the test file does not exist before starting the test | ||
| assert_false( | ||
| exists(missing_executable_file), | ||
| "Unexpected file '" + missing_executable_file + "' it should not exist", | ||
| ) | ||
|
|
||
| with assert_raises(): | ||
| _ = Process.run(missing_executable_file, List[String]()) | ||
|
|
||
|
|
||
| # CHECK-LABEL: TEST_ECHO | ||
| def main(): | ||
| test_process_run() | ||
| test_process_run_missing() |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
IMO the default should be true.