From b396ef84a13d82b1c2a362dae0a285c311b31142 Mon Sep 17 00:00:00 2001 From: Charro Gruver Date: Thu, 13 Feb 2025 18:45:03 +0000 Subject: [PATCH 1/3] Add run and serve arguments for --device and --privileged Signed-off-by: Charro Gruver --- ramalama/cli.py | 18 +++++++++++++++++- ramalama/model.py | 12 +++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/ramalama/cli.py b/ramalama/cli.py index 58677046..c7f96c0d 100644 --- a/ramalama/cli.py +++ b/ramalama/cli.py @@ -233,6 +233,12 @@ def configure_arguments(parser): help="store AI Models in the specified directory", ) parser.add_argument("-v", "--version", dest="version", action="store_true", help="show RamaLama version") + # parser.add_argument("--device", + # dest="device", + # action='append', + # type=str, + # # nargs=1, + # help="Device to leak in to the running container") def configure_subcommands(parser): @@ -801,6 +807,11 @@ def _run(parser): default=config.get('ctx_size', 2048), help="size of the prompt context (0 = loaded from model)", ) + parser.add_argument("--device", + dest="device", + action='append', + type=str, + help="Device to leak in to the running container") parser.add_argument("-n", "--name", dest="name", help="name of container in which the Model will be run") # Disable network access by default, and give the option to pass any supported network mode into # podman if needed: @@ -811,7 +822,12 @@ def _run(parser): default="none", help="set the network mode for the container", ) - + parser.add_argument( + "--privileged", + dest="privileged", + action="store_true", + help="give extended privileges to container" + ) parser.add_argument("--seed", help="override random seed") parser.add_argument( "--temp", default=config.get('temp', "0.8"), help="temperature of the response from the AI model" diff --git a/ramalama/model.py b/ramalama/model.py index 20035e42..aafcd55a 100644 --- a/ramalama/model.py +++ b/ramalama/model.py @@ -187,13 +187,11 @@ def setup_container(self, args): if hasattr(args, "port"): conman_args += ["-p", f"{args.port}:{args.port}"] - # Check for env var RAMALAMA_DEVICE to explicitly declare the GPU device path - device_override = 0 - gpu_device = os.environ.get("RAMALAMA_DEVICE") - if gpu_device: - conman_args += ["--device", gpu_device] - device_override = 1 - if device_override != 1: + if args.device: + for device_arg in args.device: + print(device_arg) + conman_args += ["--device", device_arg] + else: if (sys.platform == "darwin" and os.path.basename(args.engine) != "docker") or os.path.exists("/dev/dri"): conman_args += ["--device", "/dev/dri"] From 483f2296779b17e4f78d86acb4baa352db6c088e Mon Sep 17 00:00:00 2001 From: Charro Gruver Date: Thu, 13 Feb 2025 19:03:58 +0000 Subject: [PATCH 2/3] Add --privileged and --device args to the appropriate docs Signed-off-by: Charro Gruver --- docs/ramalama-run.1.md | 6 ++++++ docs/ramalama-serve.1.md | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/docs/ramalama-run.1.md b/docs/ramalama-run.1.md index 09d1c56a..b1831c70 100644 --- a/docs/ramalama-run.1.md +++ b/docs/ramalama-run.1.md @@ -31,6 +31,9 @@ path of the authentication file for OCI registries #### **--ctx-size**, **-c** size of the prompt context (default: 2048, 0 = loaded from model) +#### **--device** +declare host device to leak into the container + #### **--help**, **-h** show this help message and exit @@ -43,6 +46,9 @@ name of the container to run the Model in #### **--network**=*none* set the network mode for the container +#### **--privileged** +give extended privileges to container + #### **--seed**= Specify seed rather than using random seed model interaction diff --git a/docs/ramalama-serve.1.md b/docs/ramalama-serve.1.md index 837fdad5..8a391685 100644 --- a/docs/ramalama-serve.1.md +++ b/docs/ramalama-serve.1.md @@ -46,6 +46,9 @@ The default is TRUE. The --nocontainer option forces this option to False. Use the `ramalama stop` command to stop the container running the served ramalama Model. +#### **--device** +declare host device to leak into the container + #### **--generate**=type Generate specified configuration format for running the AI Model as a service @@ -70,6 +73,9 @@ set the network mode for the container #### **--port**, **-p** port for AI Model server to listen on +#### **--privileged** +give extended privileges to container + #### **--seed**= Specify seed rather than using random seed model interaction From 07bc34132b7b7bacfb3feb0af41e7b26fbeda6f1 Mon Sep 17 00:00:00 2001 From: Charro Gruver Date: Thu, 13 Feb 2025 19:54:15 +0000 Subject: [PATCH 3/3] Changes to satisfy code review Signed-off-by: Charro Gruver --- docs/ramalama-run.1.md | 25 +++++++++++++++++++++++-- docs/ramalama-serve.1.md | 25 +++++++++++++++++++++++-- ramalama/cli.py | 6 ------ ramalama/model.py | 1 - 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/docs/ramalama-run.1.md b/docs/ramalama-run.1.md index b1831c70..d845298a 100644 --- a/docs/ramalama-run.1.md +++ b/docs/ramalama-run.1.md @@ -32,7 +32,13 @@ path of the authentication file for OCI registries size of the prompt context (default: 2048, 0 = loaded from model) #### **--device** -declare host device to leak into the container +Add a host device to the container. Optional permissions parameter can +be used to specify device permissions by combining r for read, w for +write, and m for mknod(2). + +Example: --device=/dev/dri/renderD128:/dev/xvdc:rwm + +The device specifiaction is passed directly to the underlying container engine. See documentation of the supported container engine for more information. #### **--help**, **-h** show this help message and exit @@ -47,7 +53,22 @@ name of the container to run the Model in set the network mode for the container #### **--privileged** -give extended privileges to container +By default, RamaLama containers are unprivileged (=false) and cannot, for +example, modify parts of the operating system. This is because by de‐ +fault a container is only allowed limited access to devices. A "privi‐ +leged" container is given the same access to devices as the user launch‐ +ing the container, with the exception of virtual consoles (/dev/tty\d+) +when running in systemd mode (--systemd=always). + +A privileged container turns off the security features that isolate the +container from the host. Dropped Capabilities, limited devices, read- +only mount points, Apparmor/SELinux separation, and Seccomp filters are +all disabled. Due to the disabled security features, the privileged +field should almost never be set as containers can easily break out of +confinement. + +Containers running in a user namespace (e.g., rootless containers) can‐ +not have more privileges than the user that launched them. #### **--seed**= Specify seed rather than using random seed model interaction diff --git a/docs/ramalama-serve.1.md b/docs/ramalama-serve.1.md index 8a391685..a33074a6 100644 --- a/docs/ramalama-serve.1.md +++ b/docs/ramalama-serve.1.md @@ -47,7 +47,13 @@ The default is TRUE. The --nocontainer option forces this option to False. Use the `ramalama stop` command to stop the container running the served ramalama Model. #### **--device** -declare host device to leak into the container +Add a host device to the container. Optional permissions parameter can +be used to specify device permissions by combining r for read, w for +write, and m for mknod(2). + +Example: --device=/dev/dri/renderD128:/dev/xvdc:rwm + +The device specifiaction is passed directly to the underlying container engine. See documentation of the supported container engine for more information. #### **--generate**=type Generate specified configuration format for running the AI Model as a service @@ -74,7 +80,22 @@ set the network mode for the container port for AI Model server to listen on #### **--privileged** -give extended privileges to container +By default, RamaLama containers are unprivileged (=false) and cannot, for +example, modify parts of the operating system. This is because by de‐ +fault a container is only allowed limited access to devices. A "privi‐ +leged" container is given the same access to devices as the user launch‐ +ing the container, with the exception of virtual consoles (/dev/tty\d+) +when running in systemd mode (--systemd=always). + +A privileged container turns off the security features that isolate the +container from the host. Dropped Capabilities, limited devices, read- +only mount points, Apparmor/SELinux separation, and Seccomp filters are +all disabled. Due to the disabled security features, the privileged +field should almost never be set as containers can easily break out of +confinement. + +Containers running in a user namespace (e.g., rootless containers) can‐ +not have more privileges than the user that launched them. #### **--seed**= Specify seed rather than using random seed model interaction diff --git a/ramalama/cli.py b/ramalama/cli.py index c7f96c0d..eed72ceb 100644 --- a/ramalama/cli.py +++ b/ramalama/cli.py @@ -233,12 +233,6 @@ def configure_arguments(parser): help="store AI Models in the specified directory", ) parser.add_argument("-v", "--version", dest="version", action="store_true", help="show RamaLama version") - # parser.add_argument("--device", - # dest="device", - # action='append', - # type=str, - # # nargs=1, - # help="Device to leak in to the running container") def configure_subcommands(parser): diff --git a/ramalama/model.py b/ramalama/model.py index aafcd55a..73e96978 100644 --- a/ramalama/model.py +++ b/ramalama/model.py @@ -189,7 +189,6 @@ def setup_container(self, args): if args.device: for device_arg in args.device: - print(device_arg) conman_args += ["--device", device_arg] else: if (sys.platform == "darwin" and os.path.basename(args.engine) != "docker") or os.path.exists("/dev/dri"):