Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 38 additions & 5 deletions misc/python/materialize/mzcompose/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,39 @@ class ServiceDependency(TypedDict, total=False):
"""Condition under which a dependency is considered satisfied."""


class ServiceDeployResourceLimits(TypedDict, total=False):
"""Resource limits for a service deployment."""

memory: str
"""Memory limit (e.g., '6G', '512M')."""

cpus: str
"""CPU limit (e.g., '0.5', '2')."""


class ServiceDeployResources(TypedDict, total=False):
"""Resource configuration for a service deployment."""

limits: ServiceDeployResourceLimits
"""Resource limits."""

reservations: ServiceDeployResourceLimits
"""Resource reservations."""


class ServiceDeploy(TypedDict, total=False):
"""Deployment configuration for a service."""

resources: ServiceDeployResources
"""Resource configuration."""

replicas: int
"""Number of container replicas to deploy."""

restart_policy: dict[str, Any]
"""Restart policy configuration."""


class ServiceConfig(TypedDict, total=False):
"""The definition of a service in Docker Compose.

Expand Down Expand Up @@ -141,11 +174,8 @@ class ServiceConfig(TypedDict, total=False):
TODO(benesch): this should use a nested TypedDict.
"""

deploy: dict[str, dict[str, dict[str, str]]]
"""Additional deployment configuration, like resource limits.

TODO(benesch): this should use a nested TypedDict.
"""
deploy: ServiceDeploy
"""Deployment configuration, including resource limits."""

ulimits: dict[str, Any]
"""Override the default ulimits for a container."""
Expand Down Expand Up @@ -178,6 +208,9 @@ class ServiceConfig(TypedDict, total=False):
user: str | None
"""The user for the container."""

cap_add: list[str] | None
"""cap_add specifies additional container capabilities as strings."""


class Service:
"""A Docker Compose service in a `Composition`.
Expand Down
9 changes: 7 additions & 2 deletions misc/python/materialize/mzcompose/services/clusterd.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@
from materialize.mzcompose.service import (
Service,
ServiceConfig,
ServiceDeploy,
ServiceDeployResources,
ServiceDeployResourceLimits,
)


Expand Down Expand Up @@ -73,12 +76,14 @@ def __init__(
# ignored with a warning. Unfortunately no portable way of setting the
# memory limit is known.
if memory or cpu:
limits = {}
limits: ServiceDeployResourceLimits = {}
if memory:
limits["memory"] = memory
if cpu:
limits["cpus"] = cpu
config["deploy"] = {"resources": {"limits": limits}}
resources: ServiceDeployResources = {"limits": limits}
deploy: ServiceDeploy = {"resources": resources}
config["deploy"] = deploy

config.update(
{
Expand Down
9 changes: 7 additions & 2 deletions misc/python/materialize/mzcompose/services/materialized.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@
from materialize.mzcompose.service import (
Service,
ServiceConfig,
ServiceDeploy,
ServiceDeployResources,
ServiceDeployResourceLimits,
ServiceDependency,
)
from materialize.mzcompose.services.azurite import azure_blob_uri
Expand Down Expand Up @@ -296,12 +299,14 @@ def __init__(
# ignored with a warning. Unfortunately no portable way of setting the
# memory limit is known.
if memory or cpu:
limits = {}
limits: ServiceDeployResourceLimits = {}
if memory:
limits["memory"] = memory
if cpu:
limits["cpus"] = cpu
config["deploy"] = {"resources": {"limits": limits}}
resources: ServiceDeployResources = {"limits": limits}
deploy: ServiceDeploy = {"resources": resources}
config["deploy"] = deploy

# Sanity restarts are rather slow and rarely find a bug, running them
# on main and releases is enough to prevent regressions.
Expand Down
3 changes: 3 additions & 0 deletions misc/python/materialize/mzcompose/services/sql_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,16 @@ def __init__(
"MSSQL_PID=Developer",
"MSSQL_AGENT_ENABLED=True",
f"SA_PASSWORD={sa_password}",
"MSSQL_MEMORY_LIMIT_MB=2500",
*environment_extra,
],
"healthcheck": {
"test": f"/opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P '{sa_password}' -Q 'SELECT 1'",
"interval": "1s",
"start_period": "30s",
},
"deploy": {"resources": {"limits": {"memory": "3G"}}},
"cap_add": ["SYS_PTRACE"],
},
)
self.sa_password = sa_password
Expand Down
4 changes: 2 additions & 2 deletions misc/python/materialize/zippy/scenarios.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
CreateSqlServerTable,
SqlServerDML,
SqlServerStart,
SqlServerRestart,
)
from materialize.zippy.sql_server_cdc_actions import CreateSqlServerCdcTable
from materialize.zippy.mz_actions import (
Expand Down Expand Up @@ -252,8 +253,7 @@ def actions_with_weight(self) -> dict[ActionOrFactory, float]:
KillClusterd: 5,
StoragedKill: 5,
StoragedStart: 5,
# TODO: Reenable when database-issues#9624 is fixed
# SqlServerRestart: 10,
SqlServerRestart: 10,
CreateViewParameterized(): 10,
ValidateView: 20,
SqlServerDML: 100,
Expand Down