Add slurmctld functions and full slurm.conf access (#367)

Further changes:

* move cpu_freq_int_to_str to utils.helpers

* cstr: add configurable delim to to_list

* update partition unit tests

* tests: integration tests for slurmctld

* partition: use _get_memory from slurmctld module

* mkdocs.yml: add pymdownx.magiclink

* docs: add griffe_exts.py script for prettier docs

This allows to dynamically link to the Slurm Documentation with
different Slurm versions.

For example it will replace this in the docstring:

{slurm.conf#OPT_MpiParams}

with this in the finally rendered docs, when we are on pyslurm 24.11.x:

https://slurm.schedmd.com/archive/slurm-24.11-latest/slurm.conf.html#OPT_MpiParams

This way we can dynamically link to the Slurm docs.

* docs: add slurmctld reference API page

* docs: remove old config class documentation

* update common tests

* remove the setup.py clean step from builddocs.sh

* BACKPORT Changes: Fixed to work with 23.11

(cherry picked from commit ff3dc42)
(cherry picked from commit 6229e66)

Author: tazend

docs/reference/config.md

@@ -0,0 +1,8 @@
+---
+title: slurmctld
+---
+
+::: pyslurm.slurmctld
+    handler: python
+    options:
+        members: yes

docs/reference/slurmctld.md

@@ -60,6 +60,8 @@ plugins:
             show_root_heading: true
             show_symbol_type_toc: true
             show_symbol_type_heading: true
+            extensions:
+              - scripts/griffe_exts.py:DynamicDocstrings
 
 markdown_extensions:
   - admonition
@@ -73,6 +75,7 @@ markdown_extensions:
   - pymdownx.inlinehilite
   - pymdownx.superfences
   - pymdownx.details
+  - pymdownx.magiclink
 
 extra:
   version:

mkdocs.yml

@@ -50,6 +50,7 @@ from pyslurm.utils.helpers import (
     _getpwall_to_dict,
     instance_to_dict,
     _get_exit_code,
+    cpu_freq_int_to_str,
 )
 
 

pyslurm/core/job/job.pyx

@@ -34,8 +34,8 @@ from pyslurm.utils.helpers import (
     uid_to_name,
     humanize_step_id,
     dehumanize_step_id,
+    cpu_freq_int_to_str,
 )
-from pyslurm.core.job.util import cpu_freq_int_to_str
 from pyslurm.utils.ctime import (
     secs_to_timestr,
     mins_to_timestr,

pyslurm/core/job/step.pyx

@@ -277,37 +277,6 @@ def cpu_freq_str_to_int(freq):
     raise ValueError(f"Invalid cpu freq value: {freq}.")
 
 
-def cpu_freq_int_to_str(freq):
-    """Convert a numerical cpufreq value to its string representation."""
-    if freq == slurm.CPU_FREQ_LOW:
-        return "LOW"
-    elif freq == slurm.CPU_FREQ_MEDIUM:
-        return "MEDIUM"
-    elif freq == slurm.CPU_FREQ_HIGHM1:
-        return "HIGHM1"
-    elif freq == slurm.CPU_FREQ_HIGH:
-        return "HIGH"
-    elif freq == slurm.CPU_FREQ_CONSERVATIVE:
-        return "CONSERVATIVE"
-    elif freq == slurm.CPU_FREQ_PERFORMANCE:
-        return "PERFORMANCE"
-    elif freq == slurm.CPU_FREQ_POWERSAVE:
-        return "POWERSAVE"
-    elif freq == slurm.CPU_FREQ_USERSPACE:
-        return "USERSPACE"
-    elif freq == slurm.CPU_FREQ_ONDEMAND:
-        return "ONDEMAND"
-    elif freq == slurm.CPU_FREQ_SCHEDUTIL:
-        return "SCHEDUTIL"
-    elif freq & slurm.CPU_FREQ_RANGE_FLAG:
-        return None
-    elif freq == slurm.NO_VAL or freq == 0:
-        return None
-    else:
-        # This is in kHz
-        return freq
-
-
 def dependency_str_to_dict(dep):
     if not dep:
         return None

pyslurm/core/job/util.pyx

@@ -2,7 +2,7 @@
 # partition.pyx - interface to work with partitions in slurm
 #########################################################################
 # Copyright (C) 2023 Toni Harzendorf <[email protected]>
-# Copyright (C) 2023 PySlurm Developers
+# Copyright (C) 2025 PySlurm Developers
 #
 # This file is part of PySlurm
 #
@@ -31,6 +31,7 @@ from pyslurm.core.error import RPCError, verify_rpc
 from pyslurm.utils.ctime import timestamp_to_date, _raw_time
 from pyslurm.constants import UNLIMITED
 from pyslurm.settings import LOCAL_CLUSTER
+from pyslurm.core.slurmctld.config import _get_memory
 from pyslurm import xcollections
 from pyslurm.utils.helpers import (
     uid_to_name,
@@ -829,21 +830,3 @@ cdef _concat_job_default_str(typ, val, char **job_defaults_str):
         current.update({typ : _val})
 
     cstr.from_dict(job_defaults_str, current)
-
-
-def _get_memory(value, per_cpu):
-    if value != slurm.NO_VAL64:
-        if value & slurm.MEM_PER_CPU and per_cpu:
-            if value == slurm.MEM_PER_CPU:
-                return UNLIMITED
-            return u64_parse(value & (~slurm.MEM_PER_CPU))
-
-        # For these values, Slurm interprets 0 as being equal to
-        # INFINITE/UNLIMITED
-        elif value == 0 and not per_cpu:
-            return UNLIMITED
-
-        elif not value & slurm.MEM_PER_CPU and not per_cpu:
-            return u64_parse(value)
-
-    return None

pyslurm/core/partition.pyx

@@ -0,0 +1,4 @@
+# cython: c_string_type=unicode, c_string_encoding=default
+# cython: language_level=3
+
+from .config cimport Config

pyslurm/core/slurmctld.pyx

@@ -0,0 +1,27 @@
+from .config import (
+   Config,
+   MPIConfig,
+   AccountingGatherConfig,
+   CgroupConfig,
+)
+from .enums import ShutdownMode
+from .base import (
+    PingResponse,
+    ping,
+    ping_primary,
+    ping_backup,
+    ping_all,
+    shutdown,
+    reconfigure,
+    takeover,
+    add_debug_flags,
+    remove_debug_flags,
+    clear_debug_flags,
+    get_debug_flags,
+    set_log_level,
+    get_log_level,
+    enable_scheduler_logging,
+    is_scheduler_logging_enabled,
+    set_fair_share_dampening_factor,
+    get_fair_share_dampening_factor,
+)