From b3646dca96c38c54a8c8c403499cb45774d27931 Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Thu, 23 Jan 2025 17:52:13 +0100 Subject: [PATCH 01/18] feat: rework reservations --- pyslurm/__init__.py | 1 + pyslurm/core/reservation.pxd | 73 ++++++ pyslurm/core/reservation.pyx | 442 +++++++++++++++++++++++++++++++++++ 3 files changed, 516 insertions(+) create mode 100644 pyslurm/core/reservation.pxd create mode 100644 pyslurm/core/reservation.pyx diff --git a/pyslurm/__init__.py b/pyslurm/__init__.py index c6e41eb7..d34fac91 100644 --- a/pyslurm/__init__.py +++ b/pyslurm/__init__.py @@ -22,6 +22,7 @@ ) from pyslurm.core.node import Node, Nodes from pyslurm.core.partition import Partition, Partitions +from pyslurm.core.reservation import Reservation, Reservations from pyslurm.core import error from pyslurm.core.error import ( PyslurmError, diff --git a/pyslurm/core/reservation.pxd b/pyslurm/core/reservation.pxd new file mode 100644 index 00000000..7ebed106 --- /dev/null +++ b/pyslurm/core/reservation.pxd @@ -0,0 +1,73 @@ +######################################################################### +# reservation.pxd - interface to work with reservations in slurm +######################################################################### +# Copyright (C) 2025 Toni Harzendorf +# +# This file is part of PySlurm +# +# PySlurm is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# PySlurm is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with PySlurm; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# cython: c_string_type=unicode, c_string_encoding=default +# cython: language_level=3 + +from libc.string cimport memcpy, memset +from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t +from libc.stdlib cimport free +from pyslurm cimport slurm +from pyslurm.slurm cimport ( + reserve_info_t, + reserve_info_msg_t, + resv_desc_msg_t, + reservation_name_msg_t, + reserve_response_msg_t, + slurm_free_reservation_info_msg, + slurm_load_reservations, + slurm_delete_reservation, + slurm_update_reservation, + slurm_create_reservation, + slurm_init_resv_desc_msg, + xfree, + try_xmalloc, +) + +from pyslurm.utils cimport cstr +from pyslurm.utils cimport ctime +from pyslurm.utils.ctime cimport time_t +from pyslurm.utils.uint cimport * +from pyslurm.xcollections cimport MultiClusterMap + +cdef extern void slurm_free_resv_desc_msg(resv_desc_msg_t *msg) +cdef extern void slurm_free_reserve_info_members(reserve_info_t *resv) + + +cdef class Reservations(MultiClusterMap): + + cdef: + reserve_info_msg_t *info + reserve_info_t tmp_info + + +cdef class Reservation: + + cdef: + reserve_info_t *info + resv_desc_msg_t *umsg + dict passwd + dict groups + + cdef readonly cluster + + @staticmethod + cdef Reservation from_ptr(reserve_info_t *in_ptr) diff --git a/pyslurm/core/reservation.pyx b/pyslurm/core/reservation.pyx new file mode 100644 index 00000000..c3dad89b --- /dev/null +++ b/pyslurm/core/reservation.pyx @@ -0,0 +1,442 @@ +######################################################################### +# reservation.pyx - interface to work with reservations in slurm +######################################################################### +# Copyright (C) 2025 Toni Harzendorf +# +# This file is part of PySlurm +# +# PySlurm is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# PySlurm is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with PySlurm; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# cython: c_string_type=unicode, c_string_encoding=default +# cython: language_level=3 + +from typing import Union, Any +from pyslurm.utils import cstr +from pyslurm.utils import ctime +from pyslurm.utils.uint import * +from pyslurm.core.error import RPCError, verify_rpc, slurm_errno +from pyslurm.utils.ctime import timestamp_to_date, _raw_time +from pyslurm.constants import UNLIMITED +from pyslurm.settings import LOCAL_CLUSTER +from pyslurm.core.slurmctld.config import _get_memory +from datetime import datetime +from pyslurm import xcollections +from pyslurm.utils.helpers import ( + uid_to_name, + gid_to_name, + _getgrall_to_dict, + _getpwall_to_dict, + cpubind_to_num, + instance_to_dict, + dehumanize, +) +from pyslurm.utils.ctime import ( + timestr_to_mins, + timestr_to_secs, + date_to_timestamp, +) + + +cdef class Reservations(MultiClusterMap): + + def __dealloc__(self): + slurm_free_reservation_info_msg(self.info) + + def __cinit__(self): + self.info = NULL + + def __init__(self, reservations=None): + super().__init__(data=reservations, + typ="Reservations", + val_type=Reservation, + id_attr=Reservation.name, + key_type=str) + + @staticmethod + def load(): + """Load all Reservations in the system. + + Returns: + (pyslurm.Reservations): Collection of Reservation objects. + + Raises: + (pyslurm.RPCError): When getting all the Reservations from the + slurmctld failed. + """ + cdef: + Reservations reservations = Reservations() + Reservation reservation + + verify_rpc(slurm_load_reservations(0, &reservations.info)) + + # zero-out a dummy reserve_info_t + memset(&reservations.tmp_info, 0, sizeof(reserve_info_t)) + + # Put each pointer into its own instance. + for cnt in range(reservations.info.record_count): + reservation = Reservation.from_ptr(&reservations.info.reservation_array[cnt]) + + # Prevent double free if xmalloc fails mid-loop and a MemoryError + # is raised by replacing it with a zeroed-out reserve_info_t. + reservations.info.reservation_array[cnt] = reservations.tmp_info + + cluster = reservation.cluster + if cluster not in reservations.data: + reservations.data[cluster] = {} + + reservations.data[cluster][reservation.name] = reservation + + # We have extracted all pointers + reservations.info.record_count = 0 + return reservations + + +cdef class Reservation: + + def __cinit__(self): + self.info = NULL + self.umsg = NULL + + def __init__(self, name=None, **kwargs): + self._alloc_impl() + self.name = name + self.start_time = datetime.now() + self.duration = "365-00:00:00" + self.cluster = LOCAL_CLUSTER + for k, v in kwargs.items(): + setattr(self, k, v) + + def _alloc_impl(self): + self._alloc_info() + self._alloc_umsg() + + def _alloc_info(self): + if not self.info: + self.info = try_xmalloc(sizeof(reserve_info_t)) + if not self.info: + raise MemoryError("xmalloc failed for reserve_info_t") + + def _alloc_umsg(self): + if not self.umsg: + self.umsg = try_xmalloc(sizeof(resv_desc_msg_t)) + if not self.umsg: + raise MemoryError("xmalloc failed for resv_desc_msg_t") + print("we here") + slurm_init_resv_desc_msg(self.umsg) + + def _dealloc_impl(self): + slurm_free_resv_desc_msg(self.umsg) + self.umsg = NULL + slurm_free_reserve_info_members(self.info) + xfree(self.info) + + def __dealloc__(self): + self._dealloc_impl() + + def __setattr__(self, name, val): + # When a user wants to set attributes on a Reservation instance that + # was created by calling Reservations(), the "umsg" pointer is not yet + # allocated. We only allocate memory for it by the time the user + # actually wants to modify something. + self._alloc_umsg() + # Call descriptors __set__ directly + Reservation.__dict__[name].__set__(self, val) + + def __repr__(self): + return f'pyslurm.{self.__class__.__name__}({self.name})' + + @staticmethod + cdef Reservation from_ptr(reserve_info_t *in_ptr): + cdef Reservation wrap = Reservation.__new__(Reservation) + wrap._alloc_info() + wrap.passwd = {} + wrap.groups = {} + wrap.cluster = LOCAL_CLUSTER + memcpy(wrap.info, in_ptr, sizeof(reserve_info_t)) + return wrap + + def _error_or_name(self): + if not self.name: + raise RPCError(msg="No Reservation name was specified. " + "Did you set the `name` attribute on the " + "Reservation instance?") + return self.name + + def to_dict(self): + """Node information formatted as a dictionary. + + Returns: + (dict): Node information as dict + + Examples: + >>> import pyslurm + >>> mynode = pyslurm.Node.load("mynode") + >>> mynode_dict = mynode.to_dict() + """ + return instance_to_dict(self) + + @staticmethod + def load(name): + """Load information for a specific Reservation. + + Args: + name (str): + The name of the Reservation to load. + + Returns: + (pyslurm.Reservation): Returns a new Reservation instance. + + Raises: + (pyslurm.RPCError): If requesting the Reservation information from + the slurmctld was not successful. + + Examples: + >>> import pyslurm + >>> reservation = pyslurm.Reservation.load("maintenance") + """ + resv = Reservations.load().get(name) + if not resv: + raise RPCError(msg=f"Reservation '{name}' doesn't exist") + + return resv + + def create(self): + """Create a Reservation. + + Returns: + (pyslurm.Reservation): This function returns the current + Reservation instance object itself. + + Raises: + (pyslurm.RPCError): If creating the Reservation was not successful. + + Examples: + >>> import pyslurm + >>> reservation = pyslurm.Reservation("debug").create() + """ + cdef char* new_name = NULL + + self.name = self._error_or_name() + new_name = slurm_create_reservation(self.umsg) + free(new_name) + verify_rpc(slurm_errno()) + return self + + def modify(self, Reservation changes): + """Modify a Reservation. + + Args: + changes (pyslurm.Reservation): + Another Reservation object that contains all the changes to + apply. Check the `Other Parameters` of the Reservation class to + see which properties can be modified. + + Raises: + (pyslurm.RPCError): When updating the Reservation was not + successful. + + Examples: + >>> import pyslurm + >>> + >>> mynode = pyslurm.Node.load("localhost") + >>> # Prepare the changes + >>> changes = pyslurm.Node(state="DRAIN", reason="DRAIN Reason") + >>> # Modify it + >>> mynode.modify(changes) + """ + if not changes.umsg: + return + + self._error_or_name() + cstr.fmalloc(&changes.umsg.name, self.info.name) + verify_rpc(slurm_update_reservation(changes.umsg)) + + def delete(self): + """Delete a Reservation. + + Raises: + (pyslurm.RPCError): If deleting the Reservation was not successful. + + Examples: + >>> import pyslurm + >>> pyslurm.Reservation("maintenance").delete() + """ + cdef reservation_name_msg_t to_delete + memset(&to_delete, 0, sizeof(to_delete)) + to_delete.name = cstr.from_unicode(self._error_or_name()) + verify_rpc(slurm_delete_reservation(&to_delete)) + + @property + def accounts(self): + return cstr.to_list(self.info.accounts) + + @accounts.setter + def accounts(self, val): + cstr.from_list2(&self.info.accounts, &self.umsg.accounts, val) + + @property + def burst_buffer(self): + return cstr.to_unicode(self.info.burst_buffer) + + @burst_buffer.setter + def burst_buffer(self, val): + cstr.fmalloc2(&self.info.burst_buffer, &self.umsg.burst_buffer, val) + + @property + def comment(self): + return cstr.to_unicode(self.info.comment) + + @comment.setter + def comment(self, val): + cstr.fmalloc2(&self.info.comment, &self.umsg.comment, val) + + @property + def core_count(self): + return u32_parse(self.info.core_cnt, zero_is_noval=False) + + @core_count.setter + def core_count(self, val): + self.info.core_cnt = self.umsg.core_cnt = int(val) + + @property + def cores_by_node(self): + out = {} + for i in range(self.info.core_spec_cnt): + node = cstr.to_unicode(self.info.core_spec[i].node_name) + if node: + out[node] = cstr.to_unicode(self.info.core_spec[i].core_id) + + @property + def end_time(self): + return _raw_time(self.info.end_time) + + @end_time.setter + def end_time(self, val): + val = date_to_timestamp(val) + if self.start_time and val < self.info.start_time: + raise ValueError("end_time cannot be earlier then start_time.") + + self.info.end_time = self.umsg.end_time = val + + @property + def features(self): + return cstr.to_list(self.info.features) + + @features.setter + def features(self, val): + cstr.from_list2(&self.info.features, &self.umsg.features, val) + + # TODO: flags + + @property + def groups(self): + return cstr.to_list(self.info.groups) + + @groups.setter + def groups(self, val): + cstr.from_list2(&self.info.groups, &self.umsg.groups, val) + + @property + def licenses(self): + return cstr.to_list(self.info.licenses) + + @licenses.setter + def licenses(self, val): + cstr.from_list2(&self.info.licenses, &self.umsg.licenses, val) + + @property + def max_start_delay(self): + return u32_parse(self.info.max_start_delay) + + @max_start_delay.setter + def max_start_delay(self, val): + self.info.max_start_delay = self.umsg.max_start_delay = int(val) + + @property + def name(self): + return cstr.to_unicode(self.info.name) + + @name.setter + def name(self, val): + cstr.fmalloc2(&self.info.name, &self.umsg.name, val) + + @property + def node_count(self): + return u32_parse(self.info.node_cnt, zero_is_noval=False) + + @node_count.setter + def node_count(self, val): + self.info.node_cnt = self.umsg.node_cnt = int(val) + + @property + def nodes(self): + return cstr.to_unicode(self.info.node_list) + + @nodes.setter + def nodes(self, val): + cstr.fmalloc2(&self.info.node_list, &self.umsg.node_list, val) + + @property + def partition(self): + return cstr.to_unicode(self.info.partition) + + @partition.setter + def partition(self, val): + cstr.fmalloc2(&self.info.partition, &self.umsg.partition, val) + + # TODO: purge_comp_time ? + + @property + def start_time(self): + return _raw_time(self.info.start_time) + + @start_time.setter + def start_time(self, val): + self.info.start_time = self.umsg.start_time = date_to_timestamp(val) + + @property + def duration(self): + cdef time_t duration = 0 + + if self.info.end_time >= self.info.start_time: + duration = ctime.difftime(self.info.end_time, + self.info.start_time) + + return int(duration / 60) + + @duration.setter + def duration(self, val): + val = timestr_to_mins(val) + self.umsg.duration = val + self.end_time = self.start_time + (val * 60) + + @property + def is_active(self): + cdef time_t now = ctime.time(NULL) + if self.info.start_time <= now and self.info.end_time >= now: + return True + return False + + @property + def tres(self): + return cstr.to_dict(self.info.tres_str) + + @property + def users(self): + return cstr.to_list(self.info.users) + + @users.setter + def users(self, val): + cstr.from_list2(&self.info.users, &self.umsg.users, val) From dc3e8c4f79aa899d8c0436c949dd94fa8f1540da Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Fri, 24 Jan 2025 13:15:49 +0100 Subject: [PATCH 02/18] wip documentation --- pyslurm/core/reservation.pxd | 61 ++++++++++++++++++++++- pyslurm/core/reservation.pyx | 93 +++++++++++++++++++++++------------- 2 files changed, 120 insertions(+), 34 deletions(-) diff --git a/pyslurm/core/reservation.pxd b/pyslurm/core/reservation.pxd index 7ebed106..55878cdf 100644 --- a/pyslurm/core/reservation.pxd +++ b/pyslurm/core/reservation.pxd @@ -53,6 +53,12 @@ cdef extern void slurm_free_reserve_info_members(reserve_info_t *resv) cdef class Reservations(MultiClusterMap): + """A [`Multi Cluster`][pyslurm.xcollections.MultiClusterMap] collection of [pyslurm.Reservation][] objects. + + Args: + reservations (Union[list[str], dict[str, pyslurm.Reservation], str], optional=None): + Reservations to initialize this collection with. + """ cdef: reserve_info_msg_t *info @@ -60,12 +66,63 @@ cdef class Reservations(MultiClusterMap): cdef class Reservation: + """A Slurm Reservation. + + Args: + name (str, optional=None): + Name of a Reservation. + + !!! note + + All Attributes of a Reservation, except for `name` and `cpus_by_node`, + are eligible to be updated. Although the `name` attribute can be + changed on the instance, the change will not be taken into account by + `slurmctld` + Attributes: + accounts (list[str]): + List of account names that have access to the Reservation. + burst_buffer (str): + Burst Buffer specification. + comment (str): + Arbitrary comment for the Reservation. + cpus (int): + Amount of CPUs used by the Reservation + cpus_by_node (dict[str, int]): + A Mapping where each key is the node-name, and the values are a + string of CPU-IDs reserved on the specific nodes. + end_time (int): + Unix Timestamp when the Reservation ends. + features (list[str]): + List of features required by the Reservation. + groups (list[str]): + List of Groups that can access the Reservation. + licenses (list[str]): + List of licenses to be reserved. + max_start_delay (int): + TODO + name (str): + Name of the Reservation. + node_count (int): + Count of Nodes required. + nodes (str): + Nodes to be reserved. + partition (str): + Name of the partition to be used. + start_time (int): + When the Reservation starts. This is a Unix timestamp. + duration (int): + How long, in minutes, the reservation runs for. + is_active (bool): + Whether the reservation is currently active or not. + tres (dict[str, int]) + TRES for the Reservation. + users (list[str]): + List of user names permitted to use the Reservation. + """ cdef: reserve_info_t *info resv_desc_msg_t *umsg - dict passwd - dict groups cdef readonly cluster diff --git a/pyslurm/core/reservation.pyx b/pyslurm/core/reservation.pyx index c3dad89b..ee117f54 100644 --- a/pyslurm/core/reservation.pyx +++ b/pyslurm/core/reservation.pyx @@ -53,6 +53,7 @@ cdef class Reservations(MultiClusterMap): def __dealloc__(self): slurm_free_reservation_info_msg(self.info) + self.info = NULL def __cinit__(self): self.info = NULL @@ -81,15 +82,19 @@ cdef class Reservations(MultiClusterMap): verify_rpc(slurm_load_reservations(0, &reservations.info)) - # zero-out a dummy reserve_info_t + # prepare a dummy reserve_info_t struct. memset(&reservations.tmp_info, 0, sizeof(reserve_info_t)) # Put each pointer into its own instance. for cnt in range(reservations.info.record_count): reservation = Reservation.from_ptr(&reservations.info.reservation_array[cnt]) - # Prevent double free if xmalloc fails mid-loop and a MemoryError - # is raised by replacing it with a zeroed-out reserve_info_t. + # If we already parsed at least one Reservation, and if for some + # reason a MemoryError is raised after parsing subsequent + # reservations, invalid behaviour will be shown by Valgrind, since + # the Memory for the already parsed Reservation will be freed + # twice. So for all sucessfully parsed Reservations, replace it + # with a dummy struct that will be skipped in case of error. reservations.info.reservation_array[cnt] = reservations.tmp_info cluster = reservation.cluster @@ -112,8 +117,6 @@ cdef class Reservation: def __init__(self, name=None, **kwargs): self._alloc_impl() self.name = name - self.start_time = datetime.now() - self.duration = "365-00:00:00" self.cluster = LOCAL_CLUSTER for k, v in kwargs.items(): setattr(self, k, v) @@ -133,14 +136,17 @@ cdef class Reservation: self.umsg = try_xmalloc(sizeof(resv_desc_msg_t)) if not self.umsg: raise MemoryError("xmalloc failed for resv_desc_msg_t") - print("we here") slurm_init_resv_desc_msg(self.umsg) - def _dealloc_impl(self): + def _dealloc_umsg(self): slurm_free_resv_desc_msg(self.umsg) self.umsg = NULL + + def _dealloc_impl(self): + self._dealloc_umsg() slurm_free_reserve_info_members(self.info) xfree(self.info) + self.info = NULL def __dealloc__(self): self._dealloc_impl() @@ -161,8 +167,6 @@ cdef class Reservation: cdef Reservation from_ptr(reserve_info_t *in_ptr): cdef Reservation wrap = Reservation.__new__(Reservation) wrap._alloc_info() - wrap.passwd = {} - wrap.groups = {} wrap.cluster = LOCAL_CLUSTER memcpy(wrap.info, in_ptr, sizeof(reserve_info_t)) return wrap @@ -175,15 +179,16 @@ cdef class Reservation: return self.name def to_dict(self): - """Node information formatted as a dictionary. + """Reservation information formatted as a dictionary. Returns: - (dict): Node information as dict + (dict): Reservation information as dict Examples: >>> import pyslurm - >>> mynode = pyslurm.Node.load("mynode") - >>> mynode_dict = mynode.to_dict() + >>> resv = pyslurm.Reservation.load("maintenance") + >>> resv_dict = resv.to_dict() + >>> print(resv_dict) """ return instance_to_dict(self) @@ -215,6 +220,10 @@ cdef class Reservation: def create(self): """Create a Reservation. + If you did not specify atleast a `start_time` and `duration` or + `end_time`, then by default the Reservation will start effective + immediately, with a duration of one year. + Returns: (pyslurm.Reservation): This function returns the current Reservation instance object itself. @@ -224,24 +233,29 @@ cdef class Reservation: Examples: >>> import pyslurm - >>> reservation = pyslurm.Reservation("debug").create() + >>> resv = pyslurm.Reservation("debug") """ cdef char* new_name = NULL + if not self.start_time or not (self.duration and self.end_time): + raise RPCError(msg="You must atleast specify a start_time, " + " combined with an end_time or a duration.") + self.name = self._error_or_name() new_name = slurm_create_reservation(self.umsg) free(new_name) verify_rpc(slurm_errno()) return self - def modify(self, Reservation changes): + def modify(self, Reservation changes=None): """Modify a Reservation. Args: - changes (pyslurm.Reservation): + changes (pyslurm.Reservation, optional=None): Another Reservation object that contains all the changes to - apply. Check the `Other Parameters` of the Reservation class to - see which properties can be modified. + apply. This is optional - you can also directly modify a + Reservation object and just call `modify()`, and the changes + will be sent to `slurmctld`. Raises: (pyslurm.RPCError): When updating the Reservation was not @@ -250,18 +264,27 @@ cdef class Reservation: Examples: >>> import pyslurm >>> - >>> mynode = pyslurm.Node.load("localhost") - >>> # Prepare the changes - >>> changes = pyslurm.Node(state="DRAIN", reason="DRAIN Reason") - >>> # Modify it - >>> mynode.modify(changes) + >>> resv = pyslurm.Reservation.load("maintenance") + >>> # Add 60 Minutes to the reservation + >>> resv.duration += 60 + >>> + >>> # You can also add a slurm timestring. + >>> # For example, extend the duration by another day: + >>> resv.duration += pyslurm.utils.timestr_to_mins("1-00:00:00") + >>> + >>> # Now send the changes to the Controller: + >>> resv.modify() """ - if not changes.umsg: + cdef Reservation updates = changes if changes is not None else self + if not updates.umsg: return self._error_or_name() - cstr.fmalloc(&changes.umsg.name, self.info.name) - verify_rpc(slurm_update_reservation(changes.umsg)) + cstr.fmalloc(&updates.umsg.name, self.info.name) + verify_rpc(slurm_update_reservation(updates.umsg)) + + # Make sure we clean the object from any previous changes. + updates._dealloc_umsg() def delete(self): """Delete a Reservation. @@ -303,15 +326,15 @@ cdef class Reservation: cstr.fmalloc2(&self.info.comment, &self.umsg.comment, val) @property - def core_count(self): + def cpus(self): return u32_parse(self.info.core_cnt, zero_is_noval=False) - @core_count.setter - def core_count(self, val): + @cpus.setter + def cpus(self, val): self.info.core_cnt = self.umsg.core_cnt = int(val) @property - def cores_by_node(self): + def cpus_by_node(self): out = {} for i in range(self.info.core_spec_cnt): node = cstr.to_unicode(self.info.core_spec[i].node_name) @@ -410,7 +433,7 @@ cdef class Reservation: def duration(self): cdef time_t duration = 0 - if self.info.end_time >= self.info.start_time: + if self.start_time and self.info.end_time >= self.info.start_time: duration = ctime.difftime(self.info.end_time, self.info.start_time) @@ -419,7 +442,8 @@ cdef class Reservation: @duration.setter def duration(self, val): val = timestr_to_mins(val) - self.umsg.duration = val + if not self.start_time: + self.start_time = datetime.now() self.end_time = self.start_time + (val * 60) @property @@ -433,6 +457,11 @@ cdef class Reservation: def tres(self): return cstr.to_dict(self.info.tres_str) + @tres.setter + def tres(self, val): + cstr.fmalloc2(&self.info.tres_str, &self.umsg.tres_str, + cstr.dict_to_str(val)) + @property def users(self): return cstr.to_list(self.info.users) From 5c9022a58d22db24019ae95d763efee9ff23dcdf Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Sun, 26 Jan 2025 16:53:43 +0100 Subject: [PATCH 03/18] remove unused imports --- pyslurm/core/reservation.pxd | 7 +++---- pyslurm/core/reservation.pyx | 31 +++++++++++++------------------ 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/pyslurm/core/reservation.pxd b/pyslurm/core/reservation.pxd index 55878cdf..879542e0 100644 --- a/pyslurm/core/reservation.pxd +++ b/pyslurm/core/reservation.pxd @@ -45,7 +45,7 @@ from pyslurm.slurm cimport ( from pyslurm.utils cimport cstr from pyslurm.utils cimport ctime from pyslurm.utils.ctime cimport time_t -from pyslurm.utils.uint cimport * +from pyslurm.utils.uint cimport u32_parse from pyslurm.xcollections cimport MultiClusterMap cdef extern void slurm_free_resv_desc_msg(resv_desc_msg_t *msg) @@ -59,7 +59,6 @@ cdef class Reservations(MultiClusterMap): reservations (Union[list[str], dict[str, pyslurm.Reservation], str], optional=None): Reservations to initialize this collection with. """ - cdef: reserve_info_msg_t *info reserve_info_t tmp_info @@ -70,14 +69,14 @@ cdef class Reservation: Args: name (str, optional=None): - Name of a Reservation. + Name for a Reservation. !!! note All Attributes of a Reservation, except for `name` and `cpus_by_node`, are eligible to be updated. Although the `name` attribute can be changed on the instance, the change will not be taken into account by - `slurmctld` + `slurmctld` when calling `modify()`. Attributes: accounts (list[str]): diff --git a/pyslurm/core/reservation.pyx b/pyslurm/core/reservation.pyx index ee117f54..6bdae319 100644 --- a/pyslurm/core/reservation.pyx +++ b/pyslurm/core/reservation.pyx @@ -22,31 +22,24 @@ # cython: c_string_type=unicode, c_string_encoding=default # cython: language_level=3 -from typing import Union, Any from pyslurm.utils import cstr from pyslurm.utils import ctime -from pyslurm.utils.uint import * -from pyslurm.core.error import RPCError, verify_rpc, slurm_errno -from pyslurm.utils.ctime import timestamp_to_date, _raw_time -from pyslurm.constants import UNLIMITED -from pyslurm.settings import LOCAL_CLUSTER +from pyslurm.utils.uint import u32_parse +from pyslurm import settings from pyslurm.core.slurmctld.config import _get_memory from datetime import datetime from pyslurm import xcollections -from pyslurm.utils.helpers import ( - uid_to_name, - gid_to_name, - _getgrall_to_dict, - _getpwall_to_dict, - cpubind_to_num, - instance_to_dict, - dehumanize, -) +from pyslurm.utils.helpers import instance_to_dict from pyslurm.utils.ctime import ( + _raw_time, timestr_to_mins, - timestr_to_secs, date_to_timestamp, ) +from pyslurm.core.error import ( + RPCError, + verify_rpc, + slurm_errno, +) cdef class Reservations(MultiClusterMap): @@ -117,7 +110,7 @@ cdef class Reservation: def __init__(self, name=None, **kwargs): self._alloc_impl() self.name = name - self.cluster = LOCAL_CLUSTER + self.cluster = settings.LOCAL_CLUSTER for k, v in kwargs.items(): setattr(self, k, v) @@ -167,7 +160,7 @@ cdef class Reservation: cdef Reservation from_ptr(reserve_info_t *in_ptr): cdef Reservation wrap = Reservation.__new__(Reservation) wrap._alloc_info() - wrap.cluster = LOCAL_CLUSTER + wrap.cluster = settings.LOCAL_CLUSTER memcpy(wrap.info, in_ptr, sizeof(reserve_info_t)) return wrap @@ -341,6 +334,8 @@ cdef class Reservation: if node: out[node] = cstr.to_unicode(self.info.core_spec[i].core_id) + return out + @property def end_time(self): return _raw_time(self.info.end_time) From 38a6d4b2269651757e0c2fbbdda681f9cd163dfe Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Sun, 26 Jan 2025 16:53:55 +0100 Subject: [PATCH 04/18] update docs for reservation --- docs/reference/reservation.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/reference/reservation.md b/docs/reference/reservation.md index 5826fcf9..bbdefaa6 100644 --- a/docs/reference/reservation.md +++ b/docs/reference/reservation.md @@ -2,8 +2,5 @@ title: Reservation --- -!!! warning - This API is currently being completely reworked, and is subject to be - removed in the future when a replacement is introduced - -::: pyslurm.deprecated.reservation +::: pyslurm.Reservation +::: pyslurm.Reservations From 9843cbf058ddec230954f75708aebec0e96a83b2 Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Sun, 26 Jan 2025 16:54:22 +0100 Subject: [PATCH 05/18] add tests for reservations --- tests/integration/test_reservation.py | 60 +++++++++++++++++++++++++ tests/unit/test_reservation.py | 64 +++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 tests/integration/test_reservation.py create mode 100644 tests/unit/test_reservation.py diff --git a/tests/integration/test_reservation.py b/tests/integration/test_reservation.py new file mode 100644 index 00000000..de7b2cb5 --- /dev/null +++ b/tests/integration/test_reservation.py @@ -0,0 +1,60 @@ +######################################################################### +# test_reservation.py - reservation integration tests +######################################################################### +# Copyright (C) 2025 Toni Harzendorf +# +# This file is part of PySlurm +# +# PySlurm is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# PySlurm is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with PySlurm; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""test_reservation.py - integration test reservation functionalities.""" + +import pyslurm +from datetime import datetime + + +def test_api_calls(): + start = datetime.now() + duration = "1-00:00:00" + resv = pyslurm.Reservation( + name="testing", + start_time=start, + duration=duration, + users=["root"], + node_count=1, + ) + resv.create() + + reservations = pyslurm.Reservations.load() + resv = reservations["testing"] + assert len(reservations) == 1 + assert resv.name == "testing" + assert resv.to_dict() + + assert resv.start_time == int(start.timestamp()) + assert resv.duration == 60 * 24 + assert resv.end_time == resv.start_time + (60 * 60 * 24) + + resv.duration += 60 * 24 + resv.modify() + + resv = pyslurm.Reservation.load("testing") + assert resv.name == "testing" + assert resv.start_time == int(start.timestamp()) + assert resv.duration == 2 * 60 * 24 + assert resv.end_time == resv.start_time + (2 * 60 * 60 * 24) + + resv.delete() + reservations = pyslurm.Reservations.load() + assert len(reservations) == 0 diff --git a/tests/unit/test_reservation.py b/tests/unit/test_reservation.py new file mode 100644 index 00000000..59c89db2 --- /dev/null +++ b/tests/unit/test_reservation.py @@ -0,0 +1,64 @@ +######################################################################### +# test_reservation.py - reservation unit tests +######################################################################### +# Copyright (C) 2025 Toni Harzendorf +# +# This file is part of PySlurm +# +# PySlurm is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# PySlurm is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with PySlurm; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""test_reservation.py - Unit test basic reservation functionalities.""" + +import pyslurm +from datetime import datetime + + +def test_create_instance(): + resv = pyslurm.Reservation("test") + assert resv.name == "test" + assert resv.accounts == [] + assert resv.start_time == None + assert resv.end_time == None + assert resv.duration == 0 + assert resv.is_active is False + assert resv.cpus_by_node == {} + assert resv.to_dict() + + start = datetime.now() + resv.start_time = start + resv.duration = "1-00:00:00" + + assert resv.start_time == int(start.timestamp()) + assert resv.duration == 60 * 24 + assert resv.end_time == resv.start_time + (60 * 60 * 24) + + resv.duration += pyslurm.utils.timestr_to_mins("1-00:00:00") + + assert resv.start_time == int(start.timestamp()) + assert resv.duration == 2 * 60 * 24 + assert resv.end_time == resv.start_time + (2 * 60 * 60 * 24) + + start = datetime.fromisoformat("2022-04-03T06:00:00") + end = resv.end_time + resv.start_time = int(start.timestamp()) + + assert resv.start_time == int(start.timestamp()) + assert resv.end_time == end + assert resv.duration == int((resv.end_time - resv.start_time) / 60) + + duration = resv.duration + resv.end_time += 60 * 60 * 24 + assert resv.start_time == int(start.timestamp()) + assert resv.end_time == end + (60 * 60 * 24) + assert resv.duration == duration + (60 * 24) From b32383d5f88f2affc0cbc3788927d2d8d525a1d1 Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Sun, 26 Jan 2025 19:33:27 +0100 Subject: [PATCH 06/18] fix docs --- pyslurm/core/reservation.pxd | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pyslurm/core/reservation.pxd b/pyslurm/core/reservation.pxd index 879542e0..5f0ebbb3 100644 --- a/pyslurm/core/reservation.pxd +++ b/pyslurm/core/reservation.pxd @@ -70,13 +70,11 @@ cdef class Reservation: Args: name (str, optional=None): Name for a Reservation. - - !!! note - - All Attributes of a Reservation, except for `name` and `cpus_by_node`, - are eligible to be updated. Although the `name` attribute can be - changed on the instance, the change will not be taken into account by - `slurmctld` when calling `modify()`. + **kwargs (Any, optional=None): + All Attributes of a Reservation are eligible to be set, except + `cpus_by_node`. Although the `name` attribute can also be changed + on the instance, the change will not be taken into account by + `slurmctld` when calling `modify()`. Attributes: accounts (list[str]): @@ -114,7 +112,7 @@ cdef class Reservation: How long, in minutes, the reservation runs for. is_active (bool): Whether the reservation is currently active or not. - tres (dict[str, int]) + tres (dict[str, int]): TRES for the Reservation. users (list[str]): List of user names permitted to use the Reservation. From a97b8733f97bf7618756a6dcb492791458269076 Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Sun, 26 Jan 2025 22:18:17 +0100 Subject: [PATCH 07/18] rename cpus_by_node to cpu_ids_by_node --- pyslurm/core/reservation.pxd | 8 ++++---- pyslurm/core/reservation.pyx | 2 +- tests/unit/test_reservation.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyslurm/core/reservation.pxd b/pyslurm/core/reservation.pxd index 5f0ebbb3..7370ad8c 100644 --- a/pyslurm/core/reservation.pxd +++ b/pyslurm/core/reservation.pxd @@ -72,9 +72,9 @@ cdef class Reservation: Name for a Reservation. **kwargs (Any, optional=None): All Attributes of a Reservation are eligible to be set, except - `cpus_by_node`. Although the `name` attribute can also be changed - on the instance, the change will not be taken into account by - `slurmctld` when calling `modify()`. + `cpu_ids_by_node`. Although the `name` attribute can also be + changed on the instance, the change will not be taken into account + by `slurmctld` when calling `modify()`. Attributes: accounts (list[str]): @@ -85,7 +85,7 @@ cdef class Reservation: Arbitrary comment for the Reservation. cpus (int): Amount of CPUs used by the Reservation - cpus_by_node (dict[str, int]): + cpu_ids_by_node (dict[str, int]): A Mapping where each key is the node-name, and the values are a string of CPU-IDs reserved on the specific nodes. end_time (int): diff --git a/pyslurm/core/reservation.pyx b/pyslurm/core/reservation.pyx index 6bdae319..49db7c36 100644 --- a/pyslurm/core/reservation.pyx +++ b/pyslurm/core/reservation.pyx @@ -327,7 +327,7 @@ cdef class Reservation: self.info.core_cnt = self.umsg.core_cnt = int(val) @property - def cpus_by_node(self): + def cpu_ids_by_node(self): out = {} for i in range(self.info.core_spec_cnt): node = cstr.to_unicode(self.info.core_spec[i].node_name) diff --git a/tests/unit/test_reservation.py b/tests/unit/test_reservation.py index 59c89db2..79f6bcc5 100644 --- a/tests/unit/test_reservation.py +++ b/tests/unit/test_reservation.py @@ -32,7 +32,7 @@ def test_create_instance(): assert resv.end_time == None assert resv.duration == 0 assert resv.is_active is False - assert resv.cpus_by_node == {} + assert resv.cpu_ids_by_node == {} assert resv.to_dict() start = datetime.now() From b2decf5e955cb71bb68bf91bb4e606279296d507 Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Sun, 26 Jan 2025 22:21:30 +0100 Subject: [PATCH 08/18] Update docs for max_start_delay --- pyslurm/core/reservation.pxd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyslurm/core/reservation.pxd b/pyslurm/core/reservation.pxd index 7370ad8c..5e4cd9c5 100644 --- a/pyslurm/core/reservation.pxd +++ b/pyslurm/core/reservation.pxd @@ -97,7 +97,8 @@ cdef class Reservation: licenses (list[str]): List of licenses to be reserved. max_start_delay (int): - TODO + Maximum delay, in seconds, where Jobs are permitted to overlap with + the Reservation once Jobs are queued for it. name (str): Name of the Reservation. node_count (int): From db3663d25e88b175929e053c9fd5690662003739 Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Mon, 27 Jan 2025 11:03:02 +0100 Subject: [PATCH 09/18] update docs for duration --- pyslurm/core/reservation.pxd | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pyslurm/core/reservation.pxd b/pyslurm/core/reservation.pxd index 5e4cd9c5..2acbd8d2 100644 --- a/pyslurm/core/reservation.pxd +++ b/pyslurm/core/reservation.pxd @@ -110,7 +110,17 @@ cdef class Reservation: start_time (int): When the Reservation starts. This is a Unix timestamp. duration (int): - How long, in minutes, the reservation runs for. + How long, in minutes, the reservation runs for. Unless a + `start_time` has already been specified, setting this will set the + `start_time` the current time, meaning the Reservation will start + immediately. + + For setting this attribute, instead of minutes you can also specify + a time-string like this: + + duration = "1-00:00:00" + + The above means that the Reservation will last for 1 day. is_active (bool): Whether the reservation is currently active or not. tres (dict[str, int]): From 95384930d08bc42ab417c895db476eaff3921d21 Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Fri, 31 Jan 2025 16:42:33 +0100 Subject: [PATCH 10/18] wip: reservation api --- docs/reference/reservation.md | 2 + pyslurm/core/reservation.pxd | 30 +++++++++- pyslurm/core/reservation.pyx | 101 +++++++++++++++++++++++++++++++--- 3 files changed, 123 insertions(+), 10 deletions(-) diff --git a/docs/reference/reservation.md b/docs/reference/reservation.md index bbdefaa6..8fc9f401 100644 --- a/docs/reference/reservation.md +++ b/docs/reference/reservation.md @@ -4,3 +4,5 @@ title: Reservation ::: pyslurm.Reservation ::: pyslurm.Reservations +::: pyslurm.ReservationFlags +::: pyslurm.ReservationReoccurrence diff --git a/pyslurm/core/reservation.pxd b/pyslurm/core/reservation.pxd index 2acbd8d2..e5b9e62a 100644 --- a/pyslurm/core/reservation.pxd +++ b/pyslurm/core/reservation.pxd @@ -45,7 +45,12 @@ from pyslurm.slurm cimport ( from pyslurm.utils cimport cstr from pyslurm.utils cimport ctime from pyslurm.utils.ctime cimport time_t -from pyslurm.utils.uint cimport u32_parse +from pyslurm.utils.uint cimport ( + u32, + u32_parse, + u64_parse_bool_flag, + u64_set_bool_flag, +) from pyslurm.xcollections cimport MultiClusterMap cdef extern void slurm_free_resv_desc_msg(resv_desc_msg_t *msg) @@ -105,8 +110,14 @@ cdef class Reservation: Count of Nodes required. nodes (str): Nodes to be reserved. + When creating or updating a Reservation, you can also pass the + string `ALL`, when you want all nodes to be included in the + Reservation. partition (str): Name of the partition to be used. + purge_time (int): + When the Reservation is idle for this amount of seconds, it will be + removed. start_time (int): When the Reservation starts. This is a Unix timestamp. duration (int): @@ -127,10 +138,27 @@ cdef class Reservation: TRES for the Reservation. users (list[str]): List of user names permitted to use the Reservation. + flags (pyslurm.ReservationFlags): + Optional Flags for the Reservation. + For convenience, instead of using [pyslurm.ReservationFlags][] in + combination with the logical Operators, you can set this attribute + via a [list][] of [str][]: + + flags = ["MAINTENANCE", "FLEX", "MAGNETIC"] + + When setting like this, the strings must match the names of members + in [pyslurm.ReservationFlags][]. + reoccurrence (pyslurm.ReservationReoccurrence): + Describes if and when this Reservation reoccurs. + Since [pyslurm.ReservationReoccurrence] members are also just + strings, you can conveniently also set the attribute like this: + + reoccurrence = "DAILY" """ cdef: reserve_info_t *info resv_desc_msg_t *umsg + _reoccurrence cdef readonly cluster diff --git a/pyslurm/core/reservation.pyx b/pyslurm/core/reservation.pyx index 49db7c36..48a5de3d 100644 --- a/pyslurm/core/reservation.pyx +++ b/pyslurm/core/reservation.pyx @@ -30,9 +30,12 @@ from pyslurm.core.slurmctld.config import _get_memory from datetime import datetime from pyslurm import xcollections from pyslurm.utils.helpers import instance_to_dict +from pyslurm.utils.enums import SlurmEnum, SlurmFlag +from enum import auto, StrEnum from pyslurm.utils.ctime import ( _raw_time, timestr_to_mins, + timestr_to_secs, date_to_timestamp, ) from pyslurm.core.error import ( @@ -63,7 +66,8 @@ cdef class Reservations(MultiClusterMap): """Load all Reservations in the system. Returns: - (pyslurm.Reservations): Collection of Reservation objects. + (pyslurm.Reservations): Collection of [pyslurm.Reservation][] + objects. Raises: (pyslurm.RPCError): When getting all the Reservations from the @@ -75,10 +79,7 @@ cdef class Reservations(MultiClusterMap): verify_rpc(slurm_load_reservations(0, &reservations.info)) - # prepare a dummy reserve_info_t struct. memset(&reservations.tmp_info, 0, sizeof(reserve_info_t)) - - # Put each pointer into its own instance. for cnt in range(reservations.info.record_count): reservation = Reservation.from_ptr(&reservations.info.reservation_array[cnt]) @@ -96,7 +97,6 @@ cdef class Reservations(MultiClusterMap): reservations.data[cluster][reservation.name] = reservation - # We have extracted all pointers reservations.info.record_count = 0 return reservations @@ -110,6 +110,7 @@ cdef class Reservation: def __init__(self, name=None, **kwargs): self._alloc_impl() self.name = name + self._reoccurrence = ReservationReoccurrence.NO self.cluster = settings.LOCAL_CLUSTER for k, v in kwargs.items(): setattr(self, k, v) @@ -130,6 +131,7 @@ cdef class Reservation: if not self.umsg: raise MemoryError("xmalloc failed for resv_desc_msg_t") slurm_init_resv_desc_msg(self.umsg) + self.umsg.flags = 0 def _dealloc_umsg(self): slurm_free_resv_desc_msg(self.umsg) @@ -162,6 +164,9 @@ cdef class Reservation: wrap._alloc_info() wrap.cluster = settings.LOCAL_CLUSTER memcpy(wrap.info, in_ptr, sizeof(reserve_info_t)) + wrap._reoccurrence = ReservationReoccurrence.from_flag(wrap.info.flags, + default=ReservationReoccurrence.NO) + wrap.info.flags &= ~wrap.reoccurrence._flag return wrap def _error_or_name(self): @@ -226,7 +231,16 @@ cdef class Reservation: Examples: >>> import pyslurm - >>> resv = pyslurm.Reservation("debug") + >>> from pyslurm import ReservationFlags, ReservationReoccurrence + >>> resv = pyslurm.Reservation( + ... name = "debug", + ... users = ["root"], + ... nodes = "node001", + ... duration = "1-00:00:00", + ... flags = ReservationFlags.MAINTENANCE, + ... reoccurrence = ReservationReoccurrence.DAILY, + ... ) + >>> resv.create() """ cdef char* new_name = NULL @@ -356,8 +370,6 @@ cdef class Reservation: def features(self, val): cstr.from_list2(&self.info.features, &self.umsg.features, val) - # TODO: flags - @property def groups(self): return cstr.to_list(self.info.groups) @@ -414,7 +426,15 @@ cdef class Reservation: def partition(self, val): cstr.fmalloc2(&self.info.partition, &self.umsg.partition, val) - # TODO: purge_comp_time ? + @property + def purge_time(self): + return u32_parse(self.info.purge_comp_time) + + @purge_time.setter + def purge_time(self, val): + self.info.purge_comp_time = self.umsg.purge_comp_time = timestr_to_secs(val) + if ReservationFlags.PURGE not in self.flags: + self.flags |= ReservationFlags.PURGE @property def start_time(self): @@ -464,3 +484,66 @@ cdef class Reservation: @users.setter def users(self, val): cstr.from_list2(&self.info.users, &self.umsg.users, val) + + @property + def reoccurrence(self): + return self._reoccurrence + + @reoccurrence.setter + def reoccurrence(self, val): + v = ReservationReoccurrence(val) + current = self._reoccurrence + self._reoccurrence = v + if v == ReservationReoccurrence.NO: + self.umsg.flags |= current._clear_flag + else: + self.umsg.flags |= v._flag + + @property + def flags(self): + return ReservationFlags(self.info.flags) + + @flags.setter + def flags(self, val): + # TODO: What if I want to clear all flags? + flag = val + if isinstance(val, list): + flag = ReservationFlags.from_list(val) + + self.info.flags = flag.value + self.umsg.flags = flag._get_flags_cleared() + + # TODO: RESERVE_FLAG_SKIP ? + + +class ReservationFlags(SlurmFlag): + MAINTENANCE = slurm.RESERVE_FLAG_MAINT, slurm.RESERVE_FLAG_NO_MAINT + MAGNETIC = slurm.RESERVE_FLAG_MAGNETIC, slurm.RESERVE_FLAG_NO_MAGNETIC + FLEX = slurm.RESERVE_FLAG_FLEX, slurm.RESERVE_FLAG_NO_FLEX + IGNORE_RUNNING_JOBS = slurm.RESERVE_FLAG_IGN_JOBS, slurm.RESERVE_FLAG_NO_IGN_JOB + ANY_NODES = slurm.RESERVE_FLAG_ANY_NODES, slurm.RESERVE_FLAG_NO_ANY_NODES + STATIC_NODES = slurm.RESERVE_FLAG_STATIC, slurm.RESERVE_FLAG_NO_STATIC + PARTITION_NODES_ONLY = slurm.RESERVE_FLAG_PART_NODES, slurm.RESERVE_FLAG_NO_PART_NODES + USER_DELETION = slurm.RESERVE_FLAG_USER_DEL, slurm.RESERVE_FLAG_NO_USER_DEL + PURGE = slurm.RESERVE_FLAG_PURGE_COMP, slurm.RESERVE_FLAG_NO_PURGE_COMP + SPECIFIC_NODES = slurm.RESERVE_FLAG_SPEC_NODES + NO_JOB_HOLD_AFTER_END = slurm.RESERVE_FLAG_NO_HOLD_JOBS + OVERLAP = slurm.RESERVE_FLAG_OVERLAP + ALL_NODES = slurm.RESERVE_FLAG_ALL_NODES + + +class ReservationReoccurrence(SlurmEnum): + """Different reocurrences for a Reservation + + Args: + NO: + No reocurrence defined + DAILY: + Daily reocurrence. + """ + NO = auto() + DAILY = auto(), slurm.RESERVE_FLAG_DAILY, slurm.RESERVE_FLAG_NO_DAILY + HOURLY = auto(), slurm.RESERVE_FLAG_HOURLY, slurm.RESERVE_FLAG_NO_HOURLY + WEEKLY = auto(), slurm.RESERVE_FLAG_WEEKLY, slurm.RESERVE_FLAG_NO_WEEKLY + WEEKDAY = auto(), slurm.RESERVE_FLAG_WEEKDAY, slurm.RESERVE_FLAG_NO_WEEKDAY + WEEKEND = auto(), slurm.RESERVE_FLAG_WEEKEND, slurm.RESERVE_FLAG_NO_WEEKEND From 30d4893dbf48286c925f5f0150823ac3d8787ebb Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Fri, 31 Jan 2025 16:42:51 +0100 Subject: [PATCH 11/18] update reservation tests --- tests/integration/test_reservation.py | 35 +++++++++++++++++++++++++++ tests/unit/test_reservation.py | 16 ++++++++++++ 2 files changed, 51 insertions(+) diff --git a/tests/integration/test_reservation.py b/tests/integration/test_reservation.py index de7b2cb5..e1ee3482 100644 --- a/tests/integration/test_reservation.py +++ b/tests/integration/test_reservation.py @@ -21,6 +21,7 @@ """test_reservation.py - integration test reservation functionalities.""" import pyslurm +from pyslurm import ReservationFlags, ReservationReoccurrence from datetime import datetime @@ -33,6 +34,7 @@ def test_api_calls(): duration=duration, users=["root"], node_count=1, + reoccurrence="DAILY" ) resv.create() @@ -55,6 +57,39 @@ def test_api_calls(): assert resv.duration == 2 * 60 * 24 assert resv.end_time == resv.start_time + (2 * 60 * 60 * 24) + assert resv.reoccurrence == ReservationReoccurrence.DAILY + assert resv.reoccurrence == "DAILY" + # Can only remove this once the Reservation exists. Setting another + # reoccurrence doesn't work, probably a bug in slurmctld..., because it + # makes no sense why that shouldn't work. + resv.reoccurrence = ReservationReoccurrence.NO + resv.modify() + + resv = pyslurm.Reservation.load("testing") + assert resv.reoccurrence == "NO" + + resv.flags = ReservationFlags.MAINTENANCE | ReservationFlags.FLEX + resv.modify() + + resv = pyslurm.Reservation.load("testing") + assert resv.flags == ReservationFlags.MAINTENANCE | ReservationFlags.FLEX + + assert ReservationFlags.PURGE not in resv.flags + resv.purge_time = "2-00:00:00" + resv.modify() + + resv = pyslurm.Reservation.load("testing") + assert ReservationFlags.PURGE in resv.flags + assert resv.purge_time == 2 * 60 * 60 * 24 + + resv.purge_time = "3-00:00:00" + resv.modify() + + resv = pyslurm.Reservation.load("testing") + assert ReservationFlags.PURGE in resv.flags + assert resv.purge_time == 3 * 60 * 60 * 24 + + assert resv.to_dict() resv.delete() reservations = pyslurm.Reservations.load() assert len(reservations) == 0 diff --git a/tests/unit/test_reservation.py b/tests/unit/test_reservation.py index 79f6bcc5..ec6eafde 100644 --- a/tests/unit/test_reservation.py +++ b/tests/unit/test_reservation.py @@ -21,6 +21,7 @@ """test_reservation.py - Unit test basic reservation functionalities.""" import pyslurm +from pyslurm import ReservationFlags, ReservationReoccurrence from datetime import datetime @@ -62,3 +63,18 @@ def test_create_instance(): assert resv.start_time == int(start.timestamp()) assert resv.end_time == end + (60 * 60 * 24) assert resv.duration == duration + (60 * 24) + + assert resv.reoccurrence == ReservationReoccurrence.NO + assert resv.reoccurrence == "NO" + resv.reoccurrence = ReservationReoccurrence.DAILY + + assert resv.flags == resv.flags.__class__(0) + resv.flags = ReservationFlags.MAINTENANCE | ReservationFlags.FLEX + assert resv.flags == ReservationFlags.MAINTENANCE | ReservationFlags.FLEX + resv.flags |= ReservationFlags.MAGNETIC + assert resv.flags == ReservationFlags.MAINTENANCE | ReservationFlags.FLEX | ReservationFlags.MAGNETIC + + resv.flags = ["FLEX", "PURGE"] + assert resv.flags == ReservationFlags.FLEX | ReservationFlags.PURGE + + From 1926299031b47cc7417a8a7cc332f2a36635c73d Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Fri, 31 Jan 2025 16:50:05 +0100 Subject: [PATCH 12/18] add custom enums --- pyslurm/utils/enums.pyx | 88 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 pyslurm/utils/enums.pyx diff --git a/pyslurm/utils/enums.pyx b/pyslurm/utils/enums.pyx new file mode 100644 index 00000000..1a65e3d0 --- /dev/null +++ b/pyslurm/utils/enums.pyx @@ -0,0 +1,88 @@ +######################################################################### +# utils/enums.pyx - pyslurm enum helpers +######################################################################### +# Copyright (C) 2025 Toni Harzendorf +# +# This file is part of PySlurm +# +# PySlurm is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# PySlurm is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with PySlurm; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# cython: c_string_type=unicode, c_string_encoding=default +# cython: language_level=3 + +from enum import Enum, Flag + + +class SlurmEnum(str, Enum): + + def __new__(cls, name, *args): + # https://docs.python.org/3/library/enum.html + # + # 1. + # Second argument to str is encoding, third is error. We don't really + # care for that, so no need to check. + # 2. + # Python Documentation recommends to not call super().__new__, but + # the corresponding types __new__ directly, so str here. + # 3. + # Docs recommend to set _value_ + v = str(name) + new_string = str.__new__(cls, v) + new_string._value_ = v + + new_string._flag = int(args[0]) if len(args) >= 1 else 0 + new_string._clear_flag = int(args[1]) if len(args) >= 2 else 0 + return new_string + + def __str__(self): + return str(self.value) + + @staticmethod + def _generate_next_value_(name, _start, _count, _last_values): + # We just care about the name of the member to be defined. + return name.upper() + + @classmethod + def from_flag(cls, flag, default): + out = cls(default) + for item in cls: + if item._flag & flag: + return item + return out + + +class SlurmFlag(Flag): + + def __new__(cls, flag, *args): + obj = super()._new_member_(cls) + obj._value_ = int(flag) + obj._clear_flag = int(args[0]) if len(args) >= 1 else 0 + return obj + + @classmethod + def from_list(cls, inp): + out = cls(0) + for flag in cls: + if flag.name in inp: + out |= flag + + return out + + def _get_flags_cleared(self): + val = self.value + for flag in self.__class__: + if flag not in self: + val |= flag._clear_flag + return val From cc6e882bf12f63e25b28fd1462d614ce25a5d1f6 Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Fri, 31 Jan 2025 16:50:37 +0100 Subject: [PATCH 13/18] wip reservation --- pyslurm/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pyslurm/__init__.py b/pyslurm/__init__.py index d34fac91..9bf54325 100644 --- a/pyslurm/__init__.py +++ b/pyslurm/__init__.py @@ -22,7 +22,12 @@ ) from pyslurm.core.node import Node, Nodes from pyslurm.core.partition import Partition, Partitions -from pyslurm.core.reservation import Reservation, Reservations +from pyslurm.core.reservation import ( + Reservation, + Reservations, + ReservationFlags, + ReservationReoccurrence, +) from pyslurm.core import error from pyslurm.core.error import ( PyslurmError, From 078a0c9055c9d6205dccf39560ee4b14c683a321 Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Fri, 31 Jan 2025 16:50:49 +0100 Subject: [PATCH 14/18] add u8_[set|parse]_bool_flag functions --- pyslurm/utils/uint.pxd | 2 ++ pyslurm/utils/uint.pyx | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/pyslurm/utils/uint.pxd b/pyslurm/utils/uint.pxd index d886d6f3..44b2eb27 100644 --- a/pyslurm/utils/uint.pxd +++ b/pyslurm/utils/uint.pxd @@ -47,3 +47,5 @@ cdef u32_parse_bool_flag(uint32_t flags, flag) cdef u32_set_bool_flag(uint32_t *flags, boolean, true_flag, false_flag=*) cdef u16_parse_bool_flag(uint16_t flags, flag) cdef u16_set_bool_flag(uint16_t *flags, boolean, true_flag, false_flag=*) +cdef u8_parse_bool_flag(uint8_t flags, flag) +cdef u8_set_bool_flag(uint8_t *flags, boolean, true_flag, false_flag=*) diff --git a/pyslurm/utils/uint.pyx b/pyslurm/utils/uint.pyx index 23ddad48..1cdd2fb1 100644 --- a/pyslurm/utils/uint.pyx +++ b/pyslurm/utils/uint.pyx @@ -172,6 +172,10 @@ cdef u16_parse_bool(uint16_t val): return uint_parse_bool(val, slurm.NO_VAL16) +cdef u8_set_bool_flag(uint8_t *flags, boolean, true_flag, false_flag=0): + flags[0] = uint_set_bool_flag(flags[0], boolean, true_flag, false_flag) + + cdef u16_set_bool_flag(uint16_t *flags, boolean, true_flag, false_flag=0): flags[0] = uint_set_bool_flag(flags[0], boolean, true_flag, false_flag) @@ -188,6 +192,10 @@ cdef u16_parse_bool_flag(uint16_t flags, flag): return uint_parse_bool_flag(flags, flag, slurm.NO_VAL16) +cdef u8_parse_bool_flag(uint8_t flags, flag): + return uint_parse_bool_flag(flags, flag, slurm.NO_VAL8) + + cdef u32_parse_bool_flag(uint32_t flags, flag): return uint_parse_bool_flag(flags, flag, slurm.NO_VAL) From 49471cb0519592fda87be0cef3ddadabda872183 Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Sat, 1 Feb 2025 16:22:15 +0100 Subject: [PATCH 15/18] wip reservation --- pyslurm/core/reservation.pyx | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pyslurm/core/reservation.pyx b/pyslurm/core/reservation.pyx index 48a5de3d..893e9f72 100644 --- a/pyslurm/core/reservation.pyx +++ b/pyslurm/core/reservation.pyx @@ -237,7 +237,7 @@ cdef class Reservation: ... users = ["root"], ... nodes = "node001", ... duration = "1-00:00:00", - ... flags = ReservationFlags.MAINTENANCE, + ... flags = ReservationFlags.MAINTENANCE | ReservationFlags.FLEX, ... reoccurrence = ReservationReoccurrence.DAILY, ... ) >>> resv.create() @@ -505,7 +505,6 @@ cdef class Reservation: @flags.setter def flags(self, val): - # TODO: What if I want to clear all flags? flag = val if isinstance(val, list): flag = ReservationFlags.from_list(val) @@ -517,6 +516,10 @@ cdef class Reservation: class ReservationFlags(SlurmFlag): + """Flags for Reservations that can be set. + + See {scontrol#OPT_Flags} for more info. + """ MAINTENANCE = slurm.RESERVE_FLAG_MAINT, slurm.RESERVE_FLAG_NO_MAINT MAGNETIC = slurm.RESERVE_FLAG_MAGNETIC, slurm.RESERVE_FLAG_NO_MAGNETIC FLEX = slurm.RESERVE_FLAG_FLEX, slurm.RESERVE_FLAG_NO_FLEX @@ -533,13 +536,9 @@ class ReservationFlags(SlurmFlag): class ReservationReoccurrence(SlurmEnum): - """Different reocurrences for a Reservation + """Different reocurrences for a Reservation. - Args: - NO: - No reocurrence defined - DAILY: - Daily reocurrence. + See {scontrol#OPT_Flags} for more info. """ NO = auto() DAILY = auto(), slurm.RESERVE_FLAG_DAILY, slurm.RESERVE_FLAG_NO_DAILY From 064b6a0d48e3626e9e59d2139269d4c34d3dcd6a Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Sat, 1 Feb 2025 16:22:24 +0100 Subject: [PATCH 16/18] update enums --- pyslurm/utils/enums.pyx | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/pyslurm/utils/enums.pyx b/pyslurm/utils/enums.pyx index 1a65e3d0..f453d2f8 100644 --- a/pyslurm/utils/enums.pyx +++ b/pyslurm/utils/enums.pyx @@ -23,9 +23,30 @@ # cython: language_level=3 from enum import Enum, Flag +import inspect +try: + from enum import EnumMeta as EnumType +except ImportError: + from enum import EnumType -class SlurmEnum(str, Enum): + +class DocstringSupport(EnumType): + def __new__(metacls, clsname, bases, classdict): + cls = super().__new__(metacls, clsname, bases, classdict) + + # In the future, if we want to properly document enum members, + # implement this: + # source = inspect.getdoc(cls) + # docstrings = source.replace(" ", "").split("\n") + + for member in cls: + member.__doc__ = "" + + return cls + + +class SlurmEnum(str, Enum, metaclass=DocstringSupport): def __new__(cls, name, *args): # https://docs.python.org/3/library/enum.html @@ -63,7 +84,7 @@ class SlurmEnum(str, Enum): return out -class SlurmFlag(Flag): +class SlurmFlag(Flag, metaclass=DocstringSupport): def __new__(cls, flag, *args): obj = super()._new_member_(cls) From 6961c90bd01ec39fdcc9289f1fc7f656b72c94fe Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Sat, 1 Feb 2025 16:22:40 +0100 Subject: [PATCH 17/18] add hack to improve generate docs for enum members --- scripts/griffe_exts.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/griffe_exts.py b/scripts/griffe_exts.py index 905f8358..c55b9150 100644 --- a/scripts/griffe_exts.py +++ b/scripts/griffe_exts.py @@ -30,7 +30,11 @@ SLURM_DOCS_URL_BASE = "https://slurm.schedmd.com/archive" SLURM_DOCS_URL_VERSIONED = f"{SLURM_DOCS_URL_BASE}/slurm-{SLURM_VERSION}-latest" -config_files = ["acct_gather.conf", "slurm.conf", "cgroup.conf", "mpi.conf"] +config_files = ["acct_gather.conf", + "slurm.conf", + "cgroup.conf", + "mpi.conf", + "scontrol"] def replace_with_slurm_docs_url(match): @@ -94,6 +98,18 @@ def on_instance( logger.debug(f"Object {obj.path} does not have a __doc__ attribute") return + # Hack to improve generated docs for Enums. + if hasattr(obj.parent, "bases"): + for base in obj.parent.bases: + b = base.lower() + if "enum" in b and not obj.name.startswith("_"): + v = obj.value[:-1].split(" ")[-1] + obj.value = v + obj.labels = {} + + if "slurmflag" in b: + obj.value = None + if not docstring or not obj.docstring: return From 310780e9c4c681c9fbe53b12c0161713805128d1 Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Sat, 1 Feb 2025 21:41:40 +0100 Subject: [PATCH 18/18] fix typos --- pyslurm/core/reservation.pyx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyslurm/core/reservation.pyx b/pyslurm/core/reservation.pyx index 893e9f72..e208c095 100644 --- a/pyslurm/core/reservation.pyx +++ b/pyslurm/core/reservation.pyx @@ -22,6 +22,7 @@ # cython: c_string_type=unicode, c_string_encoding=default # cython: language_level=3 +from typing import Union, Any from pyslurm.utils import cstr from pyslurm.utils import ctime from pyslurm.utils.uint import u32_parse @@ -87,7 +88,7 @@ cdef class Reservations(MultiClusterMap): # reason a MemoryError is raised after parsing subsequent # reservations, invalid behaviour will be shown by Valgrind, since # the Memory for the already parsed Reservation will be freed - # twice. So for all sucessfully parsed Reservations, replace it + # twice. So for all successfully parsed Reservations, replace it # with a dummy struct that will be skipped in case of error. reservations.info.reservation_array[cnt] = reservations.tmp_info @@ -218,7 +219,7 @@ cdef class Reservation: def create(self): """Create a Reservation. - If you did not specify atleast a `start_time` and `duration` or + If you did not specify at least a `start_time` and `duration` or `end_time`, then by default the Reservation will start effective immediately, with a duration of one year. @@ -245,7 +246,7 @@ cdef class Reservation: cdef char* new_name = NULL if not self.start_time or not (self.duration and self.end_time): - raise RPCError(msg="You must atleast specify a start_time, " + raise RPCError(msg="You must at least specify a start_time, " " combined with an end_time or a duration.") self.name = self._error_or_name()