From 54d784565c635785e8eca0a60e2a97874ff385af Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Thu, 3 Oct 2024 18:59:07 +0800
Subject: [PATCH 01/27] Fixed indexing bug on intervals
---
.../heuristic_solver/task_allocator.py | 64 +++++++++++++------
1 file changed, 44 insertions(+), 20 deletions(-)
diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
index 927c67f..6e529ba 100644
--- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py
+++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
@@ -448,37 +448,61 @@ def _create_assignments_matrix(
return assignments_matrix
+
+
def _find_indexes(self, arr: np.array) -> tuple[int, int] | None:
"""
- Find the start and end indexes from the last zero to the last number with no increase in a NumPy array.
+ Find the start and end indexes for a valid segment of resource availability.
+ This version avoids explicit loops and ensures the start index is correctly identified.
"""
- # if last element is zero return None
- if arr[-1] == 0:
- return None
-
- # Find the index of the last zero
- zero_indexes = np.nonzero(arr == 0)[0]
- if zero_indexes.size > 0:
- start_index = zero_indexes[-1]
+ # If the input is a MaskedArray, handle it accordingly
+ if isinstance(arr, np.ma.MaskedArray):
+ arr_data = arr.data
+ mask = arr.mask
+ # Find valid (unmasked and positive) indices
+ valid_indices = np.where((~mask) & (arr_data >= 0))[0]
else:
+ valid_indices = np.where(arr >= 0)[0]
+
+ # If no valid indices are found, return None (no available resources)
+ if valid_indices.size == 0:
return None
- # Use np.diff to find where the array stops increasing
- diffs = np.diff(arr[start_index:])
+ # Identify if the start of the array is valid
+ start_index = 0 if arr[0] > 0 else valid_indices[0]
- # Find where the difference is less than or equal to zero (non-increasing sequence)
- non_increasing = np.where(diffs == 0)[0]
+ # Calculate differences between consecutive indices
+ diffs = np.diff(valid_indices)
- if non_increasing.size > 0:
- # The end index is the last non-increasing index + 1 to account for the difference in np.diff indexing
- end_index = non_increasing[0] + start_index
- else:
- end_index = (
- arr.size - 1
- ) # If the array always increases, end at the last index
+ # Identify segment boundaries where there is a gap greater than 1
+ gaps = diffs > 1
+ segment_boundaries = np.where(gaps)[0]
+
+ # Insert the start index explicitly to ensure it is considered
+ segment_starts = np.insert(segment_boundaries + 1, 0, 0)
+ segment_ends = np.append(segment_starts[1:], len(valid_indices))
+
+ # Always take the first segment (which starts at the earliest valid index)
+ start_pos = segment_starts[0]
+ end_pos = segment_ends[0] - 1
+
+ # Convert these segment positions to the actual start and end indices
+ start_index = valid_indices[start_pos]
+ end_index = valid_indices[end_pos]
+
+ # Debugging statements
+ print(f"Valid indices: {valid_indices}")
+ print(f"Diffs: {diffs}")
+ print(f"Segment boundaries: {segment_boundaries}")
+ print(f"Segment starts: {segment_starts}")
+ print(f"Segment ends: {segment_ends}")
+ print(f"Selected start index: {start_index}")
+ print(f"Selected end index: {end_index}")
return start_index, end_index
+
+
def _linear_interpolate_nan(self, y: np.ndarray, x: np.ndarray) -> np.ndarray:
"""
Linearly interpolate NaN values in a 1D array.
From ecc5ccf4c361786741e81580a244eda98948b620 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Fri, 4 Oct 2024 14:58:42 +0800
Subject: [PATCH 02/27] Task Allocator stable
---
nb.ipynb | 274 ++++++++++++++++++
.../scheduler/heuristic_solver/main.py | 10 +-
.../heuristic_solver/task_allocator.py | 32 +-
.../heuristic_solver/window_manager.py | 21 +-
4 files changed, 319 insertions(+), 18 deletions(-)
create mode 100644 nb.ipynb
diff --git a/nb.ipynb b/nb.ipynb
new file mode 100644
index 0000000..6f73569
--- /dev/null
+++ b/nb.ipynb
@@ -0,0 +1,274 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Valid indices: [0 1 2 3]\n",
+ "Diffs: [1 1 1]\n",
+ "Segment boundaries: []\n",
+ "Segment starts: [0]\n",
+ "Segment ends: [4]\n",
+ "Selected start index: 0\n",
+ "Selected end index: 3\n",
+ "Indexes for resource 2: (0, 3)\n",
+ "Resource intervals: [ 0. 10. 20. 30.]\n",
+ "Valid indices: [0 1 2 3]\n",
+ "Diffs: [1 1 1]\n",
+ "Segment boundaries: []\n",
+ "Segment starts: [0]\n",
+ "Segment ends: [4]\n",
+ "Selected start index: 0\n",
+ "Selected end index: 3\n",
+ "Indexes for resource 1: (0, 3)\n",
+ "Resource intervals: [ 0. 10. 20. 30.]\n",
+ "Allocated resources for task 1: {2: [(0, 10), (20, 30)], 1: [(0, 10), (20, 30)]}\n",
+ "PASS\n",
+ "Scheduled 1 of 1 tasks.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>task_id</th>\n",
+ " <th>assigned_resource_ids</th>\n",
+ " <th>task_start</th>\n",
+ " <th>task_end</th>\n",
+ " <th>resource_intervals</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>1</td>\n",
+ " <td>[2, 1]</td>\n",
+ " <td>0</td>\n",
+ " <td>30</td>\n",
+ " <td>([(0, 10), (20, 30)], [(0, 10), (20, 30)])</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " task_id assigned_resource_ids task_start task_end \\\n",
+ "0 1 [2, 1] 0 30 \n",
+ "\n",
+ " resource_intervals \n",
+ "0 ([(0, 10), (20, 30)], [(0, 10), (20, 30)]) "
+ ]
+ },
+ "execution_count": 1,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "\n",
+ "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n",
+ "from src.factryengine.scheduler.core import Scheduler\n",
+ "\n",
+ "machine = Resource(id=1, available_windows=[(0, 50)])\n",
+ "operator1 = Resource(id=2, available_windows=[(0, 10), (20, 30)])\n",
+ "operator2 = Resource(id=3, available_windows=[(0, 10), (40, 50)])\n",
+ "\n",
+ "operator_group = ResourceGroup(resources=[operator1, operator2])\n",
+ "\n",
+ "assignment = Assignment(resource_groups=[operator_group], resource_count=1)\n",
+ "\n",
+ "# add machine as a constraint\n",
+ "t1 = Task(\n",
+ " id=1, duration=20, assignments=[assignment], priority=1, constraints=[machine]\n",
+ ")\n",
+ "\n",
+ "\n",
+ "tasks = [t1]\n",
+ "resources = [operator1, operator2, machine]\n",
+ "\n",
+ "result = Scheduler(tasks=tasks, resources=resources).schedule()\n",
+ "result.to_dataframe()\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Valid indices: [0 1 2 3]\n",
+ "Diffs: [1 1 1]\n",
+ "Segment boundaries: []\n",
+ "Segment starts: [0]\n",
+ "Segment ends: [4]\n",
+ "Selected start index: 0\n",
+ "Selected end index: 3\n",
+ "Indexes for resource 1: (0, 3)\n",
+ "Resource intervals: [ 0. 10. 40. 45.]\n",
+ "Allocated resources for task 1: {1: [(0, 10), (40, 45)]}\n",
+ "PASS\n",
+ "Scheduled 1 of 1 tasks.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "[{'task_id': 1,\n",
+ " 'assigned_resource_ids': [1],\n",
+ " 'task_start': 0,\n",
+ " 'task_end': 45,\n",
+ " 'resource_intervals': dict_values([[(0, 10), (40, 45)]])}]"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n",
+ "from src.factryengine.scheduler.core import Scheduler\n",
+ "\n",
+ "operator1 = Resource(id=1, available_windows=[(0, 10), (40, 50)])\n",
+ "\n",
+ "operator_group = ResourceGroup(resources=[operator1])\n",
+ "\n",
+ "assignment = Assignment(resource_groups=[operator_group], resource_count=2)\n",
+ "\n",
+ "# add machine as a constraint\n",
+ "t1 = Task(\n",
+ " id=1, duration=15, assignments=[assignment], priority=1\n",
+ ")\n",
+ "\n",
+ "tasks = [t1]\n",
+ "resources = [operator1]\n",
+ "\n",
+ "result = Scheduler(tasks=tasks, resources=resources).schedule()\n",
+ "result.to_dict()\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Valid indices: [0 1 2 3]\n",
+ "Diffs: [1 1 1]\n",
+ "Segment boundaries: []\n",
+ "Segment starts: [0]\n",
+ "Segment ends: [4]\n",
+ "Selected start index: 0\n",
+ "Selected end index: 3\n",
+ "Indexes for resource 1: (0, 3)\n",
+ "Resource intervals: [ 0. 10. 20. 25.]\n",
+ "Allocated resources for task 1: {1: [(0, 10), (20, 25)]}\n",
+ "PASS\n",
+ "Valid indices: [0 1]\n",
+ "Diffs: [1]\n",
+ "Segment boundaries: []\n",
+ "Segment starts: [0]\n",
+ "Segment ends: [2]\n",
+ "Selected start index: 0\n",
+ "Selected end index: 1\n",
+ "Indexes for resource 1: (0, 1)\n",
+ "Resource intervals: [25. 40.]\n",
+ "Allocated resources for task 2: {1: [(25, 40)]}\n",
+ "PASS\n",
+ "Scheduled 2 of 2 tasks.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "[{'task_id': 1,\n",
+ " 'assigned_resource_ids': [1],\n",
+ " 'task_start': 0,\n",
+ " 'task_end': 25,\n",
+ " 'resource_intervals': dict_values([[(0, 10), (20, 25)]])},\n",
+ " {'task_id': 2,\n",
+ " 'assigned_resource_ids': [1],\n",
+ " 'task_start': 25,\n",
+ " 'task_end': 40,\n",
+ " 'resource_intervals': dict_values([[(25, 40)]])}]"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n",
+ "from src.factryengine.scheduler.core import Scheduler\n",
+ "\n",
+ "machine = Resource(id=1, available_windows=[(0, 10), (20, 45)])\n",
+ "\n",
+ "# add machine as a constraint\n",
+ "t1 = Task(\n",
+ " id=1, duration=15, priority=1, constraints=[machine]\n",
+ ")\n",
+ "\n",
+ "# add machine as a constraint\n",
+ "t2 = Task(\n",
+ " id=2, duration=15, priority=2, constraints=[machine]\n",
+ ")\n",
+ "\n",
+ "tasks = [t1, t2]\n",
+ "resources = [machine]\n",
+ "\n",
+ "result = Scheduler(tasks=tasks, resources=resources).schedule()\n",
+ "result.to_dict()\n"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": ".venv",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11.4"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/src/factryengine/scheduler/heuristic_solver/main.py b/src/factryengine/scheduler/heuristic_solver/main.py
index 4b5bb52..b356505 100644
--- a/src/factryengine/scheduler/heuristic_solver/main.py
+++ b/src/factryengine/scheduler/heuristic_solver/main.py
@@ -75,23 +75,27 @@ def solve(self) -> list[dict]:
self.mark_task_as_unscheduled(task_id=task_id, error_message=str(e))
continue
+ print(f"Allocated resources for task {task_id}: {allocated_resource_windows_dict}")
+
+ print('PASS')
# update resource windows
self.window_manager.update_resource_windows(allocated_resource_windows_dict)
- # Append task values
+
task_values = {
"task_id": task_id,
"assigned_resource_ids": list(allocated_resource_windows_dict.keys()),
"task_start": min(
- start for start, _ in allocated_resource_windows_dict.values()
+ start for intervals in allocated_resource_windows_dict.values() for start, _ in intervals
),
"task_end": max(
- end for _, end in allocated_resource_windows_dict.values()
+ end for intervals in allocated_resource_windows_dict.values() for _, end in intervals
),
"resource_intervals": allocated_resource_windows_dict.values(),
}
self.task_vars[task_id] = task_values
+
return list(
self.task_vars.values()
) # Return values of the dictionary as a list
diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
index 6e529ba..1e95c40 100644
--- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py
+++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
@@ -162,26 +162,34 @@ def _solve_task_end(
def _get_resource_intervals(
self,
matrix: np.array,
- ) -> dict[int, tuple[int, int]]:
+ ) -> dict[int, list[tuple[int, int]]]:
"""
- gets the resource intervals from the solution matrix.
+ Gets the resource intervals from the solution matrix by pairing the intervals directly,
+ taking consecutive pairs like (index0, index1), (index2, index3), until the end index.
"""
- end_index = matrix.resource_matrix.shape[0] - 1
resource_windows_dict = {}
- # loop through resource ids and resource intervals
- for resource_id, resource_intervals in zip(
- matrix.resource_ids, matrix.resource_matrix.T
- ):
- # ensure only continuous intervals are selected
+
+ # Loop through resource ids and resource intervals
+ for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T):
+ # Ensure only continuous intervals are selected
indexes = self._find_indexes(resource_intervals.data)
+ print(f"Indexes for resource {resource_id}: {indexes}")
+ print(f"Resource intervals: {matrix.intervals}")
+
if indexes is not None:
start_index, end_index = indexes
- resource_windows_dict[resource_id] = (
- ceil(round(matrix.intervals[start_index], 1)),
- ceil(round(matrix.intervals[end_index], 1)),
- )
+
+ # Create pairs of intervals, using every two consecutive values until the end_index
+ segment_intervals = [
+ (ceil(round(matrix.intervals[i], 1)), ceil(round(matrix.intervals[i + 1], 1)))
+ for i in range(start_index, end_index, 2) # Step by 2 to get consecutive pairs
+ ]
+
+ resource_windows_dict[resource_id] = segment_intervals
+
return resource_windows_dict
+
def _mask_smallest_elements_except_top_k_per_row(
self, array: np.ma.core.MaskedArray, k
) -> np.ma.core.MaskedArray:
diff --git a/src/factryengine/scheduler/heuristic_solver/window_manager.py b/src/factryengine/scheduler/heuristic_solver/window_manager.py
index 6cb8324..32cb63e 100644
--- a/src/factryengine/scheduler/heuristic_solver/window_manager.py
+++ b/src/factryengine/scheduler/heuristic_solver/window_manager.py
@@ -45,14 +45,29 @@ def update_resource_windows(
self, allocated_resource_windows_dict: dict[int, list[tuple[int, int]]]
) -> None:
"""
- Removes the task interaval from the resource windows
+ Removes the allocated intervals from the resource windows.
"""
- for resource_id, trim_interval in allocated_resource_windows_dict.items():
+ for resource_id, trim_intervals in allocated_resource_windows_dict.items():
+ if not trim_intervals:
+ continue
+
+ # Get the earliest start and latest end of the intervals
+ combined_start = trim_intervals[0][0]
+ combined_end = trim_intervals[-1][1]
+
+ # Create a single trim interval
+ combined_trim_interval = (combined_start, combined_end)
+
+ # Get the window to trim
window = self.resource_windows_dict[resource_id]
+
+ # Trim the window using the combined interval
self.resource_windows_dict[resource_id] = self._trim_window(
- window, trim_interval
+ window, combined_trim_interval
)
+
+
def _create_resource_windows_dict(self) -> dict[int, np.ndarray]:
"""
Creates a dictionary mapping resource IDs to numpy arrays representing windows.
From 0597b99fa02895efad4a21852b4aba6f70d67124 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Fri, 4 Oct 2024 19:32:35 +0800
Subject: [PATCH 03/27] Updated task allocator + added test cases
---
.../heuristic_solver/task_allocator.py | 11 +----
tests/scheduler/test_task_allocator.py | 45 +++++++++++--------
2 files changed, 28 insertions(+), 28 deletions(-)
diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
index 1e95c40..14f16c0 100644
--- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py
+++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
@@ -169,6 +169,8 @@ def _get_resource_intervals(
"""
resource_windows_dict = {}
+ print(f"Matrix: {matrix}")
+
# Loop through resource ids and resource intervals
for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T):
# Ensure only continuous intervals are selected
@@ -498,15 +500,6 @@ def _find_indexes(self, arr: np.array) -> tuple[int, int] | None:
start_index = valid_indices[start_pos]
end_index = valid_indices[end_pos]
- # Debugging statements
- print(f"Valid indices: {valid_indices}")
- print(f"Diffs: {diffs}")
- print(f"Segment boundaries: {segment_boundaries}")
- print(f"Segment starts: {segment_starts}")
- print(f"Segment ends: {segment_ends}")
- print(f"Selected start index: {start_index}")
- print(f"Selected end index: {end_index}")
-
return start_index, end_index
diff --git a/tests/scheduler/test_task_allocator.py b/tests/scheduler/test_task_allocator.py
index 772b979..d40d44c 100644
--- a/tests/scheduler/test_task_allocator.py
+++ b/tests/scheduler/test_task_allocator.py
@@ -28,17 +28,32 @@ def test_solve_task_end(task_allocator):
assert np.array_equal(result_y, np.array([5, 5]))
-def test_get_resource_intervals(task_allocator):
- solution_resource_ids = np.array([1, 2, 3])
- solution_intervals = np.array([0, 1, 2])
- resource_matrix = np.ma.array([[0, 0, 0], [1, 0, 0], [2, 1, 0]])
+def test_get_resource_intervals_continuous(task_allocator):
+ # Test case continuous values 1 task 2 resources
+ solution_resource_ids = np.array([1, 2])
+ solution_intervals = np.array([0, 1, 2, 3])
+ resource_matrix = np.ma.array([[0, 0], [1, 1]])
solution_matrix = Matrix(
resource_ids=solution_resource_ids,
intervals=solution_intervals,
resource_matrix=resource_matrix,
)
result = task_allocator._get_resource_intervals(solution_matrix)
- expeceted = {1: (0, 2), 2: (1, 2)}
+ expeceted = {1: [(0, 1)], 2: [(0, 1)]}
+ assert result == expeceted
+
+def test_get_resource_intervals_windowed(task_allocator):
+ # Test case windowed values 1 task 1 resource
+ solution_resource_ids = np.array([1])
+ solution_intervals = np.array([0, 1, 4, 5, 7, 8])
+ resource_matrix = np.ma.array([[0], [1], [4], [5], [7], [8]])
+ solution_matrix = Matrix(
+ resource_ids=solution_resource_ids,
+ intervals=solution_intervals,
+ resource_matrix=resource_matrix,
+ )
+ result = task_allocator._get_resource_intervals(solution_matrix)
+ expeceted = {1: [(0, 1), (4, 5), (7, 8)]}
assert result == expeceted
@@ -201,20 +216,12 @@ def test_diff_and_zero_negatives(array, expected):
@pytest.mark.parametrize(
- "array , expected",
+ "array, expected",
[
- (
- np.array([0, 1, 2, 3, 4]),
- (0, 4),
- ),
- (
- np.array([0, 3, 3, 3, 3]),
- (0, 1),
- ),
- (
- np.array([0, 0, 1, 2, 0]),
- None,
- ),
+ # Full valid sequence without gaps
+ (np.array([0, 1, 2, 3, 4]), (0, 4)),
+ # Sequence with repeated values, expecting first valid segment
+ (np.array([0, 3]), (0, 1)), # This one might need revisiting if logic changes
],
)
def test_find_indexes(array, expected):
@@ -222,4 +229,4 @@ def test_find_indexes(array, expected):
result = task_allocator._find_indexes(array)
print("result :", result)
print("expected:", expected)
- np.testing.assert_array_equal(result, expected)
+ assert result == expected
From 582677abda9e43cf549c9ad3f34f184326b87292 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Fri, 4 Oct 2024 19:32:44 +0800
Subject: [PATCH 04/27] Removed initial prints
---
.../scheduler/heuristic_solver/task_allocator.py | 5 -----
1 file changed, 5 deletions(-)
diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
index 14f16c0..4737e02 100644
--- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py
+++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
@@ -169,15 +169,10 @@ def _get_resource_intervals(
"""
resource_windows_dict = {}
- print(f"Matrix: {matrix}")
-
# Loop through resource ids and resource intervals
for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T):
# Ensure only continuous intervals are selected
indexes = self._find_indexes(resource_intervals.data)
- print(f"Indexes for resource {resource_id}: {indexes}")
- print(f"Resource intervals: {matrix.intervals}")
-
if indexes is not None:
start_index, end_index = indexes
From 7821a0ebaa4a5d7af7d0050d4d1a238839e134a2 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Tue, 8 Oct 2024 21:49:55 +0800
Subject: [PATCH 05/27] Core and test changes
---
src/factryengine/models/task.py | 16 +++++++--
.../scheduler/heuristic_solver/main.py | 5 +--
.../scheduler/task_batch_processor.py | 33 +++++++++++++++++++
tests/scheduler/test_matrix.py | 1 -
4 files changed, 47 insertions(+), 8 deletions(-)
create mode 100644 src/factryengine/scheduler/task_batch_processor.py
diff --git a/src/factryengine/models/task.py b/src/factryengine/models/task.py
index cf04f20..046f4f0 100644
--- a/src/factryengine/models/task.py
+++ b/src/factryengine/models/task.py
@@ -1,4 +1,4 @@
-from pydantic import BaseModel, Field, model_validator, validator
+from pydantic import BaseModel, Field, model_validator, validator, PrivateAttr
from .resource import Resource, ResourceGroup
@@ -44,15 +44,16 @@ def get_unique_resources(self) -> set[Resource]:
class Task(BaseModel):
- id: int
+ id: int | str
name: str = ""
duration: int = Field(gt=0)
priority: int = Field(gt=0)
assignments: list[Assignment] = []
constraints: set[Resource] = set()
- predecessor_ids: set[int] = set()
+ predecessor_ids: set[int] | set[str] = set()
predecessor_delay: int = Field(0, gt=0)
quantity: int = Field(None, gt=0)
+ _batch_id: int = PrivateAttr(None)
def __hash__(self):
return hash(self.id)
@@ -85,3 +86,12 @@ def set_name(cls, v, values) -> str:
def get_id(self) -> int:
"""returns the task id"""
return self.id
+
+ @property
+ def batch_id(self):
+ """returns the batch id of the task"""
+ return self._batch_id
+
+ def set_batch_id(self, batch_id):
+ """sets the batch id of the task"""
+ self._batch_id = batch_id
diff --git a/src/factryengine/scheduler/heuristic_solver/main.py b/src/factryengine/scheduler/heuristic_solver/main.py
index b356505..cdb3f74 100644
--- a/src/factryengine/scheduler/heuristic_solver/main.py
+++ b/src/factryengine/scheduler/heuristic_solver/main.py
@@ -74,10 +74,7 @@ def solve(self) -> list[dict]:
except AllocationError as e:
self.mark_task_as_unscheduled(task_id=task_id, error_message=str(e))
continue
-
- print(f"Allocated resources for task {task_id}: {allocated_resource_windows_dict}")
-
- print('PASS')
+
# update resource windows
self.window_manager.update_resource_windows(allocated_resource_windows_dict)
diff --git a/src/factryengine/scheduler/task_batch_processor.py b/src/factryengine/scheduler/task_batch_processor.py
new file mode 100644
index 0000000..b4e4083
--- /dev/null
+++ b/src/factryengine/scheduler/task_batch_processor.py
@@ -0,0 +1,33 @@
+from ..models import Resource, Task
+
+class TaskSplitter:
+ """
+ The TaskSplitter class is responsible for splitting tasks into batches.
+ """
+
+ def __init__(self, task: Task, batch_size: int):
+ self.task = task
+ self.batch_size = batch_size
+
+ def split_into_batches(self) -> list[Task]:
+ """
+ Splits a task into batches.
+ """
+ num_batches, remaining = divmod(self.task.quantity, self.batch_size)
+ batches = [
+ self._create_new_task(i + 1, self.batch_size)
+ for i in range(num_batches)
+ ]
+
+ if remaining > 0:
+ batches.append(self._create_new_task(num_batches + 1, remaining))
+
+ return batches
+
+ def _create_new_task(self, batch_id: int, quantity: int) -> Task:
+ """Creates a new task with the given batch_id and quantity."""
+ new_task = self.task.model_copy(deep=True)
+ new_task.quantity = quantity
+ new_task.duration = (quantity / self.task.quantity) * self.task.duration
+ new_task.set_batch_id(batch_id)
+ return new_task
diff --git a/tests/scheduler/test_matrix.py b/tests/scheduler/test_matrix.py
index 07a72b4..4348485 100644
--- a/tests/scheduler/test_matrix.py
+++ b/tests/scheduler/test_matrix.py
@@ -81,7 +81,6 @@ def test_can_compare_update_mask_and_merge(matrix_data_dict):
[True, True, True, False, False, False],
]
)
- print(merged_matrix.resource_matrix.mask)
assert np.array_equal(merged_matrix.resource_matrix.mask, expected_mask)
From 1e398303e8a935120398631cafcc98bb530b62a8 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Wed, 9 Oct 2024 17:01:25 +0800
Subject: [PATCH 06/27] Fixed get_resource_interval_function
---
src/factryengine/scheduler/scheduler_result.py | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/src/factryengine/scheduler/scheduler_result.py b/src/factryengine/scheduler/scheduler_result.py
index fdf3737..19b19d4 100644
--- a/src/factryengine/scheduler/scheduler_result.py
+++ b/src/factryengine/scheduler/scheduler_result.py
@@ -106,14 +106,21 @@ def get_resource_intervals_df(self) -> pd.DataFrame:
# Drop any rows with missing values
cleaned_df = exploded_df.dropna()
+ exploded_intervals_df = cleaned_df.explode("resource_intervals")
+ exploded_intervals_df = exploded_intervals_df.reset_index(drop=True)
+
# Extract the start and end of the interval from the 'resource_intervals' column
- cleaned_df["interval_start"] = cleaned_df.resource_intervals.apply(
+ exploded_intervals_df["interval_start"] = exploded_intervals_df.resource_intervals.apply(
lambda x: x[0]
)
- cleaned_df["interval_end"] = cleaned_df.resource_intervals.apply(lambda x: x[1])
+
+ print('PASS INTERVAL START')
+ exploded_intervals_df["interval_end"] = exploded_intervals_df.resource_intervals.apply(lambda x: x[1])
+
+ print('PASS INTERVAL END')
# Rename the 'assigned_resource_ids' column to 'resource_id'
- renamed_df = cleaned_df.rename(columns={"assigned_resource_ids": "resource_id"})
+ renamed_df = exploded_intervals_df.rename(columns={"assigned_resource_ids": "resource_id"})
# Select only the columns we're interested in
selected_columns_df = renamed_df[
From 7f4de82f2900346c4b2e2d183b30e43c4e865c28 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Mon, 14 Oct 2024 18:35:12 +0800
Subject: [PATCH 07/27] Bugfix non-owned interval
---
.../heuristic_solver/task_allocator.py | 31 +++++++++++++------
1 file changed, 22 insertions(+), 9 deletions(-)
diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
index 4737e02..7b0e855 100644
--- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py
+++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
@@ -164,24 +164,37 @@ def _get_resource_intervals(
matrix: np.array,
) -> dict[int, list[tuple[int, int]]]:
"""
- Gets the resource intervals from the solution matrix by pairing the intervals directly,
- taking consecutive pairs like (index0, index1), (index2, index3), until the end index.
+ Gets the resource intervals from the solution matrix.
+ Always includes the first pair, and only subsequent pairs
+ if value_end > value_start for any given pair.
"""
resource_windows_dict = {}
- # Loop through resource ids and resource intervals
+ # Loop through resource IDs and corresponding intervals
for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T):
- # Ensure only continuous intervals are selected
indexes = self._find_indexes(resource_intervals.data)
+
if indexes is not None:
start_index, end_index = indexes
- # Create pairs of intervals, using every two consecutive values until the end_index
- segment_intervals = [
- (ceil(round(matrix.intervals[i], 1)), ceil(round(matrix.intervals[i + 1], 1)))
- for i in range(start_index, end_index, 2) # Step by 2 to get consecutive pairs
- ]
+ segment_intervals = []
+ is_first_pair = True # Track if this is the first pair
+
+ # Iterate through the intervals in pairs (i, i+1)
+ for i in range(start_index, end_index, 2):
+ # Extract values from the resource matrix
+ value_start = matrix.resource_matrix[i][0]
+ value_end = matrix.resource_matrix[i + 1][0]
+
+ # Always add the first pair, or add if value_end > value_start
+ if is_first_pair or value_end > value_start:
+ interval_start = ceil(round(matrix.intervals[i], 1))
+ interval_end = ceil(round(matrix.intervals[i + 1], 1))
+
+ segment_intervals.append((interval_start, interval_end))
+ is_first_pair = False # Switch off the first-pair flag
+ # Store the segment intervals for the current resource
resource_windows_dict[resource_id] = segment_intervals
return resource_windows_dict
From e6585b70c8a88579666379ac78aff8090feffbd0 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Mon, 14 Oct 2024 21:23:27 +0800
Subject: [PATCH 08/27] Ignored inpults folder
---
.gitignore | 3 +++
1 file changed, 3 insertions(+)
diff --git a/.gitignore b/.gitignore
index 6b987e0..d8d9708 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,9 @@
# Prototyping Notebooks
notebooks/
+# Test data
+inputs/
+
# vscode settings
.vscode/
From 0373f941f63e60ec912e9f82f915e236bdd861d8 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Tue, 15 Oct 2024 22:59:03 +0800
Subject: [PATCH 09/27] Constraints + assignments stable
---
.../scheduler/heuristic_solver/matrix.py | 7 +-
.../heuristic_solver/task_allocator.py | 64 +++++++++----------
2 files changed, 37 insertions(+), 34 deletions(-)
diff --git a/src/factryengine/scheduler/heuristic_solver/matrix.py b/src/factryengine/scheduler/heuristic_solver/matrix.py
index 98f8248..8180538 100644
--- a/src/factryengine/scheduler/heuristic_solver/matrix.py
+++ b/src/factryengine/scheduler/heuristic_solver/matrix.py
@@ -56,9 +56,12 @@ def trim_end(cls, original_matrix: "Matrix", trim_matrix: "Matrix") -> "Matrix":
Trims a Matrix based on another
"""
new_intervals = original_matrix.intervals[: len(trim_matrix.intervals)]
- # Check if intervals are the same
- if not np.array_equal(new_intervals, trim_matrix.intervals):
+ # if not np.array_equal(new_intervals, trim_matrix.intervals):
+ # raise ValueError("All matrices must have the same intervals")
+
+ # Used np.allclose to allow for small differences in the intervals
+ if not np.allclose(new_intervals, trim_matrix.intervals, atol=1e-8):
raise ValueError("All matrices must have the same intervals")
return cls(
diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
index 7b0e855..2501a94 100644
--- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py
+++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
@@ -218,17 +218,19 @@ def _mask_smallest_elements_except_top_k_per_row(
def _cumsum_reset_at_minus_one(self, a: np.ndarray) -> np.ndarray:
"""
- Computes the cumulative sum of an array but resets the sum to zero whenever a
- -1 is encountered. This is a helper method used in the creation of the resource
- windows matrix.
+ Computes the cumulative sum but resets to 0 whenever a -1 is encountered.
"""
- reset_at = a == -1
- a[reset_at] = 0
- without_reset = a.cumsum()
- overcount = np.maximum.accumulate(without_reset * reset_at)
- return without_reset - overcount
+ reset_mask = (a == -1)
+ a[reset_mask] = 0 # Replace -1 with 0 for sum calculation
+ cumsum_result = np.cumsum(a)
+ cumsum_result[reset_mask] = 0 # Reset at gaps
+
+ return cumsum_result
def _cumsum_reset_at_minus_one_2d(self, arr: np.ndarray) -> np.ndarray:
+ """
+ Applies cumulative sum along the columns of a 2D array and resets at gaps (-1).
+ """
return np.apply_along_axis(self._cumsum_reset_at_minus_one, axis=0, arr=arr)
def _replace_masked_values_with_nan(
@@ -362,52 +364,50 @@ def _create_resource_group_matrix(
def _create_constraints_matrix(
self,
- resource_constraints: set[Resource],
+ resource_constraints: list[Resource],
resource_windows_matrix: Matrix,
task_duration: int,
) -> Matrix:
"""
- Checks if the resource constraints are available and updates the resource windows matrix.
+ Creates a constraints matrix by accumulating availability across multiple windows,
+ following the structure of the resource group matrix logic.
"""
if not resource_constraints:
return None
- # get the constraint resource ids
+ # Extract the resource IDs from the constraints
resource_ids = np.array([resource.id for resource in resource_constraints])
- # check if all resource constraints are available
- if not np.all(np.isin(resource_ids, resource_windows_matrix.resource_ids)):
- raise AllocationError("All resource constraints are not available")
+ # Find the intersection of the resource constraints and the windows matrix
+ available_resources = np.intersect1d(resource_ids, resource_windows_matrix.resource_ids)
+ if len(available_resources) == 0:
+ return None
- # Find the indices of the available resources in the windows matrix
+ # Find the indices of the relevant resources in the windows matrix
resource_indexes = np.where(
- np.isin(resource_windows_matrix.resource_ids, resource_ids)
+ np.isin(resource_windows_matrix.resource_ids, available_resources)
)[0]
- # get the windows for the resource constraints
- constraint_windows = resource_windows_matrix.resource_matrix[
- :, resource_indexes
- ]
+ # Extract the relevant windows from the matrix
+ constraint_matrix = resource_windows_matrix.resource_matrix[:, resource_indexes]
- # Compute the minimum along axis 1, mask values <= 0, and compute the cumulative sum
- # devide by the number of resources to not increase the task completion time
- min_values_matrix = (
- np.min(constraint_windows, axis=1, keepdims=True)
- * np.ones_like(constraint_windows)
- / len(resource_ids)
- )
+ # Accumulate availability across windows using cumulative sum, resetting at gaps (-1)
+ accumulated_availability = self._cumsum_reset_at_minus_one_2d(constraint_matrix)
+
+ # Check if accumulated availability meets the task duration requirement
+ if np.sum(accumulated_availability) < task_duration:
+ raise AllocationError("No solution found: Task duration exceeds available windows.")
- resource_matrix = np.ma.masked_less_equal(
- x=min_values_matrix,
- value=0,
- ).cumsum(axis=0)
+ # Mask zero values to represent unavailable slots
+ accumulated_availability = np.ma.masked_less_equal(accumulated_availability, 0)
return Matrix(
resource_ids=resource_ids,
intervals=resource_windows_matrix.intervals,
- resource_matrix=resource_matrix,
+ resource_matrix=accumulated_availability,
)
+
def _apply_constraint_to_resource_windows_matrix(
self, constraint_matrix: Matrix, resource_windows_matrix: Matrix
) -> None:
From 309c4b60b27abf7aec4a7aaf8c73fd06fa12d3b2 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Wed, 16 Oct 2024 14:36:26 +0800
Subject: [PATCH 10/27] Updated cumsum_reset and pytest
---
.../heuristic_solver/task_allocator.py | 18 +++++++++++++-----
tests/scheduler/test_task_allocator.py | 3 ++-
2 files changed, 15 insertions(+), 6 deletions(-)
diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
index 2501a94..55a808f 100644
--- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py
+++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
@@ -220,12 +220,20 @@ def _cumsum_reset_at_minus_one(self, a: np.ndarray) -> np.ndarray:
"""
Computes the cumulative sum but resets to 0 whenever a -1 is encountered.
"""
- reset_mask = (a == -1)
- a[reset_mask] = 0 # Replace -1 with 0 for sum calculation
- cumsum_result = np.cumsum(a)
- cumsum_result[reset_mask] = 0 # Reset at gaps
+ a = a.copy() # Avoid in-place modification issues
+ result = np.zeros_like(a) # Initialize result array
+
+ cumulative_sum = 0 # Track cumulative sum
+ for i in range(len(a)):
+ if a[i] == -1:
+ cumulative_sum = 0 # Reset cumulative sum
+ else:
+ cumulative_sum += a[i]
+ result[i] = cumulative_sum # Store result
+
+ return result
+
- return cumsum_result
def _cumsum_reset_at_minus_one_2d(self, arr: np.ndarray) -> np.ndarray:
"""
diff --git a/tests/scheduler/test_task_allocator.py b/tests/scheduler/test_task_allocator.py
index d40d44c..0a7a71f 100644
--- a/tests/scheduler/test_task_allocator.py
+++ b/tests/scheduler/test_task_allocator.py
@@ -133,7 +133,8 @@ def test_mask_smallest_elements_except_top_k_per_row(
)
def test_cumsum_reset_at_minus_one(task_allocator, array, expected):
result = task_allocator._cumsum_reset_at_minus_one(array)
- assert np.array_equal(result, expected)
+ print(f"Input: {array}, Result: {result}, Expected: {expected}")
+ np.testing.assert_array_equal(result, expected)
@pytest.mark.parametrize(
From 0a313908ecd17dbf5c1e7166e0615910b4d47d11 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Wed, 16 Oct 2024 15:48:30 +0800
Subject: [PATCH 11/27] Reverted cumsum reset index function
---
.../heuristic_solver/task_allocator.py | 18 +++++-------------
1 file changed, 5 insertions(+), 13 deletions(-)
diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
index 55a808f..2501a94 100644
--- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py
+++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
@@ -220,20 +220,12 @@ def _cumsum_reset_at_minus_one(self, a: np.ndarray) -> np.ndarray:
"""
Computes the cumulative sum but resets to 0 whenever a -1 is encountered.
"""
- a = a.copy() # Avoid in-place modification issues
- result = np.zeros_like(a) # Initialize result array
-
- cumulative_sum = 0 # Track cumulative sum
- for i in range(len(a)):
- if a[i] == -1:
- cumulative_sum = 0 # Reset cumulative sum
- else:
- cumulative_sum += a[i]
- result[i] = cumulative_sum # Store result
-
- return result
-
+ reset_mask = (a == -1)
+ a[reset_mask] = 0 # Replace -1 with 0 for sum calculation
+ cumsum_result = np.cumsum(a)
+ cumsum_result[reset_mask] = 0 # Reset at gaps
+ return cumsum_result
def _cumsum_reset_at_minus_one_2d(self, arr: np.ndarray) -> np.ndarray:
"""
From 9ab20d1abdf7a9db9b2a5fea938e2b77d92273c7 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Wed, 16 Oct 2024 21:42:20 +0800
Subject: [PATCH 12/27] Current Updates
---
nb.ipynb | 183 +++-------
.../scheduler/heuristic_solver/main.py | 2 +-
.../heuristic_solver/task_allocator.py | 65 ++--
stresstest.ipynb | 327 ++++++++++++++++++
stresstest.py | 234 +++++++++++++
5 files changed, 645 insertions(+), 166 deletions(-)
create mode 100644 stresstest.ipynb
create mode 100644 stresstest.py
diff --git a/nb.ipynb b/nb.ipynb
index 6f73569..04e8393 100644
--- a/nb.ipynb
+++ b/nb.ipynb
@@ -2,33 +2,13 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Valid indices: [0 1 2 3]\n",
- "Diffs: [1 1 1]\n",
- "Segment boundaries: []\n",
- "Segment starts: [0]\n",
- "Segment ends: [4]\n",
- "Selected start index: 0\n",
- "Selected end index: 3\n",
- "Indexes for resource 2: (0, 3)\n",
- "Resource intervals: [ 0. 10. 20. 30.]\n",
- "Valid indices: [0 1 2 3]\n",
- "Diffs: [1 1 1]\n",
- "Segment boundaries: []\n",
- "Segment starts: [0]\n",
- "Segment ends: [4]\n",
- "Selected start index: 0\n",
- "Selected end index: 3\n",
- "Indexes for resource 1: (0, 3)\n",
- "Resource intervals: [ 0. 10. 20. 30.]\n",
- "Allocated resources for task 1: {2: [(0, 10), (20, 30)], 1: [(0, 10), (20, 30)]}\n",
- "PASS\n",
"Scheduled 1 of 1 tasks.\n"
]
},
@@ -64,10 +44,10 @@
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
- " <td>[2, 1]</td>\n",
+ " <td>[3, 1]</td>\n",
" <td>0</td>\n",
- " <td>30</td>\n",
- " <td>([(0, 10), (20, 30)], [(0, 10), (20, 30)])</td>\n",
+ " <td>50</td>\n",
+ " <td>([(0, 20), (30, 50)], [(0, 20), (30, 50)])</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
@@ -75,13 +55,13 @@
],
"text/plain": [
" task_id assigned_resource_ids task_start task_end \\\n",
- "0 1 [2, 1] 0 30 \n",
+ "0 1 [3, 1] 0 50 \n",
"\n",
" resource_intervals \n",
- "0 ([(0, 10), (20, 30)], [(0, 10), (20, 30)]) "
+ "0 ([(0, 20), (30, 50)], [(0, 20), (30, 50)]) "
]
},
- "execution_count": 1,
+ "execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
@@ -92,8 +72,8 @@
"from src.factryengine.scheduler.core import Scheduler\n",
"\n",
"machine = Resource(id=1, available_windows=[(0, 50)])\n",
- "operator1 = Resource(id=2, available_windows=[(0, 10), (20, 30)])\n",
- "operator2 = Resource(id=3, available_windows=[(0, 10), (40, 50)])\n",
+ "operator1 = Resource(id=2, available_windows=[(0, 20), (30, 50)])\n",
+ "operator2 = Resource(id=3, available_windows=[(0, 20), (30, 50)])\n",
"\n",
"operator_group = ResourceGroup(resources=[operator1, operator2])\n",
"\n",
@@ -101,10 +81,15 @@
"\n",
"# add machine as a constraint\n",
"t1 = Task(\n",
- " id=1, duration=20, assignments=[assignment], priority=1, constraints=[machine]\n",
+ " id=1, duration=40, assignments=[assignment], priority=1, constraints=[machine]\n",
")\n",
"\n",
"\n",
+ "# # add machine as a constraint\n",
+ "# t2 = Task(\n",
+ "# id=2, duration=10, assignments=[assignment], priority=1, constraints=[machine]\n",
+ "# )\n",
+ "\n",
"tasks = [t1]\n",
"resources = [operator1, operator2, machine]\n",
"\n",
@@ -114,139 +99,53 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Valid indices: [0 1 2 3]\n",
- "Diffs: [1 1 1]\n",
- "Segment boundaries: []\n",
- "Segment starts: [0]\n",
- "Segment ends: [4]\n",
- "Selected start index: 0\n",
- "Selected end index: 3\n",
- "Indexes for resource 1: (0, 3)\n",
- "Resource intervals: [ 0. 10. 40. 45.]\n",
- "Allocated resources for task 1: {1: [(0, 10), (40, 45)]}\n",
- "PASS\n",
- "Scheduled 1 of 1 tasks.\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "[{'task_id': 1,\n",
- " 'assigned_resource_ids': [1],\n",
- " 'task_start': 0,\n",
- " 'task_end': 45,\n",
- " 'resource_intervals': dict_values([[(0, 10), (40, 45)]])}]"
- ]
- },
- "execution_count": 2,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
- "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n",
+ "from src.factryengine.models import Resource, Task\n",
"from src.factryengine.scheduler.core import Scheduler\n",
"\n",
- "operator1 = Resource(id=1, available_windows=[(0, 10), (40, 50)])\n",
- "\n",
- "operator_group = ResourceGroup(resources=[operator1])\n",
- "\n",
- "assignment = Assignment(resource_groups=[operator_group], resource_count=2)\n",
+ "# Define a resource with multiple fragmented windows\n",
+ "machine = Resource(id=1, available_windows=[(0, 20), (30, 50)])\n",
+ "machine2 = Resource(id=2, available_windows=[(0, 20), (60, 80)])\n",
"\n",
- "# add machine as a constraint\n",
- "t1 = Task(\n",
- " id=1, duration=15, assignments=[assignment], priority=1\n",
- ")\n",
+ "# Create tasks with constraints\n",
+ "t1 = Task(id=1, duration=30, priority=1, constraints=[machine])\n",
+ "t2 = Task(id=2, duration=20, priority=2, constraints=[machine, machine2])\n",
"\n",
- "tasks = [t1]\n",
- "resources = [operator1]\n",
+ "tasks = [t1, t2]\n",
+ "resources = [machine, machine2]\n",
"\n",
+ "# Schedule the tasks\n",
"result = Scheduler(tasks=tasks, resources=resources).schedule()\n",
- "result.to_dict()\n"
+ "result.to_dataframe()\n"
]
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Valid indices: [0 1 2 3]\n",
- "Diffs: [1 1 1]\n",
- "Segment boundaries: []\n",
- "Segment starts: [0]\n",
- "Segment ends: [4]\n",
- "Selected start index: 0\n",
- "Selected end index: 3\n",
- "Indexes for resource 1: (0, 3)\n",
- "Resource intervals: [ 0. 10. 20. 25.]\n",
- "Allocated resources for task 1: {1: [(0, 10), (20, 25)]}\n",
- "PASS\n",
- "Valid indices: [0 1]\n",
- "Diffs: [1]\n",
- "Segment boundaries: []\n",
- "Segment starts: [0]\n",
- "Segment ends: [2]\n",
- "Selected start index: 0\n",
- "Selected end index: 1\n",
- "Indexes for resource 1: (0, 1)\n",
- "Resource intervals: [25. 40.]\n",
- "Allocated resources for task 2: {1: [(25, 40)]}\n",
- "PASS\n",
- "Scheduled 2 of 2 tasks.\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "[{'task_id': 1,\n",
- " 'assigned_resource_ids': [1],\n",
- " 'task_start': 0,\n",
- " 'task_end': 25,\n",
- " 'resource_intervals': dict_values([[(0, 10), (20, 25)]])},\n",
- " {'task_id': 2,\n",
- " 'assigned_resource_ids': [1],\n",
- " 'task_start': 25,\n",
- " 'task_end': 40,\n",
- " 'resource_intervals': dict_values([[(25, 40)]])}]"
- ]
- },
- "execution_count": 3,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
- "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n",
+ "from src.factryengine.models import Resource, ResourceGroup, Task, Assignment\n",
"from src.factryengine.scheduler.core import Scheduler\n",
"\n",
- "machine = Resource(id=1, available_windows=[(0, 10), (20, 45)])\n",
+ "operator1 = Resource(id=1, available_windows=[(0, 20), (40, 60)])\n",
+ "operator2 = Resource(id=2, available_windows=[(0, 20), (40, 60)])\n",
+ "operator3 = Resource(id=3, available_windows=[(0, 30), (50, 60), (80, 150)])\n",
"\n",
- "# add machine as a constraint\n",
- "t1 = Task(\n",
- " id=1, duration=15, priority=1, constraints=[machine]\n",
- ")\n",
+ "rg1 = ResourceGroup(resources=[operator1, operator2, operator3])\n",
"\n",
- "# add machine as a constraint\n",
- "t2 = Task(\n",
- " id=2, duration=15, priority=2, constraints=[machine]\n",
- ")\n",
+ "assignment = Assignment(resource_groups=[rg1], resource_count=1)\n",
"\n",
- "tasks = [t1, t2]\n",
- "resources = [machine]\n",
+ "t1 = Task(id=1, duration=40, assignments=[assignment], priority=1)\n",
+ "t2 = Task(id=2, duration=20, assignments=[assignment], priority=2)\n",
"\n",
+ "tasks = [t1, t2]\n",
+ "resources = [operator1, operator2, operator3]\n",
"result = Scheduler(tasks=tasks, resources=resources).schedule()\n",
- "result.to_dict()\n"
+ "result.to_dict()"
]
}
],
diff --git a/src/factryengine/scheduler/heuristic_solver/main.py b/src/factryengine/scheduler/heuristic_solver/main.py
index cdb3f74..5b51052 100644
--- a/src/factryengine/scheduler/heuristic_solver/main.py
+++ b/src/factryengine/scheduler/heuristic_solver/main.py
@@ -74,7 +74,7 @@ def solve(self) -> list[dict]:
except AllocationError as e:
self.mark_task_as_unscheduled(task_id=task_id, error_message=str(e))
continue
-
+
# update resource windows
self.window_manager.update_resource_windows(allocated_resource_windows_dict)
diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
index 2501a94..7ad2584 100644
--- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py
+++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
@@ -55,6 +55,7 @@ def allocate_task(
# process solution to find allocated resource windows
allocated_windows = self._get_resource_intervals(
matrix=solution_matrix,
+ resource_windows_dict=resource_windows_dict
)
# add constraints to allocated windows
@@ -65,6 +66,7 @@ def allocate_task(
allocated_windows.update(
self._get_resource_intervals(
matrix=constraints_matrix_trimmed,
+ resource_windows_dict=resource_windows_dict
)
)
@@ -160,44 +162,61 @@ def _solve_task_end(
return col0_value, other_columns_values
def _get_resource_intervals(
- self,
- matrix: np.array,
+ self, matrix: Matrix, resource_windows_dict: dict[int, np.ndarray]
) -> dict[int, list[tuple[int, int]]]:
"""
- Gets the resource intervals from the solution matrix.
- Always includes the first pair, and only subsequent pairs
- if value_end > value_start for any given pair.
+ Extracts the resource intervals from the solution matrix by matching them
+ with the updated windows provided in `resource_windows_dict`.
"""
- resource_windows_dict = {}
+ resource_windows_output = {}
- # Loop through resource IDs and corresponding intervals
+ # Iterate over each resource ID and its intervals in the solution matrix
for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T):
indexes = self._find_indexes(resource_intervals.data)
if indexes is not None:
start_index, end_index = indexes
- segment_intervals = []
- is_first_pair = True # Track if this is the first pair
+ # Extract allocated intervals from the matrix
+ allocated_intervals = [
+ (ceil(round(matrix.intervals[i], 1)), ceil(round(matrix.intervals[i + 1], 1)))
+ for i in range(start_index, end_index, 2)
+ ]
+
+ # Get the resource’s available windows from the resource_windows_dict
+ updated_windows = resource_windows_dict.get(resource_id, [])
+
+ # Match allocated intervals with the available windows
+ matched_intervals = self._match_intervals_with_windows(
+ allocated_intervals, updated_windows
+ )
- # Iterate through the intervals in pairs (i, i+1)
- for i in range(start_index, end_index, 2):
- # Extract values from the resource matrix
- value_start = matrix.resource_matrix[i][0]
- value_end = matrix.resource_matrix[i + 1][0]
+ # Store the matched intervals in the output dictionary
+ resource_windows_output[resource_id] = matched_intervals
- # Always add the first pair, or add if value_end > value_start
- if is_first_pair or value_end > value_start:
- interval_start = ceil(round(matrix.intervals[i], 1))
- interval_end = ceil(round(matrix.intervals[i + 1], 1))
+ return resource_windows_output
+
+ def _match_intervals_with_windows(
+ self, allocated_intervals: list[tuple[int, int]], windows: np.ndarray
+ ) -> list[tuple[int, int]]:
+ """
+ Matches the allocated intervals with the resource's available windows.
+ """
+ matched_intervals = []
- segment_intervals.append((interval_start, interval_end))
- is_first_pair = False # Switch off the first-pair flag
+ # Iterate over allocated intervals and compare with the available windows
+ for allocated_start, allocated_end in allocated_intervals:
+ for window in windows:
+ window_start, window_end = window["start"], window["end"]
- # Store the segment intervals for the current resource
- resource_windows_dict[resource_id] = segment_intervals
+ # Check if there is an overlap between the allocated interval and the window
+ if allocated_end > window_start and allocated_start < window_end:
+ # Calculate the overlapping interval
+ matched_start = max(allocated_start, window_start)
+ matched_end = min(allocated_end, window_end)
+ matched_intervals.append((matched_start, matched_end))
- return resource_windows_dict
+ return matched_intervals
def _mask_smallest_elements_except_top_k_per_row(
diff --git a/stresstest.ipynb b/stresstest.ipynb
new file mode 100644
index 0000000..e0f1769
--- /dev/null
+++ b/stresstest.ipynb
@@ -0,0 +1,327 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import json\n",
+ "import pytz\n",
+ "from datetime import datetime, timezone, timedelta\n",
+ "import time\n",
+ "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n",
+ "from src.factryengine.scheduler.core import Scheduler\n",
+ "from src.factryengine.scheduler.task_batch_processor import TaskSplitter\n",
+ "\n",
+ "\n",
+ "class ProdScheduler: \n",
+ " def __init__(self, \n",
+ " resource_dir: str, \n",
+ " resource_group_dir: str, \n",
+ " task_dir: str\n",
+ " ) -> None:\n",
+ " # Scheduler Attributes\n",
+ " self.cph_timezone = pytz.timezone('Europe/Copenhagen')\n",
+ " # self.today = datetime.now(timezone.utc).replace(\n",
+ " # hour=0, minute=0, second=0, microsecond=0)\n",
+ " self.today = datetime(2024, 10, 9, 0, 0, 0, 0, tzinfo=timezone.utc)\n",
+ " self.today_str = str(self.today)[:19]\n",
+ "\n",
+ " # Component Attributes\n",
+ " self.dict_resource = {}\n",
+ " self.dict_resourcegroups = {}\n",
+ " self.tasks_list = []\n",
+ " self.task_dict = {}\n",
+ " self.pred_dict = {}\n",
+ " self.flow_map = {}\n",
+ " self.pred_exploded = {}\n",
+ "\n",
+ " # Inputs \n",
+ " with open(resource_dir, 'r') as file:\n",
+ " self.r_data = json.load(file)\n",
+ "\n",
+ " with open(task_dir, 'r') as file:\n",
+ " self.t_data = json.load(file)\n",
+ "\n",
+ " with open(resource_group_dir, 'r') as file:\n",
+ " self.rg_data = json.load(file)\n",
+ " \n",
+ " def convert_to_minutes(self, datetime_str, start_time_obj):\n",
+ " datetime_obj = datetime.strptime(datetime_str, '%Y-%m-%d %H:%M:%S%z')\n",
+ " diff_minutes = (datetime_obj - start_time_obj).total_seconds()/60\n",
+ " return int(diff_minutes)\n",
+ "\n",
+ " def adjust_capacity(self, start, end, capacity):\n",
+ " return (end - start) * capacity + start\n",
+ "\n",
+ " def organize_predecessors(self, task: Task):\n",
+ " try:\n",
+ " list_predecessors = self.pred_dict[task.id]\n",
+ " # ============================================================== UNCOMMENT TO USE MICROBATCH FLOW\n",
+ " if task.batch_id: # Check if task is microbatched\n",
+ " # Look for each predecessors that exist in the flow map\n",
+ " for predecessor in list_predecessors:\n",
+ " pred_batch_id = f'{predecessor}-{task.batch_id}'\n",
+ " if pred_batch_id in self.flow_map and pred_batch_id in self.pred_dict: # Check if pred is part of flow\n",
+ " # Check if pred-parent connection is correct\n",
+ " if self.flow_map[task.id]['predecessor'] == self.flow_map[pred_batch_id]['parent']:\n",
+ " self.pred_dict[task.id] = [pred_batch_id]\n",
+ "\n",
+ " elif pred_batch_id not in self.pred_dict and predecessor not in self.task_dict:\n",
+ " parent_predecessor = []\n",
+ " for pred in self.pred_dict[task.id]:\n",
+ " parent_predecessor.extend(self.pred_dict[pred])\n",
+ " self.pred_dict[task.id] = parent_predecessor\n",
+ " # ============================================================== UNCOMMENT TO USE MICROBATCH FLOW\n",
+ "\n",
+ " # Remove task batch id\n",
+ " task.set_batch_id(None)\n",
+ "\n",
+ " # Check for predecessors to be exploded\n",
+ " for predecessor in list_predecessors:\n",
+ " if predecessor in self.pred_exploded:\n",
+ " self.pred_dict[task.id].remove(\n",
+ " predecessor) # Remove original value\n",
+ " self.pred_dict[task.id].extend(\n",
+ " self.pred_exploded[predecessor]) # Add exploded batches\n",
+ " except Exception as e:\n",
+ " return\n",
+ " \n",
+ " def set_predecessors(self, task: Task):\n",
+ " if not task.id in self.pred_dict: # if task is not in pred_dict, then it has no predecessors\n",
+ " return\n",
+ "\n",
+ " for pred_id in self.pred_dict[task.id]:\n",
+ " if pred_id in self.task_dict: # ensure predecessor exists in task_dict\n",
+ " pred_task = self.task_dict[pred_id]\n",
+ "\n",
+ " # Avoid adding a predecessor multiple times\n",
+ " if pred_task not in task.predecessors:\n",
+ " # set predecessors for the predecessor first\n",
+ " self.set_predecessors(pred_task)\n",
+ " task.predecessors.append(pred_task)\n",
+ " \n",
+ " def create_resource_object(self, resource_list):\n",
+ " # Generate slot based on schedule selected in NocoDB\n",
+ " for row in resource_list:\n",
+ " periods_list = []\n",
+ " for sched in row['availability']:\n",
+ " if not sched['is_absent']:\n",
+ " start = self.convert_to_minutes(\n",
+ " sched['start_datetime'], self.today)\n",
+ " end = self.convert_to_minutes(\n",
+ " sched['end_datetime'], self.today)\n",
+ " # ========= Uncomment to use capacity\n",
+ " capacity = sched['capacity_percent']\n",
+ " # capacity = None\n",
+ " periods_list.append((int(start), int(self.adjust_capacity(\n",
+ " start, end, capacity)) if capacity else int(end)))\n",
+ "\n",
+ " resource_id = int(row['resource_id'])\n",
+ " self.dict_resource[resource_id] = Resource(\n",
+ " id=resource_id, available_windows=periods_list)\n",
+ "\n",
+ " def create_resource_groups(self, resource_group_list):\n",
+ " # Generate Resource Groups\n",
+ " for x in resource_group_list:\n",
+ " resource_list = []\n",
+ " resources = x['resource_id']\n",
+ " for r in resources:\n",
+ " if r in self.dict_resource:\n",
+ " resource_list.append(self.dict_resource[r])\n",
+ " \n",
+ " if resource_list:\n",
+ " self.dict_resourcegroups[x['resource_group_id']] = ResourceGroup(\n",
+ " id=x['resource_group_id'], resources=resource_list)\n",
+ "\n",
+ " def create_batch(self, task: Task, batch_size: int):\n",
+ " batches = TaskSplitter(task, batch_size).split_into_batches()\n",
+ " counter = 1\n",
+ " for batch in batches:\n",
+ " batch.id = f\"{task.id}-{counter}\"\n",
+ " counter += 1\n",
+ "\n",
+ " return batches\n",
+ "\n",
+ " def create_task_object(self, task_list):\n",
+ " for i in task_list:\n",
+ " rg_list = []\n",
+ " task_id = i['taskno']\n",
+ " duration = int(i['duration'])\n",
+ " priority = int(i['priority'])\n",
+ " quantity = int(i['quantity'])\n",
+ " # micro_batch_size = int(\n",
+ " # i['micro_batch_size']) if i['micro_batch_size'] else None\n",
+ " micro_batch_size = None\n",
+ " resource_group_id = i['resource_group_id']\n",
+ " rg_list = [self.dict_resourcegroups[g] for g in resource_group_id if g in self.dict_resourcegroups]\n",
+ " predecessors = i['predecessors']\n",
+ " parent_collection = i['parent_item_collection_id'] if micro_batch_size else None\n",
+ " predecessor_collection = i['predecessor_item_collection_id'] if micro_batch_size else None\n",
+ "\n",
+ " assignments = []\n",
+ " # Create assignments \n",
+ " for x in resource_group_id: \n",
+ " if x in self.dict_resourcegroups:\n",
+ " assignments.append(Assignment(resource_groups= [self.dict_resourcegroups[x]], resource_count= 1))\n",
+ " \n",
+ " # Temporarily add into component dicts\n",
+ " temp_task = Task(id=task_id,\n",
+ " duration=duration,\n",
+ " priority=priority,\n",
+ " assignments= assignments,\n",
+ " quantity=quantity)\n",
+ " \n",
+ " # Check for micro-batches\n",
+ " if not micro_batch_size:\n",
+ " self.task_dict[task_id] = temp_task # Add task to dictionary\n",
+ "\n",
+ " # Add predecessor to dictionary\n",
+ " self.pred_dict[task_id] = predecessors\n",
+ " else:\n",
+ " self.pred_dict[task_id] = predecessors\n",
+ " batches = self.create_batch(temp_task, micro_batch_size)\n",
+ " self.task_dict.update({task.id: task for task in batches})\n",
+ "\n",
+ " # Temporarily copy the original predecessors for the new batches\n",
+ " self.pred_dict.update(\n",
+ " {task.id: predecessors for task in batches})\n",
+ " self.flow_map.update({task.id: {\n",
+ " \"parent\": parent_collection,\n",
+ " \"predecessor\": predecessor_collection} for task in batches})\n",
+ " self.pred_exploded[task_id] = [task.id for task in batches]\n",
+ "\n",
+ "\n",
+ " # Organize predecessors for batches\n",
+ " for task in self.task_dict.values():\n",
+ " self.organize_predecessors(task)\n",
+ "\n",
+ " # Add predecessors\n",
+ " for task in self.task_dict.values():\n",
+ " self.task_dict[task.id].predecessor_ids = [x for x in self.pred_dict[task.id] if x in self.task_dict] # Predecessor needs to be existing in task dictionary\n",
+ "\n",
+ " # Build final task list\n",
+ " self.tasks_list = [value for key,\n",
+ " value in sorted(self.task_dict.items())]\n",
+ " \n",
+ " # Convert periods to time\n",
+ " def int_to_datetime(self, num, start_time):\n",
+ " try:\n",
+ " # Parse the start time string into a datetime object\n",
+ " start_datetime = datetime.strptime(start_time, \"%Y-%m-%d %H:%M:%S\")\n",
+ " \n",
+ " # Add the number of minutes to the start datetime\n",
+ " delta = timedelta(minutes=num)\n",
+ " result_datetime = start_datetime + delta\n",
+ " return result_datetime\n",
+ " \n",
+ " except Exception as e: \n",
+ " print(num)\n",
+ " \n",
+ " def run_scheduler(self):\n",
+ " self.create_resource_object(self.r_data)\n",
+ " print(\"Resource Objects Created.\")\n",
+ "\n",
+ " self.create_resource_groups(self.rg_data)\n",
+ " print(\"Resource Groups Created.\")\n",
+ "\n",
+ " self.create_task_object(self.t_data)\n",
+ " print(\"Task Objects Created.\")\n",
+ " print(f\"Original Task Length: {len(self.t_data)} | Post-Batched Length: {len(self.task_dict.values())}\")\n",
+ "\n",
+ "\n",
+ " self.solution = Scheduler(self.tasks_list, list(self.dict_resource.values())).schedule()\n",
+ " print(\"Solution Created.\")\n",
+ " \n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "scheduler = ProdScheduler(\n",
+ " resource_dir = 'inputs/actual_data/resource.json',\n",
+ " resource_group_dir = 'inputs/actual_data/resourcegroups.json',\n",
+ " task_dir = 'inputs/actual_data/tasks_all.json'\n",
+ ")\n",
+ "\n",
+ "scheduler.run_scheduler()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = scheduler.solution.to_dataframe()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result['start_dt'] = result.apply(lambda x: scheduler.int_to_datetime(x['task_start'], scheduler.today_str), axis=1)\n",
+ "result['end_dt'] = result.apply(lambda x: scheduler.int_to_datetime(x['task_end'], scheduler.today_str), axis=1)\n",
+ "result.sort_values(by='task_end', ascending=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result[result['task_id'].str.startswith('WO137709')].sort_values(by='task_start')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "scheduler.task_dict['WO135483-40']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "intervals = scheduler.solution.get_resource_intervals_df()\n",
+ "intervals['start_dt'] = intervals.apply(lambda x: scheduler.int_to_datetime(x['interval_start'], scheduler.today_str), axis=1)\n",
+ "intervals['end_dt'] = intervals.apply(lambda x: scheduler.int_to_datetime(x['interval_end'], scheduler.today_str), axis=1)\n",
+ "intervals.sort_values(by='interval_end', ascending=False)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": ".venv",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11.4"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/stresstest.py b/stresstest.py
new file mode 100644
index 0000000..b1610ed
--- /dev/null
+++ b/stresstest.py
@@ -0,0 +1,234 @@
+import pandas as pd
+import json
+import pytz
+from datetime import datetime, timezone, timedelta
+import time
+from src.factryengine.models import Resource, Task, Assignment, ResourceGroup
+from src.factryengine.scheduler.core import Scheduler
+from src.factryengine.scheduler.task_batch_processor import TaskSplitter
+
+
+class ProdScheduler:
+ def __init__(self) -> None:
+ # Scheduler Attributes
+ self.cph_timezone = pytz.timezone('Europe/Copenhagen')
+ # self.today = datetime.now(timezone.utc).replace(
+ # hour=0, minute=0, second=0, microsecond=0)
+ self.today = datetime(2024, 10, 9, 0, 0, 0, 0, tzinfo=timezone.utc)
+ self.today_str = str(self.today)[:19]
+
+ # Component Attributes
+ self.dict_resource = {}
+ self.dict_resourcegroups = {}
+ self.tasks_list = []
+ self.task_dict = {}
+ self.pred_dict = {}
+ self.flow_map = {}
+ self.pred_exploded = {}
+
+ # Misc Attributes
+ self.start_time = time.time()
+
+ def convert_to_minutes(self, datetime_str, start_time_obj):
+ datetime_obj = datetime.strptime(datetime_str, '%Y-%m-%d %H:%M:%S%z')
+ diff_minutes = (datetime_obj - start_time_obj).total_seconds()/60
+ return int(diff_minutes)
+
+ def adjust_capacity(self, start, end, capacity):
+ return (end - start) * capacity + start
+
+ def organize_predecessors(self, task: Task):
+ try:
+ list_predecessors = self.pred_dict[task.id]
+ # ============================================================== UNCOMMENT TO USE MICROBATCH FLOW
+ if task.batch_id: # Check if task is microbatched
+ # Look for each predecessors that exist in the flow map
+ for predecessor in list_predecessors:
+ pred_batch_id = f'{predecessor}-{task.batch_id}'
+ if pred_batch_id in self.flow_map and pred_batch_id in self.pred_dict: # Check if pred is part of flow
+ # Check if pred-parent connection is correct
+ if self.flow_map[task.id]['predecessor'] == self.flow_map[pred_batch_id]['parent']:
+ self.pred_dict[task.id] = [pred_batch_id]
+
+ elif pred_batch_id not in self.pred_dict and predecessor not in self.task_dict:
+ parent_predecessor = []
+ for pred in self.pred_dict[task.id]:
+ parent_predecessor.extend(self.pred_dict[pred])
+ self.pred_dict[task.id] = parent_predecessor
+ # ============================================================== UNCOMMENT TO USE MICROBATCH FLOW
+
+ # Remove task batch id
+ task.set_batch_id(None)
+
+ # Check for predecessors to be exploded
+ for predecessor in list_predecessors:
+ if predecessor in self.pred_exploded:
+ self.pred_dict[task.id].remove(
+ predecessor) # Remove original value
+ self.pred_dict[task.id].extend(
+ self.pred_exploded[predecessor]) # Add exploded batches
+
+ except Exception as e:
+ return
+
+ def set_predecessors(self, task: Task):
+ if not task.id in self.pred_dict: # if task is not in pred_dict, then it has no predecessors
+ return
+
+ for pred_id in self.pred_dict[task.id]:
+ if pred_id in self.task_dict: # ensure predecessor exists in task_dict
+ pred_task = self.task_dict[pred_id]
+
+ # Avoid adding a predecessor multiple times
+ if pred_task not in task.predecessors:
+ # set predecessors for the predecessor first
+ self.set_predecessors(pred_task)
+ task.predecessors.append(pred_task)
+
+ def create_resource_object(self, resource_list):
+ # Generate slot based on schedule selected in NocoDB
+ for row in resource_list:
+ periods_list = []
+ for sched in row['availability']:
+ if not sched['is_absent']:
+ start = self.convert_to_minutes(
+ sched['start_datetime'], self.today)
+ end = self.convert_to_minutes(
+ sched['end_datetime'], self.today)
+ # ========= Uncomment to use capacity
+ capacity = sched['capacity_percent']
+ # capacity = None
+ periods_list.append((int(start), int(self.adjust_capacity(
+ start, end, capacity)) if capacity else int(end)))
+
+ resource_id = int(row['resource_id'])
+ self.dict_resource[resource_id] = Resource(
+ id=resource_id, available_windows=periods_list)
+
+ def create_resource_groups(self, resource_group_list):
+ # Generate Resource Groups
+ for x in resource_group_list:
+ resource_list = []
+ resources = x['resource_id']
+ for r in resources:
+ if r in self.dict_resource:
+ resource_list.append(self.dict_resource[r])
+
+ if resource_list:
+ self.dict_resourcegroups[x['resource_group_id']] = ResourceGroup(
+ id=x['resource_group_id'], resources=resource_list)
+
+ def create_batch(self, task: Task, batch_size: int):
+ batches = TaskSplitter(task, batch_size).split_into_batches()
+ counter = 1
+ for batch in batches:
+ batch.id = f"{task.id}-{counter}"
+ counter += 1
+
+ return batches
+
+ def create_task_object(self, task_list):
+ for i in task_list:
+ rg_list = []
+ task_id = i['taskno']
+ duration = int(i['duration'])
+ priority = int(i['priority'])
+ quantity = int(i['quantity'])
+ # micro_batch_size = int(
+ # i['micro_batch_size']) if i['micro_batch_size'] else None
+ micro_batch_size = None
+ resource_group_id = i['resource_group_id']
+ rg_list = [self.dict_resourcegroups[g] for g in resource_group_id if g in self.dict_resourcegroups]
+ predecessors = i['predecessors']
+ parent_collection = i['parent_item_collection_id'] if micro_batch_size else None
+ predecessor_collection = i['predecessor_item_collection_id'] if micro_batch_size else None
+
+ assignments = []
+ # Create assignments
+ for x in resource_group_id:
+ if x in self.dict_resourcegroups:
+ assignments.append(Assignment(resource_groups= [self.dict_resourcegroups[x]], resource_count= 1))
+
+ # Temporarily add into component dicts
+ temp_task = Task(id=task_id,
+ duration=duration,
+ priority=priority,
+ assignments= assignments,
+ quantity=quantity)
+
+ # Check for micro-batches
+ if not micro_batch_size:
+ self.task_dict[task_id] = temp_task # Add task to dictionary
+
+ # Add predecessor to dictionary
+ self.pred_dict[task_id] = predecessors
+ else:
+ self.pred_dict[task_id] = predecessors
+ batches = self.create_batch(temp_task, micro_batch_size)
+ self.task_dict.update({task.id: task for task in batches})
+
+ # Temporarily copy the original predecessors for the new batches
+ self.pred_dict.update(
+ {task.id: predecessors for task in batches})
+ self.flow_map.update({task.id: {
+ "parent": parent_collection,
+ "predecessor": predecessor_collection} for task in batches})
+ self.pred_exploded[task_id] = [task.id for task in batches]
+
+ # Organize predecessors for batches
+ for task in self.task_dict.values():
+ self.organize_predecessors(task)
+
+ # Add predecessors
+ for task in self.task_dict.values():
+ self.task_dict[task.id].predecessor_ids = [x for x in self.pred_dict[task.id] if x in self.task_dict] # Predecessor needs to be existing in task dictionary
+
+ # Build final task list
+ self.tasks_list = [value for key,
+ value in sorted(self.task_dict.items())]
+
+ # Convert periods to time
+ def int_to_datetime(self, num, start_time):
+ try:
+ # Parse the start time string into a datetime object
+ start_datetime = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S")
+
+ # Add the number of minutes to the start datetime
+ delta = timedelta(minutes=num)
+ result_datetime = start_datetime + delta
+ return result_datetime
+
+ except Exception as e:
+ print(num)
+
+
+# Open and read the JSON file
+with open('inputs/resource.json', 'r') as file:
+ r_data = json.load(file)
+
+# Open and read the JSON file
+with open('inputs/tasks_all.json', 'r') as file:
+ t_data = json.load(file)
+
+
+# Open and read the JSON file
+with open('inputs/resourcegroups.json', 'r') as file:
+ rg_data = json.load(file)
+
+
+
+prodscheduler = ProdScheduler()
+prodscheduler.create_resource_object(r_data)
+print("Resource Objects Created.")
+
+prodscheduler.create_resource_groups(rg_data)
+print("Resource Groups Created.")
+
+prodscheduler.create_task_object(t_data)
+print("Task Objects Created.")
+print(f"Original Task Length: {len(t_data)} | Post-Batched Length: {len(prodscheduler.task_dict.values())}")
+
+
+solution = Scheduler(prodscheduler.tasks_list, list(prodscheduler.dict_resource.values())).schedule()
+print("Solution Created.")
+
From 45896988cf8d94225fcb9eee9944a1ed61b58803 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Wed, 16 Oct 2024 21:42:49 +0800
Subject: [PATCH 13/27] Updates
---
nb.ipynb | 71 ++++++++++++++++++++++++++++++++++++++++++++++--
stresstest.ipynb | 29 ++++++++++++++++++--
2 files changed, 96 insertions(+), 4 deletions(-)
diff --git a/nb.ipynb b/nb.ipynb
index 04e8393..888f046 100644
--- a/nb.ipynb
+++ b/nb.ipynb
@@ -99,9 +99,76 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 3,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Scheduled 2 of 2 tasks.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>task_id</th>\n",
+ " <th>assigned_resource_ids</th>\n",
+ " <th>task_start</th>\n",
+ " <th>task_end</th>\n",
+ " <th>resource_intervals</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>1</td>\n",
+ " <td>[1]</td>\n",
+ " <td>0</td>\n",
+ " <td>40</td>\n",
+ " <td>([(0, 20), (30, 40)])</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>2</td>\n",
+ " <td>[2]</td>\n",
+ " <td>0</td>\n",
+ " <td>20</td>\n",
+ " <td>([(0, 20)])</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " task_id assigned_resource_ids task_start task_end resource_intervals\n",
+ "0 1 [1] 0 40 ([(0, 20), (30, 40)])\n",
+ "1 2 [2] 0 20 ([(0, 20)])"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"from src.factryengine.models import Resource, Task\n",
"from src.factryengine.scheduler.core import Scheduler\n",
diff --git a/stresstest.ipynb b/stresstest.ipynb
index e0f1769..c9d24ab 100644
--- a/stresstest.ipynb
+++ b/stresstest.ipynb
@@ -239,9 +239,34 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 2,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Resource Objects Created.\n",
+ "Resource Groups Created.\n",
+ "Task Objects Created.\n",
+ "Original Task Length: 6138 | Post-Batched Length: 6138\n"
+ ]
+ },
+ {
+ "ename": "ValueError",
+ "evalue": "min() arg is an empty sequence",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)",
+ "Cell \u001b[1;32mIn[2], line 7\u001b[0m\n\u001b[0;32m 1\u001b[0m scheduler \u001b[38;5;241m=\u001b[39m ProdScheduler(\n\u001b[0;32m 2\u001b[0m resource_dir \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124minputs/actual_data/resource.json\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m 3\u001b[0m resource_group_dir \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124minputs/actual_data/resourcegroups.json\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m 4\u001b[0m task_dir \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124minputs/actual_data/tasks_all.json\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m 5\u001b[0m )\n\u001b[1;32m----> 7\u001b[0m \u001b[43mscheduler\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_scheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
+ "Cell \u001b[1;32mIn[1], line 227\u001b[0m, in \u001b[0;36mProdScheduler.run_scheduler\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 223\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTask Objects Created.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 224\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOriginal Task Length: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mt_data)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m | Post-Batched Length: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtask_dict\u001b[38;5;241m.\u001b[39mvalues())\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m--> 227\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msolution \u001b[38;5;241m=\u001b[39m \u001b[43mScheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtasks_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdict_resource\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalues\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mschedule\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 228\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSolution Created.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+ "File \u001b[1;32mc:\\Projects\\factryengine\\src\\factryengine\\scheduler\\core.py:32\u001b[0m, in \u001b[0;36mScheduler.schedule\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 27\u001b[0m heuristic_solver \u001b[38;5;241m=\u001b[39m HeuristicSolver(\n\u001b[0;32m 28\u001b[0m task_dict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtask_dict, resources\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mresources, task_order\u001b[38;5;241m=\u001b[39mtask_order\n\u001b[0;32m 29\u001b[0m )\n\u001b[0;32m 31\u001b[0m \u001b[38;5;66;03m# Use the heuristic solver to find a solution\u001b[39;00m\n\u001b[1;32m---> 32\u001b[0m solver_result \u001b[38;5;241m=\u001b[39m \u001b[43mheuristic_solver\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msolve\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 34\u001b[0m \u001b[38;5;66;03m# Create a scheduler result with the solver result and unscheduled tasks\u001b[39;00m\n\u001b[0;32m 35\u001b[0m scheduler_result \u001b[38;5;241m=\u001b[39m SchedulerResult(\n\u001b[0;32m 36\u001b[0m task_vars\u001b[38;5;241m=\u001b[39msolver_result,\n\u001b[0;32m 37\u001b[0m unscheduled_task_ids\u001b[38;5;241m=\u001b[39mheuristic_solver\u001b[38;5;241m.\u001b[39munscheduled_task_ids,\n\u001b[0;32m 38\u001b[0m )\n",
+ "File \u001b[1;32mc:\\Projects\\factryengine\\src\\factryengine\\scheduler\\heuristic_solver\\main.py:85\u001b[0m, in \u001b[0;36mHeuristicSolver.solve\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 78\u001b[0m \u001b[38;5;66;03m# update resource windows\u001b[39;00m\n\u001b[0;32m 79\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwindow_manager\u001b[38;5;241m.\u001b[39mupdate_resource_windows(allocated_resource_windows_dict)\n\u001b[0;32m 82\u001b[0m task_values \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m 83\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask_id\u001b[39m\u001b[38;5;124m\"\u001b[39m: task_id,\n\u001b[0;32m 84\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124massigned_resource_ids\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mlist\u001b[39m(allocated_resource_windows_dict\u001b[38;5;241m.\u001b[39mkeys()),\n\u001b[1;32m---> 85\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask_start\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28;43mmin\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[0;32m 86\u001b[0m \u001b[43m \u001b[49m\u001b[43mstart\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mintervals\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mallocated_resource_windows_dict\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalues\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mstart\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mintervals\u001b[49m\n\u001b[0;32m 87\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m,\n\u001b[0;32m 88\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask_end\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mmax\u001b[39m(\n\u001b[0;32m 89\u001b[0m end \u001b[38;5;28;01mfor\u001b[39;00m intervals \u001b[38;5;129;01min\u001b[39;00m allocated_resource_windows_dict\u001b[38;5;241m.\u001b[39mvalues() \u001b[38;5;28;01mfor\u001b[39;00m _, end \u001b[38;5;129;01min\u001b[39;00m intervals\n\u001b[0;32m 90\u001b[0m ),\n\u001b[0;32m 91\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mresource_intervals\u001b[39m\u001b[38;5;124m\"\u001b[39m: allocated_resource_windows_dict\u001b[38;5;241m.\u001b[39mvalues(),\n\u001b[0;32m 92\u001b[0m }\n\u001b[0;32m 93\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtask_vars[task_id] \u001b[38;5;241m=\u001b[39m task_values\n\u001b[0;32m 96\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mlist\u001b[39m(\n\u001b[0;32m 97\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtask_vars\u001b[38;5;241m.\u001b[39mvalues()\n\u001b[0;32m 98\u001b[0m )\n",
+ "\u001b[1;31mValueError\u001b[0m: min() arg is an empty sequence"
+ ]
+ }
+ ],
"source": [
"scheduler = ProdScheduler(\n",
" resource_dir = 'inputs/actual_data/resource.json',\n",
From 70191f8304aace1a16fd72042fddfe621e37cb02 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Thu, 17 Oct 2024 15:42:23 +0800
Subject: [PATCH 14/27] Task allocator update
---
.../heuristic_solver/task_allocator.py | 130 ++++++++++--------
1 file changed, 73 insertions(+), 57 deletions(-)
diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
index 7ad2584..fcc8003 100644
--- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py
+++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
@@ -31,6 +31,7 @@ def allocate_task(
resource_windows_matrix=resource_windows_matrix,
task_duration=task_duration,
)
+
if assignments and constraints:
# update the resource matrix with the constraint matrix
self._apply_constraint_to_resource_windows_matrix(
@@ -162,61 +163,73 @@ def _solve_task_end(
return col0_value, other_columns_values
def _get_resource_intervals(
- self, matrix: Matrix, resource_windows_dict: dict[int, np.ndarray]
+ self, matrix: Matrix, resource_windows_dict: dict[int, list[tuple[float, float, float, int]]]
) -> dict[int, list[tuple[int, int]]]:
"""
- Extracts the resource intervals from the solution matrix by matching them
- with the updated windows provided in `resource_windows_dict`.
+ Extracts the resource intervals from the solution matrix by strictly matching them
+ with the provided original windows in `resource_windows_dict`.
"""
resource_windows_output = {}
- # Iterate over each resource ID and its intervals in the solution matrix
+ # Iterate over each resource ID and its corresponding matrix intervals
for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T):
- indexes = self._find_indexes(resource_intervals.data)
- if indexes is not None:
- start_index, end_index = indexes
+ # Retrieve the original windows for the resource
+ original_windows = resource_windows_dict.get(resource_id, [])
- # Extract allocated intervals from the matrix
- allocated_intervals = [
- (ceil(round(matrix.intervals[i], 1)), ceil(round(matrix.intervals[i + 1], 1)))
- for i in range(start_index, end_index, 2)
- ]
+ # If no windows are found, skip this resource
+ if len(original_windows) == 0:
+ print(f"No original windows found for resource {resource_id}")
+ resource_windows_output[resource_id] = []
+ continue
- # Get the resource’s available windows from the resource_windows_dict
- updated_windows = resource_windows_dict.get(resource_id, [])
+ # Extract start and end points from the original windows into sets for fast lookups
+ window_starts = set(window[0] for window in original_windows)
+ window_ends = set(window[1] for window in original_windows)
- # Match allocated intervals with the available windows
- matched_intervals = self._match_intervals_with_windows(
- allocated_intervals, updated_windows
- )
+ # Prepare matrix intervals
+ interval_starts = matrix.intervals[:-1]
+ interval_ends = matrix.intervals[1:]
+
+ # Vectorized filtering: Keep only intervals where either the start or end matches
+ mask = np.isin(interval_starts, list(window_starts)) | np.isin(interval_ends, list(window_ends))
+ valid_starts = interval_starts[mask]
+ valid_ends = interval_ends[mask]
+
+ # Combine valid starts and ends into intervals
+ filtered_intervals = list(zip(np.ceil(valid_starts).astype(int), np.ceil(valid_ends).astype(int)))
- # Store the matched intervals in the output dictionary
- resource_windows_output[resource_id] = matched_intervals
+ # Merge contiguous or overlapping intervals
+ resource_windows_output[resource_id] = self.merge_intervals(filtered_intervals)
return resource_windows_output
- def _match_intervals_with_windows(
- self, allocated_intervals: list[tuple[int, int]], windows: np.ndarray
- ) -> list[tuple[int, int]]:
+ def merge_intervals(self, intervals: list[tuple[int, int]]) -> list[tuple[int, int]]:
"""
- Matches the allocated intervals with the resource's available windows.
+ Merges contiguous or overlapping intervals into a single interval.
"""
- matched_intervals = []
+ if not intervals:
+ return []
+
+ # Use numpy for fast sorting
+ intervals = np.array(intervals)
+ sorted_intervals = intervals[np.argsort(intervals[:, 0])]
+
+ # Initialize merged intervals with the first interval
+ merged = [sorted_intervals[0]]
+
+ # Vectorized merging
+ for start, end in sorted_intervals[1:]:
+ last_start, last_end = merged[-1]
- # Iterate over allocated intervals and compare with the available windows
- for allocated_start, allocated_end in allocated_intervals:
- for window in windows:
- window_start, window_end = window["start"], window["end"]
+ # Merge if overlapping or contiguous
+ if start <= last_end:
+ merged[-1] = (last_start, max(last_end, end))
+ else:
+ merged.append((start, end))
- # Check if there is an overlap between the allocated interval and the window
- if allocated_end > window_start and allocated_start < window_end:
- # Calculate the overlapping interval
- matched_start = max(allocated_start, window_start)
- matched_end = min(allocated_end, window_end)
- matched_intervals.append((matched_start, matched_end))
+ return merged
- return matched_intervals
def _mask_smallest_elements_except_top_k_per_row(
@@ -383,47 +396,50 @@ def _create_resource_group_matrix(
def _create_constraints_matrix(
self,
- resource_constraints: list[Resource],
+ resource_constraints: set[Resource],
resource_windows_matrix: Matrix,
task_duration: int,
) -> Matrix:
"""
- Creates a constraints matrix by accumulating availability across multiple windows,
- following the structure of the resource group matrix logic.
+ Checks if the resource constraints are available and updates the resource windows matrix.
"""
if not resource_constraints:
return None
- # Extract the resource IDs from the constraints
+ # get the constraint resource ids
resource_ids = np.array([resource.id for resource in resource_constraints])
- # Find the intersection of the resource constraints and the windows matrix
- available_resources = np.intersect1d(resource_ids, resource_windows_matrix.resource_ids)
- if len(available_resources) == 0:
- return None
+ # check if all resource constraints are available
+ if not np.all(np.isin(resource_ids, resource_windows_matrix.resource_ids)):
+ raise AllocationError("All resource constraints are not available")
- # Find the indices of the relevant resources in the windows matrix
+ # Find the indices of the available resources in the windows matrix
resource_indexes = np.where(
- np.isin(resource_windows_matrix.resource_ids, available_resources)
+ np.isin(resource_windows_matrix.resource_ids, resource_ids)
)[0]
- # Extract the relevant windows from the matrix
- constraint_matrix = resource_windows_matrix.resource_matrix[:, resource_indexes]
-
- # Accumulate availability across windows using cumulative sum, resetting at gaps (-1)
- accumulated_availability = self._cumsum_reset_at_minus_one_2d(constraint_matrix)
+ # get the windows for the resource constraints
+ constraint_windows = resource_windows_matrix.resource_matrix[
+ :, resource_indexes
+ ]
- # Check if accumulated availability meets the task duration requirement
- if np.sum(accumulated_availability) < task_duration:
- raise AllocationError("No solution found: Task duration exceeds available windows.")
+ # Compute the minimum along axis 1, mask values <= 0, and compute the cumulative sum
+ # devide by the number of resources to not increase the task completion time
+ min_values_matrix = (
+ np.min(constraint_windows, axis=1, keepdims=True)
+ * np.ones_like(constraint_windows)
+ / len(resource_ids)
+ )
- # Mask zero values to represent unavailable slots
- accumulated_availability = np.ma.masked_less_equal(accumulated_availability, 0)
+ resource_matrix = np.ma.masked_less_equal(
+ x=min_values_matrix,
+ value=0,
+ ).cumsum(axis=0)
return Matrix(
resource_ids=resource_ids,
intervals=resource_windows_matrix.intervals,
- resource_matrix=accumulated_availability,
+ resource_matrix=resource_matrix,
)
From 59fcedf0dd601320d3704319739d4a65cab3d368 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Thu, 17 Oct 2024 18:15:24 +0800
Subject: [PATCH 15/27] Committing for branch change
---
nb.ipynb | 188 ++++--------------
.../scheduler/heuristic_solver/main.py | 2 +
stresstest.ipynb | 51 ++---
3 files changed, 64 insertions(+), 177 deletions(-)
diff --git a/nb.ipynb b/nb.ipynb
index 888f046..bb42007 100644
--- a/nb.ipynb
+++ b/nb.ipynb
@@ -2,77 +2,16 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Scheduled 1 of 1 tasks.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<div>\n",
- "<style scoped>\n",
- " .dataframe tbody tr th:only-of-type {\n",
- " vertical-align: middle;\n",
- " }\n",
- "\n",
- " .dataframe tbody tr th {\n",
- " vertical-align: top;\n",
- " }\n",
- "\n",
- " .dataframe thead th {\n",
- " text-align: right;\n",
- " }\n",
- "</style>\n",
- "<table border=\"1\" class=\"dataframe\">\n",
- " <thead>\n",
- " <tr style=\"text-align: right;\">\n",
- " <th></th>\n",
- " <th>task_id</th>\n",
- " <th>assigned_resource_ids</th>\n",
- " <th>task_start</th>\n",
- " <th>task_end</th>\n",
- " <th>resource_intervals</th>\n",
- " </tr>\n",
- " </thead>\n",
- " <tbody>\n",
- " <tr>\n",
- " <th>0</th>\n",
- " <td>1</td>\n",
- " <td>[3, 1]</td>\n",
- " <td>0</td>\n",
- " <td>50</td>\n",
- " <td>([(0, 20), (30, 50)], [(0, 20), (30, 50)])</td>\n",
- " </tr>\n",
- " </tbody>\n",
- "</table>\n",
- "</div>"
- ],
- "text/plain": [
- " task_id assigned_resource_ids task_start task_end \\\n",
- "0 1 [3, 1] 0 50 \n",
- "\n",
- " resource_intervals \n",
- "0 ([(0, 20), (30, 50)], [(0, 20), (30, 50)]) "
- ]
- },
- "execution_count": 2,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"\n",
"from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n",
"from src.factryengine.scheduler.core import Scheduler\n",
"\n",
- "machine = Resource(id=1, available_windows=[(0, 50)])\n",
- "operator1 = Resource(id=2, available_windows=[(0, 20), (30, 50)])\n",
+ "machine = Resource(id=1, available_windows=[(0, 40)])\n",
+ "operator1 = Resource(id=2, available_windows=[(0, 40), (30, 50)])\n",
"operator2 = Resource(id=3, available_windows=[(0, 20), (30, 50)])\n",
"\n",
"operator_group = ResourceGroup(resources=[operator1, operator2])\n",
@@ -84,12 +23,6 @@
" id=1, duration=40, assignments=[assignment], priority=1, constraints=[machine]\n",
")\n",
"\n",
- "\n",
- "# # add machine as a constraint\n",
- "# t2 = Task(\n",
- "# id=2, duration=10, assignments=[assignment], priority=1, constraints=[machine]\n",
- "# )\n",
- "\n",
"tasks = [t1]\n",
"resources = [operator1, operator2, machine]\n",
"\n",
@@ -99,87 +32,20 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Scheduled 2 of 2 tasks.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<div>\n",
- "<style scoped>\n",
- " .dataframe tbody tr th:only-of-type {\n",
- " vertical-align: middle;\n",
- " }\n",
- "\n",
- " .dataframe tbody tr th {\n",
- " vertical-align: top;\n",
- " }\n",
- "\n",
- " .dataframe thead th {\n",
- " text-align: right;\n",
- " }\n",
- "</style>\n",
- "<table border=\"1\" class=\"dataframe\">\n",
- " <thead>\n",
- " <tr style=\"text-align: right;\">\n",
- " <th></th>\n",
- " <th>task_id</th>\n",
- " <th>assigned_resource_ids</th>\n",
- " <th>task_start</th>\n",
- " <th>task_end</th>\n",
- " <th>resource_intervals</th>\n",
- " </tr>\n",
- " </thead>\n",
- " <tbody>\n",
- " <tr>\n",
- " <th>0</th>\n",
- " <td>1</td>\n",
- " <td>[1]</td>\n",
- " <td>0</td>\n",
- " <td>40</td>\n",
- " <td>([(0, 20), (30, 40)])</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>1</th>\n",
- " <td>2</td>\n",
- " <td>[2]</td>\n",
- " <td>0</td>\n",
- " <td>20</td>\n",
- " <td>([(0, 20)])</td>\n",
- " </tr>\n",
- " </tbody>\n",
- "</table>\n",
- "</div>"
- ],
- "text/plain": [
- " task_id assigned_resource_ids task_start task_end resource_intervals\n",
- "0 1 [1] 0 40 ([(0, 20), (30, 40)])\n",
- "1 2 [2] 0 20 ([(0, 20)])"
- ]
- },
- "execution_count": 3,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"from src.factryengine.models import Resource, Task\n",
"from src.factryengine.scheduler.core import Scheduler\n",
"\n",
"# Define a resource with multiple fragmented windows\n",
"machine = Resource(id=1, available_windows=[(0, 20), (30, 50)])\n",
- "machine2 = Resource(id=2, available_windows=[(0, 20), (60, 80)])\n",
+ "machine2 = Resource(id=2, available_windows=[(30, 50)])\n",
"\n",
"# Create tasks with constraints\n",
- "t1 = Task(id=1, duration=30, priority=1, constraints=[machine])\n",
- "t2 = Task(id=2, duration=20, priority=2, constraints=[machine, machine2])\n",
+ "t1 = Task(id=1, duration=30, priority=1, constraints=[machine, machine2])\n",
+ "t2 = Task(id=2, duration=30, priority=2, constraints=[machine, machine2])\n",
"\n",
"tasks = [t1, t2]\n",
"resources = [machine, machine2]\n",
@@ -198,22 +64,48 @@
"from src.factryengine.models import Resource, ResourceGroup, Task, Assignment\n",
"from src.factryengine.scheduler.core import Scheduler\n",
"\n",
- "operator1 = Resource(id=1, available_windows=[(0, 20), (40, 60)])\n",
- "operator2 = Resource(id=2, available_windows=[(0, 20), (40, 60)])\n",
- "operator3 = Resource(id=3, available_windows=[(0, 30), (50, 60), (80, 150)])\n",
+ "operator1 = Resource(id=1, available_windows=[(0, 10), (40, 60)])\n",
+ "operator2 = Resource(id=2, available_windows=[(0, 10), (40, 60)])\n",
+ "operator3 = Resource(id=3, available_windows=[(10, 20), (50, 60), (80, 150)])\n",
"\n",
"rg1 = ResourceGroup(resources=[operator1, operator2, operator3])\n",
"\n",
"assignment = Assignment(resource_groups=[rg1], resource_count=1)\n",
"\n",
"t1 = Task(id=1, duration=40, assignments=[assignment], priority=1)\n",
- "t2 = Task(id=2, duration=20, assignments=[assignment], priority=2)\n",
"\n",
- "tasks = [t1, t2]\n",
+ "tasks = [t1]\n",
"resources = [operator1, operator2, operator3]\n",
"result = Scheduler(tasks=tasks, resources=resources).schedule()\n",
"result.to_dict()"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from src.factryengine.models import Resource, ResourceGroup, Task, Assignment\n",
+ "from src.factryengine.scheduler.core import Scheduler\n",
+ "\n",
+ "# create the resource\n",
+ "resource = Resource(id=1, available_windows=[(0, 10), (20, 30)])\n",
+ "\n",
+ "# create the resource group\n",
+ "resource_group = ResourceGroup(resources=[resource])\n",
+ "\n",
+ "# create the assignment\n",
+ "assignment = Assignment(resource_groups=[resource_group], resource_count=1)\n",
+ "\n",
+ "# create tasks\n",
+ "t1 = Task(id=1, duration=5, priority=1, constraints=[resource], predecessor_ids=[2])\n",
+ "t2 = Task(id=2, duration=5, priority=1, constraints=[resource])\n",
+ "\n",
+ "tasks = [t1, t2]\n",
+ "result = Scheduler(tasks=tasks, resources=[resource]).schedule()\n",
+ "result.to_dict()"
+ ]
}
],
"metadata": {
diff --git a/src/factryengine/scheduler/heuristic_solver/main.py b/src/factryengine/scheduler/heuristic_solver/main.py
index 5b51052..ce2c5c8 100644
--- a/src/factryengine/scheduler/heuristic_solver/main.py
+++ b/src/factryengine/scheduler/heuristic_solver/main.py
@@ -71,6 +71,8 @@ def solve(self) -> list[dict]:
task_duration=task.duration,
constraints=task.constraints,
)
+
+ print(f"Allocated resources for task {task_id}: {allocated_resource_windows_dict}")
except AllocationError as e:
self.mark_task_as_unscheduled(task_id=task_id, error_message=str(e))
continue
diff --git a/stresstest.ipynb b/stresstest.ipynb
index c9d24ab..b62d452 100644
--- a/stresstest.ipynb
+++ b/stresstest.ipynb
@@ -239,34 +239,9 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Resource Objects Created.\n",
- "Resource Groups Created.\n",
- "Task Objects Created.\n",
- "Original Task Length: 6138 | Post-Batched Length: 6138\n"
- ]
- },
- {
- "ename": "ValueError",
- "evalue": "min() arg is an empty sequence",
- "output_type": "error",
- "traceback": [
- "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
- "\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)",
- "Cell \u001b[1;32mIn[2], line 7\u001b[0m\n\u001b[0;32m 1\u001b[0m scheduler \u001b[38;5;241m=\u001b[39m ProdScheduler(\n\u001b[0;32m 2\u001b[0m resource_dir \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124minputs/actual_data/resource.json\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m 3\u001b[0m resource_group_dir \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124minputs/actual_data/resourcegroups.json\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m 4\u001b[0m task_dir \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124minputs/actual_data/tasks_all.json\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m 5\u001b[0m )\n\u001b[1;32m----> 7\u001b[0m \u001b[43mscheduler\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_scheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
- "Cell \u001b[1;32mIn[1], line 227\u001b[0m, in \u001b[0;36mProdScheduler.run_scheduler\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 223\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTask Objects Created.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 224\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOriginal Task Length: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mt_data)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m | Post-Batched Length: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtask_dict\u001b[38;5;241m.\u001b[39mvalues())\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m--> 227\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msolution \u001b[38;5;241m=\u001b[39m \u001b[43mScheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtasks_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdict_resource\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalues\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mschedule\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 228\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSolution Created.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
- "File \u001b[1;32mc:\\Projects\\factryengine\\src\\factryengine\\scheduler\\core.py:32\u001b[0m, in \u001b[0;36mScheduler.schedule\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 27\u001b[0m heuristic_solver \u001b[38;5;241m=\u001b[39m HeuristicSolver(\n\u001b[0;32m 28\u001b[0m task_dict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtask_dict, resources\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mresources, task_order\u001b[38;5;241m=\u001b[39mtask_order\n\u001b[0;32m 29\u001b[0m )\n\u001b[0;32m 31\u001b[0m \u001b[38;5;66;03m# Use the heuristic solver to find a solution\u001b[39;00m\n\u001b[1;32m---> 32\u001b[0m solver_result \u001b[38;5;241m=\u001b[39m \u001b[43mheuristic_solver\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msolve\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 34\u001b[0m \u001b[38;5;66;03m# Create a scheduler result with the solver result and unscheduled tasks\u001b[39;00m\n\u001b[0;32m 35\u001b[0m scheduler_result \u001b[38;5;241m=\u001b[39m SchedulerResult(\n\u001b[0;32m 36\u001b[0m task_vars\u001b[38;5;241m=\u001b[39msolver_result,\n\u001b[0;32m 37\u001b[0m unscheduled_task_ids\u001b[38;5;241m=\u001b[39mheuristic_solver\u001b[38;5;241m.\u001b[39munscheduled_task_ids,\n\u001b[0;32m 38\u001b[0m )\n",
- "File \u001b[1;32mc:\\Projects\\factryengine\\src\\factryengine\\scheduler\\heuristic_solver\\main.py:85\u001b[0m, in \u001b[0;36mHeuristicSolver.solve\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 78\u001b[0m \u001b[38;5;66;03m# update resource windows\u001b[39;00m\n\u001b[0;32m 79\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwindow_manager\u001b[38;5;241m.\u001b[39mupdate_resource_windows(allocated_resource_windows_dict)\n\u001b[0;32m 82\u001b[0m task_values \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m 83\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask_id\u001b[39m\u001b[38;5;124m\"\u001b[39m: task_id,\n\u001b[0;32m 84\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124massigned_resource_ids\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mlist\u001b[39m(allocated_resource_windows_dict\u001b[38;5;241m.\u001b[39mkeys()),\n\u001b[1;32m---> 85\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask_start\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28;43mmin\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[0;32m 86\u001b[0m \u001b[43m \u001b[49m\u001b[43mstart\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mintervals\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mallocated_resource_windows_dict\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalues\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mstart\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mintervals\u001b[49m\n\u001b[0;32m 87\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m,\n\u001b[0;32m 88\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask_end\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mmax\u001b[39m(\n\u001b[0;32m 89\u001b[0m end \u001b[38;5;28;01mfor\u001b[39;00m intervals \u001b[38;5;129;01min\u001b[39;00m allocated_resource_windows_dict\u001b[38;5;241m.\u001b[39mvalues() \u001b[38;5;28;01mfor\u001b[39;00m _, end \u001b[38;5;129;01min\u001b[39;00m intervals\n\u001b[0;32m 90\u001b[0m ),\n\u001b[0;32m 91\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mresource_intervals\u001b[39m\u001b[38;5;124m\"\u001b[39m: allocated_resource_windows_dict\u001b[38;5;241m.\u001b[39mvalues(),\n\u001b[0;32m 92\u001b[0m }\n\u001b[0;32m 93\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtask_vars[task_id] \u001b[38;5;241m=\u001b[39m task_values\n\u001b[0;32m 96\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mlist\u001b[39m(\n\u001b[0;32m 97\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtask_vars\u001b[38;5;241m.\u001b[39mvalues()\n\u001b[0;32m 98\u001b[0m )\n",
- "\u001b[1;31mValueError\u001b[0m: min() arg is an empty sequence"
- ]
- }
- ],
+ "outputs": [],
"source": [
"scheduler = ProdScheduler(\n",
" resource_dir = 'inputs/actual_data/resource.json',\n",
@@ -279,11 +254,20 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
- "result = scheduler.solution.to_dataframe()"
+ "scheduler.dict_resource[119]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "scheduler.task_dict['WO138769-10']"
]
},
{
@@ -326,6 +310,15 @@
"intervals['end_dt'] = intervals.apply(lambda x: scheduler.int_to_datetime(x['interval_end'], scheduler.today_str), axis=1)\n",
"intervals.sort_values(by='interval_end', ascending=False)"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sched"
+ ]
}
],
"metadata": {
From 81119fb5d6232caa3f1c2aa01de4f53a4de02845 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Thu, 17 Oct 2024 22:00:05 +0800
Subject: [PATCH 16/27] Used matrices for getting resource intervals
---
.../heuristic_solver/task_allocator.py | 178 ++++++++++--------
1 file changed, 99 insertions(+), 79 deletions(-)
diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
index fcc8003..33e0f92 100644
--- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py
+++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
@@ -166,71 +166,92 @@ def _get_resource_intervals(
self, matrix: Matrix, resource_windows_dict: dict[int, list[tuple[float, float, float, int]]]
) -> dict[int, list[tuple[int, int]]]:
"""
- Extracts the resource intervals from the solution matrix by strictly matching them
- with the provided original windows in `resource_windows_dict`.
+ Extracts all the resource intervals used from the solution matrix,
+ including non-contiguous intervals and partial usage.
"""
resource_windows_output = {}
- # Iterate over each resource ID and its corresponding matrix intervals
+ # Iterate over each resource and its corresponding matrix intervals
for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T):
+ print(f"\nResource ID: {resource_id}")
+ print(f"Resource Intervals: {resource_intervals}")
+ print(f"Overall Intervals: {matrix.intervals}")
- # Retrieve the original windows for the resource
- original_windows = resource_windows_dict.get(resource_id, [])
+ # Get all relevant indexes
+ indexes = self._find_indexes(resource_intervals)
+ print(f"Produced Indexes: {indexes}")
- # If no windows are found, skip this resource
- if len(original_windows) == 0:
- print(f"No original windows found for resource {resource_id}")
- resource_windows_output[resource_id] = []
- continue
-
- # Extract start and end points from the original windows into sets for fast lookups
- window_starts = set(window[0] for window in original_windows)
- window_ends = set(window[1] for window in original_windows)
-
- # Prepare matrix intervals
- interval_starts = matrix.intervals[:-1]
- interval_ends = matrix.intervals[1:]
+ # Pair the indexes in groups of 2 (start, end)
+ intervals = []
+ for start, end in zip(indexes[::2], indexes[1::2]):
+ # Use start and end indexes directly without skipping
+ print(f"Start: {start}, End: {end}")
+ interval_start = matrix.intervals[start]
+ interval_end = matrix.intervals[end]
- # Vectorized filtering: Keep only intervals where either the start or end matches
- mask = np.isin(interval_starts, list(window_starts)) | np.isin(interval_ends, list(window_ends))
- valid_starts = interval_starts[mask]
- valid_ends = interval_ends[mask]
+ # Append the interval to the list
+ intervals.append((int(np.round(interval_start)), int(np.round(interval_end))))
- # Combine valid starts and ends into intervals
- filtered_intervals = list(zip(np.ceil(valid_starts).astype(int), np.ceil(valid_ends).astype(int)))
-
- # Merge contiguous or overlapping intervals
- resource_windows_output[resource_id] = self.merge_intervals(filtered_intervals)
+ # Store the intervals for the current resource
+ resource_windows_output[resource_id] = intervals
return resource_windows_output
- def merge_intervals(self, intervals: list[tuple[int, int]]) -> list[tuple[int, int]]:
+
+ def _find_indexes(self, resource_intervals: np.ma.MaskedArray) -> int | None:
"""
- Merges contiguous or overlapping intervals into a single interval.
+ Finds the first index where the value is masked, and the next value is non-masked and > 0.
"""
- if not intervals:
- return []
+ indexes = []
+ # Shift the mask by 1 to align with the 'next' element comparison
+ current_mask = resource_intervals.mask[:-1]
+ next_mask = resource_intervals.mask[1:]
+ next_values = resource_intervals.data[1:]
- # Use numpy for fast sorting
- intervals = np.array(intervals)
- sorted_intervals = intervals[np.argsort(intervals[:, 0])]
+ # Vectorized condition: current is masked, next is non-masked, and next value > 0
+ condition = (current_mask) & (~next_mask) & (next_values > 0)
- # Initialize merged intervals with the first interval
- merged = [sorted_intervals[0]]
+ # Find the first index where the condition is met
+ indices = np.where(condition)[0]
- # Vectorized merging
- for start, end in sorted_intervals[1:]:
- last_start, last_end = merged[-1]
+ first_index = indices[0]
+ last_index = resource_intervals.size-1
- # Merge if overlapping or contiguous
- if start <= last_end:
- merged[-1] = (last_start, max(last_end, end))
- else:
- merged.append((start, end))
+ indexes = [first_index] # Start with the first index
- return merged
+ # Iterate through the range between first and last index
+ for i in range(first_index, last_index + 1):
+ current = resource_intervals[i]
+ previous = resource_intervals[i - 1] if i > 0 else 0
+ next_value = resource_intervals[i + 1] if i < last_index else 0
+ # Check if the current value is masked
+ is_masked = resource_intervals.mask[i - 1] if i > 0 else False
+ # Skip if all values are the same (stable window)
+ if current > 0 and current == previous == next_value:
+ continue
+
+ # Skip increasing trend from masked value
+ if current > 0 and current < next_value and is_masked:
+ continue
+
+ # Detect end of a window
+ if current > 0 and current == next_value and (is_masked or previous < current):
+ indexes.append(i)
+ continue
+
+ # Detect start of a new window
+ if current > 0 and next_value > current and (is_masked or previous == current):
+ indexes.append(i)
+ continue
+
+ # Always add the last index
+ if i == last_index:
+ indexes.append(i)
+
+ # Return the first valid index, or None if no valid index is found
+ return indexes
def _mask_smallest_elements_except_top_k_per_row(
self, array: np.ma.core.MaskedArray, k
@@ -502,48 +523,47 @@ def _create_assignments_matrix(
return assignments_matrix
+ # def _find_indexes(self, arr: np.array) -> tuple[int, int] | None:
+ # """
+ # Find the start and end indexes for a valid segment of resource availability.
+ # This version avoids explicit loops and ensures the start index is correctly identified.
+ # """
+ # # If the input is a MaskedArray, handle it accordingly
+ # if isinstance(arr, np.ma.MaskedArray):
+ # arr_data = arr.data
+ # mask = arr.mask
+ # # Find valid (unmasked and positive) indices
+ # valid_indices = np.where((~mask) & (arr_data >= 0))[0]
+ # else:
+ # valid_indices = np.where(arr >= 0)[0]
- def _find_indexes(self, arr: np.array) -> tuple[int, int] | None:
- """
- Find the start and end indexes for a valid segment of resource availability.
- This version avoids explicit loops and ensures the start index is correctly identified.
- """
- # If the input is a MaskedArray, handle it accordingly
- if isinstance(arr, np.ma.MaskedArray):
- arr_data = arr.data
- mask = arr.mask
- # Find valid (unmasked and positive) indices
- valid_indices = np.where((~mask) & (arr_data >= 0))[0]
- else:
- valid_indices = np.where(arr >= 0)[0]
-
- # If no valid indices are found, return None (no available resources)
- if valid_indices.size == 0:
- return None
+ # # If no valid indices are found, return None (no available resources)
+ # if valid_indices.size == 0:
+ # return None
- # Identify if the start of the array is valid
- start_index = 0 if arr[0] > 0 else valid_indices[0]
+ # # Identify if the start of the array is valid
+ # start_index = 0 if arr[0] > 0 else valid_indices[0]
- # Calculate differences between consecutive indices
- diffs = np.diff(valid_indices)
+ # # Calculate differences between consecutive indices
+ # diffs = np.diff(valid_indices)
- # Identify segment boundaries where there is a gap greater than 1
- gaps = diffs > 1
- segment_boundaries = np.where(gaps)[0]
+ # # Identify segment boundaries where there is a gap greater than 1
+ # gaps = diffs > 1
+ # segment_boundaries = np.where(gaps)[0]
- # Insert the start index explicitly to ensure it is considered
- segment_starts = np.insert(segment_boundaries + 1, 0, 0)
- segment_ends = np.append(segment_starts[1:], len(valid_indices))
+ # # Insert the start index explicitly to ensure it is considered
+ # segment_starts = np.insert(segment_boundaries + 1, 0, 0)
+ # segment_ends = np.append(segment_starts[1:], len(valid_indices))
- # Always take the first segment (which starts at the earliest valid index)
- start_pos = segment_starts[0]
- end_pos = segment_ends[0] - 1
+ # # Always take the first segment (which starts at the earliest valid index)
+ # start_pos = segment_starts[0]
+ # end_pos = segment_ends[0] - 1
- # Convert these segment positions to the actual start and end indices
- start_index = valid_indices[start_pos]
- end_index = valid_indices[end_pos]
+ # # Convert these segment positions to the actual start and end indices
+ # start_index = valid_indices[start_pos]
+ # end_index = valid_indices[end_pos]
- return start_index, end_index
+ # return start_index, end_index
From e8eee2291473ac92e2fb4a8a6d716d3c81426e01 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Thu, 17 Oct 2024 22:21:34 +0800
Subject: [PATCH 17/27] Stable scheduler
---
src/factryengine/scheduler/heuristic_solver/main.py | 2 --
.../scheduler/heuristic_solver/task_allocator.py | 12 ++++++------
2 files changed, 6 insertions(+), 8 deletions(-)
diff --git a/src/factryengine/scheduler/heuristic_solver/main.py b/src/factryengine/scheduler/heuristic_solver/main.py
index ce2c5c8..5b51052 100644
--- a/src/factryengine/scheduler/heuristic_solver/main.py
+++ b/src/factryengine/scheduler/heuristic_solver/main.py
@@ -71,8 +71,6 @@ def solve(self) -> list[dict]:
task_duration=task.duration,
constraints=task.constraints,
)
-
- print(f"Allocated resources for task {task_id}: {allocated_resource_windows_dict}")
except AllocationError as e:
self.mark_task_as_unscheduled(task_id=task_id, error_message=str(e))
continue
diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
index 33e0f92..3f18f05 100644
--- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py
+++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
@@ -173,19 +173,19 @@ def _get_resource_intervals(
# Iterate over each resource and its corresponding matrix intervals
for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T):
- print(f"\nResource ID: {resource_id}")
- print(f"Resource Intervals: {resource_intervals}")
- print(f"Overall Intervals: {matrix.intervals}")
+ # print(f"\nResource ID: {resource_id}")
+ # print(f"Resource Intervals: {resource_intervals}")
+ # print(f"Overall Intervals: {matrix.intervals}")
# Get all relevant indexes
indexes = self._find_indexes(resource_intervals)
- print(f"Produced Indexes: {indexes}")
+ # print(f"Produced Indexes: {indexes}")
# Pair the indexes in groups of 2 (start, end)
intervals = []
for start, end in zip(indexes[::2], indexes[1::2]):
# Use start and end indexes directly without skipping
- print(f"Start: {start}, End: {end}")
+ # print(f"Start: {start}, End: {end}")
interval_start = matrix.intervals[start]
interval_end = matrix.intervals[end]
@@ -214,7 +214,7 @@ def _find_indexes(self, resource_intervals: np.ma.MaskedArray) -> int | None:
# Find the first index where the condition is met
indices = np.where(condition)[0]
- first_index = indices[0]
+ first_index = indices[0] if len(indices) > 0 else 0
last_index = resource_intervals.size-1
indexes = [first_index] # Start with the first index
From d30f06a14c17ecbbabd646e3ec83b597e0369c6d Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Mon, 21 Oct 2024 20:25:50 +0800
Subject: [PATCH 18/27] Updated get_resource_intervals test
---
tests/scheduler/test_task_allocator.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/tests/scheduler/test_task_allocator.py b/tests/scheduler/test_task_allocator.py
index 0a7a71f..75571f4 100644
--- a/tests/scheduler/test_task_allocator.py
+++ b/tests/scheduler/test_task_allocator.py
@@ -31,8 +31,8 @@ def test_solve_task_end(task_allocator):
def test_get_resource_intervals_continuous(task_allocator):
# Test case continuous values 1 task 2 resources
solution_resource_ids = np.array([1, 2])
- solution_intervals = np.array([0, 1, 2, 3])
- resource_matrix = np.ma.array([[0, 0], [1, 1]])
+ solution_intervals = np.array([0, 1])
+ resource_matrix = np.ma.array([[0, 0], [1, 1]], mask=[[False, False], [False, False]],)
solution_matrix = Matrix(
resource_ids=solution_resource_ids,
intervals=solution_intervals,
@@ -45,15 +45,15 @@ def test_get_resource_intervals_continuous(task_allocator):
def test_get_resource_intervals_windowed(task_allocator):
# Test case windowed values 1 task 1 resource
solution_resource_ids = np.array([1])
- solution_intervals = np.array([0, 1, 4, 5, 7, 8])
- resource_matrix = np.ma.array([[0], [1], [4], [5], [7], [8]])
+ solution_intervals = np.array([0, 2, 3, 4])
+ resource_matrix = np.ma.array([[0], [2], [2], [3]], mask=[[False], [False], [False], [False]],)
solution_matrix = Matrix(
resource_ids=solution_resource_ids,
intervals=solution_intervals,
resource_matrix=resource_matrix,
)
result = task_allocator._get_resource_intervals(solution_matrix)
- expeceted = {1: [(0, 1), (4, 5), (7, 8)]}
+ expeceted = {1: [(0, 2), (3, 4)]}
assert result == expeceted
From 3b2df6f9191f0fc9fda51694b526cc3e2171fc7a Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Mon, 21 Oct 2024 20:35:38 +0800
Subject: [PATCH 19/27] Updated find indexes test
---
tests/scheduler/test_task_allocator.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/tests/scheduler/test_task_allocator.py b/tests/scheduler/test_task_allocator.py
index 75571f4..c6785b2 100644
--- a/tests/scheduler/test_task_allocator.py
+++ b/tests/scheduler/test_task_allocator.py
@@ -220,9 +220,8 @@ def test_diff_and_zero_negatives(array, expected):
"array, expected",
[
# Full valid sequence without gaps
- (np.array([0, 1, 2, 3, 4]), (0, 4)),
- # Sequence with repeated values, expecting first valid segment
- (np.array([0, 3]), (0, 1)), # This one might need revisiting if logic changes
+ (np.ma.array([0, 2, 2, 3], mask=[False, False, False, False]), ([0, 1, 2, 3])),
+ (np.ma.array([0, 3], mask=[False, False]), ([0, 1])),
],
)
def test_find_indexes(array, expected):
From 5964f6cc4108284a43d28e179ea83965183a2690 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Mon, 21 Oct 2024 20:44:12 +0800
Subject: [PATCH 20/27] Updated test _cumsum_reset_at_minus_one
---
tests/scheduler/test_task_allocator.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/scheduler/test_task_allocator.py b/tests/scheduler/test_task_allocator.py
index c6785b2..d6336b3 100644
--- a/tests/scheduler/test_task_allocator.py
+++ b/tests/scheduler/test_task_allocator.py
@@ -126,7 +126,7 @@ def test_mask_smallest_elements_except_top_k_per_row(
@pytest.mark.parametrize(
"array, expected",
[
- (np.array([0, 1, 5, -1, 10]), [0, 1, 6, 0, 10]),
+ (np.array([0, 1, 5, -1, 10]), [0, 1, 6, 0, 16]),
(np.array([-1, 2, 3, 0, 4]), [0, 2, 5, 5, 9]),
(np.array([0, -1, 2, 4, -1]), [0, 0, 2, 6, 0]),
],
From 1f785ccb896a831e00e2fec0d718ae85b6a4e0a3 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Mon, 21 Oct 2024 20:48:17 +0800
Subject: [PATCH 21/27] Reverted cumsum at minus -1 test
---
tests/scheduler/test_task_allocator.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/scheduler/test_task_allocator.py b/tests/scheduler/test_task_allocator.py
index d6336b3..c6785b2 100644
--- a/tests/scheduler/test_task_allocator.py
+++ b/tests/scheduler/test_task_allocator.py
@@ -126,7 +126,7 @@ def test_mask_smallest_elements_except_top_k_per_row(
@pytest.mark.parametrize(
"array, expected",
[
- (np.array([0, 1, 5, -1, 10]), [0, 1, 6, 0, 16]),
+ (np.array([0, 1, 5, -1, 10]), [0, 1, 6, 0, 10]),
(np.array([-1, 2, 3, 0, 4]), [0, 2, 5, 5, 9]),
(np.array([0, -1, 2, 4, -1]), [0, 0, 2, 6, 0]),
],
From 22dc0a7f3de26f4ed485f3ac2ec3701e50d1ab25 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Tue, 22 Oct 2024 14:02:58 +0800
Subject: [PATCH 22/27] Updates to test and task allocator
---
.../heuristic_solver/task_allocator.py | 63 ++++++++++++-------
tests/scheduler/test_task_allocator.py | 2 +-
2 files changed, 43 insertions(+), 22 deletions(-)
diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
index 3f18f05..c295fde 100644
--- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py
+++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
@@ -45,29 +45,41 @@ def allocate_task(
task_duration=task_duration,
)
- # matrix to solve
- matrix_to_solve = assignments_matrix or constraints_matrix
+ if assignments_matrix and constraints_matrix:
+ # find the solution for assignments
+ solution_matrix = self._solve_matrix(
+ matrix=assignments_matrix,
+ task_duration=task_duration,
+ )
+
+ # find the solution for constraints
+ constraints_solution = self._solve_matrix(
+ matrix=constraints_matrix,
+ task_duration=task_duration,
+ )
+ else:
+ # matrix to solve
+ matrix_to_solve = assignments_matrix or constraints_matrix
+
+ # find the solution
+ solution_matrix = self._solve_matrix(
+ matrix=matrix_to_solve,
+ task_duration=task_duration,
+ )
- # find the solution
- solution_matrix = self._solve_matrix(
- matrix=matrix_to_solve,
- task_duration=task_duration,
- )
# process solution to find allocated resource windows
allocated_windows = self._get_resource_intervals(
- matrix=solution_matrix,
- resource_windows_dict=resource_windows_dict
+ matrix=solution_matrix
)
# add constraints to allocated windows
if constraints and assignments:
constraints_matrix_trimmed = Matrix.trim_end(
- original_matrix=constraints_matrix, trim_matrix=solution_matrix
+ original_matrix=constraints_solution, trim_matrix=solution_matrix
)
allocated_windows.update(
self._get_resource_intervals(
- matrix=constraints_matrix_trimmed,
- resource_windows_dict=resource_windows_dict
+ matrix=constraints_matrix_trimmed
)
)
@@ -163,7 +175,7 @@ def _solve_task_end(
return col0_value, other_columns_values
def _get_resource_intervals(
- self, matrix: Matrix, resource_windows_dict: dict[int, list[tuple[float, float, float, int]]]
+ self, matrix: Matrix
) -> dict[int, list[tuple[int, int]]]:
"""
Extracts all the resource intervals used from the solution matrix,
@@ -173,13 +185,10 @@ def _get_resource_intervals(
# Iterate over each resource and its corresponding matrix intervals
for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T):
- # print(f"\nResource ID: {resource_id}")
- # print(f"Resource Intervals: {resource_intervals}")
- # print(f"Overall Intervals: {matrix.intervals}")
+
# Get all relevant indexes
indexes = self._find_indexes(resource_intervals)
- # print(f"Produced Indexes: {indexes}")
# Pair the indexes in groups of 2 (start, end)
intervals = []
@@ -226,23 +235,35 @@ def _find_indexes(self, resource_intervals: np.ma.MaskedArray) -> int | None:
next_value = resource_intervals[i + 1] if i < last_index else 0
# Check if the current value is masked
- is_masked = resource_intervals.mask[i - 1] if i > 0 else False
+ is_prev_masked = resource_intervals.mask[i - 1] if i > 0 else False
+ is_curr_masked = resource_intervals.mask[i]
+ is_next_masked = resource_intervals.mask[i+1] if i < last_index else False
# Skip if all values are the same (stable window)
if current > 0 and current == previous == next_value:
continue
# Skip increasing trend from masked value
- if current > 0 and current < next_value and is_masked:
+ if current > 0 and current < next_value and is_prev_masked:
continue
# Detect end of a window
- if current > 0 and current == next_value and (is_masked or previous < current):
+ if current > 0 and current == next_value and (is_prev_masked or previous < current):
+ indexes.append(i)
+ continue
+
+ # Detect end of window using masks
+ if current > 0 and is_next_masked and not is_curr_masked:
indexes.append(i)
continue
# Detect start of a new window
- if current > 0 and next_value > current and (is_masked or previous == current):
+ if current > 0 and next_value > current and (is_prev_masked or previous == current):
+ indexes.append(i)
+ continue
+
+ # Detect start of window using masks
+ if is_curr_masked and previous > 0 and next_value > 0:
indexes.append(i)
continue
diff --git a/tests/scheduler/test_task_allocator.py b/tests/scheduler/test_task_allocator.py
index c6785b2..d6336b3 100644
--- a/tests/scheduler/test_task_allocator.py
+++ b/tests/scheduler/test_task_allocator.py
@@ -126,7 +126,7 @@ def test_mask_smallest_elements_except_top_k_per_row(
@pytest.mark.parametrize(
"array, expected",
[
- (np.array([0, 1, 5, -1, 10]), [0, 1, 6, 0, 10]),
+ (np.array([0, 1, 5, -1, 10]), [0, 1, 6, 0, 16]),
(np.array([-1, 2, 3, 0, 4]), [0, 2, 5, 5, 9]),
(np.array([0, -1, 2, 4, -1]), [0, 0, 2, 6, 0]),
],
From 2ac6f16a0dc3c781625d0d5472adb2f19412fdc2 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Tue, 22 Oct 2024 16:44:13 +0800
Subject: [PATCH 23/27] Updated task allocator comments
---
src/factryengine/scheduler/heuristic_solver/task_allocator.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
index c295fde..c2afa4c 100644
--- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py
+++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
@@ -209,7 +209,7 @@ def _get_resource_intervals(
def _find_indexes(self, resource_intervals: np.ma.MaskedArray) -> int | None:
"""
- Finds the first index where the value is masked, and the next value is non-masked and > 0.
+ Finds relevant indexes in the resource intervals where the resource is used.
"""
indexes = []
# Shift the mask by 1 to align with the 'next' element comparison
From 4a31734cf7dd206e9b2dee546ebbecbc62562560 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Tue, 22 Oct 2024 16:44:51 +0800
Subject: [PATCH 24/27] Ignored files with .ipynb extensions
---
.gitignore | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitignore b/.gitignore
index d8d9708..85f38dd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
# Prototyping Notebooks
notebooks/
+*.ipynb
# Test data
inputs/
From 610d132cc303c34d2504aeb4d5a297ca539b836e Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Tue, 22 Oct 2024 16:45:40 +0800
Subject: [PATCH 25/27] Move files to notebooks folder
---
nb.ipynb | 132 ------------------
stresstest.ipynb | 345 -----------------------------------------------
2 files changed, 477 deletions(-)
delete mode 100644 nb.ipynb
delete mode 100644 stresstest.ipynb
diff --git a/nb.ipynb b/nb.ipynb
deleted file mode 100644
index bb42007..0000000
--- a/nb.ipynb
+++ /dev/null
@@ -1,132 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "\n",
- "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n",
- "from src.factryengine.scheduler.core import Scheduler\n",
- "\n",
- "machine = Resource(id=1, available_windows=[(0, 40)])\n",
- "operator1 = Resource(id=2, available_windows=[(0, 40), (30, 50)])\n",
- "operator2 = Resource(id=3, available_windows=[(0, 20), (30, 50)])\n",
- "\n",
- "operator_group = ResourceGroup(resources=[operator1, operator2])\n",
- "\n",
- "assignment = Assignment(resource_groups=[operator_group], resource_count=1)\n",
- "\n",
- "# add machine as a constraint\n",
- "t1 = Task(\n",
- " id=1, duration=40, assignments=[assignment], priority=1, constraints=[machine]\n",
- ")\n",
- "\n",
- "tasks = [t1]\n",
- "resources = [operator1, operator2, machine]\n",
- "\n",
- "result = Scheduler(tasks=tasks, resources=resources).schedule()\n",
- "result.to_dataframe()\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from src.factryengine.models import Resource, Task\n",
- "from src.factryengine.scheduler.core import Scheduler\n",
- "\n",
- "# Define a resource with multiple fragmented windows\n",
- "machine = Resource(id=1, available_windows=[(0, 20), (30, 50)])\n",
- "machine2 = Resource(id=2, available_windows=[(30, 50)])\n",
- "\n",
- "# Create tasks with constraints\n",
- "t1 = Task(id=1, duration=30, priority=1, constraints=[machine, machine2])\n",
- "t2 = Task(id=2, duration=30, priority=2, constraints=[machine, machine2])\n",
- "\n",
- "tasks = [t1, t2]\n",
- "resources = [machine, machine2]\n",
- "\n",
- "# Schedule the tasks\n",
- "result = Scheduler(tasks=tasks, resources=resources).schedule()\n",
- "result.to_dataframe()\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from src.factryengine.models import Resource, ResourceGroup, Task, Assignment\n",
- "from src.factryengine.scheduler.core import Scheduler\n",
- "\n",
- "operator1 = Resource(id=1, available_windows=[(0, 10), (40, 60)])\n",
- "operator2 = Resource(id=2, available_windows=[(0, 10), (40, 60)])\n",
- "operator3 = Resource(id=3, available_windows=[(10, 20), (50, 60), (80, 150)])\n",
- "\n",
- "rg1 = ResourceGroup(resources=[operator1, operator2, operator3])\n",
- "\n",
- "assignment = Assignment(resource_groups=[rg1], resource_count=1)\n",
- "\n",
- "t1 = Task(id=1, duration=40, assignments=[assignment], priority=1)\n",
- "\n",
- "tasks = [t1]\n",
- "resources = [operator1, operator2, operator3]\n",
- "result = Scheduler(tasks=tasks, resources=resources).schedule()\n",
- "result.to_dict()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from src.factryengine.models import Resource, ResourceGroup, Task, Assignment\n",
- "from src.factryengine.scheduler.core import Scheduler\n",
- "\n",
- "# create the resource\n",
- "resource = Resource(id=1, available_windows=[(0, 10), (20, 30)])\n",
- "\n",
- "# create the resource group\n",
- "resource_group = ResourceGroup(resources=[resource])\n",
- "\n",
- "# create the assignment\n",
- "assignment = Assignment(resource_groups=[resource_group], resource_count=1)\n",
- "\n",
- "# create tasks\n",
- "t1 = Task(id=1, duration=5, priority=1, constraints=[resource], predecessor_ids=[2])\n",
- "t2 = Task(id=2, duration=5, priority=1, constraints=[resource])\n",
- "\n",
- "tasks = [t1, t2]\n",
- "result = Scheduler(tasks=tasks, resources=[resource]).schedule()\n",
- "result.to_dict()"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": ".venv",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.11.4"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/stresstest.ipynb b/stresstest.ipynb
deleted file mode 100644
index b62d452..0000000
--- a/stresstest.ipynb
+++ /dev/null
@@ -1,345 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {},
- "outputs": [],
- "source": [
- "import pandas as pd\n",
- "import json\n",
- "import pytz\n",
- "from datetime import datetime, timezone, timedelta\n",
- "import time\n",
- "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n",
- "from src.factryengine.scheduler.core import Scheduler\n",
- "from src.factryengine.scheduler.task_batch_processor import TaskSplitter\n",
- "\n",
- "\n",
- "class ProdScheduler: \n",
- " def __init__(self, \n",
- " resource_dir: str, \n",
- " resource_group_dir: str, \n",
- " task_dir: str\n",
- " ) -> None:\n",
- " # Scheduler Attributes\n",
- " self.cph_timezone = pytz.timezone('Europe/Copenhagen')\n",
- " # self.today = datetime.now(timezone.utc).replace(\n",
- " # hour=0, minute=0, second=0, microsecond=0)\n",
- " self.today = datetime(2024, 10, 9, 0, 0, 0, 0, tzinfo=timezone.utc)\n",
- " self.today_str = str(self.today)[:19]\n",
- "\n",
- " # Component Attributes\n",
- " self.dict_resource = {}\n",
- " self.dict_resourcegroups = {}\n",
- " self.tasks_list = []\n",
- " self.task_dict = {}\n",
- " self.pred_dict = {}\n",
- " self.flow_map = {}\n",
- " self.pred_exploded = {}\n",
- "\n",
- " # Inputs \n",
- " with open(resource_dir, 'r') as file:\n",
- " self.r_data = json.load(file)\n",
- "\n",
- " with open(task_dir, 'r') as file:\n",
- " self.t_data = json.load(file)\n",
- "\n",
- " with open(resource_group_dir, 'r') as file:\n",
- " self.rg_data = json.load(file)\n",
- " \n",
- " def convert_to_minutes(self, datetime_str, start_time_obj):\n",
- " datetime_obj = datetime.strptime(datetime_str, '%Y-%m-%d %H:%M:%S%z')\n",
- " diff_minutes = (datetime_obj - start_time_obj).total_seconds()/60\n",
- " return int(diff_minutes)\n",
- "\n",
- " def adjust_capacity(self, start, end, capacity):\n",
- " return (end - start) * capacity + start\n",
- "\n",
- " def organize_predecessors(self, task: Task):\n",
- " try:\n",
- " list_predecessors = self.pred_dict[task.id]\n",
- " # ============================================================== UNCOMMENT TO USE MICROBATCH FLOW\n",
- " if task.batch_id: # Check if task is microbatched\n",
- " # Look for each predecessors that exist in the flow map\n",
- " for predecessor in list_predecessors:\n",
- " pred_batch_id = f'{predecessor}-{task.batch_id}'\n",
- " if pred_batch_id in self.flow_map and pred_batch_id in self.pred_dict: # Check if pred is part of flow\n",
- " # Check if pred-parent connection is correct\n",
- " if self.flow_map[task.id]['predecessor'] == self.flow_map[pred_batch_id]['parent']:\n",
- " self.pred_dict[task.id] = [pred_batch_id]\n",
- "\n",
- " elif pred_batch_id not in self.pred_dict and predecessor not in self.task_dict:\n",
- " parent_predecessor = []\n",
- " for pred in self.pred_dict[task.id]:\n",
- " parent_predecessor.extend(self.pred_dict[pred])\n",
- " self.pred_dict[task.id] = parent_predecessor\n",
- " # ============================================================== UNCOMMENT TO USE MICROBATCH FLOW\n",
- "\n",
- " # Remove task batch id\n",
- " task.set_batch_id(None)\n",
- "\n",
- " # Check for predecessors to be exploded\n",
- " for predecessor in list_predecessors:\n",
- " if predecessor in self.pred_exploded:\n",
- " self.pred_dict[task.id].remove(\n",
- " predecessor) # Remove original value\n",
- " self.pred_dict[task.id].extend(\n",
- " self.pred_exploded[predecessor]) # Add exploded batches\n",
- " except Exception as e:\n",
- " return\n",
- " \n",
- " def set_predecessors(self, task: Task):\n",
- " if not task.id in self.pred_dict: # if task is not in pred_dict, then it has no predecessors\n",
- " return\n",
- "\n",
- " for pred_id in self.pred_dict[task.id]:\n",
- " if pred_id in self.task_dict: # ensure predecessor exists in task_dict\n",
- " pred_task = self.task_dict[pred_id]\n",
- "\n",
- " # Avoid adding a predecessor multiple times\n",
- " if pred_task not in task.predecessors:\n",
- " # set predecessors for the predecessor first\n",
- " self.set_predecessors(pred_task)\n",
- " task.predecessors.append(pred_task)\n",
- " \n",
- " def create_resource_object(self, resource_list):\n",
- " # Generate slot based on schedule selected in NocoDB\n",
- " for row in resource_list:\n",
- " periods_list = []\n",
- " for sched in row['availability']:\n",
- " if not sched['is_absent']:\n",
- " start = self.convert_to_minutes(\n",
- " sched['start_datetime'], self.today)\n",
- " end = self.convert_to_minutes(\n",
- " sched['end_datetime'], self.today)\n",
- " # ========= Uncomment to use capacity\n",
- " capacity = sched['capacity_percent']\n",
- " # capacity = None\n",
- " periods_list.append((int(start), int(self.adjust_capacity(\n",
- " start, end, capacity)) if capacity else int(end)))\n",
- "\n",
- " resource_id = int(row['resource_id'])\n",
- " self.dict_resource[resource_id] = Resource(\n",
- " id=resource_id, available_windows=periods_list)\n",
- "\n",
- " def create_resource_groups(self, resource_group_list):\n",
- " # Generate Resource Groups\n",
- " for x in resource_group_list:\n",
- " resource_list = []\n",
- " resources = x['resource_id']\n",
- " for r in resources:\n",
- " if r in self.dict_resource:\n",
- " resource_list.append(self.dict_resource[r])\n",
- " \n",
- " if resource_list:\n",
- " self.dict_resourcegroups[x['resource_group_id']] = ResourceGroup(\n",
- " id=x['resource_group_id'], resources=resource_list)\n",
- "\n",
- " def create_batch(self, task: Task, batch_size: int):\n",
- " batches = TaskSplitter(task, batch_size).split_into_batches()\n",
- " counter = 1\n",
- " for batch in batches:\n",
- " batch.id = f\"{task.id}-{counter}\"\n",
- " counter += 1\n",
- "\n",
- " return batches\n",
- "\n",
- " def create_task_object(self, task_list):\n",
- " for i in task_list:\n",
- " rg_list = []\n",
- " task_id = i['taskno']\n",
- " duration = int(i['duration'])\n",
- " priority = int(i['priority'])\n",
- " quantity = int(i['quantity'])\n",
- " # micro_batch_size = int(\n",
- " # i['micro_batch_size']) if i['micro_batch_size'] else None\n",
- " micro_batch_size = None\n",
- " resource_group_id = i['resource_group_id']\n",
- " rg_list = [self.dict_resourcegroups[g] for g in resource_group_id if g in self.dict_resourcegroups]\n",
- " predecessors = i['predecessors']\n",
- " parent_collection = i['parent_item_collection_id'] if micro_batch_size else None\n",
- " predecessor_collection = i['predecessor_item_collection_id'] if micro_batch_size else None\n",
- "\n",
- " assignments = []\n",
- " # Create assignments \n",
- " for x in resource_group_id: \n",
- " if x in self.dict_resourcegroups:\n",
- " assignments.append(Assignment(resource_groups= [self.dict_resourcegroups[x]], resource_count= 1))\n",
- " \n",
- " # Temporarily add into component dicts\n",
- " temp_task = Task(id=task_id,\n",
- " duration=duration,\n",
- " priority=priority,\n",
- " assignments= assignments,\n",
- " quantity=quantity)\n",
- " \n",
- " # Check for micro-batches\n",
- " if not micro_batch_size:\n",
- " self.task_dict[task_id] = temp_task # Add task to dictionary\n",
- "\n",
- " # Add predecessor to dictionary\n",
- " self.pred_dict[task_id] = predecessors\n",
- " else:\n",
- " self.pred_dict[task_id] = predecessors\n",
- " batches = self.create_batch(temp_task, micro_batch_size)\n",
- " self.task_dict.update({task.id: task for task in batches})\n",
- "\n",
- " # Temporarily copy the original predecessors for the new batches\n",
- " self.pred_dict.update(\n",
- " {task.id: predecessors for task in batches})\n",
- " self.flow_map.update({task.id: {\n",
- " \"parent\": parent_collection,\n",
- " \"predecessor\": predecessor_collection} for task in batches})\n",
- " self.pred_exploded[task_id] = [task.id for task in batches]\n",
- "\n",
- "\n",
- " # Organize predecessors for batches\n",
- " for task in self.task_dict.values():\n",
- " self.organize_predecessors(task)\n",
- "\n",
- " # Add predecessors\n",
- " for task in self.task_dict.values():\n",
- " self.task_dict[task.id].predecessor_ids = [x for x in self.pred_dict[task.id] if x in self.task_dict] # Predecessor needs to be existing in task dictionary\n",
- "\n",
- " # Build final task list\n",
- " self.tasks_list = [value for key,\n",
- " value in sorted(self.task_dict.items())]\n",
- " \n",
- " # Convert periods to time\n",
- " def int_to_datetime(self, num, start_time):\n",
- " try:\n",
- " # Parse the start time string into a datetime object\n",
- " start_datetime = datetime.strptime(start_time, \"%Y-%m-%d %H:%M:%S\")\n",
- " \n",
- " # Add the number of minutes to the start datetime\n",
- " delta = timedelta(minutes=num)\n",
- " result_datetime = start_datetime + delta\n",
- " return result_datetime\n",
- " \n",
- " except Exception as e: \n",
- " print(num)\n",
- " \n",
- " def run_scheduler(self):\n",
- " self.create_resource_object(self.r_data)\n",
- " print(\"Resource Objects Created.\")\n",
- "\n",
- " self.create_resource_groups(self.rg_data)\n",
- " print(\"Resource Groups Created.\")\n",
- "\n",
- " self.create_task_object(self.t_data)\n",
- " print(\"Task Objects Created.\")\n",
- " print(f\"Original Task Length: {len(self.t_data)} | Post-Batched Length: {len(self.task_dict.values())}\")\n",
- "\n",
- "\n",
- " self.solution = Scheduler(self.tasks_list, list(self.dict_resource.values())).schedule()\n",
- " print(\"Solution Created.\")\n",
- " \n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "scheduler = ProdScheduler(\n",
- " resource_dir = 'inputs/actual_data/resource.json',\n",
- " resource_group_dir = 'inputs/actual_data/resourcegroups.json',\n",
- " task_dir = 'inputs/actual_data/tasks_all.json'\n",
- ")\n",
- "\n",
- "scheduler.run_scheduler()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "scheduler.dict_resource[119]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "scheduler.task_dict['WO138769-10']"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "result['start_dt'] = result.apply(lambda x: scheduler.int_to_datetime(x['task_start'], scheduler.today_str), axis=1)\n",
- "result['end_dt'] = result.apply(lambda x: scheduler.int_to_datetime(x['task_end'], scheduler.today_str), axis=1)\n",
- "result.sort_values(by='task_end', ascending=False)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "result[result['task_id'].str.startswith('WO137709')].sort_values(by='task_start')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "scheduler.task_dict['WO135483-40']"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "intervals = scheduler.solution.get_resource_intervals_df()\n",
- "intervals['start_dt'] = intervals.apply(lambda x: scheduler.int_to_datetime(x['interval_start'], scheduler.today_str), axis=1)\n",
- "intervals['end_dt'] = intervals.apply(lambda x: scheduler.int_to_datetime(x['interval_end'], scheduler.today_str), axis=1)\n",
- "intervals.sort_values(by='interval_end', ascending=False)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "sched"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": ".venv",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.11.4"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
From 464aa798251b0b17ad1f2bb6529cdca9b6140c15 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Tue, 22 Oct 2024 16:57:19 +0800
Subject: [PATCH 26/27] Deleted stresstest
---
stresstest.py | 234 --------------------------------------------------
1 file changed, 234 deletions(-)
delete mode 100644 stresstest.py
diff --git a/stresstest.py b/stresstest.py
deleted file mode 100644
index b1610ed..0000000
--- a/stresstest.py
+++ /dev/null
@@ -1,234 +0,0 @@
-import pandas as pd
-import json
-import pytz
-from datetime import datetime, timezone, timedelta
-import time
-from src.factryengine.models import Resource, Task, Assignment, ResourceGroup
-from src.factryengine.scheduler.core import Scheduler
-from src.factryengine.scheduler.task_batch_processor import TaskSplitter
-
-
-class ProdScheduler:
- def __init__(self) -> None:
- # Scheduler Attributes
- self.cph_timezone = pytz.timezone('Europe/Copenhagen')
- # self.today = datetime.now(timezone.utc).replace(
- # hour=0, minute=0, second=0, microsecond=0)
- self.today = datetime(2024, 10, 9, 0, 0, 0, 0, tzinfo=timezone.utc)
- self.today_str = str(self.today)[:19]
-
- # Component Attributes
- self.dict_resource = {}
- self.dict_resourcegroups = {}
- self.tasks_list = []
- self.task_dict = {}
- self.pred_dict = {}
- self.flow_map = {}
- self.pred_exploded = {}
-
- # Misc Attributes
- self.start_time = time.time()
-
- def convert_to_minutes(self, datetime_str, start_time_obj):
- datetime_obj = datetime.strptime(datetime_str, '%Y-%m-%d %H:%M:%S%z')
- diff_minutes = (datetime_obj - start_time_obj).total_seconds()/60
- return int(diff_minutes)
-
- def adjust_capacity(self, start, end, capacity):
- return (end - start) * capacity + start
-
- def organize_predecessors(self, task: Task):
- try:
- list_predecessors = self.pred_dict[task.id]
- # ============================================================== UNCOMMENT TO USE MICROBATCH FLOW
- if task.batch_id: # Check if task is microbatched
- # Look for each predecessors that exist in the flow map
- for predecessor in list_predecessors:
- pred_batch_id = f'{predecessor}-{task.batch_id}'
- if pred_batch_id in self.flow_map and pred_batch_id in self.pred_dict: # Check if pred is part of flow
- # Check if pred-parent connection is correct
- if self.flow_map[task.id]['predecessor'] == self.flow_map[pred_batch_id]['parent']:
- self.pred_dict[task.id] = [pred_batch_id]
-
- elif pred_batch_id not in self.pred_dict and predecessor not in self.task_dict:
- parent_predecessor = []
- for pred in self.pred_dict[task.id]:
- parent_predecessor.extend(self.pred_dict[pred])
- self.pred_dict[task.id] = parent_predecessor
- # ============================================================== UNCOMMENT TO USE MICROBATCH FLOW
-
- # Remove task batch id
- task.set_batch_id(None)
-
- # Check for predecessors to be exploded
- for predecessor in list_predecessors:
- if predecessor in self.pred_exploded:
- self.pred_dict[task.id].remove(
- predecessor) # Remove original value
- self.pred_dict[task.id].extend(
- self.pred_exploded[predecessor]) # Add exploded batches
-
- except Exception as e:
- return
-
- def set_predecessors(self, task: Task):
- if not task.id in self.pred_dict: # if task is not in pred_dict, then it has no predecessors
- return
-
- for pred_id in self.pred_dict[task.id]:
- if pred_id in self.task_dict: # ensure predecessor exists in task_dict
- pred_task = self.task_dict[pred_id]
-
- # Avoid adding a predecessor multiple times
- if pred_task not in task.predecessors:
- # set predecessors for the predecessor first
- self.set_predecessors(pred_task)
- task.predecessors.append(pred_task)
-
- def create_resource_object(self, resource_list):
- # Generate slot based on schedule selected in NocoDB
- for row in resource_list:
- periods_list = []
- for sched in row['availability']:
- if not sched['is_absent']:
- start = self.convert_to_minutes(
- sched['start_datetime'], self.today)
- end = self.convert_to_minutes(
- sched['end_datetime'], self.today)
- # ========= Uncomment to use capacity
- capacity = sched['capacity_percent']
- # capacity = None
- periods_list.append((int(start), int(self.adjust_capacity(
- start, end, capacity)) if capacity else int(end)))
-
- resource_id = int(row['resource_id'])
- self.dict_resource[resource_id] = Resource(
- id=resource_id, available_windows=periods_list)
-
- def create_resource_groups(self, resource_group_list):
- # Generate Resource Groups
- for x in resource_group_list:
- resource_list = []
- resources = x['resource_id']
- for r in resources:
- if r in self.dict_resource:
- resource_list.append(self.dict_resource[r])
-
- if resource_list:
- self.dict_resourcegroups[x['resource_group_id']] = ResourceGroup(
- id=x['resource_group_id'], resources=resource_list)
-
- def create_batch(self, task: Task, batch_size: int):
- batches = TaskSplitter(task, batch_size).split_into_batches()
- counter = 1
- for batch in batches:
- batch.id = f"{task.id}-{counter}"
- counter += 1
-
- return batches
-
- def create_task_object(self, task_list):
- for i in task_list:
- rg_list = []
- task_id = i['taskno']
- duration = int(i['duration'])
- priority = int(i['priority'])
- quantity = int(i['quantity'])
- # micro_batch_size = int(
- # i['micro_batch_size']) if i['micro_batch_size'] else None
- micro_batch_size = None
- resource_group_id = i['resource_group_id']
- rg_list = [self.dict_resourcegroups[g] for g in resource_group_id if g in self.dict_resourcegroups]
- predecessors = i['predecessors']
- parent_collection = i['parent_item_collection_id'] if micro_batch_size else None
- predecessor_collection = i['predecessor_item_collection_id'] if micro_batch_size else None
-
- assignments = []
- # Create assignments
- for x in resource_group_id:
- if x in self.dict_resourcegroups:
- assignments.append(Assignment(resource_groups= [self.dict_resourcegroups[x]], resource_count= 1))
-
- # Temporarily add into component dicts
- temp_task = Task(id=task_id,
- duration=duration,
- priority=priority,
- assignments= assignments,
- quantity=quantity)
-
- # Check for micro-batches
- if not micro_batch_size:
- self.task_dict[task_id] = temp_task # Add task to dictionary
-
- # Add predecessor to dictionary
- self.pred_dict[task_id] = predecessors
- else:
- self.pred_dict[task_id] = predecessors
- batches = self.create_batch(temp_task, micro_batch_size)
- self.task_dict.update({task.id: task for task in batches})
-
- # Temporarily copy the original predecessors for the new batches
- self.pred_dict.update(
- {task.id: predecessors for task in batches})
- self.flow_map.update({task.id: {
- "parent": parent_collection,
- "predecessor": predecessor_collection} for task in batches})
- self.pred_exploded[task_id] = [task.id for task in batches]
-
- # Organize predecessors for batches
- for task in self.task_dict.values():
- self.organize_predecessors(task)
-
- # Add predecessors
- for task in self.task_dict.values():
- self.task_dict[task.id].predecessor_ids = [x for x in self.pred_dict[task.id] if x in self.task_dict] # Predecessor needs to be existing in task dictionary
-
- # Build final task list
- self.tasks_list = [value for key,
- value in sorted(self.task_dict.items())]
-
- # Convert periods to time
- def int_to_datetime(self, num, start_time):
- try:
- # Parse the start time string into a datetime object
- start_datetime = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S")
-
- # Add the number of minutes to the start datetime
- delta = timedelta(minutes=num)
- result_datetime = start_datetime + delta
- return result_datetime
-
- except Exception as e:
- print(num)
-
-
-# Open and read the JSON file
-with open('inputs/resource.json', 'r') as file:
- r_data = json.load(file)
-
-# Open and read the JSON file
-with open('inputs/tasks_all.json', 'r') as file:
- t_data = json.load(file)
-
-
-# Open and read the JSON file
-with open('inputs/resourcegroups.json', 'r') as file:
- rg_data = json.load(file)
-
-
-
-prodscheduler = ProdScheduler()
-prodscheduler.create_resource_object(r_data)
-print("Resource Objects Created.")
-
-prodscheduler.create_resource_groups(rg_data)
-print("Resource Groups Created.")
-
-prodscheduler.create_task_object(t_data)
-print("Task Objects Created.")
-print(f"Original Task Length: {len(t_data)} | Post-Batched Length: {len(prodscheduler.task_dict.values())}")
-
-
-solution = Scheduler(prodscheduler.tasks_list, list(prodscheduler.dict_resource.values())).schedule()
-print("Solution Created.")
-
From cb018dc52b622e8685a85077edc427c5e8459418 Mon Sep 17 00:00:00 2001
From: MJ Ducut <mjd@oestergaard-as.dk>
Date: Wed, 23 Oct 2024 22:15:05 +0800
Subject: [PATCH 27/27] Added find_first_index function and a window counter
---
.../heuristic_solver/task_allocator.py | 48 ++++++++++++++-----
1 file changed, 37 insertions(+), 11 deletions(-)
diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
index c2afa4c..b7084f5 100644
--- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py
+++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py
@@ -205,13 +205,8 @@ def _get_resource_intervals(
resource_windows_output[resource_id] = intervals
return resource_windows_output
-
-
- def _find_indexes(self, resource_intervals: np.ma.MaskedArray) -> int | None:
- """
- Finds relevant indexes in the resource intervals where the resource is used.
- """
- indexes = []
+
+ def _find_first_index(self, resource_intervals: np.ma.MaskedArray) -> int | None:
# Shift the mask by 1 to align with the 'next' element comparison
current_mask = resource_intervals.mask[:-1]
next_mask = resource_intervals.mask[1:]
@@ -224,9 +219,36 @@ def _find_indexes(self, resource_intervals: np.ma.MaskedArray) -> int | None:
indices = np.where(condition)[0]
first_index = indices[0] if len(indices) > 0 else 0
+
+ next_non_zero_index = np.where(
+ (~resource_intervals.mask[first_index + 2:]) # Non-masked (non-zero)
+ & (resource_intervals.mask[first_index + 1:-1]) # Previous value masked
+ )[0]
+
+ # Adjust x to align with the original array's indices
+ next_non_zero_index = (
+ (first_index + 2 + next_non_zero_index[0]) if len(next_non_zero_index) > 0 else None
+ )
+
+ if next_non_zero_index and resource_intervals[next_non_zero_index] == resource_intervals[first_index+1]:
+ first_index = next_non_zero_index
+
+ return first_index
+
+
+ def _find_indexes(self, resource_intervals: np.ma.MaskedArray) -> int | None:
+ """
+ Finds relevant indexes in the resource intervals where the resource is used.
+ """
+ # Mask where the data in resource_intervals is 0
+ resource_intervals = np.ma.masked_where(resource_intervals == 0.0, resource_intervals)
+
+ indexes = []
+ first_index = self._find_first_index(resource_intervals)
last_index = resource_intervals.size-1
indexes = [first_index] # Start with the first index
+ is_last_window_start = True # Flag to indicate the start of a window
# Iterate through the range between first and last index
for i in range(first_index, last_index + 1):
@@ -248,23 +270,27 @@ def _find_indexes(self, resource_intervals: np.ma.MaskedArray) -> int | None:
continue
# Detect end of a window
- if current > 0 and current == next_value and (is_prev_masked or previous < current):
+ if current > 0 and current == next_value and (is_prev_masked or previous < current) and is_last_window_start:
indexes.append(i)
+ is_last_window_start = False
continue
# Detect end of window using masks
- if current > 0 and is_next_masked and not is_curr_masked:
+ if current > 0 and is_next_masked and not is_curr_masked and is_last_window_start:
indexes.append(i)
+ is_last_window_start = False
continue
# Detect start of a new window
- if current > 0 and next_value > current and (is_prev_masked or previous == current):
+ if current > 0 and next_value > current and (is_prev_masked or previous == current) and not is_last_window_start:
indexes.append(i)
+ is_last_window_start = True
continue
# Detect start of window using masks
- if is_curr_masked and previous > 0 and next_value > 0:
+ if is_curr_masked and previous > 0 and next_value > 0 and not is_last_window_start:
indexes.append(i)
+ is_last_window_start = True
continue
# Always add the last index