Standardization, extension of visualizations, and general updates

.github/workflows/build-test-python-package.yml

@@ -17,7 +17,7 @@ jobs:
         runs-on: ubuntu-latest
         strategy:
             matrix:
-                python-version: ["3.9", "3.10", "3.11"]
+                python-version: ["3.10", "3.11", "3.12", "3.13"]
 
         steps:
             - uses: actions/checkout@v4

.github/workflows/publish-package.yml

@@ -23,10 +23,10 @@ jobs:
         steps:
             # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
             - uses: actions/checkout@v3
-            - name: Set up Python 3.9
+            - name: Set up Python 3.12
               uses: actions/setup-python@v4
               with:
-                  python-version: 3.9
+                  python-version: 3.12
                   cache: pip
 
             # Publishes to PyPi

.gitignore

@@ -10,9 +10,16 @@ cached_data_used/kernels/*
 cached_data_used/visualizations/*
 cached_data_used/last_run/*
 cached_data_used/import_runs/*
+methodology_paper_evaluation/run/*
+hyperparametertuning/*
+hyperparametertuning_*/*
+test_run_experiment/*
+
+# ignore setup files
+*/setup/*_input.json
 
 # exceptions
-!cached_data_used/cachefiles/ktt_values_to_kerneltuner.py
+!cached_data_used/cachefiles/*.py
 
 # ignore specific experiment files
 experiment_files/milo.json

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "benchmark_hub"]
+	path = benchmark_hub
+	url = https://github.com/AutoTuningAssociation/benchmark_hub.git

README.md

@@ -22,7 +22,7 @@ Currently, the stable releases of this software package are compatible with [Ker
 ## Installation
 The package can be installed with `pip install autotuning_methodology`. 
 Alternatively, it can be installed by cloning this repository and running `pip install .` in the root of the cloned project. 
-Python >= 3.9 is supported. 
+Like most Python packages, installing in a virtual environment or with `pipx` is recommended. Python >= 3.10 is supported. 
 
 ## Notable features
 - Official software by the authors of the methodology-defining paper. 

docs/getting_started.rst

@@ -3,7 +3,7 @@ Getting Started
 
 Start out by installing the package. 
 The simplest way to do this is ``pip install autotuning_methodology``. 
-Python 3.9 and up are supported. 
+Python 3.10 and up are supported. 
 
 Defining an experiment
 ^^^^^^^^^^^^^^^^^^^^^^

experiment_files/compare_bo.json

@@ -0,0 +1,72 @@
+{
+    "version": "1.0.0",
+    "name": "Methodology paper evaluation",
+    "parent_folder": "./methodology_paper_evaluation",
+    "experimental_groups_defaults": {
+        "applications": [
+            {
+                "name": "convolution",
+                "folder": "./cached_data_used/kernels",
+                "input_file": "convolution.json"
+            },
+            {
+                "name": "pnpoly",
+                "folder": "./cached_data_used/kernels",
+                "input_file": "pnpoly.json"
+            }
+        ],
+        "gpus": [
+            "RTX_3090",
+            "RTX_2080_Ti"
+        ],
+        "pattern_for_full_search_space_filenames": {
+            "regex": "./cached_data_used/cachefiles/${applications}/${gpus}_T4.json"
+        },
+        "stochastic": true,
+        "repeats": 100,
+        "samples": 32,
+        "minimum_number_of_valid_search_iterations": 20,
+        "ignore_cache": false
+    },
+    "search_strategies": [
+        {
+            "name": "bayes_opt_og",
+            "search_method": "bayes_opt",
+            "display_name": "Bayesian Optimization",
+            "autotuner": "KernelTuner"
+        },
+        {
+            "name": "dual_annealing",
+            "search_method": "dual_annealing",
+            "display_name": "Dual Annealing",
+            "autotuner": "KernelTuner"
+        }
+    ],
+    "statistics_settings": {
+        "minimization": true,
+        "cutoff_percentile": 0.96,
+        "cutoff_percentile_start": 0.5,
+        "cutoff_type": "fevals",
+        "objective_time_keys": [
+            "all"
+        ],
+        "objective_performance_keys": [
+            "time"
+        ]
+    },
+    "visualization_settings": {
+        "x_axis_value_types": [
+            "fevals",
+            "time",
+            "aggregated"
+        ],
+        "y_axis_value_types": [
+            "normalized",
+            "baseline"
+        ],
+        "resolution": 1000.0,
+        "confidence_level": 0.95,
+        "compare_baselines": false,
+        "compare_split_times": false
+    }
+}

experiment_files/compare_simple.json

@@ -0,0 +1,132 @@
+{
+    "version": "1.2.0",
+    "name": "Compare constrained strategies pyATF vs KT",
+    "parent_folder": "/var/scratch/fjwillem/constrained_optimization",
+    "experimental_groups_defaults": {
+        "applications": [
+            {
+                "name": "dedispersion_milo",
+                "folder": "../autotuning_methodology/benchmark_hub/kernels",
+                "input_file": "dedispersion_milo.json",
+                "objective_performance_keys": [
+                    "time"
+                ]
+            }
+        ],
+        "gpus": [
+            "A100",
+            "A4000"
+        ],
+        "pattern_for_full_search_space_filenames": {
+            "regex": "./benchmark_hub/cachefiles/${applications}/${gpus}_T4.json"
+        },
+        "stochastic": true,
+        "repeats": 100,
+        "samples": 32,
+        "minimum_fraction_of_budget_valid": 0.1,
+        "minimum_number_of_valid_search_iterations": 10,
+        "ignore_cache": false
+    },
+    "search_strategies": [
+        {
+            "name": "genetic_algorithm_constrained",
+            "search_method": "genetic_algorithm",
+            "search_method_hyperparameters": [
+                {
+                    "name": "constraint_aware",
+                    "value": true
+                }
+            ],
+            "display_name": "KT GA constrained",
+            "autotuner": "KernelTuner"
+        },
+        {
+            "name": "genetic_algorithm_non_constrained",
+            "search_method": "genetic_algorithm",
+            "search_method_hyperparameters": [
+                {
+                    "name": "constraint_aware",
+                    "value": false
+                }
+            ],
+            "display_name": "KT GA non-constrained",
+            "autotuner": "KernelTuner",
+            "color_parent": "genetic_algorithm_constrained"
+        },
+        {
+            "name": "pso_constrained",
+            "search_method": "pso",
+            "search_method_hyperparameters": [
+                {
+                    "name": "constraint_aware",
+                    "value": true
+                }
+            ],
+            "display_name": "KT PSO constrained",
+            "autotuner": "KernelTuner"
+        },
+        {
+            "name": "pso_non_constrained",
+            "search_method": "pso",
+            "search_method_hyperparameters": [
+                {
+                    "name": "constraint_aware",
+                    "value": false
+                }
+            ],
+            "display_name": "KT PSO non-constrained",
+            "autotuner": "KernelTuner",
+            "color_parent": "pso_constrained"
+        },
+        {
+            "name": "simulated_annealing_constrained",
+            "search_method": "simulated_annealing",
+            "search_method_hyperparameters": [
+                {
+                    "name": "constraint_aware",
+                    "value": true
+                }
+            ],
+            "display_name": "KT SA constrained",
+            "autotuner": "KernelTuner"
+        },
+        {
+            "name": "simulated_annealing_non_constrained",
+            "search_method": "simulated_annealing",
+            "search_method_hyperparameters": [
+                {
+                    "name": "constraint_aware",
+                    "value": false
+                }
+            ],
+            "display_name": "KT SA non-constrained",
+            "autotuner": "KernelTuner",
+            "color_parent": "simulated_annealing_constrained"
+        }
+    ],
+    "statistics_settings": {
+        "cutoff_percentile": 0.95,
+        "cutoff_percentile_start": 0.01,
+        "cutoff_type": "time",
+        "objective_time_keys": [
+            "all"
+        ]
+    },
+    "visualization_settings": {
+        "plots": [
+            {
+                "scope": "aggregate",
+                "style": "head2head",
+                "comparison": {
+                    "unit": "time",
+                    "relative_time": 0.5
+                },
+                "annotate": true
+            }
+        ],
+        "resolution": 1000.0,
+        "confidence_level": 0.95,
+        "compare_baselines": false,
+        "compare_split_times": false
+    }
+}