Connect F4E gitlab to JADE

.github/workflows/pytest.yml

@@ -71,6 +71,8 @@ jobs:
           pytest --cov=. -cov-config=jade.coveragerc --cov-report xml
         env:
           ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }}
+          F4E_GITLAB_TOKEN: ${{ secrets.F4E_GITLAB_TOKEN }}
+
 
       # Activate environment and run pytest
       - name: Testing - Windows
@@ -79,6 +81,7 @@ jobs:
           pytest
         env:
           ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }}
+          F4E_GITLAB_TOKEN: ${{ secrets.F4E_GITLAB_TOKEN }}
 
       # Upload coverage to Codecov. Do it only for one case of the matrix.
       - name: Upload coverage to Codecov

.gitignore

@@ -23,4 +23,4 @@ venv/*
 .vscode/
 
 # secret file for local testing
-tests/secrets.json
+secrets.py

docs/source/usage/installation.rst

@@ -86,6 +86,15 @@ If permissions errors are encountered, try:
   | ``python -m jade``
 
 The folder structure should now look like the one described in :ref:`folders`.
+During this operation the benchmark inputs contained at `IAEA repository <https://github.com/IAEA-NDS/open-benchmarks>`_
+are fetched.
 
 A JADE instance has now been initialized and it is ready to be configured as discussed
 in the :ref:`config` section.
+
+In case the user possesses a valid access token for the F4E GitLab, they can provide it
+in the ``<root>/cfg/env_vars_cfg.yml`` file. And then use the JADE utility:
+
+  | ``python -m jade.utilitis --fetch``
+
+in order to obtain also the inputs stored at the F4E GitLab.

docs/source/usage/user_configuration.rst

@@ -22,23 +22,30 @@ Here is an example:
 
     # paths to the code executables. If the codes are not installed, just leave the field empty.
     executables:
-        - mcnp: path/to/mcnp6/executable  # it can also be just 'mcnp6' if the executable is in the PATH
-        - openmc:  path/to/openmc/executable  # it can also be just 'openmc' if the executable is in the PATH
-        - serpent: path/to/serpent/executable  # it can also be just 'sss2' if the executable is in the PATH
-        - d1s: path/to/d1s/executable  # it can also be just 'd1s' if the executable is in the PATH
+    - mcnp: path/to/mcnp6/executable  # it can also be just 'mcnp6' if the executable is in the PATH
+    - openmc:  path/to/openmc/executable  # idem
+    - serpent: path/to/serpent/executable  # idem
+    - d1s: path/to/d1s/executable  # idem
 
     # run mode is either "serial" to run locally or "job" to submit to a cluster
     run_mode: serial
-
+    # This is needed only if mpi_tasks > 1
+    mpi_prefix: srun  # the command to run jobs, it prepends your executable (e.g. mpirun, srun, etc.)
+
+    # Not needed for a windows run. If all your modules are already loaded in your environment,
+    # you can either make all these files empty or point to an empy file.
+    code_configurations: # You can either modify the files at these (relative) paths that already exist or provide your own
+    - mcnp: cfg/exe_config/mcnp_config.sh
+    - openmc: cfg/exe_config/openmc_config.sh
+    - serpent: cfg/exe_config/serpent_config.sh
+    - d1s: cfg/exe_config/d1s_config.sh
+
     # These need to be non-empty only if submitting a job to a cluster
     batch_template: batch_template/Slurmtemplate.sh  # batch template. Both relative and absolute paths should work.
     batch_system: sbatch  # the command to submit a job to the cluster (e.g. llsubmit, sbatch, etc.)
-    mpi_prefix: srun  # the command to run jobs, it prepends your executable (e.g. mpirun, srun, etc.)
-    code_configurations: # You can either modify the files at these (relative) paths that already exist or provide your own
-        - mcnp: exe_config/mcnp_config.sh
-        - openmc: exe_config/openmc_config.sh
-        - serpent: exe_config/serpent_config.sh
-        - d1s: exe_config/d1s_config.sh
+
+    # Tokens for closed benchmarks repos
+    f4e_gitlab_token:  # token for the F4E gitlab, leave empty if not needed
 
 It can be seen that in order to submit a job to a cluster, the user needs to provide the path to the batch template
 file and a config file for the code(s) to be run.

docs/source/usage/utilities.rst

@@ -17,7 +17,8 @@ or
 
 The following is a list of the available utilities:
 
-* ``--fetch``: re-download/sync benchmark inputs from the IAEA GitHub repository.
+* ``--fetch``: re-download/sync benchmark inputs from the IAEA GitHub repository and F4E GitLab (if access token
+  was provided).
 * ``--restore``: restore the default configuration settings.
 * ``--runtpe``: remove the runtpe files (.r) from MCNP simulation folders. This helps in
   reducing the storage memory occupied by the simulation outputs.

pyproject.toml

@@ -34,7 +34,8 @@ dependencies = [
 #    openmc @ git+https://github.com/openmc-dev/[email protected]#egg=openmc
     "pyyaml",
     "ttkthemes",
-    "seaborn"
+    "seaborn",
+    "python-gitlab"
 ]
 [project.optional-dependencies]
 dev = [

src/jade/app/app.py

@@ -12,7 +12,7 @@
 
 import jade.resources as res
 from jade import resources
-from jade.app.fetch import fetch_iaea_inputs
+from jade.app.fetch import fetch_f4e_inputs, fetch_iaea_inputs
 from jade.config.paths_tree import PathsTree
 from jade.config.pp_config import PostProcessConfig
 from jade.config.raw_config import ConfigRawProcessor
@@ -40,7 +40,7 @@ def __init__(self, root: PathLike | None = None, skip_init: bool = False):
         if self.tree.check_not_installed_folders(root) and not skip_init:
             self.tree.init_tree()
             self.restore_default_cfg(FIRST_INITIALIZATION)
-            self.update_inputs()
+            self.update_inputs(only_iaea=True)
             return
 
         # parse the config files
@@ -83,16 +83,34 @@ def initialize_log(self) -> None:
 
         logging.debug(JADE_TITLE)
 
-    def update_inputs(self):
+    def update_inputs(self, only_iaea: bool = False):
         """Update the benchmark inputs for the simulations.
 
         This will re-fetch inputs from the various repositories that feed JADE.
+
+        Parameters
+        ----------
+        only_iaea : bool, optional
+            If True, only the IAEA inputs are updated, by default False
         """
         success = fetch_iaea_inputs(
             self.tree.benchmark_input_templates, self.tree.exp_data
         )
         if not success:
-            logging.error("Failed to update the benchmark inputs.")
+            logging.error("Failed to update the IAEA benchmark inputs.")
+
+        if not only_iaea:
+            f4e_gitlab_token = self.run_cfg.env_vars.f4e_gitlab_token
+            if f4e_gitlab_token is not None:
+                success = fetch_f4e_inputs(
+                    self.tree.benchmark_input_templates,
+                    self.tree.exp_data,
+                    f4e_gitlab_token,
+                )
+                if not success:
+                    logging.error("Failed to update the F4E benchmark inputs.")
+            else:
+                logging.info("No F4E token found. Skipping F4E inputs update.")
 
     def restore_default_cfg(self, msg: str = ""):
         """Reset the configuration files to installation default. The session

src/jade/app/fetch.py

@@ -1,11 +1,13 @@
 from __future__ import annotations
 
+import logging
 import os
 import shutil
 import tempfile
 import zipfile
 from pathlib import Path
 
+import gitlab
 import requests
 
 from jade.helper.aux_functions import PathLike
@@ -14,7 +16,9 @@
 IAEA_URL = f"https://github.com/IAEA-NDS/open-benchmarks/archive/{BRANCH}.zip"
 
 
-def _fetch_from_git(url: str, authorization_token: str | None = None) -> str | bool:
+def _fetch_from_git(
+    url: str, authorization_token: str | None = None
+) -> PathLike | bool:
     """Download a repository from GitHub/GitLab and extract
     it to a temporary folder. It can also deal with authentication.
 
@@ -44,16 +48,64 @@ def _fetch_from_git(url: str, authorization_token: str | None = None) -> str | b
     if response.status_code != 200:
         return False
     # Save the downloaded zip file
+    extracted_folder = _extract_zip(response.content, os.path.basename(url))
+    return extracted_folder
+
+
+def _extract_zip(zip_content, dest: PathLike) -> PathLike:
     tmpdirname = tempfile.gettempdir()
-    tmp_zip = os.path.join(tmpdirname, os.path.basename(url))
+    tmp_zip = os.path.join(tmpdirname, dest)
     extracted_folder = os.path.join(tmpdirname, "extracted")
     with open(tmp_zip, "wb") as f:
-        f.write(response.content)
+        f.write(zip_content)
     # Extract the zip file
     with zipfile.ZipFile(tmp_zip, "r") as zip_ref:
+        main_folder_name = zip_ref.namelist()[0]
         zip_ref.extractall(extracted_folder)
 
-    return extracted_folder
+    return Path(extracted_folder, main_folder_name)
+
+
+def fetch_from_gitlab(
+    url: str, path: str, branch: str, authorization_token: str = None
+) -> PathLike | bool:
+    """Download a repository from GitLab and extract
+    it to a temporary folder. It can also deal with authentication. Supported
+    authentication is by token.
+    Parameters
+    ----------
+    url : str
+        path to the gitlab website (e.g. https://git.oecd-nea.org/)
+    path : str
+        path to the repository (e.g. /sinbad/sinbad.v2/sinbad-version-2-volume-1/FUS-ATN-BLK-STR-PNT-001-FNG-Osaka-Aluminium-Sphere-OKTAVIAN-oktav_al)
+    branch : str, optional
+        Branch to download. Default is jade.
+    authorization_token : str, optional
+        Authorization token to access the IAEA repository. Default is None.
+    Returns
+    -------
+    extracted_folder: Pathlike | bool
+        path to the extracted folder. False if the download was not successful.
+    """
+    gl = gitlab.Gitlab(url=url, private_token=authorization_token, ssl_verify=False)
+    try:
+        gl.auth()
+    except gitlab.exceptions.GitlabAuthenticationError:
+        logging.error("Gitlab authentication failed")
+        return False
+
+    # select the correct project
+    found = False
+    for project in gl.projects.list():
+        if path == project.path_with_namespace:
+            found = True
+            break
+    if not found:
+        logging.error(f"Successful authentication but project {path} not found")
+        return False
+
+    binary = project.repository_archive(sha=branch, format="zip")
+    return _extract_zip(binary, os.path.basename(path) + ".zip")
 
 
 def _install_data(fetch_folder: str | os.PathLike, install_folder: str | os.PathLike):
@@ -70,6 +122,28 @@ def _install_data(fetch_folder: str | os.PathLike, install_folder: str | os.Path
         )
 
 
+def _install_standard_folder_structure(
+    extracted_folder: str | os.PathLike,
+    inputs_root: PathLike,
+    exp_data_root: PathLike,
+    path_to_inputs: str | os.PathLike,
+    path_to_exp_data: str | os.PathLike,
+) -> bool:
+    if isinstance(extracted_folder, bool):
+        return False
+
+    for fetched_folder, install_folder in [
+        (path_to_inputs, inputs_root),
+        (path_to_exp_data, exp_data_root),
+    ]:
+        _install_data(fetched_folder, install_folder)
+
+    # Once done, delete the src folder
+    shutil.rmtree(extracted_folder)
+
+    return True
+
+
 def fetch_iaea_inputs(inputs_root: PathLike, exp_data_root: PathLike) -> bool:
     """Fetch IAEA benchmark inputs and experimental data and copy them to
     the correct folder in jade structure. This will always override the available
@@ -87,24 +161,55 @@ def fetch_iaea_inputs(inputs_root: PathLike, exp_data_root: PathLike) -> bool:
     bool
         True if the inputs were successfully fetched, False otherwise.
     """
-    extracted_folder = _fetch_from_git(IAEA_URL)  # no token required anymore
-    if isinstance(extracted_folder, bool):
-        return False
+    extracted_folder = str(_fetch_from_git(IAEA_URL))  # no token required anymore
 
-    path_to_inputs = Path(
-        extracted_folder, f"open-benchmarks-{BRANCH}", "jade_open_benchmarks", "inputs"
-    )
-    path_to_exp_data = os.path.join(
+    path_to_inputs = Path(extracted_folder, "jade_open_benchmarks", "inputs")
+    path_to_exp_data = Path(
         extracted_folder,
-        f"open-benchmarks-{BRANCH}",
         "jade_open_benchmarks",
         "exp_results",
     )
+    success = _install_standard_folder_structure(
+        extracted_folder, inputs_root, exp_data_root, path_to_inputs, path_to_exp_data
+    )
+    return success
 
-    for fetched_folder, install_folder in [
-        (path_to_inputs, inputs_root),
-        (path_to_exp_data, exp_data_root),
-    ]:
-        _install_data(fetched_folder, install_folder)
 
-    return True
+def fetch_f4e_inputs(
+    inputs_root: PathLike, exp_data_root: PathLike, access_token: str
+) -> bool:
+    """Fetch F4E benchmark inputs and experimental data and copy them to
+    the correct folder in jade structure. This will always override the available
+    data.
+
+    Parameters
+    ----------
+    inputs_root : PathLike
+        path to the root folder where the inputs will be stored.
+    exp_data_root : PathLike
+        path to the root folder where the experimental data will be stored.
+    access_token : str
+        Authorization token to access the F4E GitLab.
+
+    Returns
+    -------
+    bool
+        True if the inputs were successfully fetched, False otherwise.
+    """
+    extracted_folder = str(
+        fetch_from_gitlab(
+            "https://eng-gitlab.f4e.europa.eu/",
+            "f4e-projects/jade-benchmarks",
+            "main",
+            authorization_token=access_token,
+        )
+    )
+    path_to_inputs = Path(extracted_folder, "inputs")
+    path_to_exp_data = Path(
+        extracted_folder,
+        "exp_results",
+    )
+    success = _install_standard_folder_structure(
+        extracted_folder, inputs_root, exp_data_root, path_to_inputs, path_to_exp_data
+    )
+    return success

src/jade/config/run_config.py

@@ -140,14 +140,16 @@ class EnvironmentVariables:
     code_configurations : dict[CODE, PathLike] | None
         path to the configuration files for the codes. If None, the default configuration
         will be used which can be found at cfg/exe_config. By default is None.
-    batch_template : PathLike | None
+    batch_template : PathLike | None, optional
         relative path to the batch template for job submission. location is cfg/batch_templates.
         By default is None.
-    batch_system : str | None
+    batch_system : str | None, optional
         name of the batch system to use for job submission. e.g. "slurm". By default is
         None.
-    mpi_prefix : str | None
+    mpi_prefix : str | None, optional
         prefix for the mpi command. e.g. "srun", by default None
+    f4e_gitlab_token : str | None, optional
+        token to access the F4E gitlab. By default is None.
     """
 
     # parallel options
@@ -162,6 +164,7 @@ class EnvironmentVariables:
     batch_template: PathLike | None = None
     batch_system: str | None = None
     mpi_prefix: str | None = None
+    f4e_gitlab_token: str | None = None
 
     def __post_init__(self):
         if self.mpi_tasks is not None:
@@ -181,6 +184,9 @@ def __post_init__(self):
                 raise ConfigError(
                     "Batch system is needed for job submission, please provide one"
                 )
+        # make sure that the gitlab token is set to None if empty
+        if self.f4e_gitlab_token == "":
+            self.f4e_gitlab_token = None
 
     @classmethod
     def from_yaml(cls, config_file: PathLike) -> EnvironmentVariables:
@@ -208,6 +214,7 @@ def from_yaml(cls, config_file: PathLike) -> EnvironmentVariables:
             batch_template=cfg["batch_template"],
             batch_system=cfg["batch_system"],
             mpi_prefix=cfg["mpi_prefix"],
+            f4e_gitlab_token=cfg.get("f4e_gitlab_token", None),
         )