feat: integrate eso-edps pipeline into VRE

.github/workflows/env-build.yaml

@@ -3,7 +3,7 @@ name: Docker automatic build and publish
 on:
   push:
     branches: 
-      - main
+      - vre-eso_edps
 
 env:
   REGISTRY: ghcr.io

vre-singleuser-eso-edps/Dockerfile

@@ -0,0 +1,48 @@
+#FROM ghcr.io/vre-hub/vre-singleuser-py311:sha-281055c
+FROM quay.io/jupyter/scipy-notebook:python-3.11.8
+USER root
+
+RUN apt-get update \
+    && apt-get install -y build-essential \
+    procps \
+    curl \
+    file \
+    git \
+    ruby \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV XDG_RUNTIME_DIR=/var/run/adari
+RUN mkdir $XDG_RUNTIME_DIR && chmod 777 $XDG_RUNTIME_DIR
+
+# Create user
+#RUN useradd -m -d /home/linuxbrew -s /bin/bash linuxbrew
+RUN useradd -m -s /bin/bash linuxbrew
+USER linuxbrew
+ENV HOME=/home/linuxbrew
+
+# Install Homebrew
+RUN curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh | bash
+ENV PATH="${HOME}/.linuxbrew/bin:${HOME}/.linuxbrew/sbin:${PATH}"
+
+WORKDIR /home/linuxbrew
+RUN mkdir .edps EDPS_data
+
+COPY --chown=linuxbrew ./requirements.txt requirements.txt
+COPY --chown=linuxbrew ./edps-gui.py edps-gui.py
+COPY --chown=linuxbrew ./pdf_handler.py pdf_handler.py
+COPY --chown=linuxbrew ./eso-logo.jpg eso-logo.jpg
+COPY --chown=linuxbrew ./application.properties .edps/application.properties
+COPY --chown=linuxbrew ./logging.yaml .edps/logging.yaml
+
+RUN brew install [email protected]
+RUN brew tap eso/pipelines
+RUN brew install esopipe-fors
+
+RUN python3.11 -m venv venv && . venv/bin/activate && \
+    pip install --no-cache-dir --upgrade -r requirements.txt
+
+ENV VIRTUAL_ENV=/home/linuxbrew/venv
+ENV PATH=$VIRTUAL_ENV/bin:$PATH
+
+#CMD ["panel", "serve", "edps-gui.py", "--plugins", "pdf_handler", "--address", "0.0.0.0", "--port", "7860",  "--allow-websocket-origin", "*"]

vre-singleuser-eso-edps/README.md

@@ -0,0 +1,12 @@
+---
+title: Edps
+emoji: 🔥
+colorFrom: red
+colorTo: gray
+sdk: docker
+pinned: false
+license: bsd-3-clause
+short_description: ESO Data Processing System
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

vre-singleuser-eso-edps/application.properties

@@ -0,0 +1,160 @@
+[server]
+# Host or IP address to which EDPS server binds, e.g. localhost or 0.0.0.0
+host=0.0.0.0
+
+# EDPS port number, e.g. 5000
+port=5000
+
+[application]
+# Comma-separated list of directories where workflows are installed.
+# If not specified, EDPS will search for workflows in the pipeline installation tree.
+# The naming convention for workflows is: <instrument>/<instrument>_wkf.py, e.g. espresso/espresso_wkf.py
+workflow_dir=
+
+[executor]
+# esorex is the command to execute pipeline recipes and it is installed with the pipeline.
+# Please make sure that the path provided here can be located using the "which" command.
+esorex_path=esorex
+
+# Path where pipeline plugins are installed.
+# This configuration is used for ESO internal operations and can be left empty.
+pipeline_path=
+
+# genreport is the command to execute quality control plots and it is installed with the Adari package.
+genreport_path=genreport
+
+# EDPS data directory where recipe products, logs and quality control plots are saved.
+# The files are organised in a directory structure under the base directory, defined as:
+# <instrument>/<data reduction task>/<unique identifier>/<files>
+# Example: ESPRESSO/bias/fbf31155-a731-47f5-abf2-6445adce6c4b/master_bias.fits
+# Please make sure that this directory has enough disk space available for storing the pipeline products,
+# and consider enabling automatic data cleaning in the [cleanup] section.
+base_dir=EDPS_data
+
+# If true, a dummy command is executed instead of esorex
+dummy=False
+
+# If true, EDPS will attempt to execute a data reduction step even if the previous step has failed.
+continue_on_error=False
+
+# Number of concurrent data reductions processes.
+# Running concurrent data reductions will increase performance if sufficient resources are available,
+# but can also lead to pipeline crashes if not enough memory is available to execute parallel reductions.
+processes=1
+
+# Number of CPUs (cores) available for data reduction. EDPS will not exceed the number of cores when scheduling
+# data reduction tasks.
+cores=1
+
+# Pipeline recipes are parallelized using OpenMP. EDPS uses this parameter to set the number of threads when
+# running a recipe, up to the available cores: OMP_NUM_THREADS=min(default_omp_threads, cores)
+default_omp_threads=1
+
+# Execution ordering. All orderings follow topological order so parent tasks are always placed before their children.
+# Options: dfs, bfs, type, dynamic
+# dfs - depth-first, give preference to reaching final reduction target quicker
+# bfs - breadth-first, give preference to following reduction cascade level by level
+# type - same as bfs, but make sure to process same type of data together (eg. first all BIASes)
+# dynamic - immediately run whichever job is ready (has all needed inputs), no stalling but order is unpredictable
+ordering=dfs
+
+# If provided, the recipe products will be renamed according to the following scheme:
+# <prefix>.<instrument>.YYYY-MM-DDThh:mm.ss.mss.fits (Example: QC1.ESPRESSO.2023-02-09T17:30:14.326.fits),
+# where timestamp is taken from the moment of renaming the file.
+# Note that the renaming occurs in the base directory, not in the package (output) directory.
+output_prefix=
+
+# In case EDPS was stopped while some jobs were waiting to be executed, should we execute them after restart.
+resume_on_startup=False
+
+# EDPS will automatically re-execute a job if it's failed but needed as association, but only within this time window.
+reexecution_window_minutes=60
+
+[generator]
+# Path to yaml file defining locations of static calibrations for each of the workflows.
+# This configuration is used for ESO internal operations and can be left empty.
+# EDPS will automatically load static calibrations delivered with the pipeline.
+calibrations_config_file=
+
+# Path to yaml file defining locations of recipe and workflow parameters for each of the workflows.
+# This configuration is used for ESO internal operations and can be left empty.
+# EDPS will automatically load recipe and workflow parameters delivered with the pipeline.
+parameters_config_file=
+
+# In case multiple matching associated inputs (e.g. calibrations) are available, which ones should be used.
+# Options: raw, master, raw_per_quality_level, master_per_quality_level
+# raw - use reduced raw data results even if master calibrations closer in time are available
+# master - use master calibrations even if results of reduced raw data closer in time are available
+# raw_per_quality_level - use calibrations closest in time but prefer reduced raw data results
+# master_per_quality_level - use calibrations closest in time but prefer master calibrations
+association_preference=raw_per_quality_level
+
+# URL to ESO-provided list of calibration breakpoints.
+breakpoints_url=
+
+# Comma-separated list of workflows which should be combined together into one.
+# This allows to submit data from different instruments to a single workflow "edps.workflow.meta_wkf"
+meta_workflow=
+
+[repository]
+# Clear the EDPS bookkeeping database on startup.
+# This will cause all tasks to be re-executed even if they have been executed before on the same data.
+truncate=False
+
+# Should we use local database for bookkeeping (currently always True).
+local=True
+
+# Path where the bookkeeping database should be stored.
+path=db.json
+
+# Type of bookkeeping database to use.
+# Options: tiny, memory, caching
+# tiny - directly use TinyDB json-file-based database
+# memory - use fast in-memory non-persistent database
+# caching - use in-memory cache on top of persistent TinyDB database for higher performance
+type=caching
+
+# How many changes are needed to trigger TinyDB flushing data to disk.
+flush_size=10
+
+# How often automatically data should be flushed, regardless of changes.
+flush_timeout=60
+
+# Minimum amount of available disk space (in MB) required to flush data to disk.
+min_disk_space_mb=100
+
+[cleanup]
+# Should automatic cleanup of reduced data be enabled.
+enabled=False
+
+# How much time needs to pass since data got reduced to consider them for removal.
+cleanup_older_than_seconds=1209600
+
+# How often should we check if there are data to be removed.
+cleanup_check_period_seconds=3600
+
+[packager]
+# Location where selected products should be placed.
+package_base_dir=
+
+# Method to place files in the package directory. Options: link, symlink, copy.
+# link - create hardlinks
+# symlink - create symbolic links
+# copy - copy the files
+mode=symlink
+
+# Directory and filename pattern to use when placing files in the package directory.
+# The pattern can contain any string, header keywords enclosed in $ (e.g. $pro.catg$),
+# and the following predefined special variables:
+# $NIGHT - year-month-day of when the data was taken
+# $FILENAME - original name of the file
+# $EXT - original extension of the file name
+# $TASK - name of EDPS task which produced the file
+# $TIMESTAMP - timestamp when data were submitted for reduction
+# $DATASET - dataset name, derived from the first raw input file
+# Example: $DATASET/$TIMESTAMP/$object$_$pro.catg$.$EXT
+pattern=$DATASET/$TIMESTAMP/$object$_$pro.catg$.$EXT
+
+# Comma-separated list of product categories to place in the package directory.
+# Empty means all products matching reduction target.
+categories=

vre-singleuser-eso-edps/edps-gui.py

@@ -0,0 +1,104 @@
+from pathlib import Path
+
+import panel as pn
+
+from edpsgui.gui.classifier import Classifier
+from edpsgui.gui.dataset_creator import DatasetCreator
+from edpsgui.gui.edps_ctl import EDPSControl
+from edpsgui.gui.input_selector import InputSelector
+from edpsgui.gui.job_viewer import JobViewer
+from edpsgui.gui.reduction_history import ReductionHistory
+from edpsgui.gui.reduction_queue import ReductionQueue
+from edpsgui.gui.target_selector import TargetSelector
+from edpsgui.gui.workflow import Workflow
+from edpsgui.gui.workflow_selector import WorkflowSelector
+
+pn.extension('tabulator', 'tree', 'ipywidgets', 'terminal', 'codeeditor', 'modal',
+             css_files=["https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.2/css/all.min.css"],
+             disconnect_notification='Connection lost, try reloading the page!',
+             ready_notification='Application fully loaded.',
+             notifications=True)
+
+pn.state.notifications.position = 'top-center'
+
+input_selector = InputSelector()
+edps_ctl = EDPSControl()
+workflow_selector = WorkflowSelector(edps_status=edps_ctl.param.edps_status)
+workflow_graph = Workflow(edps_status=edps_ctl.param.edps_status,
+                          workflow=workflow_selector.param.workflow)
+target_selector = TargetSelector(edps_status=edps_ctl.param.edps_status,
+                                 workflow=workflow_selector.param.workflow)
+dataset_creator = DatasetCreator(edps_status=edps_ctl.param.edps_status,
+                                 workflow=workflow_selector.param.workflow,
+                                 inputs=input_selector.param.inputs,
+                                 targets=target_selector.param.targets)
+classifier = Classifier(edps_status=edps_ctl.param.edps_status,
+                        workflow=workflow_selector.param.workflow,
+                        inputs=input_selector.param.inputs)
+reduction_queue = ReductionQueue(edps_status=edps_ctl.param.edps_status,
+                                 workflow=workflow_selector.param.workflow)
+reduction_history = ReductionHistory(edps_status=edps_ctl.param.edps_status,
+                                     workflow=workflow_selector.param.workflow)
+
+data_explorer = pn.Column(
+    f'### 1. Select input data',
+    input_selector,
+    pn.layout.Divider(),
+    f'### 2. Classify and inspect input data (optional)',
+    classifier,
+    pn.layout.Divider(),
+    f'### 3. Select reduction target',
+    target_selector,
+    pn.layout.Divider(),
+    f'### 4. Configure association (optional)',
+    dataset_creator,
+)
+
+edps_aa_article = 'https://www.aanda.org/articles/aa/full_html/2024/01/aa47651-23/aa47651-23.html'
+about_edps = pn.pane.HTML(f"""
+If you make use of EDPS or its workflows for your research, we request to acknowledge
+<a href="{edps_aa_article}" target="_blank">Freudling, Zampieri, Coccato et al 2024, A&A 681, A93.</a>
+""")
+
+reduction_tabs = pn.Tabs(
+    ('Processing', reduction_queue),
+    ('Processed', reduction_history),
+    dynamic=True,
+)
+
+
+def update_tabs(event):
+    if event == 2:
+        reduction_queue.update_table()
+    elif event == 3:
+        reduction_history.update_table()
+
+
+tabs = pn.Tabs(
+    ('Workflow', workflow_graph),
+    ('Raw Data', data_explorer),
+    ('Processing Queue', reduction_queue),
+    ('Processed Data', reduction_history),
+    ('About EDPS', about_edps),
+    dynamic=True,
+)
+
+pn.bind(update_tabs, tabs.param.active, watch=True)
+
+layout = pn.Column(
+    workflow_selector,
+    pn.layout.Divider(),
+    tabs,
+    sizing_mode='scale_width'
+)
+
+logo_path = Path(__file__).resolve().parent / 'eso-logo.jpg'
+template = pn.template.FastListTemplate(title='EDPS Dashboard', sidebar_width=250, logo=str(logo_path))
+template.main.append(layout)
+template.header.append(edps_ctl)
+
+job_id = pn.state.location.query_params.get('job_id')
+if job_id:
+    JobViewer(job_id).servable()
+else:
+    template.servable()