WIP: singleuser-dev environment

.github/workflows/env-build.yaml

@@ -3,7 +3,7 @@ name: Docker automatic build and publish
 on:
   push:
     branches: 
-      - main
+      - dev_image
 
 env:
   REGISTRY: ghcr.io

vre-singleuser-dev/Dockerfile

@@ -0,0 +1,113 @@
+FROM quay.io/jupyter/scipy-notebook:python-3.11.8
+LABEL author="Image based on the work by Muhammad Aditya Hilmy - ESCAPE WP2 2020"
+LABEL maintainer="VRE Team @ CERN 23/24 - E. Garcia, G. Guerrieri"
+LABEL org.opencontainers.image.source https://github.com/vre-hub/environments
+ARG BUILD_DATE
+LABEL org.label-schema.build-date=$BUILD_DATE
+
+RUN python -m pip install --upgrade pip
+
+USER $NB_UID
+
+RUN conda install -y -n base mamba \
+    && mamba install -y -c conda-forge python-gfal2 \
+        nodejs \
+        jupyterlab">4,<5" \
+        notebook"<7" \
+        jupyterhub \
+        jsonschema>4 \
+        jupyterlab_server \
+        jupyter_server \
+        traitlets \
+        nbformat \
+        ipykernel \
+        PyJWT \
+        ipywidgets \
+    && conda clean --all -f -y
+
+# Install jupyterlab extensions
+# Branch gguerrie-API-refactoring from Soap2G - v1.3.0
+# COPY rucio_jupyterlab-1.3.0-py3-none-any.whl .
+# RUN pip install rucio_jupyterlab-1.3.0-py3-none-any.whl
+
+# v1.3.2
+COPY rucio_jupyterlab-1.3.2-py3-none-any.whl .
+RUN pip install rucio_jupyterlab-1.3.2-py3-none-any.whl
+
+#RUN jupyter server extension enable --py rucio_jupyterlab --sys-prefix 
+# RUN python -m pip install rucio-jupyterlab==1.0.0 \
+#     && jupyter server extension enable --py rucio_jupyterlab --sys-prefix 
+
+RUN python -m pip install swanoauthrenew==1.0.1 \ 
+    && jupyter server extension enable --py swanoauthrenew --sys-prefix 
+
+# RUN python -m pip install reana-jupyterlab \
+#     && jupyter server extension enable --py reana_jupyterlab --sys-prefix
+
+# RUN git clone https://github.com/vre-hub/zenodo-jupyterlab-extension.git \
+#     && cd zenodo-jupyterlab-extension \
+#     && git checkout dev \
+#     && python -m pip install . \
+#     && jupyter server extension enable --py zenodo_jupyterlab.server --sys-prefix \
+#     && cd .. \
+#     && rm -rf zenodo-jupyterlab-extension
+
+# Custom version of reana-client due to the jsonschema problem
+# RUN python -m pip install git+https://github.com/mdonadoni/reana-client.git@vre-summer-24
+
+USER root
+
+RUN apt update -y \
+    && apt install -y build-essential curl voms-clients-java software-properties-common \
+    && apt clean -y \
+    && rm /opt/conda/bin/voms-proxy-init \
+    && ln -s /usr/bin/voms-proxy-init /opt/conda/bin/voms-proxy-init
+
+# ESCAPE grid-security and VOMS setup
+RUN wget -q -O - https://dist.eugridpma.info/distribution/igtf/current/GPG-KEY-EUGridPMA-RPM-3 | apt-key add -
+
+# RUN apt update \
+#     && add-apt-repository 'deb http://repository.egi.eu/sw/production/cas/1/current egi-igtf core' \
+#     && apt -y install ca-policy-egi-core
+
+# RUN apt update \
+#     && apt install -y software-properties-common \
+#     && add-apt-repository 'deb https://lcg-ca.web.cern.ch/lcg-ca/distribution/current focal main' \
+#     && apt update \
+#     && apt -y install ca-policy-lcg
+
+
+RUN mkdir -p /etc/vomses \
+    && wget https://indigo-iam.github.io/escape-docs/voms-config/voms-escape.cloud.cnaf.infn.it.vomses -O /etc/vomses/voms-escape.cloud.cnaf.infn.it.vomses \
+    && mkdir -p /etc/grid-security/vomsdir/escape \
+    && wget https://indigo-iam.github.io/escape-docs/voms-config/voms-escape.cloud.cnaf.infn.it.lsc -O /etc/grid-security/vomsdir/escape/voms-escape.cloud.cnaf.infn.it.lsc
+
+# Setup merged CERN CA file on Ubuntu based images.
+# This file is contained in the `CERN-bundle.pem` file downloaded using 
+RUN mkdir /certs \
+    && touch /certs/rucio_ca.pem \
+    && curl -fsSL 'https://cafiles.cern.ch/cafiles/certificates/CERN%20Root%20Certification%20Authority%202.crt' | openssl x509 -inform DER -out /tmp/cernrootca2.crt \
+    && curl -fsSL 'https://cafiles.cern.ch/cafiles/certificates/CERN%20Grid%20Certification%20Authority(1).crt' -o /tmp/cerngridca.crt \
+    && curl -fsSL 'https://cafiles.cern.ch/cafiles/certificates/CERN%20Certification%20Authority.crt' -o /tmp/cernca.crt \
+    && cat /tmp/cernrootca2.crt >> /certs/rucio_ca.pem \
+    && cat /tmp/cerngridca.crt >> /certs/rucio_ca.pem \
+    && cat /tmp/cernca.crt >> /certs/rucio_ca.pem \
+    && rm /tmp/*.crt \
+    && update-ca-certificates
+
+# Setup extension Rucio instance config
+COPY configure-vre.py /usr/local/bin/
+RUN chmod +x /usr/local/bin/configure-vre.py
+COPY configure-vre.sh /usr/local/bin/before-notebook.d/
+
+RUN mkdir -p /opt/rucio/etc \
+    && chown -R $NB_UID /opt/rucio/etc 
+
+ENV JUPYTER_ENABLE_LAB=yes
+ENV RUCIO_JUPYTERLAB_LOGLEVEL=DEBUG
+ENV JUPYTERHUB_SINGLEUSER_APP=jupyter-server
+
+WORKDIR $HOME
+USER $NB_UID
+
+CMD ["start-notebook.py"]

vre-singleuser-dev/README.md

@@ -0,0 +1,3 @@
+# VRE singleuser Dev Image
+
+For Dev purposes

vre-singleuser-dev/configure-vre.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+# Derived from https://gitlab.cern.ch/escape-wp2/docker-images/-/blob/master/datalake-singleuser/bin/configure.py (restricted access).
+
+import os
+import json
+
+HOME = '/home/jovyan'
+
+def write_jupyterlab_config():
+    file_path = HOME + '/.jupyter/jupyter_server_config.json'
+    if not os.path.isfile(file_path):
+        os.makedirs(HOME + '/.jupyter/', exist_ok=True)
+    else:
+        config_file = open(file_path, 'r')
+        config_payload = config_file.read()
+        config_file.close()
+
+    try:
+        config_json = json.loads(config_payload)
+    except:
+        config_json = {}
+
+
+# Looking to the rucio-jupyterlab configuration; https://github.com/rucio/jupyterlab-extension/blob/master/rucio_jupyterlab/config/schema.py#L101
+#  either ("destination_rse", "rse_mount_path") either ("rucio_ca_cert") are required env vars, even if they are defined in the jhub manifest.
+# Adding 'rucio_base_url' too - from debugging experience
+
+    escape_config = {
+        "name": os.getenv('RUCIO_NAME', 'default'),
+        "display_name": os.getenv('RUCIO_DISPLAY_NAME', 'Default Instance'),
+        "rucio_base_url": os.getenv('RUCIO_BASE_URL', 'DEFAULT rucio base url'),
+        "rucio_auth_url": os.getenv('RUCIO_AUTH_URL'),
+        "rucio_webui_url": os.getenv('RUCIO_WEBUI_URL'),
+        "rucio_ca_cert": os.getenv('RUCIO_CA_CERT'),
+        "site_name": os.getenv('RUCIO_SITE_NAME'),
+        "vo": os.getenv('RUCIO_VO'),
+        "voms_enabled": os.getenv('RUCIO_VOMS_ENABLED', '0') == '1',
+        "voms_vomses_path": os.getenv('RUCIO_VOMS_VOMSES_PATH'),
+        "voms_certdir_path": os.getenv('RUCIO_VOMS_CERTDIR_PATH'),
+        "voms_vomsdir_path": os.getenv('RUCIO_VOMS_VOMSDIR_PATH'),
+        "destination_rse": os.getenv('RUCIO_DESTINATION_RSE', 'DEFAULT rse destination'),
+        "rse_mount_path": os.getenv('RUCIO_RSE_MOUNT_PATH', 'DEFAULT rse mount path'),
+        "replication_rule_lifetime_days": int(os.getenv('RUCIO_REPLICATION_RULE_LIFETIME_DAYS')) if os.getenv('RUCIO_REPLICATION_RULE_LIFETIME_DAYS') else None,
+        "path_begins_at": int(os.getenv('RUCIO_PATH_BEGINS_AT', '0')),
+        "mode": os.getenv('RUCIO_MODE', 'replica'),
+        "wildcard_enabled": os.getenv('RUCIO_WILDCARD_ENABLED', '0') == '1',
+        "oidc_auth": os.getenv('RUCIO_OIDC_AUTH'),
+        "oidc_env_name": os.getenv('RUCIO_OIDC_ENV_NAME'),
+        "oidc_file_name": os.getenv('RUCIO_OIDC_FILE_NAME'),
+    }
+
+    atlas_config = {
+        "name": "https://voatlasrucio-server-prod.cern.ch",
+        "display_name": "ATLAS RUCIO",
+        "rucio_base_url": "https://voatlasrucio-server-prod.cern.ch:443",
+        "rucio_auth_url": "https://atlas-rucio-auth.cern.ch:443",
+        "rucio_ca_cert": "/certs/rucio_ca.pem",
+        "site_name": "CERN",
+        "vo": "atlas",
+        "voms_enabled": os.getenv('RUCIO_VOMS_ENABLED', '0') == '1',
+        "destination_rse": "CERN-PROD_PHYS-TOP",
+        "rse_mount_path": "/eos/atlas/atlasscratchdisk/rucio",
+        "path_begins_at": 4,
+        "mode": "replica", 
+        "wildcard_enabled": os.getenv('RUCIO_WILDCARD_ENABLED', '0') == '0'
+      }
+
+    escape_config = {k: v for k,
+                    v in escape_config.items() if v is not None}
+
+    atlas_config = {k: v for k,
+                    v in atlas_config.items() if v is not None}
+
+
+    config_json['RucioConfig'] = {
+        'instances': [escape_config, atlas_config],
+        "default_instance": os.getenv('RUCIO_DEFAULT_INSTANCE', escape_config['name']),
+        "default_auth_type": os.getenv('RUCIO_DEFAULT_AUTH_TYPE', 'oidc'),
+    }
+
+    config_file = open(file_path, 'w')
+    config_file.write(json.dumps(config_json, indent=2))
+    config_file.close()
+
+def write_ipython_config():
+    file_path = HOME + '/.ipython/profile_default/ipython_kernel_config.json'
+    extension_module = 'rucio_jupyterlab.kernels.ipython'
+
+    if not os.path.isfile(file_path):
+        os.makedirs(HOME + '/.ipython/profile_default/', exist_ok=True)
+    else:
+        config_file = open(file_path, 'r')
+        config_payload = config_file.read()
+        config_file.close()
+
+    try:
+        config_json = json.loads(config_payload)
+    except:
+        config_json = {}
+
+    if 'IPKernelApp' not in config_json:
+        config_json['IPKernelApp'] = {}
+
+    ipkernel_app = config_json['IPKernelApp']
+
+    if 'extensions' not in ipkernel_app:
+        ipkernel_app['extensions'] = []
+
+    if extension_module not in ipkernel_app['extensions']:
+        ipkernel_app['extensions'].append(extension_module)
+
+    config_file = open(file_path, 'w')
+    config_file.write(json.dumps(config_json, indent=2))
+    config_file.close()    
+
+if __name__ == '__main__':
+    write_jupyterlab_config()
+    write_ipython_config()

vre-singleuser-dev/configure-vre.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -e
+python /usr/local/bin/configure-vre.py

vre-singleuser-dev/voms-atlas-auth.app.cern.ch.lsc

@@ -0,0 +1,2 @@
+/DC=ch/DC=cern/OU=computers/CN=atlas-auth.web.cern.ch
+/DC=ch/DC=cern/CN=CERN Grid Certification Authority

vre-singleuser-dev/voms-atlas-auth.cern.ch.lsc

@@ -0,0 +1,2 @@
+/DC=ch/DC=cern/OU=computers/CN=atlas-auth.cern.ch
+/DC=ch/DC=cern/CN=CERN Grid Certification Authority