diff --git a/.github/workflows/env-build.yaml b/.github/workflows/env-build.yaml index 7b9b6dd1..ca2664ee 100644 --- a/.github/workflows/env-build.yaml +++ b/.github/workflows/env-build.yaml @@ -3,7 +3,7 @@ name: Docker automatic build and publish on: push: branches: - - main + - dev_image env: REGISTRY: ghcr.io diff --git a/vre-singleuser-dev/Dockerfile b/vre-singleuser-dev/Dockerfile new file mode 100644 index 00000000..2060c891 --- /dev/null +++ b/vre-singleuser-dev/Dockerfile @@ -0,0 +1,113 @@ +FROM quay.io/jupyter/scipy-notebook:python-3.11.8 +LABEL author="Image based on the work by Muhammad Aditya Hilmy - ESCAPE WP2 2020" +LABEL maintainer="VRE Team @ CERN 23/24 - E. Garcia, G. Guerrieri" +LABEL org.opencontainers.image.source https://github.com/vre-hub/environments +ARG BUILD_DATE +LABEL org.label-schema.build-date=$BUILD_DATE + +RUN python -m pip install --upgrade pip + +USER $NB_UID + +RUN conda install -y -n base mamba \ + && mamba install -y -c conda-forge python-gfal2 \ + nodejs \ + jupyterlab">4,<5" \ + notebook"<7" \ + jupyterhub \ + jsonschema>4 \ + jupyterlab_server \ + jupyter_server \ + traitlets \ + nbformat \ + ipykernel \ + PyJWT \ + ipywidgets \ + && conda clean --all -f -y + +# Install jupyterlab extensions +# Branch gguerrie-API-refactoring from Soap2G - v1.3.0 +# COPY rucio_jupyterlab-1.3.0-py3-none-any.whl . +# RUN pip install rucio_jupyterlab-1.3.0-py3-none-any.whl + +# v1.3.2 +COPY rucio_jupyterlab-1.3.2-py3-none-any.whl . +RUN pip install rucio_jupyterlab-1.3.2-py3-none-any.whl + +#RUN jupyter server extension enable --py rucio_jupyterlab --sys-prefix +# RUN python -m pip install rucio-jupyterlab==1.0.0 \ +# && jupyter server extension enable --py rucio_jupyterlab --sys-prefix + +RUN python -m pip install swanoauthrenew==1.0.1 \ + && jupyter server extension enable --py swanoauthrenew --sys-prefix + +# RUN python -m pip install reana-jupyterlab \ +# && jupyter server extension enable --py reana_jupyterlab --sys-prefix + +# RUN git clone https://github.com/vre-hub/zenodo-jupyterlab-extension.git \ +# && cd zenodo-jupyterlab-extension \ +# && git checkout dev \ +# && python -m pip install . \ +# && jupyter server extension enable --py zenodo_jupyterlab.server --sys-prefix \ +# && cd .. \ +# && rm -rf zenodo-jupyterlab-extension + +# Custom version of reana-client due to the jsonschema problem +# RUN python -m pip install git+https://github.com/mdonadoni/reana-client.git@vre-summer-24 + +USER root + +RUN apt update -y \ + && apt install -y build-essential curl voms-clients-java software-properties-common \ + && apt clean -y \ + && rm /opt/conda/bin/voms-proxy-init \ + && ln -s /usr/bin/voms-proxy-init /opt/conda/bin/voms-proxy-init + +# ESCAPE grid-security and VOMS setup +RUN wget -q -O - https://dist.eugridpma.info/distribution/igtf/current/GPG-KEY-EUGridPMA-RPM-3 | apt-key add - + +# RUN apt update \ +# && add-apt-repository 'deb http://repository.egi.eu/sw/production/cas/1/current egi-igtf core' \ +# && apt -y install ca-policy-egi-core + +# RUN apt update \ +# && apt install -y software-properties-common \ +# && add-apt-repository 'deb https://lcg-ca.web.cern.ch/lcg-ca/distribution/current focal main' \ +# && apt update \ +# && apt -y install ca-policy-lcg + + +RUN mkdir -p /etc/vomses \ + && wget https://indigo-iam.github.io/escape-docs/voms-config/voms-escape.cloud.cnaf.infn.it.vomses -O /etc/vomses/voms-escape.cloud.cnaf.infn.it.vomses \ + && mkdir -p /etc/grid-security/vomsdir/escape \ + && wget https://indigo-iam.github.io/escape-docs/voms-config/voms-escape.cloud.cnaf.infn.it.lsc -O /etc/grid-security/vomsdir/escape/voms-escape.cloud.cnaf.infn.it.lsc + +# Setup merged CERN CA file on Ubuntu based images. +# This file is contained in the `CERN-bundle.pem` file downloaded using +RUN mkdir /certs \ + && touch /certs/rucio_ca.pem \ + && curl -fsSL 'https://cafiles.cern.ch/cafiles/certificates/CERN%20Root%20Certification%20Authority%202.crt' | openssl x509 -inform DER -out /tmp/cernrootca2.crt \ + && curl -fsSL 'https://cafiles.cern.ch/cafiles/certificates/CERN%20Grid%20Certification%20Authority(1).crt' -o /tmp/cerngridca.crt \ + && curl -fsSL 'https://cafiles.cern.ch/cafiles/certificates/CERN%20Certification%20Authority.crt' -o /tmp/cernca.crt \ + && cat /tmp/cernrootca2.crt >> /certs/rucio_ca.pem \ + && cat /tmp/cerngridca.crt >> /certs/rucio_ca.pem \ + && cat /tmp/cernca.crt >> /certs/rucio_ca.pem \ + && rm /tmp/*.crt \ + && update-ca-certificates + +# Setup extension Rucio instance config +COPY configure-vre.py /usr/local/bin/ +RUN chmod +x /usr/local/bin/configure-vre.py +COPY configure-vre.sh /usr/local/bin/before-notebook.d/ + +RUN mkdir -p /opt/rucio/etc \ + && chown -R $NB_UID /opt/rucio/etc + +ENV JUPYTER_ENABLE_LAB=yes +ENV RUCIO_JUPYTERLAB_LOGLEVEL=DEBUG +ENV JUPYTERHUB_SINGLEUSER_APP=jupyter-server + +WORKDIR $HOME +USER $NB_UID + +CMD ["start-notebook.py"] \ No newline at end of file diff --git a/vre-singleuser-dev/README.md b/vre-singleuser-dev/README.md new file mode 100644 index 00000000..f3ce8c7a --- /dev/null +++ b/vre-singleuser-dev/README.md @@ -0,0 +1,3 @@ +# VRE singleuser Dev Image + +For Dev purposes diff --git a/vre-singleuser-dev/configure-vre.py b/vre-singleuser-dev/configure-vre.py new file mode 100755 index 00000000..9de3ab95 --- /dev/null +++ b/vre-singleuser-dev/configure-vre.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# Derived from https://gitlab.cern.ch/escape-wp2/docker-images/-/blob/master/datalake-singleuser/bin/configure.py (restricted access). + +import os +import json + +HOME = '/home/jovyan' + +def write_jupyterlab_config(): + file_path = HOME + '/.jupyter/jupyter_server_config.json' + if not os.path.isfile(file_path): + os.makedirs(HOME + '/.jupyter/', exist_ok=True) + else: + config_file = open(file_path, 'r') + config_payload = config_file.read() + config_file.close() + + try: + config_json = json.loads(config_payload) + except: + config_json = {} + + +# Looking to the rucio-jupyterlab configuration; https://github.com/rucio/jupyterlab-extension/blob/master/rucio_jupyterlab/config/schema.py#L101 +# either ("destination_rse", "rse_mount_path") either ("rucio_ca_cert") are required env vars, even if they are defined in the jhub manifest. +# Adding 'rucio_base_url' too - from debugging experience + + escape_config = { + "name": os.getenv('RUCIO_NAME', 'default'), + "display_name": os.getenv('RUCIO_DISPLAY_NAME', 'Default Instance'), + "rucio_base_url": os.getenv('RUCIO_BASE_URL', 'DEFAULT rucio base url'), + "rucio_auth_url": os.getenv('RUCIO_AUTH_URL'), + "rucio_webui_url": os.getenv('RUCIO_WEBUI_URL'), + "rucio_ca_cert": os.getenv('RUCIO_CA_CERT'), + "site_name": os.getenv('RUCIO_SITE_NAME'), + "vo": os.getenv('RUCIO_VO'), + "voms_enabled": os.getenv('RUCIO_VOMS_ENABLED', '0') == '1', + "voms_vomses_path": os.getenv('RUCIO_VOMS_VOMSES_PATH'), + "voms_certdir_path": os.getenv('RUCIO_VOMS_CERTDIR_PATH'), + "voms_vomsdir_path": os.getenv('RUCIO_VOMS_VOMSDIR_PATH'), + "destination_rse": os.getenv('RUCIO_DESTINATION_RSE', 'DEFAULT rse destination'), + "rse_mount_path": os.getenv('RUCIO_RSE_MOUNT_PATH', 'DEFAULT rse mount path'), + "replication_rule_lifetime_days": int(os.getenv('RUCIO_REPLICATION_RULE_LIFETIME_DAYS')) if os.getenv('RUCIO_REPLICATION_RULE_LIFETIME_DAYS') else None, + "path_begins_at": int(os.getenv('RUCIO_PATH_BEGINS_AT', '0')), + "mode": os.getenv('RUCIO_MODE', 'replica'), + "wildcard_enabled": os.getenv('RUCIO_WILDCARD_ENABLED', '0') == '1', + "oidc_auth": os.getenv('RUCIO_OIDC_AUTH'), + "oidc_env_name": os.getenv('RUCIO_OIDC_ENV_NAME'), + "oidc_file_name": os.getenv('RUCIO_OIDC_FILE_NAME'), + } + + atlas_config = { + "name": "https://voatlasrucio-server-prod.cern.ch", + "display_name": "ATLAS RUCIO", + "rucio_base_url": "https://voatlasrucio-server-prod.cern.ch:443", + "rucio_auth_url": "https://atlas-rucio-auth.cern.ch:443", + "rucio_ca_cert": "/certs/rucio_ca.pem", + "site_name": "CERN", + "vo": "atlas", + "voms_enabled": os.getenv('RUCIO_VOMS_ENABLED', '0') == '1', + "destination_rse": "CERN-PROD_PHYS-TOP", + "rse_mount_path": "/eos/atlas/atlasscratchdisk/rucio", + "path_begins_at": 4, + "mode": "replica", + "wildcard_enabled": os.getenv('RUCIO_WILDCARD_ENABLED', '0') == '0' + } + + escape_config = {k: v for k, + v in escape_config.items() if v is not None} + + atlas_config = {k: v for k, + v in atlas_config.items() if v is not None} + + + config_json['RucioConfig'] = { + 'instances': [escape_config, atlas_config], + "default_instance": os.getenv('RUCIO_DEFAULT_INSTANCE', escape_config['name']), + "default_auth_type": os.getenv('RUCIO_DEFAULT_AUTH_TYPE', 'oidc'), + } + + config_file = open(file_path, 'w') + config_file.write(json.dumps(config_json, indent=2)) + config_file.close() + +def write_ipython_config(): + file_path = HOME + '/.ipython/profile_default/ipython_kernel_config.json' + extension_module = 'rucio_jupyterlab.kernels.ipython' + + if not os.path.isfile(file_path): + os.makedirs(HOME + '/.ipython/profile_default/', exist_ok=True) + else: + config_file = open(file_path, 'r') + config_payload = config_file.read() + config_file.close() + + try: + config_json = json.loads(config_payload) + except: + config_json = {} + + if 'IPKernelApp' not in config_json: + config_json['IPKernelApp'] = {} + + ipkernel_app = config_json['IPKernelApp'] + + if 'extensions' not in ipkernel_app: + ipkernel_app['extensions'] = [] + + if extension_module not in ipkernel_app['extensions']: + ipkernel_app['extensions'].append(extension_module) + + config_file = open(file_path, 'w') + config_file.write(json.dumps(config_json, indent=2)) + config_file.close() + +if __name__ == '__main__': + write_jupyterlab_config() + write_ipython_config() diff --git a/vre-singleuser-dev/configure-vre.sh b/vre-singleuser-dev/configure-vre.sh new file mode 100644 index 00000000..d5250679 --- /dev/null +++ b/vre-singleuser-dev/configure-vre.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -e +python /usr/local/bin/configure-vre.py \ No newline at end of file diff --git a/vre-singleuser-dev/rucio_jupyterlab-1.3.0-py3-none-any.whl b/vre-singleuser-dev/rucio_jupyterlab-1.3.0-py3-none-any.whl new file mode 100644 index 00000000..41ad3d4c Binary files /dev/null and b/vre-singleuser-dev/rucio_jupyterlab-1.3.0-py3-none-any.whl differ diff --git a/vre-singleuser-dev/rucio_jupyterlab-1.3.2-py3-none-any.whl b/vre-singleuser-dev/rucio_jupyterlab-1.3.2-py3-none-any.whl new file mode 100644 index 00000000..26774d62 Binary files /dev/null and b/vre-singleuser-dev/rucio_jupyterlab-1.3.2-py3-none-any.whl differ diff --git a/vre-singleuser-dev/voms-atlas-auth.app.cern.ch.lsc b/vre-singleuser-dev/voms-atlas-auth.app.cern.ch.lsc new file mode 100644 index 00000000..8b731083 --- /dev/null +++ b/vre-singleuser-dev/voms-atlas-auth.app.cern.ch.lsc @@ -0,0 +1,2 @@ +/DC=ch/DC=cern/OU=computers/CN=atlas-auth.web.cern.ch +/DC=ch/DC=cern/CN=CERN Grid Certification Authority \ No newline at end of file diff --git a/vre-singleuser-dev/voms-atlas-auth.cern.ch.lsc b/vre-singleuser-dev/voms-atlas-auth.cern.ch.lsc new file mode 100644 index 00000000..30794f8a --- /dev/null +++ b/vre-singleuser-dev/voms-atlas-auth.cern.ch.lsc @@ -0,0 +1,2 @@ +/DC=ch/DC=cern/OU=computers/CN=atlas-auth.cern.ch +/DC=ch/DC=cern/CN=CERN Grid Certification Authority \ No newline at end of file