Skip to content

Commit 96b286c

Browse files
Soap2GSoap2G
committed
feat: adding atlasopendata image to environments (#66)
1 parent 05643fb commit 96b286c

12 files changed

+254
-0
lines changed

‎vre-singleuser-atlasopendata/Dockerfile‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
FROM ghcr.io/atlas-outreach-data-tools/notebooks-collection-opendata:sha-1eee300
2+
LABEL author="ESCAPE WP2 2020"
3+
LABEL maintainer="VRE Team @ CERN 23/24 - E. Garcia, G. Guerrieri"
4+
LABEL org.opencontainers.image.source https://github.com/vre-hub/environments
5+
ARG BUILD_DATE
6+
LABEL org.label-schema.build-date=$BUILD_DATE
7+
8+
RUN python -m pip install --upgrade pip
9+
10+
USER $NB_UID
11+
12+
RUN mamba install -y -c conda-forge python-gfal2 \
13+
nodejs \
14+
jupyterlab">4,<5" \
15+
notebook"<7" \
16+
jupyterhub \
17+
jsonschema>4 \
18+
jupyterlab_server \
19+
jupyter_server \
20+
traitlets \
21+
nbformat \
22+
ipykernel \
23+
PyJWT \
24+
ipywidgets \
25+
&& conda clean --all -f -y
26+
27+
# Install jupyterlab extensions
28+
RUN python -m pip install rucio-jupyterlab==1.0.0 \
29+
&& jupyter server extension enable --py rucio_jupyterlab --sys-prefix
30+
31+
RUN python -m pip install swanoauthrenew==1.0.1 \
32+
&& jupyter server extension enable --py swanoauthrenew --sys-prefix
33+
34+
RUN python -m pip install reana-jupyterlab \
35+
&& jupyter server extension enable --py reana_jupyterlab --sys-prefix
36+
37+
RUN git clone https://github.com/vre-hub/zenodo-jupyterlab-extension.git \
38+
&& cd zenodo-jupyterlab-extension \
39+
&& python -m pip install . \
40+
&& jupyter server extension enable --py zenodo_jupyterlab.server --sys-prefix \
41+
&& cd .. \
42+
&& rm -rf zenodo-jupyterlab-extension
43+
44+
# Custom version of reana-client due to the jsonschema problem
45+
RUN python -m pip install git+https://github.com/mdonadoni/reana-client.git@vre-summer-24
46+
47+
USER root
48+
49+
RUN apt update -y \
50+
&& apt install -y build-essential curl voms-clients-java software-properties-common \
51+
&& apt clean -y \
52+
&& rm /opt/conda/bin/voms-proxy-init \
53+
&& ln -s /usr/bin/voms-proxy-init /opt/conda/bin/voms-proxy-init
54+
55+
# ESCAPE grid-security
56+
RUN wget -q -O - https://dist.eugridpma.info/distribution/igtf/current/GPG-KEY-EUGridPMA-RPM-3 | apt-key add -
57+
58+
RUN apt update \
59+
&& add-apt-repository 'deb http://repository.egi.eu/sw/production/cas/1/current egi-igtf core' \
60+
&& apt -y install ca-policy-egi-core
61+
62+
# VOMS setup
63+
# wget https://indigo-iam.github.io/escape-docs/voms-config/voms-escape.cloud.cnaf.infn.it.vomses -O /etc/vomses/voms-escape.cloud.cnaf.infn.it.vomses
64+
# wget https://indigo-iam.github.io/escape-docs/voms-config/voms-escape.cloud.cnaf.infn.it.lsc -O /etc/grid-security/vomsdir/escape/voms-escape.cloud.cnaf.infn.it.lsc
65+
RUN mkdir -p /etc/vomses \
66+
&& mkdir -p /etc/grid-security/vomsdir/escape \
67+
&& mkdir -p /etc/grid-security/vomsdir/atlas \
68+
&& mkdir -p /etc/grid-security/vomsdir/cms
69+
COPY voms/vomsdir/escape/* /etc/grid-security/vomsdir/escape
70+
COPY voms/vomsdir/atlas/* /etc/grid-security/vomsdir/atlas
71+
COPY voms/vomsdir/cms/* /etc/grid-security/vomsdir/cms
72+
COPY voms/vomses/* /etc/vomses
73+
74+
# Setup merged CERN CA file on Ubuntu based images.
75+
# This file is contained in the `CERN-bundle.pem` file downloaded using
76+
RUN mkdir /certs \
77+
&& touch /certs/rucio_ca.pem \
78+
&& curl -fsSL 'https://cafiles.cern.ch/cafiles/certificates/CERN%20Root%20Certification%20Authority%202.crt' | openssl x509 -inform DER -out /tmp/cernrootca2.crt \
79+
&& curl -fsSL 'https://cafiles.cern.ch/cafiles/certificates/CERN%20Grid%20Certification%20Authority(1).crt' -o /tmp/cerngridca.crt \
80+
&& curl -fsSL 'https://cafiles.cern.ch/cafiles/certificates/CERN%20Certification%20Authority.crt' -o /tmp/cernca.crt \
81+
&& cat /tmp/cernrootca2.crt >> /certs/rucio_ca.pem \
82+
&& cat /tmp/cerngridca.crt >> /certs/rucio_ca.pem \
83+
&& cat /tmp/cernca.crt >> /certs/rucio_ca.pem \
84+
&& rm /tmp/*.crt \
85+
&& update-ca-certificates
86+
87+
# Setup extension Rucio instance config
88+
COPY configure-vre.py /usr/local/bin/
89+
RUN chmod +x /usr/local/bin/configure-vre.py
90+
COPY configure-vre.sh /usr/local/bin/before-notebook.d/
91+
92+
RUN mkdir -p /opt/rucio/etc \
93+
&& chown -R $NB_UID /opt/rucio/etc
94+
95+
ENV JUPYTER_ENABLE_LAB=yes
96+
ENV JUPYTERHUB_SINGLEUSER_APP=jupyter-server
97+
98+
WORKDIR $HOME
99+
USER $NB_UID
100+
101+
CMD ["start-notebook.py"]

‎vre-singleuser-atlasopendata/README.md‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# VRE singleuser BASE python 3.11 iamge
2+
3+
This image based on the original [ESCAPE Data Lake-as-a-Service Singleuser Base Image](https://gitlab.cern.ch/escape-wp2/docker-images/-/tree/master/datalake-singleuser) (restricted acces) which is derived from [quay.io/jupyter/scipy-notebook](https://quay.io/jupyter/scipy-notebook)($\ast$) with some modifications:
4+
5+
1. Python version is `3.11.8`.
6+
2. The followinf JupyterLab extensions are installed:
7+
* [rucio-jupyterlab](https://pypi.org/project/rucio-jupyterlab) v1.0.0.
8+
* The configuration for the Rucio JupyterLab extension is preset to connect with the ESCAPE Data Lake.
9+
* [reana-jupyterlab](https://github.com/vre-hub/reana-jupyterlab-extension) v1.0.0.
10+
* [zenodo-jupyterlab](https://github.com/vre-hub/zenodo-jupyterlab-extension).
11+
* [swanoauthrenew](https://pypi.org/project/swanoauthrenew/) v1.0.1 to enable renewal of tokens.
12+
3. The image is compatible with [rucio-clients](https://pypi.org/project/rucio-clients) `release-34.6.0` - please report any possible bug or error found.
13+
4. ESCAPE `ca-certificates` and [VOMSes](https://indigo-iam.github.io/escape-docs/) files are preinstalled.
14+
5. [python-gfal2](https://anaconda.org/conda-forge/python-gfal2) and `voms-clients-java` is installed to enable direct Rucio Download functionality.
15+
16+
($\ast$) Please note that since Oct 2023, Jupyter stopped pushing images to Docker Hub and start using quay.io
17+
18+
## Using the image as a standalone instance
19+
20+
If you want to take advantage of the extension's capability in your own machine, you can do a simple Docker run:
21+
22+
```sh
23+
docker run -p 8888:8888 docker pull ghcr.io/vre-hub/vre-singleuser-py311:latest
24+
```
25+
26+
You can also override the default environment values (some of which are specific to the ESCAPE instance, such as the service URLs) specified in the [Dockerfile](Dockerfile) within the singleuser extraEnv specification inside the [zero-to-jupyterhub Helm release](https://github.com/vre-hub/vre/tree/main/infrastructure/cluster/flux) of your deployment. Take a closer look at [configure-vre.py](configure-vre.py) to see all the available envs. Refer to [this configuration guide](https://github.com/rucio/jupyterlab-extension/blob/master/CONFIGURATION.md) for details.
27+
28+
## Extending the image
29+
30+
If you want to extend the image and use it as a part of the VRE, there are some things to remember:
31+
32+
1. To custimize the image please have a look to the [Startup Docker Stack documentation Hooks](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/common.html#startup-hooks).
33+
2. If you need to add a file, make sure that the container user has the necessary permission to access the file. You can use envs `$NB_UID`, `$NB_GID`, `$NB_USER`, and `$HOME` to get the user ID, group ID, username, and home directory respectively.
34+
3. Make sure that the user and workdir values are set to their original values:
35+
36+
```
37+
WORKDIR $HOME
38+
USER $NB_UID
39+
```
40+
41+
Based on all the previous work done by the ESCAPE Data Lake-as-a-Service group since 2020.

‎vre-singleuser-atlasopendata/configure-vre.py‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
#!/usr/bin/env python
2+
# Derived from https://gitlab.cern.ch/escape-wp2/docker-images/-/blob/master/datalake-singleuser/bin/configure.py (restricted access).
3+
4+
import os
5+
import json
6+
7+
HOME = '/home/jovyan'
8+
9+
def write_jupyterlab_config():
10+
file_path = HOME + '/.jupyter/jupyter_server_config.json'
11+
if not os.path.isfile(file_path):
12+
os.makedirs(HOME + '/.jupyter/', exist_ok=True)
13+
else:
14+
config_file = open(file_path, 'r')
15+
config_payload = config_file.read()
16+
config_file.close()
17+
18+
try:
19+
config_json = json.loads(config_payload)
20+
except:
21+
config_json = {}
22+
23+
# Looking to the rucio-jupyterlab configuration; https://github.com/rucio/jupyterlab-extension/blob/master/rucio_jupyterlab/config/schema.py#L101
24+
# either ("destination_rse", "rse_mount_path") either ("rucio_ca_cert") are required env vars, even if they are defined in the jhub manifest.
25+
# Adding 'rucio_base_url' too - from debugging experience
26+
27+
instance_config = {
28+
"name": os.getenv('RUCIO_NAME', 'default'),
29+
"display_name": os.getenv('RUCIO_DISPLAY_NAME', 'Default Instance'),
30+
"rucio_base_url": os.getenv('RUCIO_BASE_URL', 'DEFAULT rucio base url'),
31+
"rucio_auth_url": os.getenv('RUCIO_AUTH_URL'),
32+
"rucio_webui_url": os.getenv('RUCIO_WEBUI_URL'),
33+
"rucio_ca_cert": os.getenv('RUCIO_CA_CERT'),
34+
"site_name": os.getenv('RUCIO_SITE_NAME'),
35+
"vo": os.getenv('RUCIO_VO'),
36+
"voms_enabled": os.getenv('RUCIO_VOMS_ENABLED', '0') == '1',
37+
"voms_vomses_path": os.getenv('RUCIO_VOMS_VOMSES_PATH'),
38+
"voms_certdir_path": os.getenv('RUCIO_VOMS_CERTDIR_PATH'),
39+
"voms_vomsdir_path": os.getenv('RUCIO_VOMS_VOMSDIR_PATH'),
40+
"destination_rse": os.getenv('RUCIO_DESTINATION_RSE', 'DEFAULT rse destination'),
41+
"rse_mount_path": os.getenv('RUCIO_RSE_MOUNT_PATH', 'DEFAULT rse mount path'),
42+
"replication_rule_lifetime_days": int(os.getenv('RUCIO_REPLICATION_RULE_LIFETIME_DAYS')) if os.getenv('RUCIO_REPLICATION_RULE_LIFETIME_DAYS') else None,
43+
"path_begins_at": int(os.getenv('RUCIO_PATH_BEGINS_AT', '0')),
44+
"mode": os.getenv('RUCIO_MODE', 'replica'),
45+
"wildcard_enabled": os.getenv('RUCIO_WILDCARD_ENABLED', '0') == '1',
46+
"oidc_auth": os.getenv('RUCIO_OIDC_AUTH'),
47+
"oidc_env_name": os.getenv('RUCIO_OIDC_ENV_NAME'),
48+
"oidc_file_name": os.getenv('RUCIO_OIDC_FILE_NAME'),
49+
}
50+
51+
instance_config = {k: v for k,
52+
v in instance_config.items() if v is not None}
53+
config_json['RucioConfig'] = {
54+
'instances': [instance_config],
55+
"default_instance": os.getenv('RUCIO_DEFAULT_INSTANCE'),
56+
"default_auth_type": os.getenv('RUCIO_DEFAULT_AUTH_TYPE'),
57+
}
58+
59+
config_file = open(file_path, 'w')
60+
config_file.write(json.dumps(config_json, indent=2))
61+
config_file.close()
62+
63+
def write_ipython_config():
64+
file_path = HOME + '/.ipython/profile_default/ipython_kernel_config.json'
65+
extension_module = 'rucio_jupyterlab.kernels.ipython'
66+
67+
if not os.path.isfile(file_path):
68+
os.makedirs(HOME + '/.ipython/profile_default/', exist_ok=True)
69+
else:
70+
config_file = open(file_path, 'r')
71+
config_payload = config_file.read()
72+
config_file.close()
73+
74+
try:
75+
config_json = json.loads(config_payload)
76+
except:
77+
config_json = {}
78+
79+
if 'IPKernelApp' not in config_json:
80+
config_json['IPKernelApp'] = {}
81+
82+
ipkernel_app = config_json['IPKernelApp']
83+
84+
if 'extensions' not in ipkernel_app:
85+
ipkernel_app['extensions'] = []
86+
87+
if extension_module not in ipkernel_app['extensions']:
88+
ipkernel_app['extensions'].append(extension_module)
89+
90+
config_file = open(file_path, 'w')
91+
config_file.write(json.dumps(config_json, indent=2))
92+
config_file.close()
93+
94+
if __name__ == '__main__':
95+
write_jupyterlab_config()
96+
write_ipython_config()

‎vre-singleuser-atlasopendata/configure-vre.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
set -e
3+
python /usr/local/bin/configure-vre.py

‎vre-singleuser-atlasopendata/voms/vomsdir/atlas/voms-atlas-auth.app.cern.ch.lsc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
/DC=ch/DC=cern/OU=computers/CN=atlas-auth.web.cern.ch
2+
/DC=ch/DC=cern/CN=CERN Grid Certification Authority

‎vre-singleuser-atlasopendata/voms/vomsdir/atlas/voms-atlas-auth.cern.ch.lsc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
/DC=ch/DC=cern/OU=computers/CN=atlas-auth.cern.ch
2+
/DC=ch/DC=cern/CN=CERN Grid Certification Authority

‎vre-singleuser-atlasopendata/voms/vomsdir/cms/voms-cms-auth.app.cern.ch.lsc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
/DC=ch/DC=cern/OU=computers/CN=cms-auth.web.cern.ch
2+
/DC=ch/DC=cern/CN=CERN Grid Certification Authority

‎vre-singleuser-atlasopendata/voms/vomsdir/cms/voms-cms-auth.cern.ch.lsc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
/DC=ch/DC=cern/OU=computers/CN=cms-auth.cern.ch
2+
/DC=ch/DC=cern/CN=CERN Grid Certification Authority

‎vre-singleuser-atlasopendata/voms/vomsdir/escape/voms-escape.cloud.cnaf.infn.it.lsc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
/DC=org/DC=terena/DC=tcs/C=IT/ST=Roma/O=Istituto Nazionale di Fisica Nucleare/CN=voms-escape.cloud.cnaf.infn.it
2+
/C=NL/O=GEANT Vereniging/CN=GEANT eScience SSL CA 4

‎vre-singleuser-atlasopendata/voms/vomses/atlas-voms-atlas-auth.app.cern.ch‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"atlas" "voms-atlas-auth.app.cern.ch" "443" "/DC=ch/DC=cern/OU=computers/CN=atlas-auth.web.cern.ch" "atlas" "24"

0 commit comments

Comments
 (0)