Skip to content

Commit 0a65e31

Browse files
Soap2GSoap2G
committed
feat(vre-swan-cern) add kernel setup and centralise configuration for the extension
This commit adds to the Rucio setup script two items: 1. Writing the kernel setup within the Rucio config script 2. Appending the Juptyer server configuration to the original one from our version in CVMFS. This removes any upstream dependency, as it's a standalone snippet. Added extensive documentation to the script, as well as logging.
1 parent 6640e4d commit 0a65e31

File tree

2 files changed

+34
-125
lines changed

2 files changed

+34
-125
lines changed

‎vre-swan-cern/python/jupyter_server_config.py‎

Lines changed: 0 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import os
2-
import json
32
from site import getsitepackages
43

54
home = os.environ.get("HOME")
@@ -45,124 +44,3 @@
4544
if os.path.isdir(candidate):
4645
c.ServerApp.extra_template_paths = [candidate]
4746
break
48-
49-
# CERN-VRE - Configure the JupyterLab server - TRIGGERED TO ALWAYS WORK
50-
use_rucio_extension = os.environ.get('USE_RUCIO_EXTENSION', 'true').lower() == 'true'
51-
if use_rucio_extension:
52-
53-
def write_jupyterlab_config():
54-
dir_config_jupyterlab = os.getenv('JUPYTER_CONFIG_DIR',
55-
os.path.join(os.getenv('HOME'), '.jupyter')
56-
)
57-
file_server_config = os.path.join(dir_config_jupyterlab, 'jupyter_server_config.json')
58-
59-
if not os.path.exists(dir_config_jupyterlab):
60-
os.makedirs( dir_config_jupyterlab, exist_ok=True)
61-
elif os.path.isfile(file_server_config) :
62-
with open(file_server_config, "r") as config_file:
63-
config_payload = config_file.read()
64-
65-
try:
66-
config_json = json.loads(config_payload)
67-
except:
68-
config_json = {}
69-
70-
escape_config = {
71-
"name": os.getenv('RUCIO_NAME', 'vre-rucio.cern.ch'),
72-
"display_name": os.getenv('RUCIO_DISPLAY_NAME', 'ESCAPE Rucio instance'),
73-
"rucio_base_url": os.getenv('RUCIO_BASE_URL', 'https://vre-rucio.cern.ch'),
74-
"rucio_auth_url": os.getenv('RUCIO_AUTH_URL', 'https://vre-rucio-auth.cern.ch'),
75-
"rucio_webui_url": os.getenv('RUCIO_WEBUI_URL', 'https://vre-rucio-ui.cern.ch'),
76-
"rucio_ca_cert": os.getenv('RUCIO_CA_CERT', '/eos/user/e/engarcia/rucio_ca_certs/rucio_ca.pem'),
77-
"site_name": os.getenv('RUCIO_SITE_NAME', 'CERN'),
78-
"vo": os.getenv('RUCIO_VO', 'escape'),
79-
"voms_enabled": os.getenv('RUCIO_VOMS_ENABLED', '0') == '1',
80-
"voms_vomses_path": os.getenv('RUCIO_VOMS_VOMSES_PATH', '/etc/vomses'),
81-
"voms_certdir_path": os.getenv('RUCIO_VOMS_CERTDIR_PATH',),
82-
"voms_vomsdir_path": os.getenv('RUCIO_VOMS_VOMSDIR_PATH', '/etc/grid-security/vomsdir'),
83-
"destination_rse": os.getenv('RUCIO_DESTINATION_RSE', 'CERN-EOSPILOT'),
84-
"rse_mount_path": os.getenv('RUCIO_RSE_MOUNT_PATH', '/eos/eulake'),
85-
"replication_rule_lifetime_days": int(os.getenv('RUCIO_REPLICATION_RULE_LIFETIME_DAYS')) if os.getenv('RUCIO_REPLICATION_RULE_LIFETIME_DAYS') else None,
86-
"path_begins_at": int(os.getenv('RUCIO_PATH_BEGINS_AT', '5')),
87-
"mode": os.getenv('RUCIO_MODE', 'replica'),
88-
"wildcard_enabled": os.getenv('RUCIO_WILDCARD_ENABLED', '1') == '1',
89-
#"oidc_auth": os.getenv('RUCIO_OIDC_AUTH'),
90-
#"oidc_env_name": os.getenv('RUCIO_OIDC_ENV_NAME'),
91-
#"oidc_file_name": os.getenv('RUCIO_OIDC_FILE_NAME'),
92-
}
93-
atlas_config = {
94-
"name": os.getenv('ATLAS_RUCIO_NAME', 'https://voatlasrucio-server-prod.cern.ch'),
95-
"display_name": os.getenv('ATLAS_RUCIO_DISPLAY_NAME', 'ATLAS RUCIO'),
96-
"rucio_base_url": os.getenv('ATLAS_RUCIO_BASE_URL', 'https://voatlasrucio-server-prod.cern.ch:443'),
97-
"rucio_auth_url": os.getenv('ATLAS_RUCIO_AUTH_URL', 'https://atlas-rucio-auth.cern.ch:443'),
98-
"rucio_webui_url": os.getenv('ATLAS_RUCIO_WEBUI_URL', 'https://rucio-ui.cern.ch'),
99-
"rucio_ca_cert": os.getenv('RUCIO_CA_CERT', '/eos/user/e/engarcia/rucio_ca_certs/rucio_ca.pem'),
100-
"site_name": os.getenv('ATLAS_RUCIO_SITE_NAME', 'CERN'),
101-
"vo": os.getenv('ATLAS_RUCIO_VO', 'atlas'),
102-
"voms_enabled": os.getenv('RUCIO_VOMS_ENABLED', '0') == '1',
103-
"voms_vomses_path": os.getenv('RUCIO_VOMS_VOMSES_PATH', '/etc/vomses'),
104-
"voms_certdir_path": os.getenv('RUCIO_VOMS_CERTDIR_PATH',),
105-
"voms_vomsdir_path": os.getenv('RUCIO_VOMS_VOMSDIR_PATH', '/etc/grid-security/vomsdir'),
106-
"destination_rse": os.getenv('ATLAS_RUCIO_DESTINATION_RSE', 'CERN-PROD_PHYS-TOP'),
107-
"rse_mount_path": os.getenv('ATLAS_RUCIO_RSE_MOUNT_PATH', '/eos/atlas/atlasgroupdisk/phys-top'),
108-
"replication_rule_lifetime_days": int(os.getenv('ATLAS_RUCIO_REPLICATION_RULE_LIFETIME_DAYS')) if os.getenv('ATLAS_RUCIO_REPLICATION_RULE_LIFETIME_DAYS') else None,
109-
"path_begins_at": int(os.getenv('ATLAS_RUCIO_PATH_BEGINS_AT', '4')),
110-
"mode": os.getenv('ATLAS_RUCIO_MODE', 'replica'),
111-
"wildcard_enabled": os.getenv('ATLAS_RUCIO_WILDCARD_ENABLED', '1') == '1',
112-
}
113-
114-
115-
escape_config = {k: v for k,
116-
v in escape_config.items() if v is not None}
117-
118-
atlas_config = {k: v for k,
119-
v in atlas_config.items() if v is not None}
120-
121-
config_json['RucioConfig'] = {
122-
'instances': [escape_config, atlas_config], # cms,
123-
"default_instance": os.getenv('RUCIO_DEFAULT_INSTANCE', escape_config['name']),
124-
"default_auth_type": os.getenv('RUCIO_DEFAULT_AUTH_TYPE', 'x509_proxy'),
125-
"log_level": os.getenv('RUCIO_LOG_LEVEL', 'debug'),
126-
}
127-
128-
config_file = open(file_server_config, 'w')
129-
config_file.write(json.dumps(config_json, indent=2))
130-
config_file.close()
131-
132-
def write_ipython_config():
133-
file_path = os.path.join(os.getenv('HOME'), '.ipython/profile_default')
134-
file_name = os.path.join(file_path, 'ipython_kernel_config.json')
135-
extension_module = 'rucio_jupyterlab.kernels.ipython'
136-
137-
if not os.path.isfile(file_name):
138-
os.makedirs(file_path, exist_ok=True)
139-
else:
140-
config_file = open(file_name, 'r')
141-
config_payload = config_file.read()
142-
config_file.close()
143-
144-
try:
145-
config_json = json.loads(config_payload)
146-
except:
147-
config_json = {}
148-
149-
if 'IPKernelApp' not in config_json:
150-
config_json['IPKernelApp'] = {}
151-
152-
ipkernel_app = config_json['IPKernelApp']
153-
154-
if 'extensions' not in ipkernel_app:
155-
ipkernel_app['extensions'] = []
156-
157-
if extension_module not in ipkernel_app['extensions']:
158-
ipkernel_app['extensions'].append(extension_module)
159-
160-
config_file = open(file_name, 'w')
161-
config_file.write(json.dumps(config_json, indent=2))
162-
config_file.close()
163-
164-
write_jupyterlab_config()
165-
write_ipython_config()
166-
167-
else:
168-
pass

‎vre-swan-cern/scripts/before-notebook.d/01_rucio.sh‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,31 +5,62 @@
55
# This script configures the environment for the usage
66
# of the Rucio JupyterLab extension.
77

8-
# RUCIO_JUPYTERLAB_VERSION is set in the Dockerfile
8+
# Check if the RUCIO_JUPYTERLAB_VERSION environment variable is set.
99
if [[ -z "$RUCIO_JUPYTERLAB_VERSION" ]]; then
1010
_log "RUCIO_JUPYTERLAB_VERSION is not set. Please set it to the desired version."
1111
exit 1
1212
fi
1313

14-
# Check if the Rucio JupyterLab version is available
14+
# Verify if the specified Rucio JupyterLab version exists in the CVMFS directory.
1515
if [[ ! -d "/cvmfs/sw.escape.eu/rucio-jupyterlab/${RUCIO_JUPYTERLAB_VERSION}" ]]; then
1616
_log "Rucio JupyterLab version ${RUCIO_JUPYTERLAB_VERSION} is not available."
1717
exit 1
1818
fi
1919

20-
# Set up the environment for Rucio JupyterLab
2120
_log "Setting up Rucio JupyterLab environment for version ${RUCIO_JUPYTERLAB_VERSION}..."
2221

22+
# Define the base path for the ESCAPE CVMFS repository.
2323
ESCAPE_CVMFS_PATH="/cvmfs/sw.escape.eu"
24+
25+
# Define the path to the specific Rucio JupyterLab version within the CVMFS repository.
2426
RUCIO_CVMFS_PATH="${ESCAPE_CVMFS_PATH}/rucio-jupyterlab/${RUCIO_JUPYTERLAB_VERSION}"
2527

28+
# Set the RUCIO_HOME environment variable to point to the Rucio JupyterLab installation directory.
2629
export RUCIO_HOME="${RUCIO_CVMFS_PATH}"
30+
31+
# Add the Rucio JupyterLab binary directory to the PATH environment variable.
2732
export USER_PATH="${RUCIO_CVMFS_PATH}/bin"
2833
export PATH="${USER_PATH}:${PATH}"
34+
35+
# Add the Rucio JupyterLab Python packages directory to the PYTHONPATH environment variable.
2936
export USER_PYTHONPATH="${RUCIO_CVMFS_PATH}/lib/python3.11/site-packages"
3037
export PYTHONPATH="${USER_PYTHONPATH}:${PYTHONPATH}"
38+
39+
# Set the path to the Rucio CA certificate used for secure communication.
3140
export RUCIO_CA_CERT="${ESCAPE_CVMFS_PATH}/etc/ssl/certs/rucio_ca.pem"
41+
42+
# Define the Python binary to be used by Rucio JupyterLab.
3243
export RUCIO_PYTHONBIN=python
3344

3445
_log "Rucio JupyterLab environment set up successfully."
3546

47+
_log "Setting up iPython kernel for Rucio JupyterLab..."
48+
49+
# Create the default iPython profile directory for the Jupyter Notebook user.
50+
mkdir -p /home/${NB_USER}/.ipython/profile_default
51+
_log "Created directory /home/${NB_USER}/.ipython/profile_default"
52+
53+
# Configure the iPython kernel to load the Rucio JupyterLab extension.
54+
# Given that this is the first script to run, we can safely overwrite any existing configuration.
55+
echo "c.IPKernelApp.extensions = ['rucio_jupyterlab.kernels.ipython']" > /home/${NB_USER}/.ipython/profile_default/ipython_kernel_config.py
56+
57+
_log "iPython kernel for Rucio JupyterLab set up successfully."
58+
59+
_log "Writing the extension configuration file..."
60+
61+
# Add the Rucio JupyterLab server configuration to the Jupyter server configuration file.
62+
# The Rusio specific file is provided by CVMFS, and the original is copied from the Dockerfile.
63+
cat "${ESCAPE_CVMFS_PATH}/etc/jupyter/jupyter_server_config.py" >> /home/${NB_USER}/.jupyter/jupyter_server_config.py
64+
65+
_log "Extension configuration file written successfully."
66+
_log "Rucio JupyterLab setup completed."

0 commit comments

Comments
 (0)