diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..298ea9e --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/conf.py b/conf.py new file mode 100644 index 0000000..5aaec32 --- /dev/null +++ b/conf.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'Documentation' +copyright = '2021, IC' +author = 'IC' + +# The short X.Y version +version = '' +# The full version, including alpha/beta/rc tags +release = 'v0.0.1' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = None + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Documentationdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'Documentation.tex', 'Documentation Documentation', + 'IC', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'documentation', 'Documentation Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'Documentation', 'Documentation Documentation', + author, 'Documentation', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Options for Epub output ------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + + +# -- Extension configuration ------------------------------------------------- diff --git a/index.rst b/index.rst new file mode 100644 index 0000000..215a6fa --- /dev/null +++ b/index.rst @@ -0,0 +1,20 @@ +.. Documentation documentation master file, created by + sphinx-quickstart on Mon Feb 8 19:50:54 2021. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Documentation's documentation! +========================================= + +.. toctree:: + :maxdepth: 3 + :caption: Contents: + + test3 + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/make.bat b/make.bat new file mode 100644 index 0000000..27f573b --- /dev/null +++ b/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/test1.rst b/test1.rst new file mode 100644 index 0000000..7607a06 --- /dev/null +++ b/test1.rst @@ -0,0 +1,370 @@ +.. contents:: + :depth: 3 +.. + +**RASCIL on command line lofar** + +**and IRIS job submission** + +**Important links:** + +- https://timcornwell.gitlab.io/rascil/ + +- https://timcornwell.gitlab.io/rascil/installation/RASCIL_docker.html#singularity + +- https://github.com/SKA-ScienceDataProcessor/rascil + +- https://gitlab.com/timcornwell/eMERLIN_RASCIL_pipeline + +This documentation follows the steps of the link below “Running RASCIL +under docker” + +https://timcornwell.gitlab.io/rascil/installation/RASCIL_docker.html but +running it under singularity. + +**LOFAR7 command line** + +**Set up environment variables:** + +**@lofar7 /raid/scratch/ > ./.bash_profile #change +to bash** + +'abrt-cli status' timed out + +**bash-4.2$ export +SINGULARITY_CACHEDIR=/raid/scratch//cache** + +OR + +bash-4.2$ export +SINGULARITY_CACHEDIR=/raid/scratch//.singularity + +bash-4.2$ TMPDIR=/raid/scratch//tmp + +bash-4.2$ mkdir -p $TMPDIR + +**Running on existing docker images:** + +The singularity containers for RASCIL are on github at: + +docker.io/timcornwell/rascil-no-data-no-root + +docker.io/timcornwell/rascil-full-no-root + +Pull the Rascil image + +bash-4.2$ singularity pull RASCIL.img +docker://timcornwell/rascil-no-data-no-root + +bash-4.2$ singularity pull RASCIL-full1.img +docker://timcornwell/rascil-full-no-root + +Running an example: + +bash-4.2$ singularity run RASCIL-full1.img + +Singularity> python3 /rascil/examples/scripts/imaging.py + +Singularity> ls + +RASCIL-full1.img RASCIL.img dask-worker-space imaging_dirty.fits +imaging_psf.fits imaging_restored.fits + +**Running notebooks** + +Singularity> jupyter notebook --no-browser --ip 0.0.0.0 +/rascil/examples/notebooks/ + +**Running RASCIL as a cluster:** + +Singularity> python3 /rascil/cluster_tests/ritoy/cluster_test_ritoy.py + +Creating scheduler and 4 workers + + + +53870592.0 + +\**\* Successfully reached end in 16.5 seconds \**\* + +OR from outside the container + +bash-4.2$ singularity exec RASCIL-full1.img python3 +/rascil/cluster_tests/ritoy/cluster_test_ritoy.py + +Creating scheduler and 4 workers + + + +53870592.0 + +\**\* Successfully reached end in 17.5 seconds \**\* + +**IRIS job submission RASCIL** + +**bash-4.2$ source bashrc** + +bash-4.2$ dirac-proxy-init -g skatelescope.eu_user -M + +**1. JDL** + +**example 1** + +bash-4.2$ dirac-wms-job-submit simpleR1.jdl + +JobID = 25260750 + +bash-4.2$ dirac-wms-job-status 25260750 + +JobID=25260750 Status=Running; MinorStatus=Input Data Resolution; +Site=LCG.UKI-NORTHGRID-MAN-HEP.uk; + +bash-4.2$ dirac-wms-job-status 25260750 + +JobID=25260750 Status=Running; MinorStatus=Application; +Site=LCG.UKI-NORTHGRID-MAN-HEP.uk; + +bash-4.2$ dirac-wms-job-status 25260750 + +JobID=25260750 Status=Done; MinorStatus=Execution Complete; +Site=LCG.UKI-NORTHGRID-MAN-HEP.uk; + +bash-4.2$ + +**bash-4.2$ cat simpleR1.jdl** + +JobName = "InputAndOuputSandbox"; + +Executable = "testR1.sh"; + +StdOutput = "StdOut"; + +StdError = "StdErr"; + +InputSandbox = {"testR1.sh"}; + +InputData = +{"LFN:/skatelescope.eu/user/c//rascil/RASCIL-full1.img"}; + +OutputSandbox = {"StdOut","StdErr"}; + +OutputData={"imaging_dirty.fits","imaging_psf.fits","imaging_restored.fits"}; + +OutputSE ="UKI-NORTHGRID-MAN-HEP-disk"; + +Site = "LCG.UKI-NORTHGRID-MAN-HEP.uk"; + +bash-4.2$ cat testR1.sh + +#!/bin/bash + +singularity exec --cleanenv -H $PWD:/srv --pwd /srv -C RASCIL-full1.img +python3 /rascil/examples/scripts/imaging.py; + +bash-4.2$ dirac-wms-job-get-output-data 25336768 + +Job 25336768 output data retrieved + +bash-4.2$ ls + +-rw-r--r--. 1 users6 2102400 May 14 17:32 imaging_dirty.fits + +-rw-r--r--. 1 users6 2102400 May 14 17:32 imaging_psf.fits + +-rw-r--r--. 1 users6 2102400 May 14 17:32 +imaging_restored.fits + +bash-4.2$ dirac-wms-job-get-output 25336768 + +Job output sandbox retrieved in +/raid/scratch//dirac_ui/tests/rascilTests/ 25336768/ + +bash-4.2$ cd 25336768 + +bash-4.2$ ls + +StdErr StdOut + +bash-4.2$ cat StdErr + +INFO: Convert SIF file to sandbox... + +INFO: Cleaning up image... + +**JDL** + +**example 2 ------ from Willice** + +**Note:** 1153+4931.ms.tar contains the calibrated ms and +RascilScriptFor1153.py is the imaging script + +cat test.jdl + +jobName = "LFNInputSandbox"; + +Executable = "trial.sh"; + +StdOutput = "StdOut"; + +StdError = "StdErr"; + +Tags = "skatelescope.eu.hmem"; + +SitesList = "LCG.UKI-NORTHGRID-MAN-HEP.uk"; + +SEList = "UKI-NORTHGRID-MAN-HEP-disk"; + +InputSandbox = +{"trial.sh","LFN:/skatelescope.eu/user/w/willice.obonyo/IRIS_RASCIL_test/1153+4931.ms.tar","LFN:/skatelescope.eu/user/w/willice.obonyo/I + +RIS_RASCIL_test/RascilScriptFor1153.py","LFN:/skatelescope.eu/user/w/willice.obonyo/IRIS_RASCIL_test/rascil.img"}; + +OutputSandbox = {"StdOut","StdErr"}; + +OutputSE = "UKI-NORTHGRID-MAN-HEP-disk"; + +OutputData = {"StdOut","outputs.tar", "job.info", "*.log","rascil.tar"}; + +cat trial.sh + +#!/bin/bash + +echo 'Checking the location of python on the grid'; + +which python; + +echo "=============================================="; + +echo "Print environment details"; + +#printenv; + +echo "=============================================="; + +singularity --version; + +tar -xzvf 1153+4931.ms.tar + +singularity exec --cleanenv -H $PWD:/srv --pwd /srv -C +/cvmfs/sw.skatelescope.eu/images/rascil.img python3 +RascilScriptFor1153.py; + +tar czf outputs.tar \*.fits + +**2.PY** + +**Set up environment variables:** + +#SET THE PATH PYTHON 2.7 INTO $PATH + +#PATH to python 2.7 added + +bash-4.2$ export PATH=/usr/local/casa/bin/python:$PATH + +bash-4.2$ cat jobpy.py + +import os + +import sys + +import time + +# setup DIRAC + +from DIRAC.Core.Base import Script + +Script.parseCommandLine(ignoreErrors=False) + +from DIRAC.Interfaces.API.Job import Job + +from DIRAC.Interfaces.API.Dirac import Dirac + +from DIRAC.Core.Security.ProxyInfo import getProxyInfo + +SitesList = ['LCG.UKI-NORTHGRID-MAN-HEP.uk'] + +SEList = ['UKI-NORTHGRID-MAN-HEP-disk'] + +dirac = Dirac() + +j = Job(stdout='StdOut', stderr='StdErr') + +j.setName('TestJob') + +j.setInputSandbox(["testR1py.sh"]) + +j.setInputData(['LFN:/skatelescope.eu/user/c//rascil/RASCIL-full1.img']) + +j.setOutputSandbox(['StdOut','StdErr']) + +j.setOutputData(['imaging_dirty.fits','imaging_psf.fits','imaging_restored.fits'],outputSE='UKI-NORTHGRID-MAN-HEP-disk') + +j.setExecutable('testR1py.sh') + +jobID = dirac.submitJob(j) + +print 'Submission Result: ', jobID + +bash-4.2$ cat testR1py.sh + +#!/bin/bash + +singularity exec --cleanenv -H $PWD:/srv --pwd /srv -C RASCIL-full1.img +python3 /rascil/examples/scripts/imaging.py + +bash-4.2$ python jobpy.py + +Submission Result: {'requireProxyUpload': False, 'OK': True, 'rpcStub': +(('WorkloadManagement/JobManag er', {'delegatedDN': None, 'timeout': +600, 'skipCACheck': False, 'keepAliveLapse': 150, 'delegatedGroup ': +None}), 'submitJob', ('[ \\n Origin = DIRAC;\n Executable = +"$DIRACROOT/scripts/dirac-jobexec"; \\n StdError = StdErr;\n LogLevel = +info;\n OutputSE = UKI-NORTHGRID-MAN-HEP-disk;\n InputSa ndbox = \\n {\n +"testR1py.sh",\n "SB:GridPPSandboxSE|/SandBox/i/iulia.c.cim +pan.skatelescope.eu_user/cf8/ca6/cf8ca689995e24c01c068eb6f34126b8.tar.bz2"\n +};\n JobName = T estJob;\n Priority = 1;\n Arguments = +"jobDescription.xml -o LogLevel=info";\n JobGroup = skat elescope.eu;\n +OutputSandbox = \\n {\n StdOut,\n StdErr,\n Sc ript1_testR1py.sh.log\n +};\n StdOutput = StdOut;\n InputData = LFN:/skatelescope.eu/user/c +//rascil/RASCIL-full1.img;\n JobType = User;\n OutputData = +\\n {\n imagin g_dirty.fits,\n imaging_psf.fits,\n +imaging_restored.fits\n };\n]',)), 'Va lue': 25344748, 'JobID': +25344748} + +bash-4.2$ dirac-wms-job-get-output 25344748 + +Job output sandbox retrieved in +/raid/scratch//dirac_ui/tests/rascilTests/25344748/ + +bash-4.2$ cd 25344748 + +bash-4.2$ ls + +Script1_testR1py.sh.log StdOut + +bash-4.2$ dirac-wms-job-get-output-data 25344748 + +Job 25344748 output data retrieved + +bash-4.2$ ls + +imaging_dirty.fits imaging_psf.fits imaging_restored.fits +Script1_testR1py.sh.log StdOut + +**Adding file to FC** + +@lofar7 /raid/scratch//dirac_ui/rascil > +dirac-dms-add-file +LFN:/skatelescope.eu/user/c//rascil/\ **RASCIL-full1.img** +**RASCIL-full1.img** UKI-NORTHGRID-MAN-HEP-disk + +Uploading /skatelescope.eu/user/c//rascil/RASCIL-full1.img + +Successfully uploaded file to UKI-NORTHGRID-MAN-HEP-disk + +@lofar7 /raid/scratch//dirac_ui/rascil > + +https://dirac.readthedocs.io/en/latest/UserGuide/Tutorials/JDLsAndJobManagementBasic/ + diff --git a/test2.rst b/test2.rst new file mode 100644 index 0000000..bc2bede --- /dev/null +++ b/test2.rst @@ -0,0 +1,11 @@ +############ +All about me +############ + +I'm Daniele Procida, a Django user and developer. + +I've contributed to: + +* django CMS +* Arkestra +* Django diff --git a/test3.rst b/test3.rst new file mode 100644 index 0000000..0eb23c5 --- /dev/null +++ b/test3.rst @@ -0,0 +1,34 @@ +#################################################### +RASCIL on command line lofar and IRIS job submission +#################################################### + + +**Important links:** + +- https://timcornwell.gitlab.io/rascil/ + +- https://timcornwell.gitlab.io/rascil/installation/RASCIL_docker.html#singularity + +- https://github.com/SKA-ScienceDataProcessor/rascil + +- https://gitlab.com/timcornwell/eMERLIN_RASCIL_pipeline + +This documentation follows the steps of the link below “Running RASCIL +under docker” + +https://timcornwell.gitlab.io/rascil/installation/RASCIL_docker.html but +running it under singularity. + +**LOFAR7 command line** + +**Set up environment variables:** + +**@lofar7 /raid/scratch/ > ./.bash_profile #change +to bash** + +'abrt-cli status' timed out + +**bash-4.2$ export +SINGULARITY_CACHEDIR=/raid/scratch//cache** + +