Skip to content

Commit 1decdf2

Browse files
mbmilliganmbmilligan
authored
Merge pull request #208 from consideRatio/pr/general-maintenance
General maintenance: autoformat, flake8, readme badges, misc in github workflows
2 parents 5924a09 + 5dde989 commit 1decdf2

15 files changed

+1088
-747
lines changed

‎.flake8

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
[flake8]
2+
# Ignore style and complexity
3+
# E: style errors
4+
# W: style warnings
5+
# C: complexity
6+
# F401: module imported but unused
7+
# F403: import *
8+
# F811: redefinition of unused `name` from line `N`
9+
# F841: local variable assigned but never used
10+
# E402: module level import not at top of file
11+
# I100: Import statements are in the wrong order
12+
# I101: Imported names are in the wrong order. Should be
13+
ignore = E, W, C, F401, F403, F811, F841, E402, I100, I101, D400

‎.github/workflows/python-publish.yml

+16-17
Original file line numberDiff line numberDiff line change
@@ -9,23 +9,22 @@ on:
99

1010
jobs:
1111
deploy:
12-
1312
runs-on: ubuntu-latest
1413

1514
steps:
16-
- uses: actions/checkout@v2
17-
- name: Set up Python
18-
uses: actions/setup-python@v2
19-
with:
20-
python-version: '3.x'
21-
- name: Install dependencies
22-
run: |
23-
python -m pip install --upgrade pip
24-
pip install setuptools wheel twine
25-
- name: Build and publish
26-
env:
27-
TWINE_USERNAME: '__token__'
28-
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
29-
run: |
30-
python setup.py sdist bdist_wheel
31-
twine upload dist/*
15+
- uses: actions/checkout@v2
16+
- name: Set up Python
17+
uses: actions/setup-python@v2
18+
with:
19+
python-version: "3.x"
20+
- name: Install dependencies
21+
run: |
22+
python -m pip install --upgrade pip
23+
pip install setuptools wheel twine
24+
- name: Build and publish
25+
env:
26+
TWINE_USERNAME: "__token__"
27+
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
28+
run: |
29+
python setup.py sdist bdist_wheel
30+
twine upload dist/*

‎.github/workflows/test.yml

+62-40
Original file line numberDiff line numberDiff line change
@@ -1,55 +1,77 @@
1+
# This is a GitHub workflow defining a set of jobs with a set of steps.
2+
# ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions
3+
#
4+
name: Test
15

2-
name: Python package
3-
4-
on: [push, pull_request]
6+
on:
7+
pull_request:
8+
push:
9+
workflow_dispatch:
510

611
jobs:
7-
build:
12+
pre-commit:
13+
name: Run pre-commit
14+
runs-on: ubuntu-20.04
15+
steps:
16+
- uses: actions/checkout@v2
17+
- uses: actions/setup-python@v2
18+
- uses: pre-commit/[email protected]
819

20+
pytest:
21+
name: "Run pytest"
922
runs-on: ubuntu-20.04
1023
continue-on-error: ${{ matrix.allow_failure }}
1124
strategy:
25+
# Keep running even if one variation of the job fail
26+
fail-fast: false
1227
matrix:
1328
python-version:
14-
- '3.5'
15-
- '3.9'
29+
- "3.5"
30+
- "3.9"
1631
JHUB_VER:
17-
- '0.9.6'
18-
- '1.0.0'
19-
- '1.1.0'
20-
- '1.2.0'
32+
- "0.9.6"
33+
- "1.0.0"
34+
- "1.1.0"
35+
- "1.2.0"
36+
- "1.3.0"
2137
allow_failure: [false]
38+
39+
exclude:
40+
# JupyterHub 1.3.0 requires python 3.6+
41+
- JHUB_VER: "1.3.0"
42+
python-version: "3.5"
2243
include:
23-
- JHUB_VER: 'master'
24-
python-version: '3.9'
44+
- JHUB_VER: "master"
45+
python-version: "3.9"
2546
allow_failure: true
2647

2748
steps:
28-
- uses: actions/checkout@v2
29-
- name: Set up Python ${{ matrix.python-version }}
30-
uses: actions/setup-python@v2
31-
with:
32-
python-version: ${{ matrix.python-version }}
33-
34-
- name: Install dependencies
35-
run: |
36-
python -m pip install --upgrade pip
37-
python -m pip install pytest
38-
pip install -r requirements.txt
39-
- name: Install nodejs dependencies
40-
run: |
41-
sudo npm install -g configurable-http-proxy
42-
# We need to check compatibility with different versions of the JH
43-
# API, including latest development. For that, we also need to
44-
# pull in the dependencies of that old JH version (but we don't
45-
# need conda/npm for our tests).
46-
- name: install JupyterHub
47-
run: |
48-
git clone --quiet --branch ${{ matrix.JHUB_VER }} https://github.com/jupyterhub/jupyterhub.git jupyterhub
49-
pip install --pre -r jupyterhub/dev-requirements.txt
50-
pip install --upgrade pytest
51-
pip install --pre -e jupyterhub
52-
53-
- name: pytest
54-
run: |
55-
pytest --lf --cov batchspawner batchspawner/tests -v
49+
- uses: actions/checkout@v2
50+
- name: Set up Python ${{ matrix.python-version }}
51+
uses: actions/setup-python@v2
52+
with:
53+
python-version: ${{ matrix.python-version }}
54+
55+
- name: Install dependencies
56+
run: |
57+
python -m pip install --upgrade pip
58+
python -m pip install pytest
59+
pip install -r requirements.txt
60+
- name: Install nodejs dependencies
61+
run: |
62+
sudo npm install -g configurable-http-proxy
63+
64+
# We need to check compatibility with different versions of the JH
65+
# API, including latest development. For that, we also need to
66+
# pull in the dependencies of that old JH version (but we don't
67+
# need conda/npm for our tests).
68+
- name: install JupyterHub
69+
run: |
70+
git clone --quiet --branch ${{ matrix.JHUB_VER }} https://github.com/jupyterhub/jupyterhub.git jupyterhub
71+
pip install --pre -r jupyterhub/dev-requirements.txt
72+
pip install --upgrade pytest
73+
pip install --pre -e jupyterhub
74+
75+
- name: pytest
76+
run: |
77+
pytest --verbose --color=yes --last-failed --cov batchspawner batchspawner/tests

‎.pre-commit-config.yaml

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# pre-commit is a tool to perform a predefined set of tasks manually and/or
2+
# automatically before git commits are made.
3+
#
4+
# Config reference: https://pre-commit.com/#pre-commit-configyaml---top-level
5+
#
6+
# Common tasks
7+
#
8+
# - Run on all files: pre-commit run --all-files
9+
# - Register git hooks: pre-commit install --install-hooks
10+
#
11+
repos:
12+
# Autoformat: Python code
13+
- repo: https://github.com/ambv/black
14+
rev: 20.8b1
15+
hooks:
16+
- id: black
17+
args: [--target-version=py36]
18+
19+
# Autoformat: markdown, yaml
20+
- repo: https://github.com/pre-commit/mirrors-prettier
21+
rev: v2.2.1
22+
hooks:
23+
- id: prettier
24+
25+
# Lint: Python code
26+
- repo: https://gitlab.com/pycqa/flake8
27+
rev: "3.8.4"
28+
hooks:
29+
- id: flake8
30+
31+
# Misc...
32+
- repo: https://github.com/pre-commit/pre-commit-hooks
33+
rev: v3.4.0
34+
# ref: https://github.com/pre-commit/pre-commit-hooks#hooks-available
35+
hooks:
36+
# Autoformat: Makes sure files end in a newline and only a newline.
37+
- id: end-of-file-fixer
38+
39+
# Autoformat: Sorts entries in requirements.txt.
40+
- id: requirements-txt-fixer
41+
42+
# Prevent giant (500kB) files from being committed.
43+
- id: check-added-large-files
44+
45+
# Lint: Check for files with names that would conflict on a
46+
# case-insensitive filesystem like MacOS HFS+ or Windows FAT.
47+
- id: check-case-conflict
48+
49+
# Lint: Checks that non-binary executables have a proper shebang.
50+
- id: check-executables-have-shebangs

‎CHANGELOG.md

+46-50
Original file line numberDiff line numberDiff line change
@@ -14,85 +14,81 @@ Fixed
1414

1515
Added (user)
1616

17-
* PR #170: SlurmSpawner: add `req_gres` to specify `-go-res`.
18-
* PR #137: GridEngineSpawner: spawner will now add the following system environment values to the spawner environment, in accordance with the Univa Admin Guide: `SGE_CELL`, `SGE_EXECD`, `SGE_ROOT`, `SGE_CLUSTER_NAME`, `SGE_QMASTER_PORT`, `SGE_EXECD_PORT`, `PATH`
17+
- PR #170: SlurmSpawner: add `req_gres` to specify `-go-res`.
18+
- PR #137: GridEngineSpawner: spawner will now add the following system environment values to the spawner environment, in accordance with the Univa Admin Guide: `SGE_CELL`, `SGE_EXECD`, `SGE_ROOT`, `SGE_CLUSTER_NAME`, `SGE_QMASTER_PORT`, `SGE_EXECD_PORT`, `PATH`
1919

2020
Added (developer)
2121

22-
* PR #187: support for unknown job state
22+
- PR #187: support for unknown job state
2323

2424
Changed
2525

26-
* PR #177: Fail on first error in batch script by setting `set -e` to script templates.
27-
* PR #165: SlurmSpawner: Update template to use `--chdir` instead of `--workdir`. Users of Slurm older than 17.11 may need to revert this locally.
28-
* PR #189: remove bashism from default script template
29-
* PR #195: fix exception handling in run_command
30-
* PR #198: change from Travis to gh-actions for testing
31-
* PR #196: documentation
32-
* PR #199: update setup.py
26+
- PR #177: Fail on first error in batch script by setting `set -e` to script templates.
27+
- PR #165: SlurmSpawner: Update template to use `--chdir` instead of `--workdir`. Users of Slurm older than 17.11 may need to revert this locally.
28+
- PR #189: remove bashism from default script template
29+
- PR #195: fix exception handling in run_command
30+
- PR #198: change from Travis to gh-actions for testing
31+
- PR #196: documentation
32+
- PR #199: update setup.py
3333

3434
## v1.0 (requires minimum JupyterHub 0.9 and Python 3.5)
3535

3636
Added (user)
3737

38-
* Add support for JupyterHub named servers. #167
39-
* Add Jinja2 templating as an option for all scripts and commands. If '{{' or `{%` is used anywhere in the string, it is used as a jinja2 template.
40-
* Add new option exec_prefix, which defaults to `sudo -E -u {username}`. This replaces explicit `sudo` in every batch command - changes in local commands may be needed.
41-
* New option: `req_keepvars_extra`, which allows keeping extra variables in addition to what is defined by JupyterHub itself (addition of variables to keep instead of replacement). #99
42-
* Add `req_prologue` and `req_epilogue` options to scripts which are inserted before/after the main jupyterhub-singleuser command, which allow for generic setup/cleanup without overriding the entire script. #96
43-
* SlurmSpawner: add the `req_reservation` option. #91
44-
* Add basic support for JupyterHub progress updates, but this is not used much yet. #86
38+
- Add support for JupyterHub named servers. #167
39+
- Add Jinja2 templating as an option for all scripts and commands. If '{{' or `{%` is used anywhere in the string, it is used as a jinja2 template.
40+
- Add new option exec_prefix, which defaults to `sudo -E -u {username}`. This replaces explicit `sudo` in every batch command - changes in local commands may be needed.
41+
- New option: `req_keepvars_extra`, which allows keeping extra variables in addition to what is defined by JupyterHub itself (addition of variables to keep instead of replacement). #99
42+
- Add `req_prologue` and `req_epilogue` options to scripts which are inserted before/after the main jupyterhub-singleuser command, which allow for generic setup/cleanup without overriding the entire script. #96
43+
- SlurmSpawner: add the `req_reservation` option. #91
44+
- Add basic support for JupyterHub progress updates, but this is not used much yet. #86
4545

4646
Added (developer)
4747

48-
* Add many more tests.
49-
* Add a new page `SPAWNERS.md` which information on specific spawners. Begin trying to collect a list of spawner-specific contacts. #97
50-
* Rename `current_ip` and `current_port` commands to `ip` and `port`. No user impact. #139
51-
* Update to Python 3.5 `async` / `await` syntax to support JupyterHub progress updates. #90
48+
- Add many more tests.
49+
- Add a new page `SPAWNERS.md` which information on specific spawners. Begin trying to collect a list of spawner-specific contacts. #97
50+
- Rename `current_ip` and `current_port` commands to `ip` and `port`. No user impact. #139
51+
- Update to Python 3.5 `async` / `await` syntax to support JupyterHub progress updates. #90
5252

5353
Changed
5454

55-
* PR #58 and #141 changes logic of port selection, so that it is selected *after* the singleuser server starts. This means that the port number has to be conveyed back to JupyterHub. This requires the following changes:
56-
- `jupyterhub_config.py` *must* explicitely import `batchspawner`
57-
- Add a new option `batchspawner_singleuser_cmd` which is used as a wrapper in the single-user servers, which conveys the remote port back to JupyterHub. This is now an integral part of the spawn process.
55+
- PR #58 and #141 changes logic of port selection, so that it is selected _after_ the singleuser server starts. This means that the port number has to be conveyed back to JupyterHub. This requires the following changes:
56+
- `jupyterhub_config.py` _must_ explicitely import `batchspawner`
57+
- Add a new option `batchspawner_singleuser_cmd` which is used as a wrapper in the single-user servers, which conveys the remote port back to JupyterHub. This is now an integral part of the spawn process.
5858
- If you have installed with `pip install -e`, you will have to re-install so that the new script `batchspawner-singleuser` is added to `$PATH`.
59-
* Update minimum requirements to JupyterHub 0.9 and Python 3.5. #143
60-
* Update Slurm batch script. Now, the single-user notebook is run in a job step, with a wrapper of `srun`. This may need to be removed using `req_srun=''` if you don't want environment variables limited.
61-
* Pass the environment dictionary to the queue and cancel commands as well. This is mostly user environment, but may be useful to these commands as well in some cases. #108, #111 If these environment variables were used for authentication as an admin, be aware that there are pre-existing security issues because they may be passed to the user via the batch submit command, see #82.
62-
59+
- Update minimum requirements to JupyterHub 0.9 and Python 3.5. #143
60+
- Update Slurm batch script. Now, the single-user notebook is run in a job step, with a wrapper of `srun`. This may need to be removed using `req_srun=''` if you don't want environment variables limited.
61+
- Pass the environment dictionary to the queue and cancel commands as well. This is mostly user environment, but may be useful to these commands as well in some cases. #108, #111 If these environment variables were used for authentication as an admin, be aware that there are pre-existing security issues because they may be passed to the user via the batch submit command, see #82.
6362

6463
Fixed
6564

66-
* Improve debugging on failed submission by raising errors including error messages from the commands. #106
67-
* Many other non-user or developer visible changes. #107 #106 #100
68-
* In Travis CI, blacklist jsonschema=3.0.0a1 because it breaks tests
65+
- Improve debugging on failed submission by raising errors including error messages from the commands. #106
66+
- Many other non-user or developer visible changes. #107 #106 #100
67+
- In Travis CI, blacklist jsonschema=3.0.0a1 because it breaks tests
6968

7069
Removed
7170

72-
7371
## v0.8.1 (bugfix release)
7472

75-
* Fix regression: single-user server binding address is overwritten by previous session server address, resulting in failure to start. Issue #76
73+
- Fix regression: single-user server binding address is overwritten by previous session server address, resulting in failure to start. Issue #76
7674

7775
## v0.8.0 (compatible with JupyterHub 0.5.0 through 0.8.1/0.9dev)
7876

79-
* SlurmSpawner: Remove `--uid` for (at least) Slurm 17.11 compatibility. If you use `sudo`, this should not be necessary, but because this is security related you should check that user management is as you expect. If your configuration does not use `sudo` then you may need to add the `--uid` option in a custom `batch_script`.
80-
* add base options `req_ngpus` `req_partition` `req_account` and `req_options`
81-
* Fix up logging
82-
* Merge `user_options` with the template substitution vars instead of having it as a separate key
83-
* Update ip/port handling for JupyterHub 0.8
84-
* Add `LICENSE` (BSD3) and `CONTRIBUTING.md`
85-
* Add `LsfSpawner` for IBM LFS
86-
* Add `MultiSlurmSpawner`
87-
* Add `MoabSpawner`
88-
* Add `condorSpawner`
89-
* Add `GridEngineSpawner`
90-
* SlurmSpawner: add `req_qos` option
91-
* WrapSpawner and ProfilesSpawner, which provide mechanisms for runtime configuration of spawners, have been split out and moved to the [`wrapspawner`](https://github.com/jupyterhub/wrapspawner) package
92-
* Enable CI testing via Travis-CI
93-
77+
- SlurmSpawner: Remove `--uid` for (at least) Slurm 17.11 compatibility. If you use `sudo`, this should not be necessary, but because this is security related you should check that user management is as you expect. If your configuration does not use `sudo` then you may need to add the `--uid` option in a custom `batch_script`.
78+
- add base options `req_ngpus` `req_partition` `req_account` and `req_options`
79+
- Fix up logging
80+
- Merge `user_options` with the template substitution vars instead of having it as a separate key
81+
- Update ip/port handling for JupyterHub 0.8
82+
- Add `LICENSE` (BSD3) and `CONTRIBUTING.md`
83+
- Add `LsfSpawner` for IBM LFS
84+
- Add `MultiSlurmSpawner`
85+
- Add `MoabSpawner`
86+
- Add `condorSpawner`
87+
- Add `GridEngineSpawner`
88+
- SlurmSpawner: add `req_qos` option
89+
- WrapSpawner and ProfilesSpawner, which provide mechanisms for runtime configuration of spawners, have been split out and moved to the [`wrapspawner`](https://github.com/jupyterhub/wrapspawner) package
90+
- Enable CI testing via Travis-CI
9491

9592
## v0.3 (tag: jhub-0.3, compatible with JupyterHub 0.3.0)
9693

97-
* initial release containing `TorqueSpawner` and `SlurmSpawner`
98-
94+
- initial release containing `TorqueSpawner` and `SlurmSpawner`

0 commit comments

Comments
 (0)