From f2b0487b0a6a59dbd73c6c9e46871c8d7b2d8975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ois=C3=ADn=20Morrison?= Date: Fri, 9 Aug 2024 14:47:27 +0000 Subject: [PATCH 1/8] simplify installation and dependencies --- .flake8 | 3 --- .github/workflows/ci.yml | 16 ++++------- .github/workflows/on-push.yml | 30 +++++++++------------ .pre-commit-config.yaml | 51 ++++++++++++++++++++++++++--------- README.md | 37 ++++++++++++++++++------- environment.yml | 29 -------------------- pyproject.toml | 30 ++++++++++++--------- 7 files changed, 103 insertions(+), 93 deletions(-) delete mode 100644 .flake8 delete mode 100644 environment.yml diff --git a/.flake8 b/.flake8 deleted file mode 100644 index b2e4bad..0000000 --- a/.flake8 +++ /dev/null @@ -1,3 +0,0 @@ -# required for flake8 and black to play nicely -[flake8] -max-line-length = 88 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17c11b7..9afc47c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: - run: pip install black flake8 isort - run: isort --check . - run: black --check . - - run: flake8 . + - run: ruff check docs: runs-on: ubuntu-latest @@ -27,17 +27,11 @@ jobs: setup_and_test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: mamba-org/setup-micromamba@v1 - with: - activate-environment: test - environment-file: environment.yml - auto-activate-base: false + - uses: actions/checkout@v3 + - uses: actions/setup-python@v2 - name: install hat package shell: bash -l {0} - run: pip install . + run: pip install .[test] - name: run tests shell: bash -l {0} - run: | - pip install pytest - pytest + run: pytest diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index e4bb583..1e67cc5 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -23,39 +23,35 @@ jobs: name: setup runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-python@v2 - - run: python -m pip install . + - name: install hat package + shell: bash -l {0} + run: pip install . test: name: test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: mamba-org/setup-micromamba@v1 - with: - activate-environment: test - environment-file: environment.yml - auto-activate-base: false - - name: install hat package + - uses: actions/checkout@v3 + - uses: actions/setup-python@v2 + - name: developer install of hat package shell: bash -l {0} - run: pip install . + run: pip install .[test] - name: run tests shell: bash -l {0} - run: | - pip install pytest - pytest + run: pytest distribution: runs-on: ubuntu-latest - needs: [setup, test] - + needs: [setup] steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v2 - name: Build distributions run: | - $CONDA/bin/python -m pip install build - $CONDA/bin/python -m build + pip install build + python build - name: Publish a Python distribution to PyPI if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5b536b2..164d454 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,40 @@ +default_language_version: + python: python3 +default_stages: + - commit + - push repos: - - repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - - repo: https://github.com/psf/black - rev: 23.3.0 - hooks: - - id: black -# - repo: https://github.com/PyCQA/flake8 -# rev: 6.0.0 -# hooks: -# - id: flake8 + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.5.6 + hooks: + - id: ruff + args: [ --fix ] + - id: ruff-format + - repo: https://github.com/psf/black + rev: 24.8.0 + hooks: + - id: black + - repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort + args: ["--profile", "black", "--filter-files"] + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: detect-private-key + - id: check-ast + - id: end-of-file-fixer + - id: mixed-line-ending + args: [--fix=lf] + - id: trailing-whitespace + - id: check-case-conflict + - repo: local + hooks: + - id: forbid-to-commit + name: Don't commit rej files + entry: | + Cannot commit .rej files. These indicate merge conflicts that arise during automated template updates. + Fix the merge conflicts manually and remove the .rej files. + language: fail + files: '.*\.rej$' \ No newline at end of file diff --git a/README.md b/README.md index 69cc149..4b76a7c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,6 @@ HAT is a suite of tools to perform data analysis on hydrological datasets. The documentation can be found at https://hydro-analysis-toolkit.readthedocs.io - **DISCLAIMER** This project is **BETA** and will be **Experimental** for the foreseeable future. Interfaces and functionality are likely to change, and the project itself may be scrapped. @@ -16,27 +15,47 @@ Clone source code repository $ git clone https://github.com/ecmwf/hat.git -Create conda python environment +Create and activate conda environment + + $ conda env create -n hat python<=3.10 + $ conda activate hat + +For default installation, run - $ cd hat - $ conda env create -f environment.yml $ pip install . -### Usage +For a developer installation (includes linting and test libraries), run -Start a hat environment + $ pip install .[dev] + $ pre-commit install - $ conda activate hat +If you plan to build a source and a wheel distribution, it is additionally required to run + + $ pip install build + +### Usage Run a command line tool $ hat-extract-timeseries --help +### Running the tests + +Tests are stored in the `tests/` folder and can be run with + + $ pytest + +### Deployment + +To build a source and a wheel distribution, run + + $ python build + ### Contributing -The main repository is hosted on [GitHub](https://github.com/ecmwf/hat), testing, bug reports and contributions are highly welcomed and appreciated. +The main repository is hosted on [GitHub](https://github.com/ecmwf/hat). Testing, bug reports and contributions are highly welcomed and appreciated. -Please report [bug](https://github.com/ecmwf/hat/issues) reports or [pull-requests](https://github.com/ecmwf/hat/pulls) on [GitHub](https://github.com/ecmwf/hat) +Please report [bug](https://github.com/ecmwf/hat/issues) reports or [pull-requests](https://github.com/ecmwf/hat/pulls) on [GitHub](https://github.com/ecmwf/hat). We want your feedback, please e-mail: user-services@ecmwf.int diff --git a/environment.yml b/environment.yml deleted file mode 100644 index 0ff4ee5..0000000 --- a/environment.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: hat -channels: - - conda-forge -dependencies: - - python<=3.10 - - netCDF4 - - eccodes - - cfgrib - - geopandas - - xarray - - plotly - - matplotlib - - jupyter - - tqdm - - typer - - humanize - - ipyleaflet - - ipywidgets - - pip - - pytest - - mkdocs - - black - - flake8 - - isort - - pre-commit - - geopy - - pandas - - pip: - - earthkit.data diff --git a/pyproject.toml b/pyproject.toml index 794cfd2..1145af5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,3 @@ - [build-system] requires = ["setuptools>=65", "setuptools_scm[toml]>=6.2"] build-backend = "setuptools.build_meta" @@ -40,7 +39,12 @@ dependencies = [ "ipyleaflet", "ipywidgets", "earthkit-data", - "cfgrib" + "cfgrib", + "netCDF4", + "eccodes", + "plotly", + "jupyter", + "geopy" ] [project.urls] @@ -50,16 +54,14 @@ dependencies = [ [project.optional-dependencies] test = [ - "pytest", + "pytest" ] - docs = [ - "mkdocs", - "mkdocs-material", - "mkdocstrings-python", - "mkdocs-autorefs", - "mkdocs-gen-files", - "pymdown-extensions", - "markdown-exec[ansi]", + dev = [ + "pytest", + "black", + "ruff", + "isort", + "pre-commit" ] [project.scripts] @@ -67,8 +69,12 @@ dependencies = [ hat-hydrostats = "hat.tools.hydrostats_cli:main" hat-stations-mapping = "hat.mapping.station_mapping:main" +# Code inspection +[tool.black] +line-length = 120 + [tool.isort] -profile="black" +profile = "black" # Testing [tool.pytest] From 24baa31e34336bc0a5064795b3d5b35a3854347b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ois=C3=ADn=20Morrison?= Date: Mon, 12 Aug 2024 07:34:01 +0000 Subject: [PATCH 2/8] fix typo in installation instructions --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4b76a7c..b9c0c0a 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Clone source code repository Create and activate conda environment - $ conda env create -n hat python<=3.10 + $ conda env create -n hat python=3.10 $ conda activate hat For default installation, run @@ -34,7 +34,7 @@ If you plan to build a source and a wheel distribution, it is additionally requi $ pip install build ### Usage - + Run a command line tool $ hat-extract-timeseries --help From 67acae911731f2c3585474d327368c18805e0b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ois=C3=ADn=20Morrison?= Date: Mon, 12 Aug 2024 07:38:06 +0000 Subject: [PATCH 3/8] fix install in ci action --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9afc47c..5479160 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,23 +7,23 @@ on: branches: [ "main", "develop" ] jobs: - + quality_assurance: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - run: pip install black flake8 isort + - run: pip install black ruff isort - run: isort --check . - run: black --check . - run: ruff check - + docs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - run: pip install -r docs/requirements.txt - run: mkdocs build - + setup_and_test: runs-on: ubuntu-latest steps: From 7c06f02f452ec37f33537713d87dd6d5fc7e182b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ois=C3=ADn=20Morrison?= Date: Mon, 12 Aug 2024 07:49:59 +0000 Subject: [PATCH 4/8] add needs statements to github workflow --- .github/workflows/on-push.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 1e67cc5..952b463 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -32,6 +32,7 @@ jobs: test: name: test runs-on: ubuntu-latest + needs: [setup] steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v2 @@ -43,16 +44,20 @@ jobs: run: pytest distribution: + name: distribution runs-on: ubuntu-latest needs: [setup] steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v2 - - name: Build distributions + - name: install hat package + shell: bash -l {0} + run: pip install . + - name: build distributions run: | pip install build python build - - name: Publish a Python distribution to PyPI + - name: publish a Python distribution to PyPI if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@release/v1 with: From 809bf9c221050cdf60a0801ea8bbdffb09247e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ois=C3=ADn=20Morrison?= Date: Mon, 12 Aug 2024 08:18:25 +0000 Subject: [PATCH 5/8] keep same linelength in linting settings --- pyproject.toml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1145af5..d281fb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,9 +70,6 @@ dependencies = [ hat-stations-mapping = "hat.mapping.station_mapping:main" # Code inspection -[tool.black] -line-length = 120 - [tool.isort] profile = "black" @@ -104,4 +101,4 @@ write_to_template = ''' __version__ = "{version}" ''' parentdir_prefix_version='hat-' # get version from GitHub-like tarballs -fallback_version='0.7.1' \ No newline at end of file +fallback_version='0.7.1' From 811dde23a6e55e5ea3b8953aaed76312a501a4da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ois=C3=ADn=20Morrison?= Date: Tue, 13 Aug 2024 13:15:47 +0000 Subject: [PATCH 6/8] update linting to use just ruff --- .github/workflows/ci.yml | 4 +--- .pre-commit-config.yaml | 11 +---------- README.md | 5 +++++ pyproject.toml | 15 +++++++++++---- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5479160..26dd3cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,9 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - run: pip install black ruff isort - - run: isort --check . - - run: black --check . + - run: pip install ruff - run: ruff check docs: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 164d454..680d886 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,15 +10,6 @@ repos: - id: ruff args: [ --fix ] - id: ruff-format - - repo: https://github.com/psf/black - rev: 24.8.0 - hooks: - - id: black - - repo: https://github.com/PyCQA/isort - rev: 5.13.2 - hooks: - - id: isort - args: ["--profile", "black", "--filter-files"] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: @@ -37,4 +28,4 @@ repos: Cannot commit .rej files. These indicate merge conflicts that arise during automated template updates. Fix the merge conflicts manually and remove the .rej files. language: fail - files: '.*\.rej$' \ No newline at end of file + files: '.*\.rej$' diff --git a/README.md b/README.md index b9c0c0a..43204cb 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Interfaces and functionality are likely to change, and the project itself may be Clone source code repository $ git clone https://github.com/ecmwf/hat.git + $ cd hat Create and activate conda environment @@ -29,6 +30,10 @@ For a developer installation (includes linting and test libraries), run $ pip install .[dev] $ pre-commit install +If you only plan to run the tests, instead run + + $ pip install .[test] + If you plan to build a source and a wheel distribution, it is additionally required to run $ pip install build diff --git a/pyproject.toml b/pyproject.toml index d281fb0..dd4f662 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,9 +58,7 @@ dependencies = [ ] dev = [ "pytest", - "black", "ruff", - "isort", "pre-commit" ] @@ -70,8 +68,17 @@ dependencies = [ hat-stations-mapping = "hat.mapping.station_mapping:main" # Code inspection -[tool.isort] -profile = "black" +[tool.ruff] +line-length = 120 +indent-width = 4 + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["E402", "F403", "F401"] +"**/{tests}/*" = ["E402"] # Testing [tool.pytest] From 6479fc293eefb9d7ecd301c921fab261a8746a3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ois=C3=ADn=20Morrison?= Date: Wed, 14 Aug 2024 08:17:06 +0000 Subject: [PATCH 7/8] update documentation --- README.md | 7 +++++-- docs/index.md | 5 ++++- docs/installation.md | 12 ++++-------- docs/usage.md | 3 +-- mkdocs.yml | 4 ++-- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 43204cb..7a97ae0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # HAT - Hydrological Analysis Toolkit -HAT is a suite of tools to perform data analysis on hydrological datasets. +The Hydrological Analysis Toolkit (HAT) is a software suite for hydrologists working with simulated and observed river discharge. HAT performs data analysis on hydrological datasets, with its main features being: +- mapping station locations into hydrological model grids +- extraction of timeseries +- statistical analysis of hydrological timeseries The documentation can be found at https://hydro-analysis-toolkit.readthedocs.io @@ -18,7 +21,7 @@ Clone source code repository Create and activate conda environment - $ conda env create -n hat python=3.10 + $ conda env create -n hat python=3.10 $ conda activate hat For default installation, run diff --git a/docs/index.md b/docs/index.md index 5069611..d1f2ff9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,3 +1,6 @@ # Welcome to HAT -Hydrological Analysis Tools (HAT) is a suite of software for hydrologists working with simulated and observed river discharge. +The Hydrological Analysis Toolkit (HAT) is a software suite for hydrologists working with simulated and observed river discharge. HAT performs data analysis on hydrological datasets, with its main features being: +- mapping station locations into hydrological model grids +- extraction of timeseries +- statistical analysis of hydrological timeseries diff --git a/docs/installation.md b/docs/installation.md index 8766a50..3b1abc8 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -2,20 +2,16 @@ Clone source code repository - git clone git@github.com:ecmwf-projects/hat.git + git clone https://github.com/ecmwf/hat.git + cd hat Create conda python environment - cd hat - # If on HPC.. # module load conda - - conda env create -f environment.yml + conda env create -n hat python=3.10 conda activate hat -Install HAT software +Installation of required dependencies pip install . - -Open a new terminal to start \ No newline at end of file diff --git a/docs/usage.md b/docs/usage.md index 40603eb..f0d998a 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -2,7 +2,7 @@ If you have already [installed](installation.md) hat then -#### Activate environment +#### Activate Environment $ conda activate hat @@ -21,4 +21,3 @@ In your python code you can import the hat module import hat For examples, please see these [jupyter notebooks](https://github.com/ecmwf-projects/hat/tree/main/notebooks) - diff --git a/mkdocs.yml b/mkdocs.yml index 8ecbd28..5ec673a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,4 @@ -site_name: Hydrological Analysis Tools +site_name: Hydrological Analysis Toolkit site_url: https://ecmwf.github.io/hat/ nav: @@ -10,4 +10,4 @@ nav: - hat-extract-timeseries: hat_extract-timeseries.md - hat-hydrostats: hat_hydrostats.md -theme: readthedocs \ No newline at end of file +theme: readthedocs From 1fa34c6d48c401bcba3b9b82f2926f670fa85a95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ois=C3=ADn=20Morrison?= Date: Thu, 15 Aug 2024 11:34:50 +0000 Subject: [PATCH 8/8] update linting settings --- pyproject.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dd4f662..f2a292f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,7 @@ dependencies = [ hat-hydrostats = "hat.tools.hydrostats_cli:main" hat-stations-mapping = "hat.mapping.station_mapping:main" -# Code inspection +# Linting settings [tool.ruff] line-length = 120 indent-width = 4 @@ -76,10 +76,6 @@ indent-width = 4 quote-style = "double" indent-style = "space" -[tool.ruff.lint.per-file-ignores] -"__init__.py" = ["E402", "F403", "F401"] -"**/{tests}/*" = ["E402"] - # Testing [tool.pytest] addopts = "--pdbcls=IPython.terminal.debugger:Pdb"