From c116c7b8d7d5bb95f064a996a6e6879cd95314aa Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 15 Feb 2024 12:02:47 -0700 Subject: [PATCH 01/10] Add some examples for coverage --- ndindex/tests/test_chunking.py | 1 + ndindex/tests/test_shapetools.py | 1 + 2 files changed, 2 insertions(+) diff --git a/ndindex/tests/test_chunking.py b/ndindex/tests/test_chunking.py index 9c02cf04..650d5375 100644 --- a/ndindex/tests/test_chunking.py +++ b/ndindex/tests/test_chunking.py @@ -205,6 +205,7 @@ def test_num_subchunks_error(): raises(ValueError, lambda: next(ChunkSize((1, 2)).num_subchunks(..., (1, 2, 3)))) +@example((2, 2), (0, False), (5, 5)) @example((5,), [0, 7], (15,)) @example((5,), [], (15,)) @given(chunk_sizes(), ndindices, chunk_shapes) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index ba8333e3..d47501c9 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -496,6 +496,7 @@ def test_asshape(): raises(TypeError, lambda: asshape(np.int64(1), allow_int=False)) raises(IndexError, lambda: asshape((2, 3), 3)) +@example([(5,)], (10,)) @example([], []) @example([()], []) @example([(0, 1)], 0) From 641b627958d405715188f4d2960c79e6914aa6e0 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 15 Feb 2024 13:40:57 -0700 Subject: [PATCH 02/10] Update the ndindex() docstring to recommend the functional syntax first --- ndindex/ndindex.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index 49b76cbb..6bb4cf13 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -12,18 +12,18 @@ class NDIndexConstructor: (generally, the same error NumPy would raise if the index were used on an array). - Indices are created by calling `ndindex` with getitem syntax: + Indices are created by calling the `ndindex` with raw index objects: >>> from ndindex import ndindex - >>> ndindex[1] - Integer(1) - >>> ndindex[0:10, :] - Tuple(slice(0, 10, None), slice(None, None, None)) + >>> ndindex(slice(0, 10)) + Slice(0, 10, None) + >>> ndindex((slice(0, 10), 0)) + Tuple(slice(0, 10, None), 0) - You can also create indices by calling `ndindex(idx)` like a function. - However, if you do this, you cannot use the `a:b` slice syntax, as it is - not syntactically valid: + Indices can also be created by calling `ndindex` with getitem syntax. + >>> ndindex[1] + Integer(1) >>> ndindex[0:10] Slice(0, 10, None) >>> ndindex(0:10) @@ -32,9 +32,10 @@ class NDIndexConstructor: ndindex(0:10) ^ SyntaxError: invalid syntax - >>> ndindex(slice(0, 10)) - Slice(0, 10, None) + The `ndindex[idx]` form should generally be preferred when creating an + index from a tuple or slice literal, since `ndindex(a:b)` is not + syntactically valid and must be typed as `ndindex(slice(a, b))`. Additionally, the `ndindex[idx]` syntax does not require parentheses when creating a tuple index: @@ -47,9 +48,6 @@ class NDIndexConstructor: >>> ndindex((0, 1)) Tuple(0, 1) - Therefore `ndindex[idx]` should generally be preferred when creating an - index from a tuple or slice literal. - """ def __getitem__(self, obj): if isinstance(obj, NDIndex): From 3fbb58e51c8ca1916d72b4be26d3a79214ba2bd1 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 15 Feb 2024 13:41:23 -0700 Subject: [PATCH 03/10] Document that skip_axes can be a list of tuples --- ndindex/shapetools.py | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 567c9d91..db518cd8 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -53,6 +53,11 @@ def broadcast_shapes(*shapes, skip_axes=()): `, except is also supports skipping axes in the shape with `skip_axes`. + `skip_axes` can be a tuple of integers which apply to all shapes, or a + list of tuples of integers, one for each shape, which apply to each + respective shape. The `skip_axes` argument works the same as in + :func:`iter_indices`. See its docstring for more details. + If the shapes are not broadcast compatible (excluding `skip_axes`), :class:`BroadcastError` is raised. @@ -66,11 +71,11 @@ def broadcast_shapes(*shapes, skip_axes=()): Axes in `skip_axes` apply to each shape *before* being broadcasted. Each shape will be broadcasted together with these axes removed. The dimensions - in skip_axes do not need to be equal or broadcast compatible with one + in `skip_axes` do not need to be equal or broadcast compatible with one another. The final broadcasted shape be the result of broadcasting all the non-skip axes. - >>> broadcast_shapes((10, 3, 2), (20, 2), skip_axes=(0,)) + >>> broadcast_shapes((10, 3, 2), (2, 20), skip_axes=[(0,), (1,)]) (3, 2) """ @@ -126,16 +131,20 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): The remaining axes will be indexed one element at a time with integer indices. - `skip_axes` should be a tuple of axes to skip. It can use negative - integers, e.g., `skip_axes=(-1,)` will skip the last axis (but note that - mixing negative and nonnegative skip axes is currently not supported). The - order of the axes in `skip_axes` does not matter. The axes in `skip_axes` - refer to the shapes *before* broadcasting (if you want to refer to the - axes after broadcasting, either broadcast the shapes and arrays first, or - refer to the axes using negative integers). For example, - `iter_indices((10, 2), (20, 1, 2), skip_axes=(0,))` will skip the size - `10` axis of `(10, 2)` and the size `20` axis of `(20, 1, 2)`. The result - is two sets of indices, one for each element of the non-skipped + `skip_axes` should be a tuple of axes to skip or a list of tuples of axes + to skip. If it is a single tuple, it applies to all shapes. Otherwise, + each tuple applies to each shape respectively. It can use negative + integers, e.g., `skip_axes=(-1,)` will skip the last axis. The order of + the axes in `skip_axes` does not matter. Mixing negative and nonnegative + skip axes is supported, but the skip axes must refer to unique dimensions + for each shape. + + The axes in `skip_axes` refer to the shapes *before* broadcasting (if you + want to refer to the axes after broadcasting, either broadcast the shapes + and arrays first, or refer to the axes using negative integers). For + example, `iter_indices((10, 2), (20, 1, 2), skip_axes=(0,))` will skip the + size `10` axis of `(10, 2)` and the size `20` axis of `(20, 1, 2)`. The + result is two sets of indices, one for each element of the non-skipped dimensions: >>> from ndindex import iter_indices @@ -194,9 +203,8 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): [110, 111, 112]]) To include an index into the final broadcasted array, you can simply - include the final broadcasted shape as one of the shapes (the NumPy - function :func:`np.broadcast_shapes() ` is - useful here). + include the final broadcasted shape as one of the shapes (the function + :func:`broadcast_shapes` is useful here). >>> np.broadcast_shapes((1, 3), (2, 1)) (2, 3) From d3798132ad39d91c922cd1224ebf7cc6c9259f07 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 15 Feb 2024 13:41:34 -0700 Subject: [PATCH 04/10] Add sphinx-autobuild to the doc requirements file --- docs/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 125d74ef..dbec7740 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,3 +3,4 @@ linkify-it-py myst-parser sphinx sphinx-copybutton +sphinx-autobuild From 4ad6f90b630ba5b09587c44d9b2486d572bb78e1 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 15 Feb 2024 13:41:49 -0700 Subject: [PATCH 05/10] Add changelog entries for 1.8 --- docs/changelog.md | 56 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 11f3c2af..69f133be 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,8 +1,62 @@ # ndindex Changelog +## Version 1.8 (2024-02-15) + +### Major Changes + +- **Breaking** {func}`~.broadcast_shapes` no longer returns `None` in the + place of skipped axes. The result is now just the non-skipped axes + broadcasted together. + +- The `skip_axes` flag to {func}`~.iter_indices` and + {func}`~.broadcast_shapes` can now be a list of tuples, of skipped axes, + which apply to each respective shape independently. + +- Mixing negative and nonnegative `skip_axes` in {func}`~.iter_indices` and + {func}`~.broadcast_shapes` is now supported. The only restriction is that + skip axes must refer to unique dimensions for each shape. + +- New index method [`selected_indices()`](Tuple.selected_indices), which + iterates indices corresponding to each element selected by the given index + on an array of a given `shape`. + +- ndindex indices can now be constructed by slicing the {func}`~.ndindex` + constructor function, like `ndindex[0:10]`. This is generally preferred for + indices with explicit slices, as this allows using the usual `:` slice + syntax instead of requiring slices to be spelled out with the `slice` + function. + +- Add a `negative_int` flag to [`reduce`](Integer.reduce), which makes it + normalize integer indices to negative integers when a shape is provided. + +- {class}`~.Slice` objects now hash to the same hash value as their + corresponding raw `slice` in Python 3.12, which now allows [native `slice` + objects to be + hashed](https://docs.python.org/3/whatsnew/3.12.html#other-language-changes). + +- Fix an incorrect result from {any}`ChunkSize.as_subchunks()` and + {any}`ChunkSize.num_subchunks()` when using multiple array indices or a + boolean array index with multiple dimensions. + +### Minor Changes + +- Drop support for Python 3.7. + +- Fully support the upcoming NumPy 2.0 release. This mostly only relevant for + the ndindex test suite, although it does also mean that one ndindex error + message has been updated to match updated wording from NumPy. + +- Specify Cython language_level, which leads to shorter code and gets rid of a + bunch of warnings. (@keszybz) + +- Rename the default git branch from `master` to `main`. + + +### Minor Changes + ## Version 1.7 (2023-04-20) -## Major Changes +### Major Changes - **Breaking:** the `skip_axes` argument {func}`~.iter_indices` function now applies the skipped axes *before* broadcasting, not after. This behavior is From 067cad5c49035bfa683465731d335d92d079c155 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 15 Feb 2024 13:45:16 -0700 Subject: [PATCH 06/10] Add a GitHub Actions workflow for trusted publishing to PyPI --- .github/workflows/publish-package.yml | 115 ++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 .github/workflows/publish-package.yml diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml new file mode 100644 index 00000000..8a4c97f2 --- /dev/null +++ b/.github/workflows/publish-package.yml @@ -0,0 +1,115 @@ +name: publish distributions +on: + push: + branches: + - main + tags: + - '[0-9]+.[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+' + pull_request: + branches: + - main + release: + types: [published] + workflow_dispatch: + inputs: + publish: + type: choice + description: 'Publish to TestPyPI?' + options: + - false + - true + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: Build Python distribution + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install python-build and twine + run: | + python -m pip install --upgrade pip setuptools + python -m pip install build twine + python -m pip list + + - name: Install dependencies + run: python -m pip install -r requirements-dev.txt + + - name: Build a wheel and a sdist + run: | + CYTHONIZE_NDINDEX=0 PYTHONWARNINGS=error,default::DeprecationWarning python -m build . + + - name: Verify the distribution + run: twine check --strict dist/* + + - name: List contents of sdist + run: python -m tarfile --list dist/ndindex-*.tar.gz + + - name: List contents of wheel + run: python -m zipfile --list dist/ndindex-*.whl + + - name: Upload distribution artifact + uses: actions/upload-artifact@v4 + with: + name: dist-artifact + path: dist + + publish: + name: Publish Python distribution to (Test)PyPI + if: github.event_name != 'pull_request' && github.repository == 'data-apis/ndindex' + needs: build + runs-on: ubuntu-latest + # Mandatory for publishing with a trusted publisher + # c.f. https://docs.pypi.org/trusted-publishers/using-a-publisher/ + permissions: + id-token: write + contents: write + # Restrict to the environment set for the trusted publisher + environment: + name: publish-package + + steps: + - name: Download distribution artifact + uses: actions/download-artifact@v4 + with: + name: dist-artifact + path: dist + + - name: List all files + run: ls -lh dist + + - name: Publish distribution 📦 to Test PyPI + # Publish to TestPyPI on tag events of if manually triggered + # Compare to 'true' string as booleans get turned into strings in the console + if: >- + (github.event_name == 'push' && startsWith(github.ref, 'refs/tags')) + || (github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true') + uses: pypa/gh-action-pypi-publish@v1.8.11 + with: + repository-url: https://test.pypi.org/legacy/ + print-hash: true + + - name: Create GitHub Release from a Tag + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + files: dist/* + + - name: Publish distribution 📦 to PyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@v1.8.11 + with: + print-hash: true From eea704984e39b1f7c434fd8f162554f6e05e4f3d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 15 Feb 2024 13:53:11 -0700 Subject: [PATCH 07/10] Remove rever --- rever.xsh | 62 ------------------------------------------------------- 1 file changed, 62 deletions(-) delete mode 100644 rever.xsh diff --git a/rever.xsh b/rever.xsh deleted file mode 100644 index 33e66b3b..00000000 --- a/rever.xsh +++ /dev/null @@ -1,62 +0,0 @@ -from tempfile import mkdtemp - -from rever.activity import activity -from rever.conda import run_in_conda_env - -@activity -def mktmp(): - curdir = $(pwd).strip() - tmpdir = mkdtemp(prefix=$GITHUB_REPO + "_") - print(f"Running in {tmpdir}") - cd @(tmpdir) - git clone @(curdir) - cd $GITHUB_REPO - -@activity -def run_tests(): - # Don't use the built-in pytest action because that uses Docker, which is - # overkill and requires installing Docker - with run_in_conda_env(['python=3.10', 'pytest', 'hypothesis', 'numpy', - 'pyflakes', 'pytest-cov', 'pytest-flakes', 'mkl']): - pyflakes ndindex - python -We:invalid -We::SyntaxWarning -m compileall -f -q ndindex/ - ./run_doctests - pytest - -@activity -def build_docs(): - with run_in_conda_env(['python=3.10', 'sphinx', 'myst-parser', 'numpy', - 'sphinx-copybutton', 'furo']): - cd docs - make html - cd .. - -@activity -def annotated_tag(): - # https://github.com/regro/rever/issues/212 - git tag -a -m "$GITHUB_REPO $VERSION release" $VERSION - -# Ensure the wheels do not build with Cythonization. -$CYTHONIZE_NDINDEX = 0 - -$PROJECT = 'ndindex' -$ACTIVITIES = [ - # 'mktmp', - 'run_tests', - 'build_docs', - 'annotated_tag', # Creates a tag for the new version number - 'pypi', # Sends the package to pypi - 'push_tag', # Pushes the tag up to the $TAG_REMOTE - 'ghrelease', # Creates a Github release entry for the new tag - # 'ghpages', # Update GitHub Pages -] - -$PUSH_TAG_REMOTE = 'git@github.com:Quansight-Labs/ndindex.git' # Repo to push tags to - -$GITHUB_ORG = 'Quansight-Labs' # Github org for Github releases and conda-forge -$GITHUB_REPO = 'ndindex' # Github repo for Github releases and conda-forge - -$GHPAGES_REPO = 'git@github.com:Quansight-Labs/ndindex.git' -$GHPAGES_COPY = $GHPAGES_COPY = ( - ('docs/_build/html', '$GHPAGES_REPO_DIR'), -) From 5967ff78d0ea96be7160df91e0a7ba8237bef7c2 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 15 Feb 2024 13:53:39 -0700 Subject: [PATCH 08/10] Test installation on CI with pip install --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d290600a..3b915189 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -44,7 +44,7 @@ jobs: pytest $PYTEST_FLAGS ./run_doctests # Make sure it installs - python setup.py install + pip install . # Coverage. This also sets the failing status if the # coverage is not 100%. Travis sometimes cuts off the last command, which is # why we print stuff at the end. From b5f1742108c4988dcad26bc1e25f91f18b514219 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 15 Feb 2024 17:47:19 -0700 Subject: [PATCH 09/10] Fix GitHub org name in publish-package --- .github/workflows/publish-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml index 8a4c97f2..69cb5580 100644 --- a/.github/workflows/publish-package.yml +++ b/.github/workflows/publish-package.yml @@ -69,7 +69,7 @@ jobs: publish: name: Publish Python distribution to (Test)PyPI - if: github.event_name != 'pull_request' && github.repository == 'data-apis/ndindex' + if: github.event_name != 'pull_request' && github.repository == 'Quansight-Labs/ndindex' needs: build runs-on: ubuntu-latest # Mandatory for publishing with a trusted publisher From b6a515242e33cfcc8d7eb30f3c502e7de0a023c1 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 16 Feb 2024 01:34:30 -0700 Subject: [PATCH 10/10] Add an example for coverage --- ndindex/tests/test_booleanarray.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/tests/test_booleanarray.py b/ndindex/tests/test_booleanarray.py index 171140b6..75a1b8e7 100644 --- a/ndindex/tests/test_booleanarray.py +++ b/ndindex/tests/test_booleanarray.py @@ -51,6 +51,7 @@ def test_booleanarray_reduce_no_shape_hypothesis(idx, shape, kwargs): check_same(a, index.raw, ndindex_func=lambda a, x: a[x.reduce(**kwargs).raw]) +@example(full((1, 9), True), 3, {}) @example(full((1, 9), True), (3, 3), {}) @example(full((1, 9), False), (3, 3), {}) @given(boolean_arrays, one_of(short_shapes, integers(0, 10)), reduce_kwargs)