diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0ffef95e..6ff0f982 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -5,7 +5,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.10'] + python-version: ['3.12'] fail-fast: false steps: - uses: actions/checkout@v2 @@ -16,39 +16,10 @@ jobs: run: | set -x set -e - # $CONDA is an environment variable pointing to the root of the miniconda directory - echo $CONDA/bin >> $GITHUB_PATH - conda config --set always_yes yes --set changeps1 no - conda config --add channels conda-forge - conda update -q conda - conda info -a - conda create -n test-environment python=${{ matrix.python-version }} --file docs/requirements.txt - conda init + python -m pip install -r docs/requirements.txt - name: Build Docs run: | - # Copied from .bashrc. We can't just source .bashrc because it exits - # when the shell isn't interactive. - - # >>> conda initialize >>> - # !! Contents within this block are managed by 'conda init' !! - __conda_setup="$('/usr/share/miniconda/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" - if [ $? -eq 0 ]; then - eval "$__conda_setup" - else - if [ -f "/usr/share/miniconda/etc/profile.d/conda.sh" ]; then - . "/usr/share/miniconda/etc/profile.d/conda.sh" - else - export PATH="/usr/share/miniconda/bin:$PATH" - fi - fi - unset __conda_setup - # <<< conda initialize <<< - - set -x - set -e - - conda activate test-environment cd docs make html diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b5791b11..c85f403c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,13 +25,15 @@ jobs: if [[ ${{ matrix.numpy-version }} == 'latest' ]]; then python -m pip install --upgrade numpy elif [[ ${{ matrix.numpy-version }} == 'dev' ]]; then - python -m pip install --pre --upgrade --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy + python -m pip install --pre --upgrade --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy -r requirements-dev.txt else python -m pip install --upgrade numpy==${{ matrix.numpy-version }}.* fi - name: Run Doctests run: | ./run_doctests + # A NumPy 2.0 compatible skimage doesn't support 3.9. Easiest to just skip this for now. + if: matrix.numpy-version != 'dev' && matrix.python-version != '3.9' - name: Test Installation run: | python -m pip install . diff --git a/docs/_static/arrow-short-curved.svg b/docs/_static/arrow-short-curved.svg new file mode 100644 index 00000000..12fb3476 --- /dev/null +++ b/docs/_static/arrow-short-curved.svg @@ -0,0 +1,65 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Based on https://commons.wikimedia.org/wiki/File:U%2B2192.svg --> + +<svg + inkscape:version="1.2.2 (b0a8486, 2022-12-01)" + sodipodi:docname="arrow-short-curved.svg" + id="svg3555" + version="1.1" + height="47" + width="191" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns="http://www.w3.org/2000/svg" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:dc="http://purl.org/dc/elements/1.1/"> + <defs + id="defs3559" /> + <sodipodi:namedview + id="namedview3557" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + inkscape:pageshadow="2" + inkscape:pageopacity="0.0" + inkscape:pagecheckerboard="0" + showgrid="false" + inkscape:snap-object-midpoints="true" + inkscape:snap-others="true" + inkscape:snap-midpoints="false" + inkscape:snap-smooth-nodes="false" + inkscape:snap-intersection-paths="false" + inkscape:object-paths="true" + inkscape:zoom="4.5156423" + inkscape:cx="91.570584" + inkscape:cy="1.6608933" + inkscape:window-width="1536" + inkscape:window-height="907" + inkscape:window-x="0" + inkscape:window-y="25" + inkscape:window-maximized="0" + inkscape:current-layer="svg3555" + inkscape:showpageshadow="2" + inkscape:deskcolor="#d1d1d1" /> + <metadata + id="metadata5724"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + </cc:Work> + </rdf:RDF> + </metadata> + <path + d="m 0.8403006,14.930942 c 2.2381554,2.331925 3.5765507,4.99012 4.015159,7.974572 l 1.1660439,0.57124 C 6.2629854,21.653644 6.1057255,19.743775 5.549692,17.747165 5.6914242,17.483858 5.8255938,17.217008 5.9558046,16.948226 6.0860155,16.679444 6.2122674,16.40873 6.3381642,16.137693 8.256588,15.353319 9.864745,14.301537 11.162655,12.982359 l -1.166044,-0.57124 c -2.6270153,1.482643 -5.547718,2.054338 -8.7620719,1.715083 l -0.3942385,0.80474" + id="arrow-head" + sodipodi:nodetypes="ccccsccccc" /> + <path + style="fill:none;stroke:#000000;stroke-width:1.2581;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + d="M 190.0811,16.914227 A 130.26695,63.944771 0 0 1 97.968449,35.643217 130.26695,63.944771 0 0 1 5.8558046,16.914226" + id="arrow-tail" /> +</svg> diff --git a/docs/_static/arrow-short.svg b/docs/_static/arrow-short.svg new file mode 100644 index 00000000..7083a34e --- /dev/null +++ b/docs/_static/arrow-short.svg @@ -0,0 +1,28 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Based on https://commons.wikimedia.org/wiki/File:U%2B2192.svg --> +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="190" + height="14.33"> + <metadata + id="metadata5724"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + </cc:Work> + </rdf:RDF> + </metadata> +<g> +<path d="M 0.3654500000000098,7.613060000000019 C 3.4012800000000425,8.722540000000095 5.77263999999991,10.520860000000084 7.479499999999916,13.008020000000101 L 8.777949999999919,13.008020000000101 C 8.192749999999933,11.264580000000024 7.211299999999937,9.618650000000002 5.833579999999984,8.070260000000076 L 5.833579999999984,6.278030000000058 C 7.211299999999937,4.729649999999992 8.192749999999933,3.077629999999999 8.777949999999919,1.3219700000000785 L 7.479499999999916,1.3219700000000785 C 5.77263999999991,3.809150000000045 3.4012800000000425,5.60748000000001 0.3654500000000098,6.716940000000022 L 0.3654500000000098,7.613060000000019" id="arrow-head"/> +<path d="M 5.83358,8.07026 H 190 V 6.27803 H 5.83358" id="arrow-tail"/> +</g> +</svg> diff --git a/docs/_static/arrow.svg b/docs/_static/arrow.svg new file mode 100644 index 00000000..047f47ee --- /dev/null +++ b/docs/_static/arrow.svg @@ -0,0 +1,28 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Based on https://commons.wikimedia.org/wiki/File:U%2B2192.svg --> +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="1054" + height="14.33" + preserveAspectRatio="none"> + <metadata + id="metadata5724"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + </cc:Work> + </rdf:RDF> + </metadata> +<g> +<path d="M 0.3654500000000098,7.613060000000019 C 3.4012800000000425,8.722540000000095 5.77263999999991,10.520860000000084 7.479499999999916,13.008020000000101 L 8.777949999999919,13.008020000000101 C 8.192749999999933,11.264580000000024 7.211299999999937,9.618650000000002 5.833579999999984,8.070260000000076 L 1054.0,8.070260000000076 L 1054.0,6.278030000000058 L 5.833579999999984,6.278030000000058 C 7.211299999999937,4.729649999999992 8.192749999999933,3.077629999999999 8.777949999999919,1.3219700000000785 L 7.479499999999916,1.3219700000000785 C 5.77263999999991,3.809150000000045 3.4012800000000425,5.60748000000001 0.3654500000000098,6.716940000000022 L 0.3654500000000098,7.613060000000019 "/> +</g> +</svg> diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 089432d5..991b7230 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -19,6 +19,7 @@ --color-sidebar-current: white; --color-sidebar-background-current: var(--color-brand-medium-blue); --color-sidebar--hover: var(--color-brand-dark-blue); + --color-sidebar-arrow-current: white; } @media (prefers-color-scheme: dark) { @@ -26,12 +27,245 @@ --color-sidebar-background-current: var(--color-brand-dark-blue); --color-brand-bg: var(--color-brand-dark-bg); --color-sidebar--hover: white; + --color-sidebar-arrow-current: var(--color-sidebar-link-text--top-level); } } [data-theme='dark'] { --color-sidebar-background-current: var(--color-brand-dark-blue); --color-brand-bg: var(--color-brand-dark-bg); --color-sidebar--hover: white; + --color-sidebar-arrow-current: var(--color-sidebar-link-text--top-level); +} + +/* Slice diagram stuff from the indexing guide */ + +.slice-diagram { + text-align: center; + display: flex; + justify-content: center; + flex-wrap: wrap; + flex-direction: column; + align-content: center; + align-items: center; + padding-left: 2em; + padding-right: 2em; + overflow: auto; + --color-slice-diagram-selected: #3030FF; + --color-slice-diagram-not-selected: #EE0000; + --color-slice-diagram-slice: var(--color-slice-diagram-selected); +} +[data-theme="dark"] .slice-diagram { + /* We could also use --color-brand-green here */ + --color-slice-diagram-selected: var(--color-brand-light-blue); + --color-slice-diagram-not-selected: #FF5E5E; + --color-slice-diagram-slice: var(--color-slice-diagram-selected); +} +.slice-diagram>code { + padding-top: 10px; + padding-bottom: 10px; +} + +.centered-text { + position: absolute; + left: 50%; + transform: translateX(-50%) translateY(-50%); + background-color: var(--color-background-primary); + padding: 0 10px; + z-index: 1; +} + +.horizontal-line { + position: absolute; + top: 50%; + left: 0; + right: 0; + border-top: 1.5px solid var(--color-foreground-primary); + margin-left: 10px; + margin-right: 10px; + z-index: 0; +} +.slice-diagram table { + border-collapse: collapse; +} + +.slice-diagram th { + white-space: nowrap; + text-align: right; +} + +.slice-diagram td { + border: none; + padding: 0.5em; + text-align: center; + position: relative; + width: 0.8em; + height: 0.8em; + white-space: nowrap; +} + +.underline-cell { + position: relative; +} + +.underline-cell::after { + content: ''; + position: absolute; + left: 0; + right: 0; + bottom: 0.3em; /* Position of the underline relative to the cell */ + height: 2px; /* Thickness of the underline */ + background-color: var(--color-slice-diagram-selected); +} + +.vertical-bar-red:before { + content: ''; + display: block; + position: absolute; + top: 0; + bottom: 0; + left: 50%; + transform: translateX(-50%); + width: 0; + border-left: 2px dotted var(--color-slice-diagram-not-selected); +} + +.vertical-bar-blue:before { + content: ''; + display: block; + position: absolute; + top: 0; + bottom: 0; + left: 50%; + transform: translateX(-50%); + width: 0; + border-left: 2px solid var(--color-slice-diagram-selected); +} + +.slice-diagram td:first-child { + border-left: none; +} + +.slice-diagram td:last-child:before { + border-right: none; +} + +.overflow-content { + white-space: nowrap; + overflow: visible; + width: 100%; + max-width: 0px; +} + +.slice-diagram-slice { + color: var(--color-slice-diagram-slice); +} +.slice-diagram-selected { + color: var(--color-slice-diagram-selected); + /* text-decoration: underline; */ +} +.slice-diagram-not-selected { + color: var(--color-slice-diagram-not-selected); +} +.slice-diagram-index-label-selected { + line-height: 0em; + vertical-align: top; + color: var(--color-slice-diagram-selected); +} +.slice-diagram-index-label-not-selected { + line-height: 0em; + vertical-align: top; + color: var(--color-slice-diagram-not-selected); +} +.circle-red, +.circle-blue { + display: inline-block; + width: 25px; + height: 25px; + line-height: 25px; + border-radius: 50%; + text-align: center; +} +.circle-red { + border: 1px solid var(--color-slice-diagram-not-selected); +} +.circle-blue { + border: 1px solid var(--color-slice-diagram-selected); +} +.left-arrow-cell::before { + content: ''; + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-image: url('./arrow.svg'); + background-repeat: no-repeat; + background-position: 0px center; +} +[data-theme="dark"] .left-arrow-cell::before { + filter: invert(1); +} + +.right-arrow-cell::before { + content: ''; + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-image: url('./arrow.svg'); + background-repeat: no-repeat; + background-position: 0px center; + transform: scaleX(-1); +} +[data-theme="dark"] .right-arrow-cell::before { + filter: invert(1); +} + +.left-arrow-curved-cell::before { + content: ''; + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 80px; + background-image: url('./arrow-short-curved.svg'); + background-repeat: no-repeat; + background-position: 0px center; + background-size: contain; + transform: translate(0px, -41px); +} +[data-theme="dark"] .left-arrow-curved-cell::before { + filter: invert(1); +} + +.right-arrow-curved-cell::before { + content: ''; + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 80px; + background-image: url('./arrow-short-curved.svg'); + background-repeat: no-repeat; + background-position: 0px center; + background-size: contain; + transform: translate(0px, -41px) scaleX(-1); +} +[data-theme="dark"] .right-arrow-curved-cell::before { + filter: invert(1); +} + +/* Styling for the GitHub link in the sidebar */ +.sidebar-tree .sidebar-extra { + box-sizing: border-box; + display: inline-block; + height: 100%; + line-height: var(--sidebar-item-line-height); + overflow-wrap: anywhere; + padding: var(--sidebar-item-spacing-vertical) var(--sidebar-item-spacing-horizontal); + width: 100%; + font-weight: bold !important } /* The furo theme uses only-light and only-dark for light/dark-mode only @@ -76,6 +310,15 @@ html body .only-dark-inline { color: var(--color-sidebar--hover); } +.sidebar-tree .current-page>label:not(:hover) .icon { + color: var(--color-sidebar-arrow-current); +} + +.sidebar-tree .reference:hover ~ label .icon { + color: var(--color-sidebar-link-text--top-level) !important; +} + + /* The "hide search matches" text after doing a search. Defaults to the same color as the search icon which is illegible on the colored background. */ .highlight-link a { @@ -126,3 +369,14 @@ h4, h5, h6 { html { scroll-behavior: auto; } + +/* Hide the annoying "back to top" button */ +.show-back-to-top .back-to-top { + display: none; +} + +/* Highlight footnotes when they are linked to */ +.footnote:target { + background-color: var(--color-highlight-on-target); + color: var(--color-foreground-primary); +} diff --git a/docs/_templates/sidebar/github.html b/docs/_templates/sidebar/github.html new file mode 100644 index 00000000..a5b0dcec --- /dev/null +++ b/docs/_templates/sidebar/github.html @@ -0,0 +1,8 @@ +<div class="sidebar-tree sidebar-extra"> + <ul> + <li class="toctree-l1"> + <a class="sidebar-extra" + href="https://github.com/Quansight-Labs/ndindex">GitHub<svg version="1.1" width="1.0em" height="1.0em" class="sd-octicon sd-octicon-link-external" viewBox="0 0 16 16" aria-hidden="true"><path fill-rule="evenodd" d="M10.604 1h4.146a.25.25 0 01.25.25v4.146a.25.25 0 01-.427.177L13.03 4.03 9.28 7.78a.75.75 0 01-1.06-1.06l3.75-3.75-1.543-1.543A.25.25 0 0110.604 1zM3.75 2A1.75 1.75 0 002 3.75v8.5c0 .966.784 1.75 1.75 1.75h8.5A1.75 1.75 0 0014 12.25v-3.5a.75.75 0 00-1.5 0v3.5a.25.25 0 01-.25.25h-8.5a.25.25 0 01-.25-.25v-8.5a.25.25 0 01.25-.25h3.5a.75.75 0 000-1.5h-3.5z"></path></svg></a> + </li> + </ul> +</div> diff --git a/docs/api.rst b/docs/api.rst deleted file mode 100644 index 97162042..00000000 --- a/docs/api.rst +++ /dev/null @@ -1,118 +0,0 @@ -=============== - API Reference -=============== - -The ndindex API consists of classes representing the different types of index -objects (integers, slices, etc.), as well as some helper functions for dealing -with indices. - -ndindex -======= - -.. autofunction:: ndindex.ndindex - -Index Types -=========== - -The following classes represent different types of indices. - - -.. autoclass:: ndindex.Integer - :members: - :special-members: - -.. _slice-api: - -.. autoclass:: ndindex.Slice - :members: - :special-members: - -.. autoclass:: ndindex.ellipsis - :members: - -.. autoclass:: ndindex.Newaxis - :members: - -.. autoclass:: ndindex.Tuple - :members: - -.. autoclass:: ndindex.IntegerArray - :members: - :inherited-members: - :exclude-members: dtype - - .. autoattribute:: dtype - :annotation: - -.. autoclass:: ndindex.BooleanArray - :members: - :inherited-members: - :exclude-members: dtype - - .. autoattribute:: dtype - :annotation: - -Index Helpers -============= - -The functions here are helpers for working with indices that aren't methods of -the index objects. - -.. autofunction:: ndindex.iter_indices - -.. autofunction:: ndindex.broadcast_shapes - -Exceptions -========== - -These are some custom exceptions that are raised by a few functions in -ndindex. Note that most functions in ndindex will raise `IndexError` -(if the index would be invalid), or `TypeError` or `ValueError` (if the input -types or values are incorrect). - -.. autoexception:: ndindex.BroadcastError - -.. autoexception:: ndindex.AxisError - -Chunking -======== - -ndindex contains objects to represent chunking an array. - -.. autoclass:: ndindex.ChunkSize - :members: - -Internal API -============ - -These classes are only intended for internal use in ndindex. They shouldn't -relied on as they may be removed or changed. - -.. autoclass:: ndindex.ndindex.ImmutableObject - :members: - -.. autoclass:: ndindex.ndindex.NDIndex - :members: - -.. autoclass:: ndindex.array.ArrayIndex - :members: - :exclude-members: dtype - - .. autoattribute:: dtype - :annotation: Subclasses should redefine this - -.. autoclass:: ndindex.slice.default - -.. autofunction:: ndindex.ndindex.operator_index - -.. autofunction:: ndindex.shapetools.asshape - -.. autofunction:: ndindex.shapetools.ncycles - -.. autofunction:: ndindex.shapetools.associated_axis - -.. autofunction:: ndindex.shapetools.remove_indices - -.. autofunction:: ndindex.shapetools.unremove_indices - -.. autofunction:: ndindex.shapetools.normalize_skip_axes diff --git a/docs/api/chunking.rst b/docs/api/chunking.rst new file mode 100644 index 00000000..38703d65 --- /dev/null +++ b/docs/api/chunking.rst @@ -0,0 +1,9 @@ +Chunking +======== + +The :class:`~.ChunkSize` class represents the shape of a chunk for a chunked +array. It contains methods for manipulating these chunks and indices on a +chunked array. + +.. autoclass:: ndindex.ChunkSize + :members: diff --git a/docs/api/index-types.rst b/docs/api/index-types.rst new file mode 100644 index 00000000..d6f4d628 --- /dev/null +++ b/docs/api/index-types.rst @@ -0,0 +1,51 @@ +.. _index-types: + +Index Types +=========== + +The ndindex API consists of classes to represent the different kinds of NumPy +indices, :class:`~.Integer`, :class:`~.Slice`, :class:`~.ellipsis`, +:class:`~.Newaxis`, :class:`~.Tuple`, :class:`~.IntegerArray`, and +:class:`~.BooleanArray`. Typical usage of ndindex consists of constructing one +of these classes, typically with the :func:`~.ndindex` constructor, then using +the methods on the objects. With a few exceptions, all index classes have the +same set of methods, so that they can be used uniformly regardless of the +actual index type. Consequently, many of the method docstrings below are +duplicated across all the classes. For classes where there is are particular +things of note for a given method, the docstring will be different (for +example, :meth:`.Slice.reduce` notes the specific invariants that the +:meth:`~.NDIndex.reduce()` method applies to :class:`~.Slice` objects). Such +methods will be noted by their "See Also" sections. + +.. autoclass:: ndindex.Integer + :members: + :special-members: + +.. autoclass:: ndindex.Slice + :members: + :special-members: + +.. autoclass:: ndindex.ellipsis + :members: + +.. autoclass:: ndindex.Newaxis + :members: + +.. autoclass:: ndindex.Tuple + :members: + +.. autoclass:: ndindex.IntegerArray + :members: + :inherited-members: + :exclude-members: dtype + + .. autoattribute:: dtype + :annotation: + +.. autoclass:: ndindex.BooleanArray + :members: + :inherited-members: + :exclude-members: dtype + + .. autoattribute:: dtype + :annotation: diff --git a/docs/api/index.rst b/docs/api/index.rst new file mode 100644 index 00000000..2da80322 --- /dev/null +++ b/docs/api/index.rst @@ -0,0 +1,20 @@ +=============== + API Reference +=============== + +The ndindex API consists of classes representing the different types of index +objects (integers, slices, etc.), as well as some helper functions for dealing +with indices. + + +API Reference Index +=================== + +.. toctree:: + :titlesonly: + + ndindex.rst + index-types.rst + shapetools.rst + chunking.rst + internal.rst diff --git a/docs/api/internal.rst b/docs/api/internal.rst new file mode 100644 index 00000000..f0ca068b --- /dev/null +++ b/docs/api/internal.rst @@ -0,0 +1,49 @@ +============ +Internal API +============ + +These classes are only intended for internal use in ndindex. They shouldn't +relied on as they may be removed or changed. + +Note that the documentation for methods on ndindex classes will sometimes link +to this page because the methods are defined on the on the +:class:`~.ImmutableObject`, :class:`~.NDIndex`, or :class:`~.ArrayIndex` base +classes. These classes are not designed to be used directly. Such methods are +present on all `ndindex classes <index-types.rst>`_, which are what should be +actually be constructed. Remember that the primary entry-point API for +constructing ndindex index classes is the :func:`~.ndindex` function. + +Base Classes +============ + +.. autoclass:: ndindex.ndindex.ImmutableObject + :members: + +.. autoclass:: ndindex.ndindex.NDIndex + :members: + +.. autoclass:: ndindex.array.ArrayIndex + :members: + :exclude-members: dtype + + .. autoattribute:: dtype + :annotation: Subclasses should redefine this + +Other Internal Functions +======================== + +.. autoclass:: ndindex.slice.default + +.. autofunction:: ndindex.ndindex.operator_index + +.. autofunction:: ndindex.shapetools.asshape + +.. autofunction:: ndindex.shapetools.ncycles + +.. autofunction:: ndindex.shapetools.associated_axis + +.. autofunction:: ndindex.shapetools.remove_indices + +.. autofunction:: ndindex.shapetools.unremove_indices + +.. autofunction:: ndindex.shapetools.normalize_skip_axes diff --git a/docs/api/ndindex.rst b/docs/api/ndindex.rst new file mode 100644 index 00000000..5306ec60 --- /dev/null +++ b/docs/api/ndindex.rst @@ -0,0 +1,8 @@ +======= +ndindex +======= + +The primary entry-point to the ndindex API is the `ndindex()` function, which +converts Python index objects into ndindex objects. + +.. autofunction:: ndindex.ndindex diff --git a/docs/api/shapetools.rst b/docs/api/shapetools.rst new file mode 100644 index 00000000..19a8cb95 --- /dev/null +++ b/docs/api/shapetools.rst @@ -0,0 +1,24 @@ +Shape Tools +=========== + +ndindex contains several helper functions for working with and manipulating +array shapes. + +Functions +--------- + +.. autofunction:: ndindex.iter_indices + +.. autofunction:: ndindex.broadcast_shapes + +Exceptions +---------- + +These are some custom exceptions that are raised by the above functions. Note +that most of the other functions in ndindex will raise `IndexError` (if the +index would be invalid), or `TypeError` or `ValueError` (if the input types or +values are incorrect). + +.. autoexception:: ndindex.BroadcastError + +.. autoexception:: ndindex.AxisError diff --git a/docs/changelog.md b/docs/changelog.md index 69f133be..15ced93b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -101,7 +101,7 @@ - Minor improvements to some documentation. -- Fix a typo in the [type confusion](type-confusion) docs. (@ruancomelli) +- Fix a typo in the [type confusion](type-confusion.md) docs. (@ruancomelli) ## Version 1.6 (2022-01-24) @@ -153,6 +153,7 @@ ### Major Changes - ndindex now has a logo:  + Thanks to [Irina Fumarel](mailto:ifumarel@quansight.com) for the logo design. - Improve {any}`ChunkSize.as_subchunks()` to never use the slow fallback @@ -289,10 +290,10 @@ run the ndindex test suite due to the way ndindex tests itself against NumPy. for these, please [open an issue](https://github.com/Quansight-Labs/ndindex/issues) to let me know. -- Add a new document to the documentation on [type confusion](type-confusion). - The document stresses that ndindex types should not be confused with the - built-in/NumPy types that they wrap, and outlines some pitfalls and best - practices to avoid them when using ndindex. +- Add a new document to the documentation on [type + confusion](type-confusion.md). The document stresses that ndindex types + should not be confused with the built-in/NumPy types that they wrap, and + outlines some pitfalls and best practices to avoid them when using ndindex. ### Minor Changes @@ -368,7 +369,7 @@ run the ndindex test suite due to the way ndindex tests itself against NumPy. - SymPy is now a hard dependency of ndindex. -- Added extensive documentation on [slice semantics](slices) to the +- Added extensive documentation on [slice semantics](indexing-guide/slices) to the documentation. ### Minor Changes diff --git a/docs/conf.py b/docs/conf.py index c520ebc1..6089f8e1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,8 @@ # import os import sys +import subprocess +import inspect sys.path.insert(0, os.path.abspath('..')) @@ -32,12 +34,25 @@ 'sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', + 'sphinx.ext.linkcode', 'sphinx_copybutton', + 'sphinx_reredirects', + 'sphinx_design', + 'matplotlib.sphinxext.plot_directive', ] +plot_html_show_source_link = False +plot_include_source = False +plot_html_show_formats = False +plot_formats = ['svg'] + intersphinx_mapping = { 'numpy': ('https://numpy.org/doc/stable/', None), + 'pandas': ('https://pandas.pydata.org/docs/', None), } +# Require :external: to reference intersphinx. Prevents accidentally linking +# to something from numpy. +intersphinx_disabled_reftypes = ['*'] # # From # # https://stackoverflow.com/questions/56062402/force-sphinx-to-interpret-markdown-in-python-docstrings-instead-of-restructuredt @@ -78,6 +93,19 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] +templates_path = ['_templates'] + +html_sidebars = { + "**": [ + "sidebar/scroll-start.html", + "sidebar/brand.html", + "sidebar/search.html", + "sidebar/navigation.html", + "sidebar/scroll-end.html", + "sidebar/github.html", + ], +} + # These are defined in _static/custom.css light_blue = "var(--color-brand-light-blue)" green = "var(--color-brand-green)" @@ -110,10 +138,14 @@ "code-font-size": "var(--font-size--small)", + "color-highlight-on-target": "var(--color-highlighted-background)", } html_theme_options = { 'light_logo': 'ndindex_logo_white_bg.svg', 'dark_logo': 'ndindex_logo_dark_bg.svg', + "source_repository": "https://github.com/Quansight-Labs/ndindex/", + "source_branch": "main", + "source_directory": "docs/", "light_css_variables": { **theme_colors_common, "color-brand-primary": dark_blue, @@ -178,7 +210,9 @@ }, } -myst_update_mathjax=False +myst_update_mathjax = False + +myst_footnote_transition = False # Lets us use single backticks for code default_role = 'code' @@ -193,3 +227,102 @@ href="https://github.com/Quansight-Labs/ndindex/pull/{PR_NUMBER}/commits/{SHA1}">{SHA1[:7]}</a>. If you aren't looking for a PR preview, go to <a href="https://quansight-labs.github.io/ndindex//">the main ndindex documentation</a>. """ + +# Add redirects here. This should be done whenever a page that is in the +# existing release docs is moved somewhere else so that the URLs don't break. +# The format is + +# "page/path/without/extension": "../relative_path_with.html" + +# Note that the html path is relative to the redirected page. Always test the +# redirect manually (they aren't tested automatically). See +# https://documatt.gitlab.io/sphinx-reredirects/usage.html + +redirects = { + "slices": "indexing-guide/slices.html", + "api": "api/index.html", +} + + +# Required for linkcode extension. +# Get commit hash from the external file. + +# Based on code from SymPy's conf.py + +commit_hash_filepath = '../commit_hash.txt' +commit_hash = None +if os.path.isfile(commit_hash_filepath): + with open(commit_hash_filepath) as f: + commit_hash = f.readline() + +# Get commit hash from the external file. +if not commit_hash: + try: + commit_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD']) + commit_hash = commit_hash.decode('ascii') + commit_hash = commit_hash.rstrip() + except: + import warnings + warnings.warn( + "Failed to get the git commit hash as the command " \ + "'git rev-parse HEAD' is not working. The commit hash will be " \ + "assumed as the ndindex master, but the lines may be misleading " \ + "or nonexistent as it is not the correct branch the doc is " \ + "built with. Check your installation of 'git' if you want to " \ + "resolve this warning.") + commit_hash = 'master' + +fork = 'Quansight-Labs' +blobpath = \ + "https://github.com/{}/ndindex/blob/{}/ndindex/".format(fork, commit_hash) + + +def linkcode_resolve(domain, info): + """Determine the URL corresponding to Python object.""" + import ndindex + + if domain != 'py': + return + + modname = info['module'] + fullname = info['fullname'] + + submod = sys.modules.get(modname) + if submod is None: + return + + obj = submod + for part in fullname.split('.'): + try: + obj = getattr(obj, part) + except Exception: + return + + # strip decorators, which would resolve to the source of the decorator + # possibly an upstream bug in getsourcefile, bpo-1764286 + try: + unwrap = inspect.unwrap + except AttributeError: + pass + else: + obj = unwrap(obj) + + try: + fn = inspect.getsourcefile(obj) + except Exception: + fn = None + if not fn: + return + + try: + source, lineno = inspect.getsourcelines(obj) + except Exception: + lineno = None + + if lineno: + linespec = "#L%d-L%d" % (lineno, lineno + len(source) - 1) + else: + linespec = "" + + fn = os.path.relpath(fn, start=os.path.dirname(ndindex.__file__)) + return blobpath + fn + linespec diff --git a/docs/index.md b/docs/index.md index 16806dab..e5ce7234 100644 --- a/docs/index.md +++ b/docs/index.md @@ -16,12 +16,11 @@ are - Give 100% correct semantics as defined by numpy's ndarray. This means that ndindex will not make a transformation on an index object unless it is correct for all possible input array shapes. The only exception to this rule - is that ndindex assumes that any given index will not raise IndexError (for - instance, from an out of bounds integer index or from too few dimensions). - For those operations where the array shape is known, there is a - [`reduce()`](NDIndex.reduce) - method to reduce an index to a simpler index that is equivalent for the - given shape. + is that ndindex assumes that any given index will not raise `IndexError` + (for instance, from an out of bounds integer index or from too few + dimensions). For those operations where the array shape is known, there is a + [`reduce()`](NDIndex.reduce) method to reduce an index to a simpler index + that is equivalent for the given shape. - Enable useful transformation and manipulation functions on index objects. @@ -42,10 +41,11 @@ their limitations: invalid slices like `slice(0.5)` or `slice(0, 10, 0)` are allowed. Also slices that would always be equivalent like `slice(None, 10)` and `slice(0, 10)` are unequal. To contrast, ndindex objects always assume they are - indices to numpy arrays and type check their input. The `reduce` method can - be used to put the arguments into canonical form. + indices to numpy arrays and type check their input, and the + [`reduce()`](Slice.reduce) method can be used to put the arguments into + canonical form. -- Once you generalizing `slice` objects to more general indices, it is +- Once you start generalizing `slice` objects to more general indices, it is difficult to work with them in a uniform way. For example, `a[i]` and `a[(i,)]` are always equivalent for numpy arrays, but `tuple`, `slice`, `int`, etc. are not related to one another. To contrast, all ndindex types @@ -54,21 +54,22 @@ their limitations: - The above limitations can be annoying, but you might consider them worth living with. The real pain comes when you start trying to do slice - arithmetic. Slices in Python behave fundamentally differently depending on - whether the step is positive or negative and the start and stop are - positive, negative, or None. Consider, for example, the meaning of the slice - `a[4:-2:-2]`, where `a` is a one-dimensional array. This slices every other - element from the fifth element to the second from the last, but not - including the second from last. The resulting array will have shape `(0,)` - if the original shape is less than 1 or greater than 5, and shape `(1,)` - otherwise. In ndindex, one can use `len(Slice(4, -2, -2))` to compute the - maximum length of this slice (`1`), or `len(Slice(4, -2, -2).reduce(shape))` - to compute the length for a specific array shape. See - {meth}`ndindex.Slice.__len__` and {meth}`ndindex.Slice.reduce`. + arithmetic. [Slices in Python](indexing-guide/slices.md) behave + fundamentally differently depending on whether the step is positive or + negative and the start and stop are positive, negative, or None. Consider, + for example, the meaning of the slice `a[4:-2:-2]`, where `a` is a + one-dimensional array. This slices every other element from the fifth + element to the second from the last, but not including the second from last. + The resulting array will have shape `(0,)` if the original shape is less + than 1 or greater than 5, and shape `(1,)` otherwise. In ndindex, one can + use `len(Slice(4, -2, -2))` to compute the maximum length of this slice + (`1`), or `len(Slice(4, -2, -2).reduce(shape))` to compute the length for a + specific array shape. See {meth}`ndindex.Slice.__len__` and + {meth}`ndindex.Slice.reduce`. ndindex pre-codes common slice arithmetic into useful abstractions so you don't have to try to figure out all the different cases yourself. And due to - extensive testing (see below), you can be assured that ndindex is correct. + [extensive testing](testing), you can be assured that ndindex is correct. (installation)= ## Installation @@ -85,12 +86,13 @@ The environment variable `CYTHONIZE_NDINDEX` is used to explicitly control this default behavior: - `CYTHONIZE_NDINDEX=0`: disables Cythonization (even if a - working Cython environment is available) + working Cython environment is available). - `CYTHONIZE_NDINDEX=1`: force Cythonization (will fail when Cython or a - compiler isn't present) + compiler isn't present). -- `CYTHONIZE_NDINDEX` not set: the default behavior +- `CYTHONIZE_NDINDEX` not set: the default behavior (Cythonize if Cython is + installed and working). Cythonization is still experimental, and is only enabled for direct source installations. The pip and conda packages are not Cythonized. Future versions @@ -109,23 +111,27 @@ implemented: checking). Objects can be put into canonical form by calling [`reduce()`](NDIndex.reduce). - >>> from ndindex import Slice - >>> Slice(None, 12) - Slice(None, 12, None) - >>> Slice(None, 12).reduce() - Slice(0, 12, 1) + ```py + >>> from ndindex import Slice + >>> Slice(None, 12) + Slice(None, 12, None) + >>> Slice(None, 12).reduce() + Slice(0, 12, 1) + ``` `reduce()` can also be called with a `shape` argument. [`idx.reduce(shape)`](NDIndex.reduce) reduces an index to an equivalent index over an array with the given shape. - >>> from numpy import arange - >>> Slice(2, -1).reduce((10,)) - Slice(2, 9, 1) - >>> arange(10)[2:-1] - array([2, 3, 4, 5, 6, 7, 8]) - >>> arange(10)[2:9:1] - array([2, 3, 4, 5, 6, 7, 8]) + ```py + >>> from numpy import arange + >>> Slice(2, -1).reduce((10,)) + Slice(2, 9, 1) + >>> arange(10)[2:-1] + array([2, 3, 4, 5, 6, 7, 8]) + >>> arange(10)[2:9:1] + array([2, 3, 4, 5, 6, 7, 8]) + ``` `reduce()` simplifies all index types, but for [slice indices](Slice.reduce) in particular, it always puts them into canonical form, so that @@ -135,27 +141,36 @@ implemented: array shapes. This can be used to test slice equality without indexing an array. - >>> Slice(2, 4, 3).reduce() - Slice(2, 3, 1) - >>> Slice(2, 5, 3).reduce() - Slice(2, 3, 1) - - >>> Slice(-2, 5, 3).reduce(3) - Slice(1, 2, 1) - >>> Slice(-2, -1).reduce(3) - Slice(1, 2, 1) - >>> a = [0, 1, 2] - >>> a[-2:5:3] - [1] - >>> a[-2:-1] - [1] - >>> a[1:2:1] - [1] + ```py + >>> Slice(2, 4, 3).reduce() + Slice(2, 3, 1) + >>> Slice(2, 5, 3).reduce() + Slice(2, 3, 1) + ``` + + ```py + >>> Slice(-2, 5, 3).reduce(3) + Slice(1, 2, 1) + >>> Slice(-2, -1).reduce(3) + Slice(1, 2, 1) + >>> a = [0, 1, 2] + >>> a[-2:5:3] + [1] + >>> a[-2:-1] + [1] + >>> a[1:2:1] + [1] + ``` - Object arguments can be accessed with `idx.args` - >>> Slice(1, 3).args - (1, 3, None) + ``` + >>> Slice(1, 3).args + (1, 3, None) + ``` + + ndindex objects can always be recreated exactly from their args, so that + `type(idx)(idx.args) == idx`. - All ndindex objects are immutable/hashable and can be used as dictionary keys. @@ -165,67 +180,111 @@ implemented: test if two indices index the same elements as each other (n.b. pure canonicalization is currently only guaranteed for slice indices). - >>> from ndindex import Tuple - >>> from numpy import array - >>> # This would fail with ValueError for a Python tuple containing an array - >>> Tuple(array([1, 2]), 0) == Tuple(array([1, 2]), 0) - True - >>> Slice(0, 10).reduce(5) == Slice(0, 5).reduce(5) - True + ```py + >>> from ndindex import Tuple + >>> from numpy import array + >>> # This would fail with ValueError for a Python tuple containing an array + >>> Tuple(array([1, 2]), 0) == Tuple(array([1, 2]), 0) + True + >>> Slice(0, 10).reduce(5) == Slice(0, 5).reduce(5) + True + ``` - A real index object can be accessed with [`idx.raw`](NDIndex.raw). Use this to use an ndindex index to index an array. - >>> s = Slice(0, 2) - >>> arange(4)[s.raw] - array([0, 1]) + ```py + >>> s = Slice(0, 2) + >>> arange(4)[s.raw] + array([0, 1]) + ``` - [`len()`](Slice.__len__) computes the maximum length of a slice index (over the first axis). - >>> len(Slice(2, 10, 3)) - 3 - >>> len(arange(10)[2:10:3]) - 3 + ```py + >>> len(Slice(2, 10, 3)) + 3 + >>> len(arange(10)[2:10:3]) + 3 + ``` - [`idx.isempty()`](NDIndex.isempty) returns True if an index always indexes to an empty array (an array with a 0 in its shape). `isempty` can also be called with a shape like `idx.isempty(shape)`. - >>> Slice(0, 0).isempty() - True + ```py + >>> Slice(0, 0).isempty() + True + ``` - [`idx.expand(shape)`](NDIndex.expand) expands an index so that it is as explicit as possible. An expanded index is always a [`Tuple`](ndindex.Tuple) where each indexed axis is indexed explicitly. - >>> from ndindex import Tuple - >>> Tuple(Slice(0, 10), ..., Slice(1, None)).expand((10, 11, 12)) - Tuple(slice(0, 10, 1), slice(0, 11, 1), slice(1, 12, 1)) + ```py + >>> from ndindex import Tuple + >>> Tuple(Slice(0, 10), ..., Slice(1, None)).expand((10, 11, 12)) + Tuple(slice(0, 10, 1), slice(0, 11, 1), slice(1, 12, 1)) + ``` - [`idx.newshape(shape)`](NDIndex.newshape) returns the shape of `a[idx]`, assuming `a` has shape `shape`. - >>> Tuple(0, ..., Slice(0, 5)).newshape((10, 10, 10)) - (10, 5) + ```py + >>> Tuple(0, ..., Slice(0, 5)).newshape((10, 10, 10)) + (10, 5) + ``` - [`idx.broadcast_arrays()`](NDIndex.broadcast_arrays) broadcasts the array indices in `idx`, and converts boolean arrays into equivalent integer arrays. - >>> Tuple(array([[True, False], [True, False]]), array([0, 1])).broadcast_arrays() - Tuple([0, 1], [0, 0], [0, 1]) + ```py + >>> Tuple(array([[True, False], [True, False]]), array([0, 1])).broadcast_arrays() + Tuple([0, 1], [0, 0], [0, 1]) + ``` - [`i.as_subindex(j)`](NDIndex.as_subindex) produces an index `k` such that `a[j][k]` gives all the elements of `a[j]` that are also in `a[i]` (see the [documentation](NDIndex.as_subindex) for more information). This is useful for re-indexing an index onto chunks of an array. - >>> chunks = [Slice(0, 100), Slice(100, 200)] - >>> idx = Slice(50, 160) - >>> idx.as_subindex(chunks[0]) - Slice(50, 100, 1) - >>> idx.as_subindex(chunks[1]) - Slice(0, 60, 1) + ```py + >>> chunks = [Slice(0, 100), Slice(100, 200)] + >>> idx = Slice(50, 160) + >>> idx.as_subindex(chunks[0]) + Slice(50, 100, 1) + >>> idx.as_subindex(chunks[1]) + Slice(0, 60, 1) + ``` + +- The [`ChunkSize`](ndindex.ChunkSize) object abstracts splitting an array + into uniform chunks and contains methods to compute things about these + chunks: + + ```py + >>> from ndindex import ChunkSize + >>> chunk_size = ChunkSize((100, 200)) + >>> shape = (10000, 10001) + >>> chunk_size.num_chunks(shape) + 5100 + >>> chunk_size.containing_block((slice(450, 1050), slice(100, 200)), shape) + Tuple(slice(400, 1100, 1), slice(0, 200, 1)) + >>> for idx in chunk_size.as_subchunks(_, shape): + ... print(idx) + Tuple(slice(400, 500, 1), slice(0, 200, 1)) + Tuple(slice(500, 600, 1), slice(0, 200, 1)) + Tuple(slice(600, 700, 1), slice(0, 200, 1)) + Tuple(slice(700, 800, 1), slice(0, 200, 1)) + Tuple(slice(800, 900, 1), slice(0, 200, 1)) + Tuple(slice(900, 1000, 1), slice(0, 200, 1)) + Tuple(slice(1000, 1100, 1), slice(0, 200, 1)) + ``` + +- Additionally, the ndindex documentation contains an extensive [guide to + NumPy indexing](indexing-guide/index.md), which goes over how all NumPy + index types work, and should be useful to all users of NumPy even if you + aren't a user of ndindex. The following things are not yet implemented, but are planned. @@ -267,22 +326,24 @@ input arrays `a` and ndindex objects `idx`. There are two primary types of tests that we employ to verify this: -- Exhaustive tests. These test every possible value in some range. For +- **Exhaustive tests**. These test every possible value in some range. For example, slice tests test all possible `start`, `stop`, and `step` values in - the range [-10, 10], as well as `None`, on `numpy.arange(n)` for `n` in the - range [0, 10]. This is the best type of test, because it checks every + the range $[-10, 10]$, as well as `None`, on `numpy.arange(n)` for `n` in + the range $[0, 10]$. This is the best type of test, because it checks every possible case. Unfortunately, it is often impossible to do full exhaustive - testing due to combinatorial explosion. - -- Hypothesis tests. Hypothesis is a library that can intelligently check a - combinatorial search space of inputs. This requires writing hypothesis - strategies that can generate all the relevant types of indices (see + testing due to combinatorial explosion. For example, it would be impossible + to exhaustively test tuple indices, even with a severely restricted search + space, and the situation is even worse with integer array indices. + +- **Hypothesis tests**. + [Hypothesis](https://hypothesis.readthedocs.io/en/latest/index.html) is a + library that can intelligently check a combinatorial search space of inputs. + This requires writing hypothesis strategies that can generate all the + relevant types of indices (see [ndindex/tests/helpers.py](https://github.com/Quansight-Labs/ndindex/blob/main/ndindex/tests/helpers.py)). - For more information on hypothesis, see - <https://hypothesis.readthedocs.io/en/latest/index.html>. All tests have - hypothesis tests, even if they are also tested exhaustively. + All tests have hypothesis tests, even if they are also tested exhaustively. -Why bother with hypothesis if the same thing is already tested exhaustively? +Why bother with Hypothesis if the same thing is already tested exhaustively? The main reason is that hypothesis is much better at producing human-readable failure examples. When an exhaustive test fails, the failure will always be from the first set of inputs in the loop that produces a failure. Hypothesis @@ -325,13 +386,12 @@ Jupyter. <a class="reference external image-reference" href="https://www.deshaw.com"><img alt="https://www.deshaw.com/" class="only-dark-inline" src="https://www.deshaw.com/assets/svg/embedded/logo-white.svg" style="width: 200px;"></a> </div> -## Table of Contents - ```{toctree} :titlesonly: +:hidden: -api.md -slices.md +api/index.md +indexing-guide/index.md type-confusion.md changelog.md style-guide.md diff --git a/docs/indexing-guide/index.md b/docs/indexing-guide/index.md new file mode 100644 index 00000000..0800b3e4 --- /dev/null +++ b/docs/indexing-guide/index.md @@ -0,0 +1,59 @@ +# Guide to NumPy Indexing + +This section of the ndindex documentation discusses the semantics of NumPy +indices. This really is more of a documentation of NumPy itself than of +ndindex. However, understanding the underlying semantics of indices is +critical to making the best use of ndindex, as well as for making the best use +of NumPy arrays themselves. Furthermore, the sections on [integer +indices](integer-indices.md) and [slices](slices.md) also apply to the built-in +Python sequence types like `list` and `str`. + +This guide is aimed for people who are new to NumPy indexing semantics, but it +also tries to be as complete as possible and at least mention all the various +corner cases. Some of these technical points can be glossed over if you are a +beginner. + +## Table of Contents + +This guide is split into four sections. + +After a short [introduction](intro.md), the first two sections cover the basic +single-axis index types: [integer indices](integer-indices.md), and +[slices](slices.md). These are the indices that only work on a single axis of +an array at a time. These are also the indices that work on built-in sequence +types such as `list` and `str`. The semantics of these index types on `list` +and `str` are exactly the same as on NumPy arrays, so even if you do not care +about NumPy or array programming, these sections of this document can be +informative just as a general Python programmer. Slices in particular are oft +confused and the guide on slicing clarifies their exact rules and debunks some +commonly spouted false beliefs about how they work. + +The third section covers [multidimensional +indices](multidimensional-indices/index.md). These indices will not work on +the built-in Python sequence types like `list` and `str`; they are only +defined for NumPy arrays. This section is itself split into six subsections. +First is a basic introduction to [what a NumPy array +is](multidimensional-indices/what-is-an-array.md). Following this are pages +for each of the remaining index types, the basic indices: +[tuples](multidimensional-indices/tuples.md), +[ellipses](multidimensional-indices/ellipses.md), and +[newaxis](multidimensional-indices/newaxis.md); and the advanced indices: +[integer arrays](multidimensional-indices/integer-arrays.md) and [boolean +arrays](multidimensional-indices/boolean-arrays.md). + +Finally, a page on [other topics relevant to indexing](other-topics.md) covers +a set of miscellaneous topics about NumPy arrays that are useful for +understanding how indexing works, such as [broadcasting](broadcasting), +[views](views-vs-copies), [strides](strides), and +[ordering](c-vs-fortran-ordering). + +```{toctree} +:titlesonly: +:includehidden: + +intro.md +integer-indices.md +slices.md +multidimensional-indices/index.md +other-topics.md +``` diff --git a/docs/indexing-guide/integer-indices.md b/docs/indexing-guide/integer-indices.md new file mode 100644 index 00000000..a9d78f60 --- /dev/null +++ b/docs/indexing-guide/integer-indices.md @@ -0,0 +1,295 @@ +# Integer Indices + +The simplest possible index type is an integer index, that is, `a[i]` where `i` +is an integer like `0`, `3`, or `-2`. + +Integer indexing operates on the familiar Python data types `list`, `tuple`, +and `str`, as well as NumPy arrays. + +(prototype-example)= +Let us consider the following prototype list as an example: + +<div class="slice-diagram"> + <table> + <tr> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>['a',</pre></td> + <td><pre>'b',</pre></td> + <td><pre>'c',</pre></td> + <td><pre>'d',</pre></td> + <td><pre>'e',</pre></td> + <td><pre>'f',</pre></td> + <td><pre>'g']</pre></td> + </tr> + </table> +</div> + +The list `a` has 7 elements. + +The elements of `a` are strings, but the indices and slices on the list `a` +will always use integers. As with [all other index types](intro.md), **the +result of an integer index is never based on the values of the elements; it is +based instead on their positions in the list.**[^dict-footnote] + +[^dict-footnote]: If you are looking for something that allows non-integer +indices or that indexes by value, you may want a `dict`. + +An integer index selects a single element from the list `a`. + +> **The key thing to remember about indexing in Python, both for integer and + slice indexing, is that it is 0-based.** + +(fourth-sentence)= +This means that indices start at 0 ("0, 1, 2, ..."). For example, +`a[3]` selects the *fourth* element of `a`, in this case, `'d'`: + +<div class="slice-diagram"> +<code style="font-size: 16pt;">a[<span class="slice-diagram-slice">3</span>] == 'd'</code> + <table> + <tr> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>['a',</pre></td> + <td><pre> 'b',</pre></td> + <td><pre> 'c',</pre></td> + <td class="underline-cell"><pre> 'd',</pre></td> + <td><pre> 'e',</pre></td> + <td><pre> 'f',</pre></td> + <td><pre> 'g']</pre></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td class="slice-diagram-not-selected">0</td> + <td class="slice-diagram-not-selected">1</td> + <td class="slice-diagram-not-selected">2</td> + <td class="slice-diagram-selected">3</td> + <td class="slice-diagram-not-selected">4</td> + <td class="slice-diagram-not-selected">5</td> + <td class="slice-diagram-not-selected">6</td> + </tr> + </table> +</div> + +```py +>>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] +>>> a[3] +'d' +``` + +0-based indexing is different from how people typically count things, which is +1-based ("1, 2, 3, ..."). Thinking in terms of 0-based indexing requires some +practice, but doing so is essential to becoming an effective Python +programmer, especially if you are planning to work with arrays. + +For *negative* integers, indices index from the end of the list. These indices +are necessarily 1-based (or rather, −1-based), since `0` already refers +to the first element of the list. `-1` selects the last element, `-2` the +second-to-last, and so on. For example, `a[-3]` selects the *third-to-last* +element of `a`, in this case, `'e'`: + + +<div class="slice-diagram"> +<code style="font-size: 16pt;">a[<span class="slice-diagram-slice">-3</span>] == 'e'</code> + <table> + <tr> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>['a',</pre></td> + <td><pre> 'b',</pre></td> + <td><pre> 'c',</pre></td> + <td><pre> 'd',</pre></td> + <td class="underline-cell"><pre> 'e',</pre></td> + <td><pre> 'f',</pre></td> + <td><pre> 'g']</pre></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td class="slice-diagram-not-selected">−7</td> + <td class="slice-diagram-not-selected">−6</td> + <td class="slice-diagram-not-selected">−5</td> + <td class="slice-diagram-not-selected">−4</td> + <td class="slice-diagram-selected">−3</td> + <td class="slice-diagram-not-selected">−2</td> + <td class="slice-diagram-not-selected">−1</td> + </tr> + </table> +</div> + +```py +>>> a[-3] +'e' +``` + +An equivalent way to think about negative indices is that an index +`a[-i]` selects `a[len(a) - i]`, that is, you can subtract the negative +index off of the size of `a` (for a NumPy array, replace `len(a)` +with the size of the axis being sliced). For example, `len(a)` is `7`, so +`a[-3]` is the same as `a[7 - 3]`: + +```py +>>> len(a) +7 +>>> a[7 - 3] +'e' +``` + +Therefore, negative indices are primarily a syntactic convenience that +allows one to specify parts of a list that would otherwise need to be +specified in terms of the size of the list. + +If an integer index is greater than or equal to the size of the list, or less +than negative the size of the list (`i >= len(a)` or `i < -len(a)`), then it +is out of bounds and will raise an `IndexError`. + +```py +>>> a[7] +Traceback (most recent call last): +... +IndexError: list index out of range +>>> a[-8] +Traceback (most recent call last): +... +IndexError: list index out of range +``` + +For NumPy arrays, `i` is bounded by the size of the axis being indexed (not +the total size of the array): + + +```py +>>> import numpy as np +>>> a = np.ones((2, 3)) # A has 6 elements but the first axis has size 2 +>>> a[2] +Traceback (most recent call last): +... +IndexError: index 2 is out of bounds for axis 0 with size 2 +>>> a[-3] +Traceback (most recent call last): +... +IndexError: index -3 is out of bounds for axis 0 with size 2 +``` + +Fortunately, NumPy arrays give more helpful `IndexError` error messages than +Python lists do. + +The second important fact about integer indexing is that it reduces the +dimensionality of the container being indexed. For a `list` or `tuple`, this +means that an integer index returns an element of the list, which is in +general a different type than `list` or `tuple`. For instance, above we saw +that indexing `a` with an integer resulted in a `str`, because `a` is a list +that contains strings. This is in contrast with [slices](slices.md), which +always [return the same container type](subarray). + +(strings-integer-indexing)= +The exception to this rule is when integer indexing a +`str`, the result is also a `str`. This is because there is no `char` class in +Python. A single character is just represented as a string of length 1. + +```py +>>> 'abc'[0] +'a' +>>> type('abc'[0]) +<class 'str'> +``` + +For NumPy arrays, an integer index always indexes a single axis of the array. +By default, it indexes the first axis, unless it is part of a larger +[multidimensional index](multidimensional-indices/index). The result is always +an array with the dimensionality reduced by 1, namely, the axis being indexed +is removed from the resulting shape. This is in contrast with +[slices](slices.md), which always [maintain the dimension being +sliced](subarray). + +```py +>>> a = np.ones((2, 3, 4)) +>>> a.shape +(2, 3, 4) +>>> a[0].shape +(3, 4) +>>> a[-1].shape +(3, 4) +>>> a[..., 0].shape # Index the last axis, see the section on ellipses +(2, 3) +``` + +If `a` has only a single dimension, the result is a 0-D array, i.e., a single +scalar element (just as if `a` were a list): + +```py +>>> a = np.asarray(['a', 'b', 'c', 'd', 'e', 'f', 'g']) +>>> a[3] # doctest: +SKIPNP1 +np.str_('d') +``` + +In general, the resulting array is a subarray corresponding to the `i`-th +position along the given axis, using the 0- and −1-based rules discussed +above. For example: + +```py +>>> a = np.arange(4).reshape((2, 2)) +>>> a +array([[0, 1], + [2, 3]]) +>>> a[0] # The first subarray along the first axis +array([0, 1]) +>>> a[1] # The second subarray along the first axis +array([2, 3]) +>>> a[:, 0] # The first subarray along the second axis +array([0, 2]) +>>> a[:, 1] # The second subarray along the second axis +array([1, 3]) +``` + +A helpful analogy for understanding integer indexing on NumPy arrays is to +consider it in terms of a [list of +lists](multidimensional-indices/what-is-an-array.md). An integer index on the +first axis `a[i]` selects the `i`-th sub-list at the top level of sub-list +nesting. And in general, an integer index `i` on axis `k` selects the `i`-th +sub-lists at the `k`-th nesting level.[^nesting-level] For example, if `l` is +a nested list of lists + +[^nesting-level]: Thinking about the `k`-th level of nesting can get + confusing. For instance, it is unclear whether `k` should be counted with + 0-based or 1-based numbering, or which level counts as which, considering + that at the outermost "level," there is always a single list. + List-of-lists is a good analogy for thinking about why one might want to + use an nd-array in the first place, but as you actually use NumPy arrays + in practice, you'll find it's much better to think about dimensions and + axes directly, not "levels of nesting." + +```py +>>> l = [[0, 1], [2, 3]] +``` + +And `a` is the corresponding array: + +``` +>>> a = np.array(l) +``` + +Then `a[0]` is the same thing as `l[0]`, the first sub-list: + +``` +>>> a[0] +array([0, 1]) +>>> l[0] +[0, 1] +``` + +If we instead index the second axis, like `a[:, 0]`, this is the same as +indexing `0` in each list inside of `l`, like + +``` +>>> [x[0] for x in l] +[0, 2] +>>> a[:, 0] +array([0, 2]) +``` + +```{rubric} Footnotes +``` +<!-- Footnotes are written inline above but markdown will put them here at the +end of the document. --> diff --git a/docs/indexing-guide/intro.md b/docs/indexing-guide/intro.md new file mode 100644 index 00000000..1472fd7e --- /dev/null +++ b/docs/indexing-guide/intro.md @@ -0,0 +1,105 @@ +# Introduction: What is an Index? + +Nominally, an index is any object that can be placed between the square +brackets after an array. That is, if `a` is a NumPy array, then in `a[x]`, +*`x`* is an *index* of `a`.[^index-vs-slice-footnote] This also applies to +built-in sequence types in Python, such as `list`, `tuple`, and `str`; +however, be careful not to confuse this with the similar notation used in +Python dictionaries. If `d` is a Python dictionary, it uses the same notation +`d[x]`, but the meaning of `x` is completely different from what is discussed +in this document. This document also does not apply to indexing Pandas +DataFrame or Series objects, except insofar as they reuse the same semantics +as NumPy. Finally, note that some other Python array libraries (e.g., PyTorch +or Jax) have similar indexing rules, but they generally implement only a +subset of the full NumPy semantics outlined here. + +[^index-vs-slice-footnote]: Some people call `x` a *slice* of `a`, but we + avoid this confusing nomenclature, using *slice* to refer only to the + [slice index type](slices.md). The term "index" is used in the Python + language itself (e.g., in the built-in exception type `IndexError`). + +Semantically, an index `x` selects, or *indexes*[^indexes-footnote], some +subset of the elements of `a`. An index `a[x]` always either returns a new +array containing a subset of the elements of `a` or raises an `IndexError`. +When it comes to indexing, the most important rule, which applies to all types +of indices, is this: + +[^indexes-footnote]: For clarity, in this document and throughout the ndindex + documentation, the plural of *index* is *indices*. *Indexes* is always a + verb. For example, + + > In `a[i, j]`, the *indices* are `i` and `j`. They represent a single + tuple index `(i, j)`, which *indexes* the array `a`. + +> **Indices do not in any way depend on the *values* of the elements they + select. They only depend on their *positions* in the array `a`.** + +For example, consider `a`, an array of integers with the shape `(2, 3, 2)`: + +```py +>>> import numpy as np +>>> a = np.array([[[0, 1], [2, 3], [4, 5]], [[6, 7], [8, 9], [10, 11]]]) +>>> a.shape +(2, 3, 2) +``` + +Let's take as an example the index `0, ..., 1:`. We'll investigate how +exactly this index works later. For now, just notice that `a[0, ..., 1:]` +returns a new array with some of the elements of `a`. + +```py +>>> a[0, ..., 1:] +array([[1], + [3], + [5]]) +``` + +Now consider another array, `b`, with the exact same shape `(2, 3, 2)`, but +containing completely different entries, such as strings. If we apply the same +index `0, ..., 1:` to `b`, it will choose the exact same corresponding +elements. + +```py +>>> b = np.array([[['A', 'B'], ['C', 'D'], ['E', 'F']], [['G', 'H'], ['I', 'J'], ['K', 'L']]]) +>>> b[0, ..., 1:] +array([['B'], + ['D'], + ['F']], dtype='<U1') +``` + +Notice that `'B'` is in the same place in `b` as `1` was in `a`, `'D'` as `3`, +and `'F'` as `5`. Furthermore, the shapes of the resulting arrays are the +same: + +```py +>>> a[0, ..., 1:].shape +(3, 1) +>>> b[0, ..., 1:].shape +(3, 1) +``` + +Therefore, the following statements are always true about any index: + +- **An index on an array always produces a new array with the same dtype (unless + it raises `IndexError`).** + +- **Each element of the new array corresponds to some element of the original + array.** + +- **These elements are chosen by their position in the original array only. + The values of these elements are irrelevant.** + +- **As such, the same index applied to any other array with the same shape will + produce an array with the exact same resulting shape with elements in the + exact same corresponding places.** + +The full range of valid indices allows the generation of more or less +arbitrary new arrays whose elements come from the indexed array `a`. In +practice, the most commonly desired indexing operations are represented by +basic indices such as [integer indices](integer-indices.md), +[slices](slices.md), and [ellipses](multidimensional-indices/ellipses.md). + +```{rubric} Footnotes +``` +<!-- Footnotes are written inline above but markdown will put them here at the +end of the document. --> diff --git a/docs/indexing-guide/multidimensional-indices/boolean-arrays.md b/docs/indexing-guide/multidimensional-indices/boolean-arrays.md new file mode 100644 index 00000000..e1eae0ec --- /dev/null +++ b/docs/indexing-guide/multidimensional-indices/boolean-arrays.md @@ -0,0 +1,730 @@ +# Boolean Array Indices + +The final index type is boolean arrays. Boolean array indices are also +sometimes called *masks*,[^mask-footnote] because they "mask out" elements of +the array. + +```{note} +In this section, as with [the previous](integer-arrays.md), do not confuse the +*array being indexed* with the *array that is the index*. The former can be +anything and have any dtype. It is only the latter that is restricted to being +integer or boolean. +``` + +[^mask-footnote]: Not to be confused with {external+numpy:std:doc}`NumPy + masked arrays <reference/maskedarray>`. + +A boolean array index specifies which elements of an array should be selected +and which should not be selected. + +The simplest and most common case is where a boolean array index has the same +shape as the array being indexed, and is the sole index (i.e., not part of a +larger [tuple index](tuples.md)). + +Consider the array: + +```py +>>> import numpy as np +>>> a = np.arange(9).reshape((3, 3)) +>>> a +array([[0, 1, 2], + [3, 4, 5], + [6, 7, 8]]) +``` + +Suppose we want to select the elements `1`, `3`, and `4`: to do so, we create a +boolean array of the same shape as `a` which is `True` in the positions where +those elements are and `False` everywhere else. + +```py +>>> idx = np.array([ +... [False, True, False], +... [ True, True, False], +... [False, False, False]]) +>>> a[idx] +array([1, 3, 4]) +``` + +From this we can see a few things: + +- The result of indexing with the boolean mask is a 1-D array. If we think + about it, this is the only possibility. A boolean index could select any + number of elements. In this case, it selected 3 elements, but it could + select as few as 0 and as many as 9 elements from `a`. So there would be no + way to return a higher dimensional shape or for the shape of the result to + be somehow related to the shape of `a`. + +- The selected elements are "in order" ([more on what this means + later](boolean-array-c-order)). + +However, these details are usually not important. This is because an array +indexed by a boolean array is typically used indirectly, such as on the +left-hand side of an assignment. + +A typical use case of boolean indexing involves creating a boolean mask using +the array itself with operators that return boolean arrays, such as relational +operators (`<`, `<=`, `==`, `>`, `>=`, `!=`), logical operators (`&` (and), +`|` (or), `~` (not), `^` (xor)), and boolean functions (e.g., +{external+numpy:py:data}`isnan() <numpy.isnan>` or +{external+numpy:py:data}`isinf() <numpy.isinf>`). + +Consider an array of the integers from -10 to 10: + +```py +>>> a = np.arange(-10, 11) +>>> a +array([-10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, + 3, 4, 5, 6, 7, 8, 9, 10]) +``` + +Say we want to select the elements of `a` that are both positive and odd. The +boolean array `a > 0` represents which elements are positive and the boolean +array `a % 2 == 1` represents which elements are odd. So our mask would be + +```py +>>> mask = (a > 0) & (a % 2 == 1) +``` + +Note the careful use of parentheses to match [Python operator +precedence](https://docs.python.org/3/reference/expressions.html#operator-precedence). +Masks must use the logical operators `&`, `|`, and `~` so that they can +operate on arrays. They cannot use the Python keywords `and`, `or`, and `not`, +because they don't work on arrays. + +Our `mask` is just an array of booleans: + +```py +>>> mask +array([False, False, False, False, False, False, False, False, False, + False, False, True, False, True, False, True, False, True, + False, True, False]) +``` + +To get the actual matching elements, we need to index `a` with the mask: + +```py +>>> a[mask] +array([1, 3, 5, 7, 9]) +``` + +Often, one will see the `mask` written directly in the index, like + +```py +>>> a[(a > 0) & (a % 2 == 1)] +array([1, 3, 5, 7, 9]) +``` + +Suppose we want to set these elements of `a` to `-100` (i.e., to "mask" them +out). This can be done easily with an indexing +assignment[^indexing-assignment-footnote]: + +[^indexing-assignment-footnote]: All the indexing rules discussed in this + guide apply when the indexed array is on the left-hand side of an `=` + assignment. The elements of the array that are selected by the index are + assigned in-place to the array or number on the right-hand side. + +``` +>>> a[(a > 0) & (a % 2 == 1)] = -100 +>>> a +array([ -10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0, + -100, 2, -100, 4, -100, 6, -100, 8, -100, 10]) +``` + +One common use case of this sort of thing is to mask out `nan` entries with a +finite number, like `0`: + +``` +>>> a = np.linspace(-5, 5, 10) +>>> b = np.log(a) +>>> b +array([ nan, nan, nan, nan, nan, + -0.58778666, 0.51082562, 1.02165125, 1.35812348, 1.60943791]) +>>> b[np.isnan(b)] = 0. +>>> b +array([ 0. , 0. , 0. , 0. , 0. , + -0.58778666, 0.51082562, 1.02165125, 1.35812348, 1.60943791]) +``` + +Here `np.isnan(x)` returns a boolean array of the same shape as `x` that is +`True` if the corresponding element is `nan` and `False` otherwise. + +Note that for this kind of use case, the actual shape of `a[mask]` is +irrelevant. The important thing is that it is some subset of `a`, which is +then assigned to, mutating only those elements of `a`. + +It's important to not be fooled by this way of constructing a mask. Even +though the *expression* `(a > 0) & (a % 2 == 1)` depends on `a`, the resulting +*array itself* does not---it is just an array of booleans. **Boolean array +indexing, as with [all other types of indexing](../intro.md), does not depend +on the values of the array, only in the positions of its elements.** + +This distinction might feel overly pedantic, but it matters once you realize +that a mask created with one array can be used on another array, so long as it +has the same shape. It is common to have multiple arrays representing +different data about the same set of points. You may want to select a subset +of one array based on the values of the corresponding points in another array. + +For example, suppose we want to plot the function $f(x) = 4x\sin(x) - +\frac{x^2}{4} - 2x$ on $[-10,10]$. We can set `x = np.linspace(-10, 10)` and +compute the array expression: + +<!-- myst doesn't work with ```{plot}, and furthermore, if the two plot +directives are put in separate eval-rst blocks, the same plot is copied to +both. --> + +```{eval-rst} +.. plot:: + :context: reset + :include-source: True + :output-base-name: plot-{counter} + :alt: A plot of 4*x*np.sin(x) - x**2/4 - 2*x from -10 to 10. The curve crosses the x-axis several times at irregular intervals. + + >>> import matplotlib.pyplot as plt + >>> x = np.linspace(-10, 10, 10000) # 10000 evenly spaced points between -10 and 10 + >>> y = 4*x*np.sin(x) - x**2/4 - 2*x # our function + >>> plt.scatter(x, y, marker=',', s=1) + <matplotlib.collections.PathCollection object at ...> + +If we want to show only those x values that are positive, we could easily do +this by modifying the ``linspace`` call that created ``x``. But what if we +want to show only those ``y`` values that are positive? The only way to do +this is to select them using a mask: + +.. plot:: + :context: close-figs + :include-source: True + :output-base-name: plot-{counter} + :alt: A plot of only the parts of 4*x*np.sin(x) - x**2/4 - 2*x that are above the x-axis. + + >>> plt.scatter(x[y > 0], y[y > 0], marker=',', s=1) + <matplotlib.collections.PathCollection object at ...> + +``` + +Here we are using the mask `y > 0` to select the corresponding values from +*both* the `x` and the `y` arrays. Since the same mask is used on both arrays, +the values corresponding to this mask in both arrays will be selected. With +`x[y > 0]`, even though the mask itself is not strictly created *from* `x`, it +still makes sense as a mask for the array `x`. In this case, the mask selects +a nontrivial subset of `x`. + +Using a boolean array mask created from a different array is very common. For +example, in [scikit-image](https://scikit-image.org/), an image is represented +as an array of pixel values. Masks can be used to select a subset of the +image. A mask based on the pixel values (e.g., all red pixels) would depend on +the array, but a mask based on a geometric shape independent of the pixel +values, such as a +[circle](https://scikit-image.org/docs/stable/auto_examples/numpy_operations/plot_camera_numpy.html), +would not. In that case, the mask would just be a circular arrangement of +`True`s and `False`s. As another example, in machine learning, if `group` is +an array with group numbers and `X` is an array of features with repeated +measurements per group, one can select the features for a single group to do +cross-validation like `X[group == 0]`. + +## Advanced Notes + +As [with integer array indices](integer-arrays-advanced-notes), the above +section provides the basic gist of boolean array indexing, but there are some +advanced semantics described below, which can be skipped by new NumPy users. + +(boolean-array-result-shape)= +### Result Shape + +> **A boolean array index will remove as many dimensions as the index has, and +> replace them with a single flat dimension, which has size equal to the +> number of `True` elements in the index.** + +The shape of the boolean array index must exactly match the dimensions being +replaced, or the index will result in an `IndexError`. + +For example: + +```py +>>> a = np.arange(24).reshape((2, 3, 4)) +>>> idx = np.array([[True, False, True], +... [True, True, True]]) +>>> a.shape +(2, 3, 4) +>>> idx.shape # Matches the first two dimensions of a +(2, 3) +>>> np.count_nonzero(idx) # The number of True elements in idx +5 +>>> a[idx].shape # The (2, 3) in a.shape is replaced with count_nonzero(idx) +(5, 4) +``` + +This means that the final shape of an array indexed with a boolean mask +depends on the value of the mask, specifically, the number of `True` values in +it. It is easy to construct array expressions with boolean masks where the +size of the array cannot be determined until runtime. For example: + +```py +>>> rng = np.random.default_rng(11) # Seeded so this example reproduces +>>> a = rng.integers(0, 2, (3, 4)) # A shape (3, 4) array of 0s and 1s +>>> a[a==0].shape # Could be any size from 0 to 12 +(7,) +``` + +However, even if the number of elements in an indexed array is not +determinable until runtime, the *number of dimensions* is determinable. This +is because a boolean mask acts as a flattening operation. All the dimensions +of the boolean array index are removed from the indexed array and replaced +with a single dimension. Only the *size* of this dimension cannot be +determined, unless the number of `True` elements in the index is known. + +This detail means that sometimes code that uses boolean array indexing can be +difficult to reason about statically, because the array shapes are inherently +unknowable until runtime and may depend on data. For this reason, array +libraries that build computational graphs from array expressions without +evaluating them, such as +[JAX](https://jax.readthedocs.io/en/latest/index.html) or [Dask +Array](https://docs.dask.org/en/stable/array.html), may have limited or no +support for boolean array indexing. + +(boolean-array-c-order)= +### Result Order + +> **The order of the elements selected by a boolean array index `idx` +> corresponds to the elements being iterated in C order.** + +C order iterates the array `a` so that the last axis varies the fastest, +like `(0, 0, 0)`, `(0, 0, 1)`, `(0, 0, 2)`, `(0, 1, 0)`, `(0, 1, 1)`, etc. + +For example: + +```py +>>> a = np.arange(12).reshape((3, 4)) +>>> a +array([[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11]]) +>>> idx = np.array([[ True, False, True, True], +... [False, True, False, False], +... [ True, True, False, True]]) +>>> a[idx] +array([ 0, 2, 3, 5, 8, 9, 11]) +``` + +In this example, the elements of `a` are ordered `0 1 2 ...` in C order, which +is why in the final indexed array `a[idx]`, they are still in sorted order. C +order also corresponds to reading the elements of the array in the order that +NumPy prints them, from left to right, ignoring the brackets and commas. + +C ordering is always used, even when the underlying memory is not C-ordered +(see [](c-vs-fortran-ordering) for more details on C array ordering). + +### Masking a Subset of Dimensions + +It is possible to use a boolean mask to select only a subset of the dimensions +of `a`. For example, let's take a shape `(2, 3, 4)` array `a`: + +```py +>>> a = np.arange(24).reshape((2, 3, 4)) +>>> a +array([[[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11]], +<BLANKLINE> + [[12, 13, 14, 15], + [16, 17, 18, 19], + [20, 21, 22, 23]]]) +``` + +Say we want to select the elements of `a` that are greater than 5, but only in +the first subarray along the first dimension (only the elements from 0 to 11). +We can create a mask on only that subarray: + +```py +>>> mask = a[0] > 5 +>>> mask.shape +(3, 4) +``` + +Then, apply it to that same subarray: + +```py +>>> a[0, mask] +array([ 6, 7, 8, 9, 10, 11]) +``` + +The [tuple](tuples.md) index `(0, mask)` works just like any other tuple +index: it selects the subarray `a[0]` along the first axis, then applies the +`mask` to the remaining dimensions. The shape of `mask`, `(3, 4)`, matches +those remaining dimensions (by construction), so the index is valid. + +Masking a subset of dimension is not as common as masking the entire array +`a`, but it does happen. Remember that we can always think of an array as an +"array of subarrays". For instance, suppose we have a video with 1920 x 1080 +pixels and 500 frames. This might be represented as an array of shape `(500, +1080, 1920, 3)`, where the final dimension, 3, represents the 3 RGB color +values of a pixel. We can think of this array as 500 `(1080, 1920, 3)` +"frames". Or as 500 x 1080 x 1920 3-tuple "pixels". Or we could slice along +the last dimension and think of it as 3 `(500, 1080, 1920)` video "channels", +one for each primary color. + +In each case, we imagine that our array is really an array (or a stack or +batch) of subarrays, where some of our dimensions are the "stacking" +dimensions and some of them are the array dimensions. This way of thinking is +also common when doing linear algebra on arrays. The last two dimensions +(typically) are considered matrices, and the leading dimensions are batch +dimensions. An array of shape `(10, 5, 4)` might be thought of as ten 5 x 4 +matrices. NumPy linear algebra functions like `solve` and the `@` matmul +operator will automatically operate on the last two dimensions of an array. + +So, how does this relate to using a boolean array index to select only a +subset of the array dimensions? Well, we might want to use a boolean index to +select only along the inner "subarray" dimensions, and pretend like the outer +"batching" dimensions are our "array". + +For example, say we have an image represented in +[scikit-image](https://scikit-image.org/) as a 3-D array: + +```{eval-rst} +.. plot:: + :context: reset + :include-source: True + :output-base-name: astronaut-{counter} + :alt: An image of an astronaut, which is represented as a shape (512, 512, 3) array. + + >>> def imshow(image, title): + ... import matplotlib.pyplot as plt + ... plt.axis('off') + ... plt.title(title) + ... plt.imshow(image) + >>> from skimage.data import astronaut + >>> image = astronaut() + >>> image.shape + (512, 512, 3) + >>> imshow(image, "Original Image") + +Now, suppose we want to increase the saturation of this image. We can do this +by converting the image to `HSV space +<https://en.wikipedia.org/wiki/HSL_and_HSV>`_ and increasing the saturation +value (the second value in the last dimension, which should always be between +0 and 1): + +.. plot:: + :context: close-figs + :include-source: True + :output-base-name: astronaut-{counter} + :alt: An image of an astronaut with increased saturation. The lighter parts of the image appear washed out. + + >>> from skimage import color + >>> hsv_image = color.rgb2hsv(image) + >>> # Add 0.3 to the saturation, clipping the values to the range [0, 1] + >>> hsv_image[..., 1] = np.clip(hsv_image[..., 1] + 0.3, 0, 1) + >>> # Convert back to RGB + >>> saturated_image = color.hsv2rgb(hsv_image) + >>> imshow(saturated_image, "Saturated Image (Naive)") + +However, this ends up looking bad and washed out, because the whole image now +has a minimum saturation of 0.3. A better approach would be to select the +pixels that already have a saturation above some threshold, and increase the +saturation of only those pixels: + +.. plot:: + :context: close-figs + :include-source: True + :output-base-name: astronaut-{counter} + :alt: An image of an astronaut with increased saturation. The image does not appear washed out. + + >>> hsv_image = color.rgb2hsv(image) + >>> # Mask only those pixels whose saturation is > 0.6 + >>> high_sat_mask = hsv_image[:, :, 1] > 0.6 + >>> # Increase the saturation of those pixels by 0.3 + >>> hsv_image[high_sat_mask, 1] = np.clip(hsv_image[high_sat_mask, 1] + 0.3, 0, 1) + >>> # Convert back to RGB + >>> enhanced_color_image = color.hsv2rgb(hsv_image) + >>> imshow(enhanced_color_image, "Saturated Image") + +``` + +Here, `hsv_image.shape` is `(512, 512, 3)`, so our mask `hsv_image[:, :, 1] > +0.6` has shape `(512, 512)`, i.e., the shape of the first two dimensions. In +other words, the mask has one value for each pixel, either `True` if the +saturation is `> 0.6` or `False` if it isn't. To add 0.3 to only those pixels +above the threshold, we mask the original array with `hsv_image[high_sat_mask, +1]`. The `high_sat_mask` part of the index selects only those pixel values +that have high saturation, and the `1` in the final dimension selects the +saturation channel for those pixels. + +(nonzero-equivalence)= +### `nonzero()` Equivalence + +Another way to think about boolean array indices is based on the +`np.nonzero()` function. `np.nonzero(x)` returns a tuple of arrays of integer +indices where `x` is nonzero, or in the case where `x` is boolean, where `x` +is True. For example: + +```py +>>> idx = np.array([[ True, False, True, True], +... [False, True, False, False], +... [ True, True, False, True]]) +>>> np.nonzero(idx) +(array([0, 0, 0, 1, 2, 2, 2]), array([0, 2, 3, 1, 0, 1, 3])) +``` + +The first array in the tuple corresponds to indices for the first dimension; +the second array to the second dimension, and so on. If this seems familiar, +it's because this is exactly how we saw that [multidimensional integer array +indices](multidimensional-integer-indices) worked. Indeed, there is a basic +equivalence between the two: + +> **A boolean array index `idx` is the same as if you replaced `idx` with the +result of {external+numpy:func}`np.nonzero(idx) <numpy.nonzero>` (unpacking +the tuple), using the rules for [integer array indices](integer-arrays.md) +outlined previously.** + +Note, however, that this rule *does not* apply to [0-dimensional boolean +indices](0-d-boolean-index). + +```py +>>> a = np.arange(12).reshape((3, 4)) +>>> a[idx] +array([ 0, 2, 3, 5, 8, 9, 11]) +>>> np.nonzero(idx) +(array([0, 0, 0, 1, 2, 2, 2]), array([0, 2, 3, 1, 0, 1, 3])) +>>> idx0, idx1 = np.nonzero(idx) +>>> a[idx0, idx1] # this is the same as a[idx] +array([ 0, 2, 3, 5, 8, 9, 11]) +``` + +Here `np.nonzero(idx)` returns two integer array indices, one for each +dimension of `idx`. These indices each have `7` elements, one for each +`True` element of `idx`, and they select (in C order), the corresponding +elements. Another way to think of this is that `idx[np.nonzero(idx)]` will +always return an array of `np.count_nonzero(idx)` `True`s, because +`np.nonzero(idx)` is exactly the integer array indices that select the +`True` elements of `idx`: + +```py +>>> idx[np.nonzero(idx)] +array([ True, True, True, True, True, True, True]) +``` + +What this all means is that all the rules that are outlined previously about +[integer array indices](integer-arrays.md), e.g., [how they +broadcast](integer-array-broadcasting) or [combine together with +slices](integer-arrays-combined-with-basic-indices), all also apply to boolean +array indices after this transformation. This also specifies how boolean array +indices and integer array indices combine +together.[^combining-integer-and-boolean-indices-footnote] + +[^combining-integer-and-boolean-indices-footnote]: Combining an integer array + and boolean array index together is not common, as the shape of the + integer array index would have to be broadcast compatible with the number + of `True` elements in the boolean array. + + ```py + >>> a = np.arange(10).reshape((2, 5)) + >>> a[np.array([0, 1, 0]), np.array([True, False, True, True, False])] + array([0, 7, 3]) + >>> a[np.array([0, 1, 0]), np.array([True, False, True, True, True])] + Traceback (most recent call last): + ... + IndexError: shape mismatch: indexing arrays could not be broadcast together with shapes (3,) (4,) + ``` + + It's not impossible for this to come up in practice, but like many of the + advanced indexing semantics discussed here, it's mostly supported for the + sake of completeness. + +Effectively, a boolean array index can be combined with other boolean or +integer array indices by first converting the boolean index into integer +indices (one for each dimension of the boolean index) that select each `True` +element of the index, and then broadcasting them all to a common shape. + +The ndindex method +[`Tuple.broadcast_arrays()`](ndindex.Tuple.broadcast_arrays) (as well as +[`expand()`](ndindex.Tuple.expand)) will convert boolean array indices into +integer array indices via {external+numpy:func}`numpy.nonzero` and broadcast +array indices together into a canonical form. + +(0-d-boolean-index)= +### Boolean Scalar Indices + +A 0-dimensional boolean index (i.e., just the scalar `True` or `False`) is a +little special. The [`np.nonzero` rule](nonzero-equivalence) stated above does +not actually apply. This is because `np.nonzero` exhibits odd behavior with +0-D arrays. `np.nonzero(a)` usually returns a tuple with as many arrays as +dimensions of `a`: + +```py +>>> np.nonzero(np.array([True, False])) +(array([0]),) +>>> np.nonzero(np.array([[True, False]])) +(array([0]), array([0])) +``` + +But for a 0-D array, `np.nonzero(a)` doesn't return an empty tuple, but +rather the same thing as +`np.nonzero(np.array([a]))`:[^nonzero-deprecated-footnote] + +<!-- TODO: Update this text when NumPy 2.0 is released. --> +[^nonzero-deprecated-footnote]: In NumPy 2.0, calling `nonzero()` on a 0-D + array is deprecated, and in NumPy 2.1 it will result in an error, + precisely due to this odd behavior. + + +```py +>>> np.nonzero(np.array(False)) # doctest: +SKIP +(array([], dtype=int64),) +>>> np.nonzero(np.array(True)) # doctest: +SKIP +(array([0]),) +``` + +However, the key point---that a [boolean array index removes `idx.ndim` +dimensions from `a` and replaces them with a single dimension with size equal +to the number of `True` elements](boolean-array-result-shape)---remains true. +Here, `idx.ndim` is `0`, because `array(True)` and `array(False)` have shape +`()`. Thus, these indices "remove" 0 dimensions and add a single dimension of +size 1 for `True` or 0 for `False`. Hence, if `a` has shape `(s1, ..., sn)`, +then `a[True]` has shape `(1, s1, ..., sn)`, and `a[False]` has shape `(0, s1, +..., sn)`. + +```py +>>> a.shape # as above +(2, 5) +>>> a[True].shape +(1, 2, 5) +>>> a[False].shape +(0, 2, 5) +``` + +This is different from what `a[np.nonzero(True)]` would +return:[^nonzero-scalar-footnote] + +[^nonzero-scalar-footnote]: But note that this also wouldn't work if + `np.nonzero(True)` returned the empty tuple `()`. In fact, there's no + generic index that `np.nonzero()` could return that would be equivalent + to the actual indexing behavior of a boolean scalar, especially for + `False`. + +<!-- TODO: Update this when NumPy 2.0 is released. --> +```py +>>> a[np.nonzero(True)].shape # doctest: +SKIP +(1, 5) +>>> a[np.nonzero(False)].shape # doctest: +SKIP +(0, 5) +``` + +The scalar boolean behavior may seem like an odd corner case. You might wonder +why NumPy supports using a `True` or `False` as an index, especially since it +has slightly different semantics than higher dimensional boolean arrays. + +The reason scalar booleans are supported is that they are a natural +generalization of n-D boolean array indices. While the `np.nonzero()` rule +does not hold for them, the more general rule about replacing +`idx.ndim` dimensions a single dimension does. + +Consider the most common case of using a boolean index: masking some subset of +the entire array. This typically looks something like +`a[some_boolean_expression_on_a] = mask_value`. For example: + +```py +>>> a = np.asarray([[0, 1], [1, 0]]) +>>> a[a == 0] = -1 +>>> a +array([[-1, 1], + [ 1, -1]]) +``` + +Here, we set all the `0` elements of `a` to `-1`. We do this by creating the +boolean mask `a == 0`, which is a boolean expression created from `a`. Our +mask might be a lot more complicated in general, but it still is usually the +case that our mask is constructed from `a`, and thus has the exact same shape +as `a`. Therefore, `a[mask]` is a 1 dimensional array with +`np.count_nonzero(mask)` elements. In this example, this doesn't actually +matter because we are using the mask as the left-hand side of an assignment. +As long as the right-hand side is broadcast compatible with `a[mask]`, it will +be fine. In this case, it works because `-1` is a scalar, which is always +broadcast compatible with everything, but more generally we could index the +right-hand side with the exact same mask index to ensure it is exactly the +same shape as the left-hand side. + +In particular, note that `a[a == 0] = -1` works no matter what the shape or +dimensionality of `a` is, and no matter how many `0` entries it has. Above +it had 2 dimensions and two `0`s, but it would also work if it were +1-dimensional: + +```py +>>> a = np.asarray([0, 1, 0, 1]) +>>> a[a == 0] = -1 +>>> a +array([-1, 1, -1, 1]) +``` + +Or if it had no actual `0`s:[^0-d-mask-footnote] + +[^0-d-mask-footnote]: In this example, `a == 0` is `array([False, False, + False])`, and `a[a == 0]` is an empty array of shape `(0,)`. The reason + this works is that the right-hand side of the assignment is a scalar, + i.e., NumPy casts it to an array of shape `()`. The shape `()` broadcasts + with the shape `(0,)` to the shape `(0,)`, and so this is what gets + assigned, i.e., "nothing" (of shape `(0,)`) gets assigned to "nothing" (of + matching shape `(0,)`). This is one reason why [broadcasting + rules](broadcasting) apply even to dimensions of size 0. + +```py +>>> a = np.asarray([1, 1, 2]) +>>> a[a == 0] = -1 +>>> a +array([1, 1, 2]) +``` + +But even if `a` is a 0-D array, i.e., a single scalar value, we would expect +this sort of thing to still work, since, as we said, `a[a == 0] = -1` should +work for *any* array. And indeed, it does: + +```py +>>> a = np.asarray(0) +>>> a.shape +() +>>> a[a == 0] = -1 +>>> a +array(-1) +``` + +Consider what happened here. `a == 0` is the a 0-D array `array(True)`. +`a[True]` is a 1-D array containing the single True value corresponding to +the mask, i.e., `array([0])`. + +```py +>>> a = np.asarray(0) +>>> a[a == 0] +array([0]) +``` + +This then gets assigned the value `-1`, which as a scalar, gets broadcasted +to the entire array, thereby replacing this single `0` value with `-1`. The +`0` in the masked array corresponds to the same `0` in memory as `a`, so the +assignment mutates it to `-1`. + +If our 0-D `a` was not `0`, then `a == 0` would be `array(False)`. Then `a[a == +0]` would be a 1-D array containing no values, i.e., a shape `(0,)` array: + +```py +>>> a = np.asarray(1) +>>> a[a == 0] +array([], dtype=int64) +>>> a[a == 0].shape +(0,) +``` + +In this case, `a[a == 0] = -1` would assign `-1` to all the values in `a[a +== 0]`, which would be no values, so `a` would remain unchanged: + +```py +>>> a[a == 0] = -1 +>>> a +array(1) +``` + +The point is that the underlying logic works out so that `a[a == 0] = -1` +always does what you'd expect: every `0` value in `a` is replaced with `-1` +*regardless* of the shape of `a`, including if that shape is `()`. + +```{rubric} Footnotes +``` +<!-- Footnotes are written inline above but markdown will put them here at the +end of the document. --> diff --git a/docs/indexing-guide/multidimensional-indices/ellipses.md b/docs/indexing-guide/multidimensional-indices/ellipses.md new file mode 100644 index 00000000..fdd2d1fb --- /dev/null +++ b/docs/indexing-guide/multidimensional-indices/ellipses.md @@ -0,0 +1,149 @@ +# Ellipses + +Now that we understand how [tuple indices](tuples.md) work, the remaining +basic index types are relatively straightforward. The first type of index we +will look at is the ellipsis. An ellipsis is written as literally three dots: +`...`.[^ellipsis-footnote] + +[^ellipsis-footnote]: You can also write out the word `Ellipsis`, but this is + discouraged. In older versions of Python, the three dots `...` were not + valid syntax outside of the square brackets of an index, but as of Python + 3, `...` is valid anywhere, making it unnecessary to use the spelled out + `Ellipsis` in any context. The only reason I mention this is that if you + type `...` at the interpreter, it will print "Ellipsis", and this explains + why. + + ```py + >>> ... + Ellipsis + ``` + + This is also why the type name for the [ndindex `ellipsis`](ellipsis) + object is lowercase, since `Ellipsis` is already a built-in name. + +Consider an array with three dimensions: + +```py +>>> import numpy as np +>>> a = np.arange(24).reshape((3, 2, 4)) +>>> a +array([[[ 0, 1, 2, 3], + [ 4, 5, 6, 7]], +<BLANKLINE> + [[ 8, 9, 10, 11], + [12, 13, 14, 15]], +<BLANKLINE> + [[16, 17, 18, 19], + [20, 21, 22, 23]]]) +``` + +In [one of the examples in the previous section](tuples-slices-example), we +wanted to select only the first element of the last axis, and we saw that we +could use the index `:, :, 0`: + +```py +>>> a[:, :, 0] +array([[ 0, 4], + [ 8, 12], + [16, 20]]) +``` + +However, this index only works for our specific array, because it has 3 +dimensions. If it had 5 dimensions instead, we would need to use `a[:, :, :, +:, 0]`. This is not only tedious to type, but also makes it impossible to +write an index that works for any number of dimensions. To contrast, if we +want the first element of the *first* axis, we could write `a[0]`, which +works if `a` has 3 dimensions or 5 dimensions or any number of dimensions. + +The ellipsis solves this problem. An ellipsis index skips all the axes of an +array to the end, so that the indices after it select the last axes of the +array. + +```py +>>> a[..., 0] +array([[ 0, 4], + [ 8, 12], + [16, 20]]) +``` + +You can also place indices before the ellipsis. The indices before the +ellipsis will select the first axes of the array, and the indices after it +will select the last axes. The ellipsis automatically skips all the +intermediate axes. For example, to select the first element of the first axis +and the last element of the last axis, we could use + +```py +>>> a[0, ..., -1] +array([3, 7]) +``` + +An ellipsis can also skip zero axes if all the axes of the array are already +accounted for. For example, these are the same because `a` has 3 dimensions: + +```py +>>> a[1, 0:2, 2] +array([10, 14]) +>>> a[1, 0:2, ..., 2] +array([10, 14]) +``` + +Indeed, the index `1, 0:2, ..., 2` will work with any array that has *at +least* three dimensions (assuming of course that the first dimension is at +least size `2` and the last dimension is at least size `3`). + +Previously, we saw that a [tuple index](tuples.md) implicitly ends in some +number of trivial `:` slices. We can also see here that a tuple index always +implicitly ends with an ellipsis, serving the same purpose. In other words: + +> **An ellipsis automatically serves as a stand-in for the "correct" number of +trivial `:` slices to select the intermediate axes of an array**. + +And just as with the +empty tuple index `()`, which we saw is the same as writing the right number +of trivial `:` slices, a single ellipsis and nothing else is the same as +selecting every axis of the array, i.e., it leaves the array +intact.[^tuple-ellipsis-footnote] + +[^tuple-ellipsis-footnote]: See [footnote 2](tuple-ellipsis-footnote-ref) in + the [](tuples.md) section. + +```py +>>> a[...] +array([[[ 0, 1, 2, 3], + [ 4, 5, 6, 7]], +<BLANKLINE> + [[ 8, 9, 10, 11], + [12, 13, 14, 15]], +<BLANKLINE> + [[16, 17, 18, 19], + [20, 21, 22, 23]]]) +``` + +Finally, only one ellipsis is allowed (otherwise it would be ambiguous which +axes are being indexed): + +```py +>>> a[0, ..., 1, ..., 2] +Traceback (most recent call last): + File "<stdin>", line 1, in <module> +IndexError: an index can only have a single ellipsis ('...') +``` + +In summary, the rules for an ellipsis index are + +- **An ellipsis index is written with three dots: `...`.** + +- **`...` automatically selects 0 or more intermediate axes in an array.** + +- **Every index before `...` operates on the first axes of the array. Every + index after `...` operates on the last axes of the array.** + +- **Every tuple index that does not have an ellipsis in it implicitly ends in + `...`.** + +- **At most one `...` is allowed in a tuple index.** + +```{rubric} Footnotes +``` +<!-- Footnotes are written inline above but markdown will put them here at the +end of the document. --> diff --git a/docs/indexing-guide/multidimensional-indices/index.md b/docs/indexing-guide/multidimensional-indices/index.md new file mode 100644 index 00000000..bd69990f --- /dev/null +++ b/docs/indexing-guide/multidimensional-indices/index.md @@ -0,0 +1,60 @@ +# Multidimensional Indices + +This section of the indexing guide deals with indices that only operate on +NumPy arrays. Unlike [integers](../integer-indices.md) and +[slices](../slices.md), which also work on built-in Python sequence types such +as `list`, `tuple`, and `str`, the remaining index types do not work at all on +built-in sequence types. For example, if you try to use a [tuple +index](tuples.md) on a `list`, you will get an `IndexError` The semantics of +these indices are defined by the NumPy library, not the Python language. + +To begin, we should be sure we understand what an array is: + +- [](what-is-an-array.md) + +(basic-indices)= +## Basic Multidimensional Indices + +There are two types of multidimensional indices, basic and advanced indices. +Basic indices are so-called because they are simpler and the most common. They +also are notable because they always return a view (see [](views-vs-copies)). + +We've already learned about two types of basic indices in previous sections: + +- [](../integer-indices.md) +- [](../slices.md) + +There are three others: + +- [](tuples.md) +- [](ellipses.md) +- [](newaxis.md) + +(advanced-indices)= +## Advanced Indices + +Lastly are the so-called advanced indices. These are "advanced" in the sense +that they are more complex. They are also distinct from "basic" indices in +that they always return a copy (see [](views-vs-copies)). Advanced indices +allow selecting arbitrary parts of an array, in ways that are impossible with +the basic index types. Advanced indexing is also sometimes called "fancy +indexing" or indexing by arrays, as the indices themselves are arrays. + +Using an array that does not have an integer or boolean dtype as an index +results in an error. + +- [](integer-arrays.md) +- [](boolean-arrays.md) + +```{toctree} +:titlesonly: +:hidden: + +what-is-an-array.md +tuples.md +ellipses.md +newaxis.md +integer-arrays.md +boolean-arrays.md + +``` diff --git a/docs/indexing-guide/multidimensional-indices/integer-arrays.md b/docs/indexing-guide/multidimensional-indices/integer-arrays.md new file mode 100644 index 00000000..c95d45e9 --- /dev/null +++ b/docs/indexing-guide/multidimensional-indices/integer-arrays.md @@ -0,0 +1,796 @@ +# Integer Array Indices + +```{note} +In this section, and [the next](boolean-arrays), do not confuse the *array +being indexed* with the *array that is the index*. The former can be anything +and have any dtype. It is only the latter that is restricted to being integer +or boolean. +``` + +Integer array indices are very powerful. Using them, you can effectively +construct arbitrary new arrays consisting of elements from the original +indexed array. + + +To start, let's consider a simple one-dimensional array: + +```py +>>> import numpy as np +>>> a = np.array([100, 101, 102, 103]) +``` + +Now suppose we wish to construct the following 2-D array from this array using +only indexing operations: + +``` +[[ 100, 102, 100 ], + [ 103, 100, 102 ]] +``` + +It should hopefully be clear that there's no way we could possibly construct +this array as `a[idx]` using only the index types we've discussed so far. For +one thing, [integer indices](../integer-indices.md), [slices](../slices.md), +[ellipses](ellipses.md), and [newaxes](newaxis.md) all only select +elements of the array in order (or possibly [reversed order](negative-steps) +for slices), whereas this array has elements completely shuffled from `a`, and +some are even repeated. + +However, we could "cheat" a bit here, and do something like + +```py +>>> new_array = np.array([[a[0], a[2], a[0]], +... [a[3], a[0], a[2]]]) +>>> new_array +array([[100, 102, 100], + [103, 100, 102]]) +``` + +This is the array we want. We sort of constructed it using only indexing +operations, but we didn't actually do `a[idx]` for some index `idx`. Instead, +we just listed the index of each individual element. + +An integer array index is essentially this "cheating" method, but as a single +index. Instead of listing out `a[0]`, `a[2]`, and so on, we just create a +single integer array with those [integer indices](../integer-indices.md): + +```py +>>> idx = np.array([[0, 2, 0], +... [3, 0, 2]]) +``` + +If we then index `a` with this array, it works just like `new_array` above: + +```py +>>> a[idx] +array([[100, 102, 100], + [103, 100, 102]]) +``` + +(multidimensional-integer-indices)= +This is how integer array indices work: + +> **An integer array index can construct *arbitrary* new arrays with elements +from `a`, with the elements in any order and even repeated, simply by +enumerating the integer index positions where each element of the new array +comes from.** + +Note that `a[idx]` above is not the same size as `a` at all. `a` has 4 +elements and is 1-dimensional, whereas `a[idx]` has 6 elements and is +2-dimensional. `a[idx]` also contains some duplicate elements from `a`, and +there are some elements which aren't selected at all. Indeed, we could take +*any* integer array of any shape, and as long as the elements are between 0 +and 3, `a[idx]` would create a new array with the same shape as `idx` with +corresponding elements selected from `a`. + +A useful way to think about integer array indexing is that it generalizes +[integer indexing](../integer-indices.md). With integer indexing, we are +effectively indexing using a 0-dimensional integer array, that is, a single +integer.[^integer-scalar-footnote] This always selects the corresponding +element from the given axis and removes the dimension. That is, it replaces +that dimension in the shape with `()`, the "shape" of the integer index. + +Similarly, + +> **an integer array index `a[idx]` selects elements from the specified axis +> and replaces the dimension in the shape with the shape of the index array +> `idx`.** + + +For example: + +``` +>>> a = np.empty((3, 4)) +>>> idx = np.zeros((2, 2), dtype=int) +>>> a[idx].shape # (3,) is replaced with (2, 2) +(2, 2, 4) +>>> a[:, idx].shape # Indexing the second dimension, (4,) is replaced with (2, 2) +(3, 2, 2) +``` + +In particular, even when the index array `idx` has more than one dimension, an +integer array index still only selects elements from a single axis of `a`. + +``` +>>> a = np.array([[100, 101, 102], +... [103, 104, 105]]) +>>> idx = np.array([0, 0, 1]) +>>> a[idx] # Index the first dimension +array([[100, 101, 102], + [100, 101, 102], + [103, 104, 105]]) +>>> a[:, idx] # Index the second dimension +array([[100, 100, 101], + [103, 103, 104]]) +``` + +It would appear that this limits the ability to arbitrarily shuffle elements +of `a` using integer indexing. For instance, suppose we want to create the +array `[105, 100]` from the above 2-D `a`. Based on the above examples, it +might not seem possible. The elements `105` and `100` are not in the same row +or column of `a`. + +However, this is doable by providing multiple integer array +indices: + +(multiple-integer-arrays)= +> **When multiple integer array indices are provided, the elements of each +> index are selected correspondingly for that axis.** + +It's perhaps most illustrative to +show this as an example. Given the above `a`, we can produce the array `[105, +100]` using. + +``` +>>> idx = (np.array([1, 0]), np.array([2, 0])) +>>> a[idx] +array([105, 100]) +``` + +Let's break this down. `idx` is a [tuple index](tuples.md) with two arrays, +which are both the same shape. The first element of our desired result, `105` +corresponds to index `(1, 2)` in `a`: + +```py +>>> a[1, 2] # doctest: +SKIPNP1 +np.int64(105) +``` + +So we write `1` in the first array and `2` in the second array. Similarly, the +next element, `100` corresponds to index `(0, 0)`, so we write `0` in the +first array and `0` in the second. In general, the first array contains the +indices for the first axis, the second array contains the indices for the +second axis, and so on. If we were to +[zip](https://docs.python.org/3/library/functions.html#zip) up our two index +arrays, we would get the set of indices for each corresponding element, `(1, +2)` and `(0, 0)`. + +The resulting array has the same shape as our two index arrays. As before, +this shape can be arbitrary. Suppose we want to create the array + +``` +[[[ 102, 103], + [ 102, 101]], + [[ 100, 105], + [ 102, 102]]] +``` + +Recall our array `a`: + +``` +>>> a +array([[100, 101, 102], + [103, 104, 105]]) +``` + +Noting the index for each element in our desired array, we get + +``` +>>> idx0 = np.array([[[0, 1], [0, 0]], [[0, 1], [0, 0]]]) +>>> idx1 = np.array([[[2, 0], [2, 1]], [[0, 2], [2, 2]]]) +>>> a[idx0, idx1] +array([[[102, 103], + [102, 101]], +<BLANKLINE> + [[100, 105], + [102, 102]]]) +``` + +Again, reading across, the first element, `102` corresponds to index `(0, 2)`, +the next element, `103`, corresponds to index `(1, 0)`, and so on. + +## Use Cases + +A common use case for integer array indexing is sampling. For example, to +sample $k$ elements from a 1-D array of size $n$ with replacement, we can +simply construct an a random integer index in the range $[0, n)$ with $k$ +elements (see the +{external+numpy:meth}`numpy.random.Generator.integers` +documentation):[^random-integers-footnote] + +[^random-integers-footnote]: Note that `np.random` also supports this + operation directly with + {external+numpy:meth}`numpy.random.Generator.choice`. + +``` +>>> k = 10 +>>> a = np.array([100, 101, 102, 103]) # as above +>>> rng = np.random.default_rng(11) # Seeded so this example reproduces +>>> idx = rng.integers(0, a.size, k) # rng.integers() excludes the upper bound +>>> idx +array([0, 0, 3, 1, 2, 2, 2, 0, 1, 0]) +>>> a[idx] +array([100, 100, 103, 101, 102, 102, 102, 100, 101, 100]) +``` + +(permutation-example)= + +Another common use case of integer array indexing is to permute an array. An +array can be randomly permuted with +{external+numpy:meth}`numpy.random.Generator.permutation`. But what if we want +to permute two arrays with the same permutation? We can compute a permutation +index and apply it to both arrays. For a 1-D array `a` of size $n$, a +permutation index is just a permutation of the integer array index +`np.arange(n)`, which itself is the [identity +permutation](https://en.wikipedia.org/wiki/identity_permutation) on `a`: + +```py +>>> a = np.array([100, 101, 102, 103]) # as above +>>> b = np.array([200, 201, 202, 203]) # another array +>>> identity = np.arange(a.size) +>>> a[identity] # arange by itself is the identity permutation index +array([100, 101, 102, 103]) +>>> rng = np.random.default_rng(11) # Seeded so this example reproduces +>>> random_permutation = rng.permutation(identity) +>>> a[random_permutation] +array([103, 101, 100, 102]) +>>> b[random_permutation] # The same permutation on b +array([203, 201, 200, 202]) +``` + +(integer-arrays-advanced-notes)= +## Advanced Notes + +The information above provides the basic gist of integer array indexing, but +there are also many subtleties and advanced behaviors involved with them. The +subsections here are included for completeness; however, if you are a beginner +NumPy user, you may wish to skip them. + +### Negative Indices + +> **Indices in the integer array can also be negative. Negative indices work +the same as they do with [integer indices](../integer-indices.md).** + +Negative and +nonnegative indices can be mixed arbitrarily. + +```py +>>> a = np.array([100, 101, 102, 103]) # as above +>>> idx = np.array([0, 1, -1]) +>>> a[idx] +array([100, 101, 103]) +``` + +If you want to convert an index containing negative indices into an index +without any negative indices, you can use the ndindex +[`reduce()`](ndindex.IntegerArray.reduce) method with a `shape` argument. + +### Python Lists + +> **You can use a list of integers instead of an array to represent an integer +array index.[^lists-footnote]** + +Using a list is useful when writing an array index by hand; however, in all +other cases, using an actual array is preferable. In most real-world +scenarios, an array index is constructed from some other array methods. + +[^lists-footnote]: Beware that [versions of NumPy prior to + 1.23](https://numpy.org/doc/stable/release/1.23.0-notes.html#expired-deprecations) + treated a single list as a [tuple index](tuples.md) rather than as an + array. + + +```py +>>> a = np.array([100, 101, 102, 103]) # as above +>>> a[[0, 1, -1]] +array([100, 101, 103]) +>>> idx = np.array([0, 1, -1]) +>>> a[idx] # this is the same +array([100, 101, 103]) +``` + +(integer-array-broadcasting)= +### Broadcasting + +> **The integer arrays in an index must either be the same shape or be able to +> be [broadcast](broadcasting) together to the same shape.** + +If the arrays are not the same shape, they are first broadcast together, and +those broadcasted arrays are used as the indices. This broadcasting behavior +is useful if the index array would otherwise be repeated in a given dimension, +and provides a convenient way to do outer indexing (see the [next +section](outer-indexing)). + +This also means that mixing an integer array index with a single [integer +index](../integer-indices.md) is the same as replacing the single integer +index with an array of the same shape filled with that integer (because +remember, a single integer index is the same thing as an integer array index +of shape `()`). + +For example: + +```py +>>> a = np.array([[100, 101, 102], # as above +... [103, 104, 105]]) +>>> idx0 = np.array([1, 0]) +>>> idx0.shape +(2,) +>>> idx1 = np.array([[0], [1], [2]]) +>>> idx1.shape +(3, 1) +>>> # idx0 and idx1 broadcast to shape (3, 2), which will +>>> # be the shape of a[idx0, idx1] +>>> a[idx0, idx1] +array([[103, 100], + [104, 101], + [105, 102]]) +>>> a[idx0, idx1].shape +(3, 2) +>>> idx0_broadcasted = np.array([[1, 0], [1, 0], [1, 0]]) +>>> idx1_broadcasted = np.array([[0, 0], [1, 1], [2, 2]]) +>>> idx0_broadcasted.shape +(3, 2) +>>> idx1_broadcasted.shape +(3, 2) +>>> a[idx0_broadcasted, idx1_broadcasted] # The same thing as a[idx0, idx1] +array([[103, 100], + [104, 101], + [105, 102]]) +``` + +(mixing-array-and-integer)= +And mixing an array and an integer index: + +```py +>>> a +array([[100, 101, 102], + [103, 104, 105]]) +>>> idx0 = np.array([1, 0, 0]) +>>> a[idx0, 2] +array([105, 102, 102]) +>>> idx1_broadcasted = np.array([2, 2, 2]) # The 0-D array '2' broadcasted to shape (3,) +>>> a[idx0, idx1_broadcasted] # The same thing as a[idx0, 2] +array([105, 102, 102]) +``` + +Here the `idx0` array specifies the indices along the first dimension, `1`, +`0`, and `0`, and the `2` specifies to always use index `2` along the second +dimension. This is the same as using the array `[2, 2, 2]` for the second +dimension, since this is the scalar `2` broadcasted to the shape of `[1, 0, +0]`. + +The ndindex methods +[`Tuple.broadcast_arrays()`](ndindex.Tuple.broadcast_arrays) and +[`expand()`](ndindex.Tuple.expand) will broadcast array indices together into +a canonical form. + +[^integer-scalar-footnote]: + <!-- This is the only way to cross reference a footnote across documents --> + (integer-scalar-footnote-ref)= + + In fact, if the integer array index itself has + shape `()`, then the behavior is identical to simply using an `int` with + the same value. So it's a true generalization. In ndindex, + [`IntegerArray.reduce()`](ndindex.IntegerArray.reduce) will always convert + a 0-D array index into an [`Integer`](ndindex.integer.Integer). + + However, there is one difference between `a[0]` and `a[asarray(0)]`. The + latter is considered an advanced index, so it does not create a + [view](views-vs-copies): + + ```py + >>> a = np.empty((2, 3)) + >>> a[0].base is a + True + >>> print(a[np.array(0)].base) + None + ``` + +(outer-indexing)= +#### Outer Indexing + +The broadcasting behavior for multiple integer indices may seem odd, but it +serves a useful purpose. [As we saw above](multiple-integer-arrays), multiple +integer array indices are required to select elements from higher dimensional +arrays, one array for each dimension. These integer arrays enumerate the +indices of the selected elements along these dimensions. For example, as +above: + +```py +>>> a = np.array([[100, 101, 102], +... [103, 104, 105]]) +>>> a[[1, 0], [2, 0]] # selects elements (1, 2) and (0, 0) +array([105, 100]) +``` + +However, you might have noticed that this behavior is somewhat unusual +compared to other index types. For all other index types we've discussed so +far, such as [slices](../slices.md) and [integer indices](../integer-indices.md), +each index applies "independently" along each dimension. For example, `x[0:3, +0:2]` applies the slice `0:3` to the first dimension of `x` and `0:2` to the +second dimension. The resulting array has `3*2 = 6` elements, because there +are 3 subarrays selected from the first dimension with 2 elements each. But in +the above example, `a[[1, 0], [2, 0]]` only has 2 elements, not 4. And +something like `a[[1, 0], [2, 0, 1]]` is an error. + +The integer array equivalent of the way slices work is called "outer +indexing".[^vectorized-indexing-footnote] An outer index "`a[[1, 0], [2, 0, 1]]`" would have 6 elements: rows +1 and 0, with elements from columns 2, 0, and 1, in that order. However, the +index `a[[1, 0], [2, 0, 1]]` doesn't actually work like +this.[^outer-indexing-footnote] + +[^vectorized-indexing-footnote]: The type of integer array indexing that NumPy + uses where arrays are broadcasted and "zipped" together is sometimes called + "vectorized indexing" or "inner indexing". The "outer" and "inner" are + because they act like an outer- or inner-product. + +[^outer-indexing-footnote]: Outer indexing is how integer array indexing works + in many other languages such as MATLAB, Fortran, and R. There is a proposed + [NEP](https://numpy.org/neps/nep-0021-advanced-indexing.html) to add more + direct support for outer indexing like this to NumPy, but it hasn't been + accepted yet. + +Strictly speaking, though, NumPy's integer array indexing rules do allow for +outer indexing. This is because, as we saw above, integer array indexing +allows for creating *arbitrary* new arrays from a given input array. And as it +turns out, the integer arrays required to represent an outer array index are +quite simple to construct. They are simply the outer index arrays broadcasted +together. + +To see why this is, consider the above example, `a[[1, 0], [2, 0, 1]]`. We +want our end result to be + +``` +[[105, 103, 104], + [102, 101, 100]] +``` + +That is, the rows of `a` should be in the order `[0, 1]`, and the columns +should be in the order `[2, 0, 1]`. The end result should be an array of shape +`(2, 3)` (which happens to be the same shape as `a`, but that's just a +coincidence; an outer-indexed array constructed from `a` could have any 2-D +shape). So using the integer array indexing rules above, we need to index `a` +by integer arrays of shape `(2, 3)`. Since `a` has two dimensions, we will +need two index arrays, one for each dimension. Let's consider what these +arrays should be. For the first dimension, we want to select row `1` three +times and then row `0` three times: + +``` +[[1, 1, 1], + [0, 0, 0]] +``` + +And for the second dimension, we want to select the columns `2`, `0`, and `1`, +in that order, regardless of which row we are in: + +``` +[[2, 0, 1], + [2, 0, 1]] +``` + +In general, we want to repeat each outer selection array along the +corresponding dimension so as to fill an array with the final desired shape. +This is exactly what broadcasting does! If we reshape our first array to have +shape `(2, 1)` and the second array to have shape `(1, 3)`, then broadcasting +them together will repeat the first dimension of the first array along the +second axis, and the second dimension of the second array along the first +axis, i.e., exactly the arrays we want. + +This is why NumPy automatically broadcasts integer array indices together. + +> **Outer indexing arrays can be constructed by inserting size-1 dimensions +> into the desired "outer" integer array indices so that the non-size-1 +> dimension for each is in the indexing dimension.** + +For example, + +```py +>>> idx0 = np.array([1, 0]) +>>> idx1 = np.array([2, 0, 1]) +>>> a[idx0[:, np.newaxis], idx1[np.newaxis, :]] +array([[105, 103, 104], + [102, 100, 101]]) +``` + +Here, we use [newaxis](newaxis.md) along with `:` to turn `idx0` and +`idx1` into shape `(2, 1)` and `(1, 3)` arrays, respectively. These then +automatically broadcast together to give the desired outer index. + +This "insert size-1 dimensions" operation can also be performed automatically +with the {external+numpy:func}`numpy.ix_` function.[^ix-footnote] + +[^ix-footnote]: `ix_()` is currently limited to only support 1-D input arrays + and can't be mixed with other index types. In the general case you will + need to apply the reshaping operation manually. There is an [open + issue](https://github.com/Quansight-Labs/ndindex/issues/29) to implement + this more generally in ndindex. + +```py +>>> np.ix_(idx0, idx1) +(array([[1], + [0]]), array([[2, 0, 1]])) +>>> a[np.ix_(idx0, idx1)] +array([[105, 103, 104], + [102, 100, 101]]) +``` + +Outer indexing can be thought of as a generalization of slicing. With a +[slice](../slices.md), you can really only select a "regular" sequence of +elements from a dimension, that is, either a contiguous chunk, or a contiguous +chunk split by a regular step value. It's impossible, for instance, to use a +slice to select the indices `[0, 1, 2, 3, 5, 6, 7]`, because `4` is omitted. +For instance, say the first dimension of your array represents time steps and +you want to select time steps 0--7, but time step 4 is invalid for some reason +and you want to ignore it for your analysis. If you just care about the first +dimension, you can just use the integer index `[0, 1, 2, 3, 5, 6, 7]`. But +suppose you also wanted select some other non-contiguous "slice" from the +second dimension. Using just basic indices, you'd have to index the array with +normal slices then either remove or ignore the non-desired indices, neither of +which is ideal. And it would be even more complicated if you also wanted the +indices out-of-order or repeated for some reason. + +With outer indexing, you would just construct your "slice" of non-contiguous +indices as integer arrays, turn them into "outer" indices using `ix_` or +manual reshaping, then use that outer index to construct the desired array +directly. + +Conversely, a slice like `2:9` is equivalent to the outer index `[2, 3, +4, 5, 6, 7, 8]`.[^slice-outer-index-footnote] + +[^slice-outer-index-footnote]: They aren't actually equivalent, because [a + slice creates a view and an integer array index creates a + copy](views-vs-copies). If your index can be represented as a slice, it's + better to use an actual `slice`. + +### Assigning to an Integer Array Index + +As with all index types discussed in this guide, an integer array index can be +used on the left-hand side of an assignment. This is useful because it allows +you to surgically inject new elements into your array. + +```py +>>> a = np.array([100, 101, 102, 103]) # as above +>>> idx = np.array([0, 3]) +>>> a[idx] = np.array([200, 203]) +>>> a +array([200, 101, 102, 203]) +``` + +However, exercise caution, as this is inherently ambiguous if the index array +contains duplicate elements. For example, suppose we attempted to +set index `0` to both `1` and `3`: + +```py +>>> a = np.array([100, 101, 102, 103]) # as above +>>> idx = np.array([0, 1, 0]) +>>> a[idx] = np.array([1, 2, 3]) +>>> a +array([ 3, 2, 102, 103]) +``` + +The end result was `3`. This happened because `3` corresponded to the last `0` +in the index array. But importantly, this is just an implementation detail. +**NumPy makes no guarantees regarding the order in which index elements are +assigned.**[^cupy-assignment-footnote] If you are using an integer array as an +assignment index, be careful to avoid duplicate entries in the index or, at +the very least, ensure that duplicate entries are always assigned the same +value. + +[^cupy-assignment-footnote]: For example, in [CuPy](https://cupy.dev/), + which implements the NumPy API on top of GPUs, [the behavior of this sort + of thing is + undefined](https://docs.cupy.dev/en/stable/reference/generated/cupy.ndarray.html#cupy.ndarray.__setitem__). + This is because CuPy parallelizes the assignment operation on the GPU, and + the element that gets assigned to the duplicate index "last" becomes + dependent on a race condition. + +(integer-arrays-combined-with-basic-indices)= +### Combining Integer Arrays Indices with Basic Indices + +If any [slice](../slices.md), [ellipsis](ellipses.md), or +[newaxis](newaxis.md) indices precede or follow all the +[integer](../integer-indices) and integer array indices in an index, the two +sets of indices operate independently. Slices and ellipses select the +corresponding axes, newaxes add new axes to these locations, and the integer +array indices select the elements on their respective axes, as previously +described. + +For example, consider: + +```py +>>> a = np.array([[[100, 101, 102], # Like above, but with an extra dimension +... [103, 104, 105]]]) +``` + +This is the same `a` as in the above examples, except it has an extra size-1 +dimension: + +```py +>>> a.shape +(1, 2, 3) +``` + +We can select this first dimension with a slice `:`, then use the exact same +index as in the example [shown previously](mixing-array-and-integer): + +```py +>>> idx0 +array([1, 0]) +>>> a[:, idx0, 2] +array([[105, 102]]) +>>> a[:, idx0, 2].shape +(1, 2) +``` + +The primary point of this behavior is that you can use `...` at the beginning +of an index to select the last axes of an array using integer array indices, +or several `:`s to select some middle axes. This lets you do with indexing +what you can also do with the {external+numpy:func}`numpy.take` function. + +To be sure though, this index could use any slice, not just `:`, and could +also include newaxes. This behavior is mainly implemented for the sake of +semantic completeness, although it could potentially allow combining two +sequential indexing operations into a single step. + +### Integer Array Indices Separated by Basic Indices + +Finally, if the [slices](../slices.md), [ellipses](ellipses.md), or +[newaxes](newaxis.md) are *in between* the integer array indices, then +something more strange happens. The two index types still operate +"independently"; however, unlike the previous case, the shape derived from the +array indices is *prepended* to the shape derived from the non-array indices. +This is because in these cases there is inherent ambiguity in where these +dimensions should be placed in the final shape. + +An example demonstrates this most clearly: + +```py +>>> a = np.empty((2, 3, 4, 5)) +>>> a.shape +(2, 3, 4, 5) +>>> idx = np.zeros((10, 20), dtype=int) +>>> idx.shape +(10, 20) +>>> a[idx, :, :, idx].shape +(10, 20, 3, 4) +``` + +Here the integer array index shape `(10, 20)` comes first in the result array +and the shape corresponding to the rest of the index, `(3, 4)`, comes last. + +If you find yourself running into this behavior, chances are you would be +better off rewriting the indexing operation to be simpler, for instance, by +first reshaping the array so that the integer array indices are together in +the index. This is considered a design flaw in +NumPy[^advanced-indexing-design-flaw-footnote], and no other Python array +library has replicated it. ndindex will raise a `NotImplementedError` +exception on indices like these, because I don't want to deal with +implementing this obscure +logic.[^ndindex-advanced-indexing-design-flaw-footnote] + +[^advanced-indexing-design-flaw-footnote]: Travis Oliphant, the original + creator of NumPy, told me privately that "somebody should have slapped me + with a wet fish" when he designed this. + +[^ndindex-advanced-indexing-design-flaw-footnote]: I might accept a pull + request implementing it, but I'm not going to do it myself. + +## Exercise + +Based on the above sections, you should be able to complete the following +exercise: How might you randomly permute a 2-D array using +{external+numpy:meth}`numpy.random.Generator.permutation` and indexing, so +that each axis is permuted independently? This operation might correspond to +multiplying the array by random [permutation +matrices](https://en.wikipedia.org/wiki/Permutation_matrix) on the left and +right, like $P_1AP_2$. + +For example, the array + + +```py +a = array([[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11]]) +``` + +Might be permuted to + +```py +a_perm = array([[ 5, 4, 6, 7], + [ 1, 0, 2, 3], + [ 9, 8, 10, 11]]) +``` + +(Note that this is not a full permutation of the array. For instance, the +first row `[5, 4, 7, 6]` contains only elements from the second row of `a`.) + +~~~~{dropdown} Click here to show the solution + +Suppose we start with the following 2-D array `a`: + +```py +>>> a = np.arange(12).reshape((3, 4)) +>>> a +array([[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11]]) +``` + +We can generate permutations for the two axes using +{external+numpy:meth}`numpy.random.Generator.permutation` [as +above](permutation-example): + +```py +>>> rng = np.random.default_rng(11) # Seeded so this example reproduces +>>> idx0 = rng.permutation(np.arange(3)) +>>> idx1 = rng.permutation(np.arange(4)) +``` + +However, we cannot do `a[idx0, idx1]` as this will fail. + +```py +>>> a[idx0, idx1] +Traceback (most recent call last): +... +IndexError: shape mismatch: indexing arrays could not be broadcast together +with shapes (3,) (4,) +``` + +Remember that we want a permutation of `a`, so the result array should have +the same shape as `a` (`(3, 4)`). This should therefore be the broadcasted +shape of `idx0` and `idx1`, which are currently shapes `(3,)`, and `(4,)`. We +can use [`newaxis`](newaxis.md) to insert dimensions so that they are +shape `(3, 1)` and `(1, 4)` so that they broadcast together to this shape. + +```py +>>> a[idx0[:, np.newaxis], idx1[np.newaxis]] +array([[ 5, 4, 6, 7], + [ 1, 0, 2, 3], + [ 9, 8, 10, 11]]) +``` + +You can check that this is a permutation of `a` where each axis is permuted +independently. + +We can also interpret this as an [outer indexing](outer-indexing) operation. +In this case, our non-contiguous "slices" that we are outer indexing by are a +full slice along each axis, just permuted. We can use the `ix_()` helper to +construct the same index as above + +``` +>>> a[np.ix_(idx0, idx1)] +array([[ 5, 4, 6, 7], + [ 1, 0, 2, 3], + [ 9, 8, 10, 11]]) +``` + +As an extra bonus, here's how we can interpret this as a multiplication by +permutation matrices, using the same indices (but of course, simply permuting +`a` directly with the indices is more efficient): + +```py +>>> P1 = np.eye(3, dtype=int)[idx0] +>>> P2 = np.eye(4, dtype=int)[idx1] +>>> P1 @ a @ P2.T +array([[ 5, 4, 6, 7], + [ 1, 0, 2, 3], + [ 9, 8, 10, 11]]) +``` + +Can you see why this works? +~~~~ + +```{rubric} Footnotes +``` +<!-- Footnotes are written inline above but markdown will put them here at the +end of the document. --> diff --git a/docs/indexing-guide/multidimensional-indices/newaxis.md b/docs/indexing-guide/multidimensional-indices/newaxis.md new file mode 100644 index 00000000..9d83b186 --- /dev/null +++ b/docs/indexing-guide/multidimensional-indices/newaxis.md @@ -0,0 +1,264 @@ +# newaxis + +The final basic multidimensional index type is `newaxis`. `np.newaxis` is an +alias for `None`. Both `np.newaxis` and `None` function identically; however, +`np.newaxis` is often more explicit than `None`, which may appear odd in an +index, so it is generally preferred. However, some people do use `None` +directly instead of `np.newaxis`, so it's important to remember that they are +the same thing. + +```py +>>> import numpy as np +>>> print(np.newaxis) +None +>>> np.newaxis is None # They are exactly the same thing +True +``` + +`newaxis`, as the name suggests, adds a new axis. This new axis has size `1`. +The new axis is added at the corresponding location within the array. A size +`1` axis neither adds nor removes any elements from the array. Using the +[nested lists analogy](what-is-an-array.md), it essentially adds a new "layer" +to the list of lists. + + +```py +>>> b = np.arange(4) +>>> b +array([0, 1, 2, 3]) +>>> b[np.newaxis] +array([[0, 1, 2, 3]]) +>>> b.shape +(4,) +>>> b[np.newaxis].shape +(1, 4) +``` + +Including `newaxis` alongside other indices in a [tuple index](tuples.md) does +not affect which axes those indices select. You can think of the `newaxis` +index as inserting the new axis in-place in the index, so that the other +indices still select the same corresponding axes they would select if it +weren't there. + +Take our example array, which has shape `(3, 2, 4)`: + +```py +>>> a = np.arange(24).reshape((3, 2, 4)) +>>> a.shape +(3, 2, 4) +``` + +The index `a[0, :2]` results in a shape of `(2, 4)`: the integer index `0` +removes the first axis, the slice `:2` selects 2 elements from the second +axis, and the third axis is not selected at all, so it remains intact with 4 +elements. + +```py +>>> a[0, :2] +array([[0, 1, 2, 3], + [4, 5, 6, 7]]) +>>> a[0, :2].shape +(2, 4) +``` + +Now, observe the shape of `a` when we insert `newaxis` at various points +within the index `a[0, :2]`: + +```py +>>> a[np.newaxis, 0, :2].shape +(1, 2, 4) +>>> a[0, np.newaxis, :2].shape +(1, 2, 4) +>>> a[0, :2, np.newaxis].shape +(2, 1, 4) +>>> a[0, :2, ..., np.newaxis].shape +(2, 4, 1) +``` + +In each case, the exact same elements are selected: `0` always targets the +first axis, and `:2` always targets the second axis. The only difference is +where the size-1 axis is inserted: + +```py +>>> a[np.newaxis, 0, :2] +array([[[0, 1, 2, 3], + [4, 5, 6, 7]]]) +>>> a[0, np.newaxis, :2] +array([[[0, 1, 2, 3], + [4, 5, 6, 7]]]) +>>> a[0, :2, np.newaxis] +array([[[0, 1, 2, 3]], +<BLANKLINE> + [[4, 5, 6, 7]]]) +>>> a[0, :2, ..., np.newaxis] +array([[[0], + [1], + [2], + [3]], +<BLANKLINE> + [[4], + [5], + [6], + [7]]]) +``` + + +- `a[np.newaxis, 0, :2]`: the new axis is inserted before the first axis, but +the `0` and `:2` still index the original first and second axes. The resulting +shape is `(1, 2, 4)`. + +- `a[0, np.newaxis, :2]`: the new axis is inserted after the first axis, but +because the `0` removes this axis when it indexes it, the resulting shape is +still `(1, 2, 4)` (and the resulting array is the same). + +- `a[0, :2, np.newaxis]`: the new axis is inserted after the second axis, +because the `newaxis` comes right after the `:2`, which indexes the second +axis. The resulting shape is `(2, 1, 4)`. Remember that the `4` in the shape +corresponds to the last axis, which isn't represented in the index at all. +That's why in this example, the `4` still comes at the end of the resulting +shape. + +- `a[0, :2, ..., np.newaxis]`: the `newaxis` is after an ellipsis, so the new +axis is inserted at the end of the shape. The resulting shape is `(2, 4, 1)`. + +In general, in a tuple index, the axis that each index selects corresponds to +its position in the tuple index after removing any `newaxis` indices +Equivalently, `newaxis` indices can be though of as adding new axes *after* +the existing axes are indexed. + +A size-1 axis can always be inserted anywhere in an array's shape without +changing the underlying elements. + +An array index can include multiple instances of `newaxis` (or `None`). Each +will add a size-1 axis in the corresponding location. + +**Exercise:** Can you determine the shape of this array, given that `a.shape` +is `(3, 2, 4)`? + +```py +a[np.newaxis, 0, newaxis, :2, newaxis, ..., newaxis] +``` + +~~~~{dropdown} Click here to show the solution + +```py +>>> a[np.newaxis, 0, np.newaxis, :2, np.newaxis, ..., np.newaxis].shape +(1, 1, 2, 1, 4, 1) +``` + +~~~~ + +In summary, + +> **`np.newaxis` (which is just an alias for `None`) inserts a new size-1 axis + in the corresponding location in the tuple index. The remaining, + non-`newaxis` indices in the tuple index are indexed as if the `newaxis` + indices were not there.** + +## Where `newaxis` is Used + +What we haven't said yet is why you would want to do such a thing in the first +place. One use case is to explicitly convert a 1-D vector into a 2-D matrix +representing a row or column vector. For example, + +```py +>>> v = np.array([0, 1, -1]) +>>> v.shape +(3,) +>>> v[np.newaxis] +array([[ 0, 1, -1]]) +>>> v[np.newaxis].shape +(1, 3) +>>> v[..., np.newaxis] +array([[ 0], + [ 1], + [-1]]) +>>> v[..., np.newaxis].shape +(3, 1) +``` + +`v[newaxis]` inserts an axis at the beginning of the shape, making `v` a `(1, +3)` row vector and `v[..., newaxis]` inserts an axis at the end, making it a +`(3, 1)` column vector. + +But the most common usage is due to [broadcasting](broadcasting). The key idea +of broadcasting is that size-1 dimensions are not directly useful, in the +sense that they could be removed without actually changing anything about the +underlying data in the array. So they are used as a signal that that dimension +can be repeated in operations. `newaxis` is therefore useful for inserting +these size-1 dimensions in situations where you want to force your data to be +repeated. For example, suppose we have the two arrays + +```py +>>> x = np.array([1, 2, 3]) +>>> y = np.array([100, 200]) +``` + +and suppose we want to compute an "outer" sum of `x` and `y`, that is, we want +to compute every combination of `i + j` where `i` is from `x` and `j` is from +`y`. The key realization here is that what we want is simply to +repeat each entry of `x` 3 times, to correspond to each entry of `y`, and +respectively repeat each entry of `y` 3 times, to correspond to each entry of +`x`. And this is exactly the sort of thing broadcasting does! We only need to +make the shapes of `x` and `y` match in such a way that the broadcasting will +do that. Since we want both `x` and `y` to be repeated, we will need to +broadcast both arrays. We want to compute + +```py +[[ x[0] + y[0], x[0] + y[1] ], + [ x[1] + y[0], x[1] + y[1] ], + [ x[2] + y[0], x[2] + y[1] ]] +``` + +That way the first dimension of the resulting array will correspond to values +from `x`, and the second dimension will correspond to values from `y`, i.e., +`a[i, j]` will be `x[i] + y[j]`. Thus the resulting array will have shape `(3, +2)`. So to make `x` (which is shape `(3,)`) and `y` (which is shape `(2,)`) +broadcast to this, we need to make them `(3, 1)` and `(1, 2)`, respectively. +This can easily be done with `np.newaxis`. + +```py +>>> x[:, np.newaxis].shape +(3, 1) +>>> y[np.newaxis, :].shape +(1, 2) +``` + +Once we have the desired shapes, we just perform the operation, and NumPy will +do the broadcasting automatically.[^outer-footnote] + +[^outer-footnote]: We could have also used the + [`outer`](https://numpy.org/doc/stable/reference/generated/numpy.ufunc.outer.html) + method of the `add` ufunc to achieve this, but using this for more a more + complicated function than just `x + y` would be tedious, and it would not + work in situations where you want to only repeat certain dimensions. + Broadcasting is a more general way to do this, and `newaxis` is an + important tool for making shapes align properly to make broadcasting do + what you want. + +```py +>>> x[:, np.newaxis] + y[np.newaxis, :] +array([[101, 201], + [102, 202], + [103, 203]]) +``` + +Note: broadcasting automatically prepends shape `1` dimensions, so the +`y[np.newaxis, :]` operation is unnecessary. + +```py +>>> x[:, np.newaxis] + y +array([[101, 201], + [102, 202], + [103, 203]]) +``` + +As we saw [before](single-axis-tuple), size-1 dimensions may seem redundant, +but they are not a bad thing. Not only do they allow indexing an array +uniformly, they are also very important in the way they interact with NumPy's +broadcasting rules. + +```{rubric} Footnotes +``` +<!-- Footnotes are written inline above but markdown will put them here at the +end of the document. --> diff --git a/docs/indexing-guide/multidimensional-indices/tuples.md b/docs/indexing-guide/multidimensional-indices/tuples.md new file mode 100644 index 00000000..0c906c0a --- /dev/null +++ b/docs/indexing-guide/multidimensional-indices/tuples.md @@ -0,0 +1,419 @@ +# Tuple Indices + +The basic building block of multidimensional indexing is the `tuple` index. A +tuple index doesn't select elements on its own. Instead, it contains other +indices that themselves select elements. The general rule for tuples is that + +> **each element of a tuple index selects the corresponding elements for the + corresponding axis of the array** + +(this rule is modified a little bit in the presence of [ellipses](ellipses.md) +or [newaxis](newaxis.md), as we will see later). + +For example, suppose we have a three-dimensional array `a` with the +shape `(3, 2, 4)`. For simplicity, we'll define `a` as a reshaped `arange`, so +that each element is distinct and we can easily see which elements are +selected. + +```py +>>> import numpy as np +>>> a = np.arange(24).reshape((3, 2, 4)) +>>> a +array([[[ 0, 1, 2, 3], + [ 4, 5, 6, 7]], +<BLANKLINE> + [[ 8, 9, 10, 11], + [12, 13, 14, 15]], +<BLANKLINE> + [[16, 17, 18, 19], + [20, 21, 22, 23]]]) +``` + +If we use a basic single axis index on `a` such as an integer or slice, it +will operate on the first dimension of `a`: + +```py +>>> a[0] # The first row of the first axis +array([[0, 1, 2, 3], + [4, 5, 6, 7]]) +>>> a[2:] # The elements that are not in the first or second rows of the first axis +array([[[16, 17, 18, 19], + [20, 21, 22, 23]]]) +``` + +We also observe that integer indices remove the axis, and slices keep the axis +(even when the resulting axis has size-1): + +```py +>>> a[0].shape +(2, 4) +>>> a[2:].shape +(1, 2, 4) +``` + +The indices in a tuple index target the corresponding elements of the +corresponding axis. So for example, the index `(1, 0, 2)` selects the second +element of the first axis, the first element of the second axis, and the third +element of the third axis (remember that indexing is 0-based, so index `0` +corresponds to the first element, index `1` to the second, and so on). Looking +at the list of lists representation of `a` that was printed by NumPy: + +```py +>>> a +array([[[ 0, 1, 2, 3], + [ 4, 5, 6, 7]], +<BLANKLINE> + [[ 8, 9, 10, 11], + [12, 13, 14, 15]], +<BLANKLINE> + [[16, 17, 18, 19], + [20, 21, 22, 23]]]) +``` + +The first index is `1`, so we should take the second element of the outermost list, giving + +```py +[[ 8, 9, 10, 11], + [12, 13, 14, 15]] + +``` + +The next index is `0`, so we get the first element of this list, which is the list + + +```py +[ 8, 9, 10, 11] +``` + +Finally, the last index is `2`, giving the third element of this list: + +```py +10 +``` + +And indeed: + +```py +>>> a[(1, 0, 2)] # doctest: +SKIPNP1 +np.int64(10) +``` + +If we had stopped at an intermediate tuple, instead of getting an element, we +would have gotten the subarray that we accessed. For example, just `(1,)` +gives us the first intermediate array we looked at: + +```py +>>> a[(1,)] +array([[ 8, 9, 10, 11], + [12, 13, 14, 15]]) +``` + +And `(1, 0)` gives us the second intermediate array we looked at: + +```py +>>> a[(1, 0)] +array([ 8, 9, 10, 11]) +``` + +In each case, the integers remove the corresponding axes from the array shape: + +```py +>>> a.shape +(3, 2, 4) +>>> a[(1,)].shape +(2, 4) +>>> a[(1, 0)].shape +(4,) +``` + +We can actually think of the final element, `10`, as being an array with shape +`()` (0 dimensions). Indeed, NumPy agrees with this idea: + +```py +>>> a[(1, 0, 2)].shape +() +``` + +Now, it's important to note a key point about tuple indices: **the parentheses +in a tuple index are completely optional.** Instead of writing `a[(1, 0, 2)]`, +we could simply write `a[1, 0, 2]`. + +```py +>>> a[1, 0, 2] # doctest: +SKIPNP1 +np.int64(10) +``` + +These are exactly the same. When the parentheses are omitted, Python +automatically treats the index as a tuple. From here on, we will always omit +the parentheses, as is common practice. Not only is this cleaner, but it is +also important for another reason: syntactically, Python does not allow slices +in a tuple index if the parentheses are included: + + +```py +>>> a[(1:, :, :-1)] # doctest: +SKIP + File "<stdin>", line 1 + a[(1:, :, :-1)] + ^ +SyntaxError: invalid syntax +>>> a[1:, :, :-1] +array([[[ 8, 9, 10], + [12, 13, 14]], +<BLANKLINE> + [[16, 17, 18], + [20, 21, 22]]]) +``` + +Now, let's go back and look at an example we just showed: + +```py +>>> a[(1,)] # or just a[1,] +array([[ 8, 9, 10, 11], + [12, 13, 14, 15]]) +``` + +You might have noticed something about this. It is selecting the second element +of the first axis. But from what we said earlier, we can also do this just by +using the basic index `1`, which will operate on the first axis: + +```py +>>> a[1] # Exactly the same thing as a[(1,)] +array([[ 8, 9, 10, 11], + [12, 13, 14, 15]]) +``` + +This illustrates the first important fact about tuple indices: + +> **A tuple index with a single index, `a[i,]`, is exactly the same as that + single index, `a[i]`.** + +The reason is that in both cases, the index `i` operates over the +first axis of the array. This is true no matter what kind of index `i` is. `i` +can be an integer index, a slice, an ellipsis, and so on. With one exception, +that is: `i` cannot itself be a tuple index! Nested tuple indices are not +allowed. + +In practice, this means that when working with NumPy arrays, you can think of +every index type as a single element tuple index. An integer index `0` is +"actually" the tuple index `(0,)`. The slice `a[0:3]` is actually a tuple +`a[0:3,]`. This is a good way to think about indices because it will help you +remember that non-tuple indices operate as if they were the first element of a +single-element tuple index, namely, they operate on the first axis of the +array. Remember, however, that this does not apply to Python built-in types; +for example, `l[0,]` and `l[0:3,]` will both produce errors if `l` is a +`list`, `tuple`, or `str`. + +Up to now, we looked at the tuple index `(1, 0, 2)`, which selected a single +element. And we considered sub-tuples of this, `(1,)` and `(1, 0)`, which +selected subarrays. What if we want to select other subarrays? For example, +`a[1, 0]` selects the subarray with the second element of the first axis and +the first element of the second axis. What if instead we wanted the first +element of the *last* axis (axis 3)? + +(tuples-slices-example)= +We can do this with slices. In particular, the trivial slice `:` will select +every single element of an axis (remember that the `:` slice means ["select +everything"](omitted)). So we want to select every element from the first and +second axis, and only the first element of the last axis, meaning our index is +`:, :, 0`: + +```py +>>> a[:, :, 0] +array([[ 0, 4], + [ 8, 12], + [16, 20]]) +``` + +`:` serves as a convenient way to "skip" axes. It is one of the most common +types of indices that you will see in practice for this reason. However, it is +important to remember that `:` is not special. It is just a slice, which selects +every element of the corresponding axis. We could also replace `:` with `0:n`, +where `n` is the size of the corresponding axis. + +```py +>>> a[0:3, 0:2, 0] +array([[ 0, 4], + [ 8, 12], + [16, 20]]) +``` + +Of course, in practice using `:` is better because we might not know or care +what the actual size of the axis is, and it's less typing anyway. + +When we used the indices `(1,)` and `(1, 0)`, we observed that they targeted +the first and the first two axes, respectively, leaving the remaining axes +intact and producing subarrays. Another way of saying this is that the each +tuple index implicitly ended with `:` slices, one for each axis we didn't +index: + +```py +>>> a[1,] +array([[ 8, 9, 10, 11], + [12, 13, 14, 15]]) +>>> a[1, :, :] +array([[ 8, 9, 10, 11], + [12, 13, 14, 15]]) +>>> a[1, 0] +array([ 8, 9, 10, 11]) +>>> a[1, 0, :] +array([ 8, 9, 10, 11]) +``` + +This is a rule in general: + +> **A tuple index implicitly ends in as many slices `:` as there are remaining + dimensions of the array.** + +(single-axis-tuple)= +The [slices](../slices.md) page stressed the point that [slices always keep +the axis they index](subarray), but it wasn't clear why that is important +until now. Suppose we slice the first axis of `a`, then later, we take that +array and want to get the first element of the last row. + + +```py +>>> n = 2 +>>> b = a[:n] +>>> b[-1, -1, 0] # doctest: +SKIPNP1 +np.int64(12) +``` + +Here `b = a[:2]` has shape `(2, 2, 4)` + +``` +>>> b.shape +(2, 2, 4) +``` + +But suppose we used a slice that only selected one element from the first axis +instead + +```py +>>> n = 1 +>>> b = a[:n] +>>> b[-1, -1, 0] # doctest: +SKIPNP1 +np.int64(4) +``` + +It still works. Here `b` has shape `(1, 2, 4)`: + +```py +>>> b.shape +(1, 2, 4) +>>> b +array([[[0, 1, 2, 3], + [4, 5, 6, 7]]]) +``` + +Even though the slice `a[:1]` only produces a single element in the first +axis, that axis is maintained as size `1`. We might think this array is +"equivalent" to the same array with shape `(2, 4)`, since the first axis is +redundant (the outermost list only has one element, so we don't really need +it). + +```py +>>> # c is kind of the same as b above +>>> c = np.array([[0, 1, 2, 3], +... [4, 5, 6, 7]]) +``` + +This is true in the sense that the elements are the same, but the +resulting array has different properties. Namely, the index we used for `b` +will not work for it. + +```py +>>> c[-1, -1, 0] +Traceback (most recent call last): + File "<stdin>", line 1, in <module> +IndexError: too many indices for array: array is 2-dimensional, but 3 were +indexed +``` + +Here we tried to use the same index on `c` that we used on `b`, but it didn't +work, because our index assumed three axes, but `c` only has two: + +```py +>>> c.shape +(2, 4) +``` + +Thus, when it comes to indexing, all axes, even "trivial" axes, matter. It's +sometimes a good idea to maintain the same number of dimensions in an array +throughout a computation, even if one of them sometimes has size 1, simply +because it means that you can index the array +uniformly.[^size-1-dimension-footnote] And this doesn't apply just to +indexing. Many NumPy functions reduce the number of dimensions of their output +(for example, {external+numpy:func}`numpy.sum`), but they have a `keepdims` +argument to retain the dimension as a size-1 dimension instead. + +[^size-1-dimension-footnote]: In this example, if we knew that we were always + going to select exactly one element (say, the second one) from the first + dimension, we could equivalently use `a[1, np.newaxis]` (see + [](../integer-indices.md) and [](newaxis.md)). The advantage of this is + that we would get an error if the first dimension of `a` didn't actually + have `2` elements, whereas `a[1:2]` would just silently give a size-0 + array. + +There are two final facts about tuple indices that should be noted before we +move on to the other basic index types. First, as we noticed above, + +> **if a tuple index has more elements than there are dimensions in an array, + it raises an `IndexError`.** + +Secondly, an array can be indexed by an empty tuple `()`. If we think about it +for a moment, we said that every tuple index implicitly ends in enough trivial +`:` slices to select the remaining axes of an array. That means that for an +array `a` with $n$ dimensions, an empty tuple index `a[()]` should be the same +as `a[:, :, … (n times)]`. This would select every element of every axis. In +other words, + +> **the empty tuple index `a[()]` always just returns the entire array `a` + unchanged.**[^tuple-ellipsis-footnote] + +[^tuple-ellipsis-footnote]: + <!-- This is the only way to cross reference a footnote across documents --> + (tuple-ellipsis-footnote-ref)= + + There is one important distinction between the + empty tuple index (`a[()]`) and a single ellipsis index (`a[...]`). NumPy + makes a distinction between scalars and 0-D (i.e., shape `()`) arrays. On + either, an empty tuple index `()` will always produce a scalar, and a + single ellipsis `...` will always produce a 0-D array: + + + ```py + >>> s = np.int64(0) # scalar + >>> x = np.array(0) # 0-D array + >>> s[()] # doctest: +SKIPNP1 + np.int64(0) + >>> x[()] # doctest: +SKIPNP1 + np.int64(0) + >>> s[...] + array(0) + >>> x[...] + array(0) + ``` + + This also applies for tuple indices that select a single element. If the + tuple contains a (necessarily redundant) ellipsis, the result is a 0-D + array. Otherwise, the result is a scalar. With the example array: + + ```py + >>> a[1, 0, 2] # scalar # doctest: +SKIPNP1 + np.int64(10) + >>> a[1, 0, 2, ...] # 0-D array + array(10) + ``` + + The difference between scalars and 0-D arrays in NumPy is subtle. In most + contexts, they will both work identically, but, rarely, you may need one + and not the other, and the above trick can be used to convert between + them. See [footnote 1](integer-scalar-footnote-ref) in the [](integer-arrays.md) + section and [footnote 1](view-scalar-footnote-ref) in the + [](../other-topics.md) section for two important + differences between the scalars and 0-D arrays which are related to indexing. + +```{rubric} Footnotes +``` +<!-- Footnotes are written inline above but markdown will put them here at the +end of the document. --> diff --git a/docs/indexing-guide/multidimensional-indices/what-is-an-array.md b/docs/indexing-guide/multidimensional-indices/what-is-an-array.md new file mode 100644 index 00000000..f0454198 --- /dev/null +++ b/docs/indexing-guide/multidimensional-indices/what-is-an-array.md @@ -0,0 +1,192 @@ +# What is an Array? + +Before we look at indices, let's take a step back and look at the NumPy array. +Just what is it that makes NumPy arrays so ubiquitous and NumPy one of the +most successful numerical tools ever? The answer is quite a few things, which +come together to make NumPy a fast and easy to use library for array +computations. But one feature in particular stands out: multidimensional +indexing. + +Let's consider pure Python for a second. Suppose we have a list of values, for +example, a list of your bowling scores. + +```py +>>> scores = [70, 65, 71, 80, 73] +``` + +From what we learned before, we can now index this list with +[integers](../integer-indices.md) or [slices](../slices.md) to get some +subsets of it. + +```py +>>> scores[0] # The first score +70 +>>> scores[-3:] # The last three scores +[71, 80, 73] +``` + +You can imagine all sorts of different things you'd want to do with your +scores that might involve selecting individual scores or ranges of scores (for +example, with the above examples, we could easily compute the average score of +our last three games, and see how it compares to our first game). So hopefully +you are convinced that at least the types of indices we have learned so far +are useful. + +Now suppose your bowling buddy Bob learns that you are keeping track of scores +and wants you to add his scores as well. He bowls with you, so his scores +correspond to the same games as yours. You could make a new list, +`bob_scores`, but this means storing a new variable. You've got a feeling you +are going to end up keeping track of a lot of people's scores. So instead, you +change your `scores` list into a list of lists. The first inner list is your +scores, and the second will be Bob's: + +```py +>>> scores = [[70, 65, 71, 80, 73], [100, 93, 111, 104, 113]] +``` + +Now you can easily get your scores: + +```py +>>> scores[0] +[70, 65, 71, 80, 73] +``` + +and Bob's scores: + +```py +>>> scores[1] +[100, 93, 111, 104, 113] +``` + +But now there's a problem (aside from the obvious problem that Bob is a better +bowler than you). If you want to see what everyone's scores are for the first +game, you have to do something like this: + +```py +>>> [p[0] for p in scores] +[70, 100] +``` + +That's a mess. Clearly, you should have inverted the way you constructed your +list of lists, so that each inner list corresponds to a game, and each element +of that list corresponds to the person (for now, just you and Bob): + +```py +>>> scores = [[70, 100], [65, 93], [71, 111], [80, 104], [73, 113]] +``` + +Now you can much more easily get the scores for the first game + +```py +>>> scores[0] +[70, 100] +``` + +Except now if you want to look at just your scores for all games (that was +your original purpose after all, before Bob got involved), it's the same +problem again. To extract that you have to do + +```py +>>> [game[0] for game in scores] +[70, 65, 71, 80, 73] +``` + +which is the same mess as above. What are you to do? + +The NumPy array provides an elegant solution to this problem. Our idea of +storing the scores as a list of lists was a good one, but unfortunately, it +pushed the limits of what the Python `list` type was designed to do. Python +`list`s can store anything, be it numbers, strings, or even other lists. +If we want to tell Python to index a list that is inside of another list, we +have to do it manually, because the elements of the outer list might not even +be lists. For example, `l = [1, [2, 3]]` is a perfectly valid Python `list`, but +the expression `[i[0] for i in l]` is invalid, because not every element +of `l` is a list. + +NumPy arrays function like a list of lists, but are restricted so that these +kinds of operations always "make sense". More specifically, if you have a +"list of lists", each element of the "outer list" must be a list. `[1, [2, +3]]` is not a valid NumPy array. Furthermore, each inner list must have the +same length, or more precisely, the lists at each level of nesting must have +the same length. + +Lists of lists can be nested more than just two levels deep. For example, you +might want to take your scores and create a new outer list, splitting them by +season. Then you would have a list of lists of lists, and your indexing +operations would look like `[[game[0] for game in season] for season in +scores]`. + +In NumPy, these different levels of nesting are called *axes* or *dimensions*. +The number of axes---the level of nesting---is called the +*rank*[^rank-footnote] or *dimensionality* of the array. Together, the lengths +of these lists at each level are called the *shape* of the array (remember +that the lists at each level have to have the same number of elements). + +[^rank-footnote]: Not to be confused with [mathematical definitions of + rank](https://en.wikipedia.org/wiki/Rank_(linear_algebra)). Because of + this ambiguity, the term "dimensionality" or "number of dimensions" is + generally preferred to "rank", and is what we use in this guide. + +A NumPy array of our scores (using the last representation) looks like this + +```py +>>> import numpy as np +>>> scores = np.array([[70, 100], [65, 93], [71, 111], [80, 104], [73, 113]]) +``` + +Except for the `np.array()` call, it looks exactly the same as the list of +lists. But the difference is indexing. If we want the first game, as before, +we use `scores[0]`: + +```py +>>> scores[0] +array([ 70, 100]) +``` + +But if we want to find only our scores, instead of using a list comprehension, +we can simply use + +```py +>>> scores[:, 0] +array([70, 65, 71, 80, 73]) +``` + +This index contains two components: the slice `:` and the integer index `0`. +The slice `:` says to take everything from the first axis (which represents +games), and the integer index `0` says to take the first element of the second +axis (which represents people). + +The shape of our array is a tuple with the number of games (the outer axis) +and the number of people (the inner axis). + +```py +>>> scores.shape +(5, 2) +``` + +This is the power of multidimensional indexing in NumPy arrays. If we have a +list of lists of numbers, or a list of lists of lists of numbers, or a list of +lists or lists of lists..., we can index things at any "nesting level" equally +easily. There is a small reasonable restriction, namely that each "level" of +lists (axis) must have the same number of elements. This restriction is +reasonable because in the real world, data tends to be tabular, like bowling +scores, meaning each axis will naturally have the same number of elements. +Even if this weren't the case, for instance, if Bob were out sick for a game, +we could easily use a sentinel value like `-1` or `nan` for a missing value to +maintain uniform lengths. + +The indexing semantics are only a small part of what makes NumPy arrays so +powerful. They also have many other advantages that are unrelated to indexing. +They operate on contiguous memory using native machine data types, which makes +them very fast. They can be manipulated using array expressions with +[broadcasting semantics](broadcasting); for example, you can easily add a +handicap to the scores array with something like `scores + np.array([124, +95])`, which would itself be a nightmare using the list of lists +representation. This, along with the powerful ecosystem of libraries like +`scipy`, `matplotlib`, and thousands of others, are what have made NumPy such +a popular and essential tool. + +```{rubric} Footnotes +``` +<!-- Footnotes are written inline above but markdown will put them here at the +end of the document. --> diff --git a/docs/indexing-guide/other-topics.md b/docs/indexing-guide/other-topics.md new file mode 100644 index 00000000..e8d1a5fb --- /dev/null +++ b/docs/indexing-guide/other-topics.md @@ -0,0 +1,663 @@ +# Other Topics Relevant to Indexing + +There is a great deal of functionality in NumPy, and most of it is +out-of-scope for this guide. However, a few additional features are useful to +understand in order to fully utilize indexing to its fullest potential, or as +important caveats to be aware of when doing indexing operations. + +(broadcasting)= +## Broadcasting + +Broadcasting is a powerful abstraction that applies to all operations in +NumPy. It allows arrays with mismatched shapes to be combined together as if +one or more of their dimensions were simply repeated the appropriate number of +times. + +Normally, when we perform an operation on two arrays with the same shape, it +does what we'd expect, i.e., the operation is applied to each corresponding +element of each array. For example, if `x` and `y` are both `(2, 2)` arrays, +`x + y` results in a `(2, 2)` array with the sum of the corresponding +elements. + +```py +>>> import numpy as np +>>> a = np.array([[1, 2], +... [3, 4]]) +>>> b = np.array([[101, 102], +... [103, 104]]) +>>> a + b +array([[102, 104], + [106, 108]]) +``` + +However, you may have noticed that `x` and `y` doesn't always have to be two +arrays with the same shape. For example, you can add a single scalar element +to an array, and it will add it to each element. + +```py +>>> a + 1 +array([[2, 3], + [4, 5]]) +``` + +In the above example, we can think of the scalar `1` as a shape `()` array, +whereas `x` is a shape `(2, 2)` array. Thus, `x` and `1` do not have the same +shape, but `x + 1` is allowed via repeating `1` across every element of `x`. +This means taking `1` and treating it as if it were the shape `(2, 2)` array +`[[1, 1], [1, 1]]`. + +Broadcasting is a generalization of this behavior. Specifically, instead of +repeating just a single number into an array, we can repeat just some +dimensions of an array into a bigger array. For example, here we multiply `x`, +a shape `(3, 2)` array, with `y`, a shape `(2,)` array. `y` is virtually +repeated into a shape `(3, 2)` array with each element of the last dimension +repeated 3 times. + +```py +>>> x = np.array([[1, 2], +... [3, 4], +... [5, 6]]) +>>> x.shape +(3, 2) +>>> y = np.array([0, 2]) +>>> x*y +array([[ 0, 4], + [ 0, 8], + [ 0, 12]]) +``` + +We can see how broadcasting works using `np.broadcast_to` + +```py +>>> np.broadcast_to(y, x.shape) +array([[0, 2], + [0, 2], + [0, 2]]) +``` + +This is what the array `y` looks like before it is combined with `x`, except +the power of broadcasting is that the repeated entries are not literally +repeated in memory. Broadcasting is implemented efficiently so that the +"repeated" elements actually refer to the same objects in memory. See +[](views-vs-copies) and [](strides) below. + +Broadcasting always happens automatically in NumPy whenever two arrays with +different shapes are combined using any function or operator, assuming those +shapes are broadcast compatible. The rule for broadcast compatibility is that +the shorter of the shapes are prepended with length 1 dimensions so that they +have the same number of dimensions. Then any dimensions that are size 1 in a +shape are replaced with the corresponding size in the other shape. The other +non-1 sizes must be equal or broadcasting is not allowed. + +In the above example, we broadcast `(3, 2)` with `(2,)` by first extending +`(2,)` to `(1, 2)` then broadcasting the size `1` dimension to the +corresponding size in the other shape, `3`, giving a broadcasted shape of `(3, +2)`. In more advanced examples, both shapes may have broadcasted dimensions. +For instance, `(3, 1)` can broadcast with `(2,)` to create `(3, 2)`. The first +shape repeats the first axis 2 times along the second axis, while the second +shape inserts a new axis at the beginning that repeats 3 times. + +We can think of `(3, 2)` as a "stack" of 3 shape `(2,)` arrays. Just as the +scalar 1 got repeated to match the full shape of `a` above, the shape `(2,)` +array `y` gets repeated into a `(3,)` "stack" so it matches the shape of `x`. + +Additionally, size `1` dimensions are a signal to NumPy that that dimension is +allowed to be repeated, or broadcasted, when used with another array where +that dimension is not size `1`. + +It can be useful to think of broadcasting as repeating "stacks" of smaller +arrays in this way. The size `1` dimension rule allows these "stacks" to be +along any dimensions of the array, not just the last ones. + +See the [NumPy +documentation](https://numpy.org/doc/stable/user/basics.broadcasting.html) for +more examples of broadcasting. + +(views-vs-copies)= +## Views vs. Copies + +There is an important distinction between basic indices (i.e., +[integers](integer-indices.md), [slices](slices.md), +[ellipses](multidimensional-indices/ellipses.md), +[newaxis](multidimensional-indices/newaxis.md)) and [advanced +indices](advanced-indices) (i.e., [integer array +indices](multidimensional-indices/integer-arrays.md) and [boolean array +indices](multidimensional-indices/boolean-arrays.md)) in NumPy that must be +noted in some situations. Specifically, basic indices always create a **view** +into an array[^view-scalar-footnote], while advanced indices always create a +**copy** of the underlying array. [Tuple](multidimensional-indices/tuples.md) +indices (i.e., multidimensional indices) will create a view if they do not +contain an advanced index and a copy if they do. + + +[^view-scalar-footnote]: + <!-- This is the only way to cross reference a footnote across documents --> + (view-scalar-footnote-ref)= + + There is one exception to this rule, which is that an + index that would return a scalar returns a copy, since scalars are + supposed to be immutable. + + ```py + >>> a = np.arange(20) + >>> a[0] # doctest: +SKIPNP1 + np.int64(0) + >>> print(a[0].base) + None + >>> a[0, ...] + array(0) + >>> a[0, ...].base is a + True + ``` + +A **view** is a special type of array whose data points to another array. This +means that if you mutate the data in one array, the other array will also have +that data mutated as well. For example, + +```py +>>> a = np.arange(24).reshape((3, 2, 4)) +>>> b = a[:, 0] # b is a view of a +>>> a +array([[[ 0, 1, 2, 3], + [ 4, 5, 6, 7]], +<BLANKLINE> + [[ 8, 9, 10, 11], + [12, 13, 14, 15]], +<BLANKLINE> + [[16, 17, 18, 19], + [20, 21, 22, 23]]]) +>>> b[:] = 0 # Mutating b also mutates a +>>> a +array([[[ 0, 0, 0, 0], + [ 4, 5, 6, 7]], +<BLANKLINE> + [[ 0, 0, 0, 0], + [12, 13, 14, 15]], +<BLANKLINE> + [[ 0, 0, 0, 0], + [20, 21, 22, 23]]]) +``` + +Mutating `b` also changed `a`, because both arrays point to the same memory. + +Note that this behavior is exactly the opposite of the way Python lists work. +With Python lists, `a[:]` is a shorthand to copy `a`. But with NumPy, `a[:]` +creates a view into `a` (to copy an array with NumPy, use `a.copy()`). Python +lists do not have a notion of views. + +```py +>>> a = [1, 2, 3] # list +>>> b = a[:] # a copy of a +>>> b[0] = 0 # Modifies b but not a +>>> b +[0, 2, 3] +>>> a +[1, 2, 3] +>>> a = np.array([1, 2, 3]) # NumPy array +>>> b = a[:] # A view of a +>>> b[0] = 0 # Modifies both b and a +>>> b +array([0, 2, 3]) +>>> a +array([0, 2, 3]) +>>> c = a.copy() # A copy of a +>>> c[0] = -1 # Only modifies c +>>> c +array([-1, 2, 3]) +>>> a +array([0, 2, 3]) +``` + +Views don't just come from indexing. For instance, when you reshape an array, +that also creates a view. + +```py +>>> a = np.arange(24) +>>> b = a.reshape((3, 2, 4)) +>>> b[0] = 0 +>>> b +array([[[ 0, 0, 0, 0], + [ 0, 0, 0, 0]], +<BLANKLINE> + [[ 8, 9, 10, 11], + [12, 13, 14, 15]], +<BLANKLINE> + [[16, 17, 18, 19], + [20, 21, 22, 23]]]) +>>> a +array([ 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23]) +``` + +Many other operations also create views, for example +[`np.transpose`](https://numpy.org/doc/stable/reference/generated/numpy.transpose.html), +[`a.T`](https://numpy.org/doc/stable/reference/generated/numpy.ndarray.T.html), +[`np.ravel`](https://numpy.org/doc/stable/reference/generated/numpy.ravel.html), +[`broadcast`](https://numpy.org/doc/stable/reference/generated/numpy.broadcast.html), +and +[`a.view`](https://numpy.org/doc/stable/reference/generated/numpy.ndarray.view.html).[^view-functions-footnote] + +[^view-functions-footnote]: Some of these functions will sometimes return a + copy because returning a view is not possible. For example, it is not + always possible to represent a reshape as a [strides](strides) + manipulation if the strides are already non-contiguous. + +To check if an array is a view, check `a.base`. It will be `None` if it is not +a view and will point to the base array otherwise. A view of a view will have +the same base as the original array. + +```py +>>> a = np.arange(24) +>>> b = a[:] +>>> print(a.base) # a is not a view +None +>>> b.base is a # b is a view of a +True +>>> c = b[::2] +>>> c.base is a # c is a further view of a +True +``` + +In contrast, an advanced index will always create a copy (even if it would be +possible to represent it as a view). This includes any [tuple +index](multidimensional-indices/tuples.md) (i.e., multidimensional index) that +contains at least one array index. + +```py +>>> a = np.arange(10) +>>> b = a[::4] +>>> c = a[[0, 4, 8]] +>>> b +array([0, 4, 8]) +>>> c +array([0, 4, 8]) +>>> b.base is a # b is a view of a +True +>>> print(c.base) # c is not (it is a copy) +None +``` + +Whether an array is a view or a copy matters for two reasons: + +1. If you ever mutate the array, if it is a view, it will also mutate the data + in the base array, as shown above. Be aware that views affect mutations in + both directions: if `a` is a view, mutating it will also mutate whichever + array it is a view on, but conversely, even if `a` is not a view, mutating + it will modify any other arrays which are views into `a`. And while you can + check if `a` is a view by looking at `a.base`, there is no easy way to + check if `a` has other views pointing at it. The only way to know is to + analyze the program and check any array which is created from `a`. + + It's best to minimize mutations in the presence of views, or to restrict + them to a controlled part of the code, to avoid unexpected "[action at a + distance](https://en.wikipedia.org/wiki/Action_at_a_distance_(computer_programming))" + bugs. + + Note that you can always ensure that `a` is a new array that isn't a view + and doesn't have any views pointing to it by copying it, using `a.copy()`. + +2. Even if you don't mutate data, views are important because they are more + efficient. A view is a relatively cheap thing to make, even if the array is + large. It also saves on memory usage. + + For example, here we have an array with about 800 kB of data, and it takes + over 200x longer to copy it than to create a view (using + [IPython](https://ipython.org/)'s `%timeit`): + + ``` + In [1]: import numpy as np + + In [2]: a = np.arange(100000, dtype=np.float64) + + In [3]: %timeit a.copy() + 25.9 µs ± 645 ns per loop (mean ± std. dev. of 7 runs, 10,000 loops each) + + In [4]: %timeit a[:] + 127 ns ± 1.62 ns per loop (mean ± std. dev. of 7 runs, 10,000,000 loops each) + ``` + +(strides)= +## Strides + +The reason so many types of indexing into arrays can be represented as a +[view](views-vs-copies) without creating a copy is that NumPy arrays aren't +merely a pointer to a blob of memory. They are a pointer along with something +called *strides*. The strides tell NumPy how many bytes to skip in memory +along each axis to get to the next element of the array along that dimension. +This along with the *memory offset* (the address in physical memory of the +first byte of data), the *shape*, and the *itemsize* (the number of bytes each +element takes up, which depends on the *dtype*) exactly determines how the +corresponding memory is organized into an array. For example, let's start with +a flat 1-dimensional array with `24` elements whose itemsize is 8 (an `int64` +takes up 8 bytes). Its strides is `(8,)`: + +```py +>>> a = np.arange(24) +>>> a.itemsize +8 +>>> a.strides +(8,) +>>> a.shape +(24,) +``` + +Now let's create a view `b`, which is `a` reshaped to shape `(3, 2, 4)`. `b` +uses the exact same memory as `a` (which is just `0 1 2 ... 23`). Its itemsize +is the same because it has the same dtype, but its strides and shape are +different. + +```py +>>> b = a.reshape((3, 2, 4)) +>>> b +array([[[ 0, 1, 2, 3], + [ 4, 5, 6, 7]], +<BLANKLINE> + [[ 8, 9, 10, 11], + [12, 13, 14, 15]], +<BLANKLINE> + [[16, 17, 18, 19], + [20, 21, 22, 23]]]) +>>> b.itemsize +8 +>>> b.strides +(64, 32, 8) +>>> b.shape +(3, 2, 4) +``` + +This tells NumPy that to get the next element in the first dimension of `b`, +it needs to skip 64 bytes. That's because the first dimension contains 2\*4=8 +items each, corresponding to the sizes of the second and third dimensions, and +each item is 8 bytes, so 8\*8=64. For example, the next element in the first +dimension after `0` (index `(0, 0, 0)`) is `8` (index `(1, 0, 0)`), which sits +exactly 64 bytes after it in memory. Similarly, to get the next element in the +second dimension, it should skip 32 bytes (4 elements). + +The memory offset of an array can be accessed using `a.ctypes.data`. This is +the address in physical memory where the data (`0 1 2 ... 23`) lives. `a` +and `b` have the same memory offset because they both start with the same +first element: + +```py +>>> a.ctypes.data # doctest: +SKIP +105553170825216 +>>> b.ctypes.data # doctest: +SKIP +105553170825216 +``` + +When we slice off the beginning of the array, we can see that all this does is +move the memory offset forward, and adjust the shape correspondingly. + +```py +>>> a[2:].ctypes.data # doctest: +SKIP +105553170825232 +>>> a[2:].shape +(22,) +>>> a[2:].strides # the strides are the same +(8,) +``` + +Specifically, it moves it by `2*8` (where remember `8` is `a.itemsize`): + +```py +>>> a[2:].ctypes.data - a.ctypes.data +16 +``` + +Here the strides are the same. Similarly, if we slice off the end of the +array, all it needs to do is adjust the shape. The memory offset is the same, +because it still starts at the same place in memory. + +```py +>>> a[:2].ctypes.data # the memory offset is the same # doctest: +SKIP +105553170825216 +>>> a[:2].shape +(2,) +>>> a[:2].strides # the strides are the same +(8,) +``` + +If we instead slice with a step, this adjusts the strides. For instance +`a[::2]` will double the strides, making it skip every other element (but the +offset will be again unchanged because it still starts at the first element of +`a`): + +```py +>>> a[::2].strides +(16,) +>>> a[::2].ctypes.data # the memory offset is the same # doctest: +SKIP +105553170825216 +>>> a[::2].shape +(12,) +``` + +If we use a *negative* step, the strides will become negative. This will cause +NumPy to work backwards in memory as it accesses the elements of the array. +The memory offset also changes here so that it starts with the last element of +`a`: + +```py +>>> a[::-2].strides +(-16,) +>>> a[::-2].ctypes.data # doctest: +SKIP +105553170825400 +``` + +From this, you are hopefully convinced that every possible slice is just a +manipulation of the memory offset, shape, and strides. It's not hard to see +that this also applies to integer indices (which just removes the stride for +the corresponding axis, adjusting the shape and memory offset accordingly) and +newaxis (which just adds `0` to the strides): + +```py +>>> b.strides +(64, 32, 8) +>>> b[2].strides +(32, 8) +>>> b[np.newaxis].strides +(0, 64, 32, 8) +``` + +This is why [basic indexing](basic-indices) always produces a +[view](views-vs-copies): because it can always be represented as a +manipulation of the strides (plus shape and offset). + +Another important fact about strides is that [broadcasting](broadcasting) can +be achieved by manipulating the strides, namely by using a `0` stride to +repeat the same data along a given axis. + +```py +>>> c = a.reshape((1, 12, 2)) +>>> d = np.broadcast_to(c, (5, 12, 2)) +>>> c.shape +(1, 12, 2) +>>> c.strides +(192, 16, 8) +>>> d.shape +(5, 12, 2) +>>> d.strides +(0, 16, 8) +>>> c.base is a +True +>>> d.base is a +True +``` + +If you've ever used `broadcast_to()`, you might have noticed that it returns a +read-only array. That's because writing into it would not do what you'd +expect, since the repeated elements literally refer to the same memory. + +This shows why [broadcasting](broadcasting) is so powerful: it can be done +without any actual copy of the data. When you perform an operation on two +arrays, the broadcasting is implicit, but even explicitly creating a +broadcasted array is cheap, because all it does is create a view with +different strides. + +You can also manually create a view with any strides you want using [stride +tricks](https://numpy.org/doc/stable/reference/generated/numpy.lib.stride_tricks.as_strided.html). +Most views that you would want to create can be made with a combination of +indexing, `reshape`, `broadcast_to`, and `transpose`, but it's possible to use +strides to represent some things which are not so easy to do with just these +functions, for example, [sliding +windows](https://numpy.org/doc/stable/reference/generated/numpy.lib.stride_tricks.sliding_window_view.html) +and [convolutions](https://stackoverflow.com/a/43087507/161801). However, if +you do use stride tricks, be careful of the caveats (see the [notes section of +the `as_strided` +docs](https://numpy.org/doc/stable/reference/generated/numpy.lib.stride_tricks.as_strided.html)). + +(c-vs-fortran-ordering)= +## C vs. Fortran Ordering + +NumPy has an internal distinction between C order and Fortran +order.[^c-order-footnote] C-ordered arrays are stored in memory such that the +last axis varies the fastest. For example, if `a` has 3 dimensions, then its +elements are stored in memory like `a[0, 0, 0], a[0, 0, 1], a[0, 0, 2], ..., +a[0, 1, 0], a[0, 1, 1], a[0, 1, 2], ...`. Fortran ordering is the opposite: +the elements are stored in memory so that the first axis varies fastest, like +`a[0, 0, 0], a[1, 0, 0], a[2, 0, 0], ..., a[0, 1, 0], a[1, 1, 0], a[2, 1, 0] +...`. + +[^c-order-footnote]: C ordering and Fortran ordering are also sometimes + "row-major" and "column-major" ordering, respectively. However, this + terminology is confusing when the array has more than two axes or when it + does not represent a mathematical matrix. It's better to think of ordering + in terms of which axes vary the fastest---the last for C ordering and the + first for Fortran ordering. Also, I don't know about you, but I can never + remember which is supposed to be "row-" and "column-" major, but I do + remember how array indexing works in C, so just thinking about that requires + no mnemonic. + +The most important thing to note about C and Fortran ordering for our purposes +is that + +> **the internal ordering of an array does not change any indexing +> semantics.** + +The same index will select the same elements on `a` regardless of whether it +uses C or Fortran ordering internally. + +More generally, the actual memory layout of an array has no bearing on +indexing semantics. Indexing operates on the logical abstraction of the array +as presented to the user, even if the true memory doesn't look anything like +that because the array is a [view](views-vs-copies) or has some other layout +due to [stride tricks](strides). + +In particular, this also applies to [boolean array +indices](multidimensional-indices/boolean-arrays.md). A boolean mask always +[selects the elements in C order](boolean-array-c-order), even if the +underlying arrays use Fortran ordering. + +```py +>>> a = np.arange(9).reshape((3, 3)) +>>> a +array([[0, 1, 2], + [3, 4, 5], + [6, 7, 8]]) +>>> a_f = np.asarray(a, order='F') +>>> a_f # a_f looks the same as a, but the internal memory is ordered differently +array([[0, 1, 2], + [3, 4, 5], + [6, 7, 8]]) +>>> idx = np.array([ +... [False, True, False], +... [ True, True, False], +... [False, False, False]]) +>>> idx_f = np.asarray(idx, order='F') # It doesn't matter if the index array is Fortran-ordered either +>>> a[idx] # These are all the same +array([1, 3, 4]) +>>> a_f[idx] +array([1, 3, 4]) +>>> a[idx_f] +array([1, 3, 4]) +>>> a_f[idx_f] +array([1, 3, 4]) +``` + +~~~~{admonition} Aside +If you read the previous section on [strides](strides), you may have guessed +that the difference between C-ordered and Fortran-ordered arrays is a +difference of...strides! + +```py +>>> a.strides +(24, 8) +>>> a_f.strides +(8, 24) +``` + +In a C-ordered array the strides decrease and in a Fortran-ordered array they +increase, because a smaller stride corresponds to "faster varying". +~~~~ + +**What ordering *does* affect is the performance of certain operations.** In +particular, the ordering determines whether it is more optimal to index along +the first or last axes of an array. For example, `a[0]` selects the first +subarray along the first axis (recall that `a[0]` is a [view](views-vs-copies) +into `a`, so it references the exact same memory as `a`). For a C-ordered +array, which is the default, this subarray is contiguous in memory. This is +because the indices on the last axes vary the fastest (i.e., are next to each +other in memory), so selecting a subarray of the first axis picks elements +which are still contiguous. Conversely, for a Fortran-ordered array, `a[0]` is +not contiguous, but `a[..., 0]` is. + +``` +>>> a[0].data.contiguous +True +>>> a_f[0].data.contiguous +False +>>> a_f[..., 0].data.contiguous +True +``` + +Operating on contiguous memory allows the CPU to place the entire memory block +in the cache at once, and is more performant as a result. The performance +difference won't be noticeable for our small example `a` above, as it is small +enough to fit entirely in the cache, but it becomes significant for larger +arrays. Compare the time to sum along `a[0]` or `a[..., 0]` for a +3-dimensional array `a` with a million elements using C and Fortran ordering +(using [IPython](https://ipython.org/)'s `%timeit`): + +``` +In [1]: import numpy as np + +In [2]: a = np.ones((100, 100, 100)) # a has C order (the default) + +In [3]: %timeit np.sum(a[0]) +8.57 µs ± 121 ns per loop (mean ± std. dev. of 7 runs, 100,000 loops each) + +In [4]: %timeit np.sum(a[..., 0]) +24.2 µs ± 1.29 µs per loop (mean ± std. dev. of 7 runs, 10,000 loops each) + +In [5]: a_f = np.asarray(a, order='F') + +In [6]: %timeit np.sum(a_f[0]) +26.3 µs ± 952 ns per loop (mean ± std. dev. of 7 runs, 10,000 loops each) + +In [7]: %timeit np.sum(a_f[..., 0]) +8.6 µs ± 130 ns per loop (mean ± std. dev. of 7 runs, 100,000 loops each) +``` + +Summing along contiguous memory (`a[0]` for C ordering and `a[..., 0]` for +Fortran ordering) is about 3x faster. + +NumPy indexing semantics generally favor using C ordering, as it does not +require an ellipsis to select contiguous subarrays. C ordering also matches +the [list-of-lists intuition](multidimensional-indices/what-is-an-array.md) of +an array, since an array like `[[0, 1], [2, 3]]` is stored in memory as +literally `0, 1, 2, 3` with C ordering. It also aligns well with NumPy's +[broadcasting](broadcasting) rules, where broadcasted dimensions are prepended +by default, allowing one to think of an array as a "stack" of contiguous +subarrays. + +C ordering is the default in NumPy when creating arrays with functions like +`asarray`, `ones`, `arange`, and so on. One typically only switches to +Fortran ordering when calling certain Fortran codes, or when creating an +array from another memory source that produces Fortran-ordered data. + +Regardless of which ordering you are using, it is worth structuring your data +so that operations are done on contiguous memory when possible. + +```{rubric} Footnotes +``` +<!-- Footnotes are written inline above but markdown will put them here at the +end of the document. --> diff --git a/docs/indexing-guide/slices.md b/docs/indexing-guide/slices.md new file mode 100644 index 00000000..c01250cd --- /dev/null +++ b/docs/indexing-guide/slices.md @@ -0,0 +1,2674 @@ +# Slices + +Python's slice syntax is one of the more confusing parts of the language, even +for experienced developers. This page carefully breaks down the rules for +slicing, and examines just what it is that makes them so confusing. + +There are two primary aspects of slices that make them difficult to +understand: confusing conventions, and discontinuous definitions. By confusing +conventions, we mean that slice semantics have definitions that are often +difficult to reason about mathematically. These conventions were chosen for +syntactic convenience, and one can easily see for most of them how they lead +to concise notation for very common operations; nonetheless, it remains true +that they can complicate figuring out the *right* slice to use in the first +place. By discontinuous definitions, we mean that the definition of a slice +takes on fundamentally different meanings if the start, stop, or step are +negative, nonnegative, or omitted. This is again done for syntactic +convenience, but it means that as a user, you must switch your mode of +thinking about slices depending on the value of the arguments. There are no +uniform formulas that apply to all slices. + +The [ndindex](../index) library can help with much of this, especially for +developers of libraries that consume slices. However, for end-users, the +challenge is often just to write down a slice. + +Even though this page is a part of a larger [guide](index) on indexing NumPy +arrays, and indeed, the [ndindex](../index) library focuses on NumPy array +index semantics, this page can be treated as a standalone guide to slicing, +which should be useful for any Python programmer, even those who do not +regularly use array libraries such as NumPy. This is because everything on +this page also applies to the built-in Python sequence types like `list`, +`tuple`, and `str`, and slicing these objects is a common operation across all +types of Python code. + +## What is a slice? + +In Python, a slice is a special syntax that is allowed only in an index, that +is, inside of square brackets proceeding an expression. A slice consists of +one or two colons, with either an expression or nothing on either side of each +colon. For example, the following are all valid slices on the object +`a`:[^slice-name-footnote] + +[^slice-name-footnote]: Sometimes people call any kind of index `a[idx]` a +*slice*. However, this sort of nomenclature is confusing, since there are many +[valid possibilities](index) of `idx` that are not `slice` objects. It's +better to use the word *index* to refer to an arbitrary object that can index +an array or sequence, and reserve the word *slice* for `slice` instances, which +are just one type of *index*. + +```py +a[x:y] +a[x:y:z] +a[:] +a[x::] +a[x::z] +``` + +Furthermore, for a slice `a[x:y:z]`, `x`, `y`, and `z` must be +integers.[^non-integer-footnote] + +[^non-integer-footnote]: Non-integer `start`, `stop`, and `step` are + syntatically allowed by Python, but the built-in types (`list`, `tuple`, + `str`) and NumPy arrays do not allow them. There are other libraries that + make use of this feature. For instance, the Pandas + {external+pandas:attr}`~pandas.DataFrame.loc` attribute allows + slicing with strings corresponding to labels. The semantics of such + extensions to slicing may not necessarily correspond to the semantics + outlined in this guide. + +The three arguments to a slice are traditionally called `start`, `stop`, and +`step`: + +```py +a[start:stop:step] +``` + +We will use these names throughout this guide. + +At a high level, **a slice is a convenient way to select a sequential subset +of `a`** (roughly, "every `step` elements between the `start` and `stop`"). +The exact way in which this occurs is outlined throughout this guide. + +It is worth noting that the `x:y:z` syntax is not valid outside of square +brackets. However, slice objects can be created manually using the `slice()` +builtin (`a[x:y:z]` is the same as `a[slice(x, y, z)]`). If you want to +perform more advanced operations like arithmetic on slices, consider using +the [`ndindex.Slice()`](ndindex.slice.Slice) object. + +(rules)= +## Rules + +These are the rules to keep in mind to understand how slices work. Each of +these is explained in detail below. Many of the detailed descriptions below +also outline several *wrong* rules, which are bad ways of thinking about +slices but which you may be tempted to think about as rules. The below 7 rules +are always correct. + +In this document, "*nonnegative*" means $\geq 0$ and "*negative*" means $< 0$. + +For a slice `a[start:stop:step]`: + +1. Slicing something never raises an `IndexError`, even if the slice is empty. + For a NumPy array, a slice always keeps the axis being sliced, even if that + means the resulting dimension will be 0 or 1. (See section {ref}`subarray`) + +2. The `start` and `stop` use *0-based indexing* from the *beginning* of `a` when + they are *nonnegative*, and *−1-based indexing* from *end* of `a` when they + are *negative*. (See sections {ref}`0-based` and {ref}`negative-indices`) + +3. The `stop` is never included in the slice. (See section {ref}`half-open`) + +4. The `start` and `stop` are clipped to the bounds of `a`. (See section + {ref}`clipping`) + +5. The slice starts at the `start` and successively adds `step` until it + reaches an index that is at or past the `stop`, and then stops without + including that `stop` index. (See sections {ref}`steps` and + {ref}`negative-steps`) + +6. If the `step` is omitted it defaults to `1`. (See section {ref}`omitted`) + +7. If the `start` or `stop` are omitted they extend to the beginning or end of + `a` in the direction being sliced. Slices like `a[:i]` or `a[i:]` should be + thought of as the `start` or `stop` being omitted, not as a colon to the + left or right of an index. (See section {ref}`omitted`) + +Throughout this guide, we will use as an example the same prototype list as we +used in the [integer indexing section](prototype-example): + +<div class="slice-diagram"> + <table> + <tr> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>['a',</pre></td> + <td><pre>'b',</pre></td> + <td><pre>'c',</pre></td> + <td><pre>'d',</pre></td> + <td><pre>'e',</pre></td> + <td><pre>'f',</pre></td> + <td><pre>'g']</pre></td> + </tr> + </table> +</div> + +The list `a` has 7 elements. + +As a reminder, the elements of `a` are strings, but the slices on the list `a` +will always use integers. Like [all other index types](intro.md), +**the result of a slice is never based on the values of the elements, but +rather on the position of the elements in the list.**[^dict-footnote] + +[^dict-footnote]: If you are looking for something that allows non-integer +indices or that indexes by value, you may want a `dict`. Despite using similar +syntax, `dict`s do not allow slicing. + +(slices-points-of-confusion)= +## Points of Confusion + +Before running through this guide, ensure you have a solid understanding of +how integer indexing works.. See the previous section, [](integer-indices). + +Now, let us come back to slices. The full definition of a slice could be +written down in a couple of sentences, although the discontinuous definitions +would necessitate several "if" conditions. The [NumPy +docs](https://numpy.org/doc/stable/user/basics.indexing.html#slicing-and-striding) on slices +say + +(numpy-definition)= + +> The basic slice syntax is `i:j:k` where *i* is the starting index, *j* is +> the stopping index, and *k* is the step ( $k\neq 0$ ). This selects the `m` +> elements (in the corresponding dimension) with index values *i, i + k, ..., +> i + (m - 1) k* where $m = q + (r\neq 0)$ and *q* and *r* are the quotient and +> remainder obtained by dividing *j - i* by *k*: *j - i = q k + r*, so that +> *i + (m - 1) k \< j*. + +While these definitions may give a technically accurate description of slices, +they aren't especially helpful to someone who is trying to construct a slice +from a higher level of abstraction such as "I want to select this particular +subset of my array."[^numpy-definition-footnote] + +[^numpy-definition-footnote]: This formulation actually isn't particularly + helpful for formulating higher level slice formulas such as the ones used + by ndindex either. Plus it fails to account for [some of the + details](clipping) discussed on this page. + +Instead, we shall examine slices by carefully reviewing all the various +aspects of their syntax and semantics that can lead to confusion, and +attempting to demystify them through simple [rules](rules). + +(subarray)= +### Subarray + +> **A slice always produces a subarray (or sub-list, sub-tuple, sub-string, +etc.). For NumPy arrays, this means that a slice will always *preserve* the +dimension that is sliced.** + +(empty-slice)= +This holds true even if the slice selects only a single element, or even if it +selects no elements at all (a slice that selects no elements is called an +*empty slice*, and produces an size-0 array. This is also true for lists, +tuples, and strings, in the sense that a slice on a list, tuple, or string +will always produce a list, tuple, or string. This behavior is different from +[integer indices](integer-indices), which always remove the dimension that +they index. + +For example + +```py +>>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] +>>> a[3] # An element of the list +'d' +>>> a[3:4] # A sub-list +['d'] +>>> a[5:2] # Empty slice +[] +>>> import numpy as np +>>> arr = np.array([[1, 2], [3, 4]]) +>>> arr.shape +(2, 2) +>>> arr[0].shape # Integer index removes the first dimension +(2,) +>>> arr[0:1].shape # Slice preserves the first dimension +(1, 2) +``` + +One consequence of this is that, unlike integer indices, **slices will never +raise `IndexError`, even if the slice is empty or extends past the bounds of +the array**.[^slice-error-footnote] Therefore, you cannot rely on runtime +errors to alert you to coding mistakes relating to slice bounds that are too +large. A slice cannot be "out of bounds." See also the section on +[clipping](clipping) below. + +[^slice-error-footnote]: A slice might raise another exception, though, if it +is completely invalid, e.g., `a[1.0:]` and `a[::0]` raise `TypeError` and +`ValueError`, respectively. + +(0-based)= +### 0-based + +For the slice `a[start:stop]`, where the `start` and `stop` are nonnegative +integers, the indices `start` and `stop` are 0-based, as in [integer +indexing](integer-indices). However, note that although the `stop` is 0-based, +[it is not included in the slice](wrong-rule-4). + +For example: + +<div class="slice-diagram"> +<code style="font-size: 16pt;">a[<span class="slice-diagram-slice">3:5</span>] == ['d', 'e']</code> + <table> + <tr> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>['a',</pre></td> + <td><pre> 'b',</pre></td> + <td><pre> 'c',</pre></td> + <td class="underline-cell"><pre> 'd',</pre></td> + <td class="underline-cell"><pre> 'e',</pre></td> + <td><pre> 'f',</pre></td> + <td><pre> 'g']</pre></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td class="slice-diagram-not-selected">0</td> + <td class="slice-diagram-not-selected">1</td> + <td class="slice-diagram-not-selected">2</td> + <td class="slice-diagram-selected">3</td> + <td class="slice-diagram-selected">4</td> + <td class="slice-diagram-not-selected">5</td> + <td class="slice-diagram-not-selected">6</td> + </tr> + </table> +</div> + +```py +>>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] +>>> a[3:5] +['d', 'e'] +``` + +Do not be worried if you find 0-based indexing hard to get used to, or if you +find yourself forgetting about it. Even experienced Python developers (this +author included) still find themselves writing `a[3]` instead of `a[2]` from +time to time. The best way to learn to use 0-based indexing is to practice +using it enough that you use it automatically without thinking about it. + +(half-open)= +### Half-open + +Slices behave like half-open intervals. What this means is that + +> **the `stop` in `a[start:stop]` is *never* included in the slice** + +(the exception is if [the `stop` is omitted](omitted)). + +For example, `a[3:5]` slices the indices `3` and `4`, but not `5` +([0-based](0-based)). + +<div class="slice-diagram"> +<code style="font-size: 16pt;">a[<span class="slice-diagram-slice">3:5</span>] == ['d', 'e']</code> + <div> + <table> + <tr> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>['a',</pre></td> + <td><pre> 'b',</pre></td> + <td><pre> 'c',</pre></td> + <td class="underline-cell"><pre> 'd',</pre></td> + <td class="underline-cell"><pre> 'e',</pre></td> + <td><pre> 'f',</pre></td> + <td><pre> 'g']</pre></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td class="slice-diagram-not-selected">0</td> + <td class="slice-diagram-not-selected">1</td> + <td class="slice-diagram-not-selected">2</td> + <td><div class="circle-blue slice-diagram-selected">3</div></td> + <td><div class="circle-blue slice-diagram-selected">4</div></td> + <td><div class="circle-red slice-diagram-not-selected">5</div></td> + <td class="slice-diagram-not-selected">6</td> + </tr> + </table> + </div> +</div> + +```py +>>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] +>>> a[3:5] +['d', 'e'] +``` + +The half-open nature of slices means that you must always remember that the +`stop` slice element is not included in the slice. However, it has a few +advantages: + +(sanity-check)= + +- The maximum length of a slice `a[start:stop]`, when the `start` and `stop` + are nonnegative, is always `stop - start`. For example, `a[i:i+n]` slices + `n` elements from `a`. The caveat "maximum" is here because if `stop` + extends beyond the end of `a`, then `a[start:stop]` will only slice up to + `len(a) - start` (see {ref}`clipping` below). Also be careful that this is + not true when the `start` or `stop` are negative (see + {ref}`negative-indices` below). However, given those caveats, this is often + a very useful sanity check that a slice is correct. If you expect a slice to + have length `n` but `stop - start` is clearly different from `n`, then the + slice is likely wrong. Length calculations are more complicated when `step + != 1`; in those cases, {meth}`len(ndindex.Slice(...)) + <ndindex.Slice.__len__>` can be useful. + +- `len(a)` can be used as a `stop` value to slice to the end of `a`. For + example, `a[1:len(a)]` slices from the second element to the end of `a` + (this is equivalent to `a[1:]`, see {ref}`omitted`) + + ```py + >>> a[1:len(a)] + ['b', 'c', 'd', 'e', 'f', 'g'] + >>> a[1:] + ['b', 'c', 'd', 'e', 'f', 'g'] + ``` + +- Consecutive slices can be concatenated to one another by making each successive + slice's `start` the same as the previous slice's `stop`. For example, for our + list `a`, `a[2:3] + a[3:5]` is the same as `a[2:5]`. + + ```py + >>> a[2:3] + a[3:5] + ['c', 'd', 'e'] + >>> a[2:5] + ['c', 'd', 'e'] + ``` + + A common usage of this is to split a slice into two slices. For example, the + slice `a[i:j]` can be split as `a[i:k]` and `a[k:j]`. + +If the `start` is on or after the `stop`, the resulting list will be empty. +That is to say, the `stop` *not* being included takes precedence over the +`start` being included. + +```py +>>> a[3:3] +[] +>>> a[5:2] +[] +``` + +Recall that for NumPy arrays, a slice always [preserves the axis being +sliced](subarray). This applies even if the size of the resulting axis is 0 or +1. + +```py +>>> import numpy as np +>>> arr = np.array([[1, 2], [3, 4]]) +>>> arr.shape +(2, 2) +>>> arr[0].shape # Integer index removes the first dimension +(2,) +>>> arr[0:1].shape # Slice preserves the first dimension +(1, 2) +>>> arr[0:0].shape # Slice preserves the first dimension as an empty dimension +(0, 2) +``` + +#### Wrong Ways of Thinking about Half-open Semantics + +> **The proper rule to remember for half-open semantics is "the `stop` is not + included."** + +There are several alternative interpretations of the half-open rule, but they +are all wrong in subtle ways. To be sure, for each of these, one could "fix" +the rule by adding some conditions, "it's this in the case where such and such +is nonnegative and that when such and such is negative, and so on." But that's +not the point. The goal here is to *understand* slices. Remember that one of +the reasons that slices are difficult to understand is these branching rules. +By trying to remember a rule that has branching conditions, you open yourself +up to confusion. The rule becomes much more complicated than it appears at +first glance, making it hard to remember. You may forget the "uncommon" cases +and get things wrong when they come up in practice. You might as well think +about slices using the [definition from the NumPy docs](numpy-definition). + +Rather, it is best to remember the simplest possible rule that is *always* +correct. That rule is "the `stop` is not included." This rule is extremely +simple, and is always right, regardless of what the values of `start`, `stop`, +or `step` are (the only exception is if the `stop` is omitted, in which case, +the rule obviously doesn't apply as-is, and so you can fallback to [the rule +about omitted `start`/`stop`](omitted)). + +(wrong-rule-1)= +##### Wrong Rule 1: "A slice `a[start:stop]` slices the half-open interval $[\text{start}, \text{stop})$." + +(or equivalently, "a slice `a[start:stop]` selects the elements $i$ such that +$\text{start} <= i < \text{stop}$") + +This is *only* the case if the `step` is positive. It also isn't directly true +when the `start` or `stop` are negative. For example, with a `step` of `-1`, +`a[start:stop:-1]` slices starting at `start` going in reverse order to +`stop`, but not including `stop`. Mathematically, this creates a half open +interval $(\text{stop}, \text{start}]$ (except reversed). + +For example, say we believed that `a[5:3:-1]` sliced the half-open interval +$[3, 5)$ but in reverse order. + +<div class="slice-diagram"> + <code style="font-size: 16pt;">a[<span class="slice-diagram-slice">5:3:-1</span>] "==" ['e', 'd']</code> + <table> + <tr> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>['a',</pre></td> + <td><pre> 'b',</pre></td> + <td><pre> 'c',</pre></td> + <td class="underline-cell"><pre> 'd',</pre></td> + <td class="underline-cell"><pre> 'e',</pre></td> + <td><pre> 'f',</pre></td> + <td><pre> 'g']</pre></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td class="slice-diagram-not-selected">0</td> + <td class="slice-diagram-not-selected">1</td> + <td class="slice-diagram-not-selected">2</td> + <td style="background-color: > + <div style="position: relative;"> + <span class="math notranslate nohighlight" style="position: absolute; display: flex; height: 100%; top: 0; align-items: center;">\([\)</span> + <span class="slice-diagram-selected">3</span + </div> + </td> + <td class="slice-diagram-selected">4</td> + <td> + <div style="position: relative;"> + <span class="slice-diagram-not-selected">5</span> + <span class="math notranslate nohighlight" style="position: absolute; display: flex; height: 100%; align-items: center; top: 0; right: 0;">\()\)</span> + </div> + </td> + <td class="slice-diagram-not-selected">6</td> + </tr> + <tr> + <th></th> + <td></td> + <td></td> + <td></td> + <td></td> + <td colspan="3"><div class="centered-text">(reversed)</div><div class="horizontal-line"></div></td> + <td></td> + </tr> + <tr> + <th></th> + <td></td> + <td></td> + <td></td> + <td></td> + <td colspan="3" class="slice-diagram-not-selected"><div class="centered-text"><b>THIS IS WRONG!</b></div></td> + <td></td> + </tr> + </table> +</div> + + +We might assume we would get + +```py +>>> a[5:3:-1] # doctest:+SKIP +['e', 'd'] # WRONG +``` + +Actually, what we really get is + +```py +>>> a[5:3:-1] +['f', 'e'] +``` + +This is because the slice `5:3:-1` starts at index `5` and steps backwards to +index `3`, but not including `3` (see [](negative-steps) below). + +<div class="slice-diagram"> +<code style="font-size: 16pt;">a[<span class="slice-diagram-slice">5:3:-1</span>] == ['f', 'e']</code> +<table> + <tr> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>['a',</pre></td> + <td></td> + <td><pre> 'b',</pre></td> + <td></td> + <td><pre> 'c',</pre></td> + <td></td> + <td><pre> 'd',</pre></td> + <td></td> + <td class="underline-cell"><pre> 'e',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre> 'f',</pre></td> + <td></td> + <td><pre> 'g']</pre></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td class="slice-diagram-not-selected">0</td> + <td></td> + <td class="slice-diagram-not-selected">1</td> + <td></td> + <td class="slice-diagram-not-selected">2</td> + <td></td> + <td><div class="circle-red slice-diagram-not-selected">3</div></td> + <td class="left-arrow-cell"><div style="font-size: smaller; transform: +translateY(-12px) translateX(3px)">−1</div></td> + <td><div class="circle-blue slice-diagram-selected">4</div></td> + <td class="left-arrow-cell"><div style="font-size: smaller; transform: +translateY(-12px) translateX(3px)">−1</div></td> + <td><div class="circle-blue slice-diagram-selected">5</div></td> + <td></td> + <td class="slice-diagram-not-selected">6</td> + </tr> +</table> +</div> + +(wrong-rule-2)= +##### Wrong Rule 2: "A slice works like `range()`." + +There are many similarities between the behaviors of slices and the behavior +of `range()`. However, they are not exactly the same. A slice +`a[start:stop:step]` only acts like `range(start, stop, step)` if the `start` +and `stop` are **nonnegative**. If either of them are negative, the slice +wraps around and slices from the end of the list (see {ref}`negative-indices` +below). `range()` on the other hand treats negative numbers as the actual +start and stop values for the range. For example: + +```py +>>> list(range(3, 5)) +[3, 4] +>>> b = list(range(7)) +>>> b[3:5] # b is range(7), and these are the same +[3, 4] +>>> list(range(3, -2)) # Empty, because -2 is less than 3 +[] +>>> b[3:-2] # Indexes from 3 to the second to last (5), but not including 5 +[3, 4] +``` + +This rule is appealing because `range()` simplifies some computations. For +example, you can index or take the `len()` of a range. If you need to perform +computations on slices, we recommend using [ndindex](ndindex.slice.Slice). +This is what it was designed for. + +Note however, that the reverse does work: if you have a `range()` object, you +can slice it to get another `range()` object. This works without every +actually computing the range values, so it is efficient even if the actual +range would be huge. + +```py +>>> range(2, 1000000000, 3)[-1:0:-5] +range(999999998, 2, -15) +``` + +So slices can be used to compute transformations on `range` objects, but +`range` objects should not be used to compute things about slices. If you want +to do that, use [ndindex](../index). + +(wrong-rule-3)= +##### Wrong Rule 3: "Slices index the spaces between the elements of the list." + +This is a very common rule that is taught for both slices and integer +indexing. The reasoning goes as follows: 0-based indexing is confusing, where +the first element of a list is indexed by 0, the second by 1, and so on. +Rather than thinking about that, consider the spaces between the elements: + +<div class="slice-diagram"> +<code style="font-size: 16pt;">a[<span class="slice-diagram-slice">3:5</span>] == ['d', 'e']</code> + +<div> + <table> + <tr> + <td><pre>a =</pre></td> + <td><pre>[</pre></td> + <td></td> + <td><pre>'a',</pre></td> + <td></td> + <td><pre>'b',</pre></td> + <td></td> + <td><pre>'c',</pre></td> + <td></td> + <td class="underline-cell"><pre>'d',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre>'e',</pre></td> + <td></td> + <td><pre>'f',</pre></td> + <td></td> + <td><pre>'g'</pre></td> + <td></td> + <td><pre>]</pre></td> + </tr> + <tr> + <th></th> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td class="slice-diagram-not-selected">0</td> + <td></td> + <td class="slice-diagram-not-selected">1</td> + <td></td> + <td class="slice-diagram-not-selected">2</td> + <td></td> + <td class="slice-diagram-selected">3</td> + <td></td> + <td class="slice-diagram-selected">4</td> + <td></td> + <td class="slice-diagram-selected">5</td> + <td></td> + <td class="slice-diagram-not-selected">6</td> + <td></td> + <td class="slice-diagram-not-selected">7</td> + </tr> + </table> +</div> +<i>(not a great way of thinking about 0-based indexing)</i> +</div> + + +Using this way of thinking, the first element of `a` is to the left of the +"1-divider". An integer index `i` produces the element to the right of the +"`i`-divider", and a slice `a[i:j]` selects the elements between the `i` and +`j` dividers. + +At first glance, this seems like a rather clever way to think about the +half-open rule. For instance, between the `3` and `5` dividers is the subarray +`['d', 'e']`, which is indeed what we get for `a[3:5]`. However, there are several +reasons why this way of thinking creates more confusion than it removes. + +- As with [wrong rule 1](wrong-rule-1), it works well enough if the `step` is + positive, but falls apart when it is negative. + + Consider again the slice `a[5:3:-1]`. Looking at the above figure, we might + imagine it to give the same incorrect subarray that we imagined before. + + <div class="slice-diagram"> + <code style="font-size: 16pt;">a[<span class="slice-diagram-slice">5:3:-1</span>] "==" ['e', 'd']</code> + + <div> + <table> + <tr> + <td><pre>a =</pre></td> + <td><pre>[</pre></td> + <td></td> + <td><pre>'a',</pre></td> + <td></td> + <td><pre>'b',</pre></td> + <td></td> + <td><pre>'c',</pre></td> + <td></td> + <td class="underline-cell"><pre>'d',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre>'e',</pre></td> + <td></td> + <td><pre>'f',</pre></td> + <td></td> + <td><pre>'g'</pre></td> + <td></td> + <td><pre>]</pre></td> + </tr> + <tr> + <th></th> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td class="slice-diagram-not-selected">0</td> + <td></td> + <td class="slice-diagram-not-selected">1</td> + <td></td> + <td class="slice-diagram-not-selected">2</td> + <td></td> + <td class="slice-diagram-selected">3</td> + <td></td> + <td class="slice-diagram-selected">4</td> + <td></td> + <td class="slice-diagram-selected">5</td> + <td></td> + <td class="slice-diagram-not-selected">6</td> + <td></td> + <td class="slice-diagram-not-selected">7</td> + </tr> + </table> + </div> + <div class="slice-diagram-not-selected"><b>THIS IS WRONG!</b></div> + </div> + + As before, we might assume we would get + + ```py + >>> a[5:3:-1] # doctest: +SKIP + ['e', 'd'] # WRONG + ``` + + but this is incorrect! What we really get is + + ```py + >>> a[5:3:-1] + ['f', 'e'] + ``` + + If you've ever espoused the "spaces between elements" way of thinking about + indices, this should give you serious pause. As the above diagram + illustrates, this rule is flat out wrong when the `step` is negative, and + there's no clear way to salvage it. Contrast thinking about this same slice + as simply [stepping backwards](negative-steps) from index `5` to index `3`, + but not including index `3`: + + <div class="slice-diagram"> + <code style="font-size: 16pt;">a[<span class="slice-diagram-slice">5:3:-1</span>] == ['f', 'e']</code> + <table> + <tr> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>['a',</pre></td> + <td></td> + <td><pre> 'b',</pre></td> + <td></td> + <td><pre> 'c',</pre></td> + <td></td> + <td><pre> 'd',</pre></td> + <td></td> + <td class="underline-cell"><pre> 'e',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre> 'f',</pre></td> + <td></td> + <td><pre> 'g']</pre></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td class="slice-diagram-not-selected">0</td> + <td></td> + <td class="slice-diagram-not-selected">1</td> + <td></td> + <td class="slice-diagram-not-selected">2</td> + <td></td> + <td><div class="circle-red slice-diagram-not-selected">3</div></td> + <td class="left-arrow-cell"><div style="font-size: smaller; transform: + translateY(-12px) translateX(3px)">−1</div></td> + <td><div class="circle-blue slice-diagram-selected">4</div></td> + <td class="left-arrow-cell"><div style="font-size: smaller; transform: + translateY(-12px) translateX(3px)">−1</div></td> + <td><div class="circle-blue slice-diagram-selected">5</div></td> + <td></td> + <td class="slice-diagram-not-selected">6</td> + </tr> + <tr> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td class="slice-diagram-index-label-not-selected">stop</td> + <td></td> + <td></td> + <td></td> + <td class="slice-diagram-index-label-selected">start</td> + <td></td> + <td></td> + </tr> + </table> + </div> + +- The rule does work when the `start` or `stop` are negative, but only if you + think about it correctly. The correct way to think about it is to reverse + the dividers: + + <div class="slice-diagram"> + <code style="font-size: 16pt;">a[<span class="slice-diagram-slice">-4:-2</span>] == ['d', 'e']</code> + <div> + <table> + <tr> + <td><pre>a =</pre></td> + <td><pre>[</pre></td> + <td></td> + <td><pre>'a',</pre></td> + <td></td> + <td><pre>'b',</pre></td> + <td></td> + <td><pre>'c',</pre></td> + <td></td> + <td class="underline-cell"><pre>'d',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre>'e',</pre></td> + <td></td> + <td><pre>'f',</pre></td> + <td></td> + <td><pre>'g'</pre></td> + <td></td> + <td><pre>]</pre></td> + </tr> + <tr> + <th></th> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td class="slice-diagram-not-selected">−7</td> + <td></td> + <td class="slice-diagram-not-selected">−6</td> + <td></td> + <td class="slice-diagram-not-selected">−5</td> + <td></td> + <td class="slice-diagram-selected">−4</td> + <td></td> + <td class="slice-diagram-selected">−3</td> + <td></td> + <td class="slice-diagram-selected">−2</td> + <td></td> + <td class="slice-diagram-not-selected">−1</td> + <td></td> + <td class="slice-diagram-not-selected">0</td> + </tr> + </table> + </div> + <i>(not a great way of thinking about negative indices)</i> + </div> + + For example, `a[-4:-2]` will give `['d', 'e']` + + ```py + >>> a[-4:-2] + ['d', 'e'] + ``` + + However, it would be quite easy to get confused here, as the "other" way of + thinking about negative indices (the way we are recommending) is that the + end starts at -1. So you might mistakenly imagine something like this: + + + <div class="slice-diagram"> + <code style="font-size: 16pt;">a[<span class="slice-diagram-slice">-4:-2</span>] "==" ['e', 'f']</code> + <div> + <table> + <tr> + <td><pre>a =</pre></td> + <td><pre>[</pre></td> + <td></td> + <td><pre>'a',</pre></td> + <td></td> + <td><pre>'b',</pre></td> + <td></td> + <td><pre>'c',</pre></td> + <td></td> + <td><pre>'d',</pre></td> + <td></td> + <td class="underline-cell"><pre>'e',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre>'f',</pre></td> + <td></td> + <td><pre>'g'</pre></td> + <td></td> + <td><pre>]</pre></td> + </tr> + <tr> + <th></th> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-red"></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td class="slice-diagram-not-selected">−8</td> + <td></td> + <td class="slice-diagram-not-selected">−7</td> + <td></td> + <td class="slice-diagram-not-selected">−6</td> + <td></td> + <td class="slice-diagram-not-selected">−5</td> + <td></td> + <td class="slice-diagram-selected">−4</td> + <td></td> + <td class="slice-diagram-selected">−3</td> + <td></td> + <td class="slice-diagram-selected">−2</td> + <td></td> + <td class="slice-diagram-not-selected">−1</td> + </tr> + </table> + </div> + <div class="slice-diagram-not-selected"><b>THIS IS WRONG!</b></div> + </div> + + But things are even worse than that. If we combine a negative `start` and + `stop` with a negative `step`, things get even more confusing. Consider the + slice `a[-2:-4:-1]`. This gives `['f', 'e']`: + + ```py + >>> a[-2:-4:-1] + ['f', 'e'] + ``` + + To get this with the "spacers" idea, we have to use the above "wrong" + diagram! + + <div class="slice-diagram"> + <code style="font-size: 16pt;">a[<span class="slice-diagram-slice">-2:-4:-1</span>] == ['f', 'e']</code> + <div> + <table> + <tr> + <td><pre>a =</pre></td> + <td><pre>[</pre></td> + <td></td> + <td><pre>'a',</pre></td> + <td></td> + <td><pre>'b',</pre></td> + <td></td> + <td><pre>'c',</pre></td> + <td></td> + <td><pre>'d',</pre></td> + <td></td> + <td class="underline-cell"><pre>'e',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre>'f',</pre></td> + <td></td> + <td><pre>'g'</pre></td> + <td></td> + <td><pre>]</pre></td> + </tr> + <tr> + <th></th> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-red"></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td class="slice-diagram-not-selected">−8</td> + <td></td> + <td class="slice-diagram-not-selected">−7</td> + <td></td> + <td class="slice-diagram-not-selected">−6</td> + <td></td> + <td class="slice-diagram-not-selected">−5</td> + <td></td> + <td class="slice-diagram-selected">−4</td> + <td></td> + <td class="slice-diagram-selected">−3</td> + <td></td> + <td class="slice-diagram-selected">−2</td> + <td></td> + <td class="slice-diagram-not-selected">−1</td> + </tr> + </table> + </div> + <span style="color:var(--color-slice-diagram-selected);"><b>NOW RIGHT!</b></span> + </div> + + <div class="slice-diagram"> + <code style="font-size: 16pt;">a[<span class="slice-diagram-slice">-2:-4:-1</span>] "==" ['e', 'd']</code> + <div> + <table> + <tr> + <td><pre>a =</pre></td> + <td><pre>[</pre></td> + <td></td> + <td><pre>'a',</pre></td> + <td></td> + <td><pre>'b',</pre></td> + <td></td> + <td><pre>'c',</pre></td> + <td></td> + <td class="underline-cell"><pre>'d',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre>'e',</pre></td> + <td></td> + <td><pre>'f',</pre></td> + <td></td> + <td><pre>'g'</pre></td> + <td></td> + <td><pre>]</pre></td> + </tr> + <tr> + <th></th> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-blue"></td> + <td></td> + <td class="vertical-bar-red"></td> + <td></td> + <td class="vertical-bar-red"></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td class="slice-diagram-not-selected">−7</td> + <td></td> + <td class="slice-diagram-not-selected">−6</td> + <td></td> + <td class="slice-diagram-not-selected">−5</td> + <td></td> + <td class="slice-diagram-selected">−4</td> + <td></td> + <td class="slice-diagram-selected">−3</td> + <td></td> + <td class="slice-diagram-selected">−2</td> + <td></td> + <td class="slice-diagram-not-selected">−1</td> + <td></td> + <td class="slice-diagram-not-selected">0</td> + </tr> + </table> + </div> + <div class="slice-diagram-not-selected"><b>THIS IS WRONG!</b></div> + </div> + + In other words, the "right" way to think of spacers when the `start` and + `stop` are negative depends if the `step` is positive or negative. + + This is because the correct half-open rule is based on not including the + `stop`. It *isn't* based on not including the larger end of the interval. If + the `step` is positive, the `stop` will be larger, but if it is + [negative](negative-steps), the `stop` will be smaller. + +- The rule "works" for slices, but is harder to conceptualize for integer + indices. In the divider way of thinking, an integer index `n` corresponds to + the entry to the *right* of the `n` divider. Rules that involve remembering + left or right aren't great when it comes to memorability. + +(fencepost)= +- This rule can lead to off-by-one errors due to "the fencepost problem." The + fencepost problem is this: say you want to build a fence that is 100 feet + long with posts spaced every 10 feet. How many fenceposts do you need? + + The naive answer is 10, but the correct answer is 11. The reason is the + fenceposts go in between the 10 feet divisions, including at the end points. + So there is an "extra" fencepost compared to the number of fence sections. + + + ```{figure} ../imgs/jeff-burak-lPO0VzF_4s8-unsplash.jpg + A section of a fence that has 6 segments and 7 fenceposts.[^fencepost-jeff-burbak-footnote] + + [^fencepost-jeff-burbak-footnote]: Image credit [Jeff Burak via + Unsplash](https://unsplash.com/photos/lPO0VzF_4s8). The image is of + Chautauqua Park in Boulder, Colorado. + ``` + + Fencepost problems are a leading cause of off-by-one errors. To think about + slices in this way is to think about lists as separated by fenceposts, and + is only begging for problems. This will especially be the case if you also + find yourself otherwise thinking about indices as pointing to list elements + themselves, rather than the divisions between them. And of course you will + think of them this way, because that's what they actually are. + +Rather than trying to think about dividers between elements, it's much simpler +to just think about the elements themselves, but being counted starting at 0. +To be sure, 0-based indexing also leads to off-by-one errors, since it is not +the usual way humans are taught to count things. Nonetheless, this is the +better way to think about things, especially as you gain practice in counting +starting at 0. As long as you apply the rule "the `stop` is not included," you +will get the correct results. + +(wrong-rule-4)= +##### Wrong Rule 4: "The `stop` of a slice `a[start:stop]` is 1-based." + +You might try to get clever and say `a[3:5]` indexes from the 3rd element with +0-based indexing to the 5th element with 1-based indexing. Don't do this. It +is confusing. Moreover, this rule must necessarily be reversed for negative +indices. `a[-5:-3]` indexes from the (−5)th element with −1-based +indexing to the (−3)rd element with 0-based indexing (and of course, +negative and nonnegative starts and stops can be mixed, like `a[3:-3]`). Don't +get cute here. It isn't worth it. The `stop` *is* 0-based; it just isn't +included. + +(negative-indices)= +### Negative Indices + +Negative indices in slices work the same way they do with [integer +indices](integer-indices). + +> **For `a[start:stop:step]`, negative `start` or `stop` use −1-based indexing + from the end of `a`.** + +However, the `start` or `stop` being negative does *not* change the order of +the slicing---only the [`step` does that](negative-steps). The other +[rules](rules) of slicing remain unchanged when the `start` or `stop` are +negative. [The `stop` is still not included](half-open), values less than +`-len(a)` still [clip](clipping), and so on. + +Positive and negative `start` and `stop` can be mixed. The following slices of +`a` all produce `['d', 'e']`: + +<div class="slice-diagram" style="padding-left: 1em; padding-right: 1em;"> +<div style="font-size: 16pt;"><code>a[<span class="slice-diagram-slice">3:5</span>] == a[<span class="slice-diagram-slice">-4:-2</span>] == a[<span class="slice-diagram-slice">3:-2</span>] == a[<span class="slice-diagram-slice">-4:5</span>] == ['d', 'e']</code></div> + <div> + <table> + <tr> + <th></th> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>['a',</pre></td> + <td><pre> 'b',</pre></td> + <td><pre> 'c',</pre></td> + <td class="underline-cell"><pre> 'd',</pre></td> + <td class="underline-cell"><pre> 'e',</pre></td> + <td><pre> 'f',</pre></td> + <td><pre> 'g']</pre></td> + </tr> + <tr> + <th>nonnegative index</th> + <td></td> + <td></td> + <td class="slice-diagram-not-selected">0</td> + <td class="slice-diagram-not-selected">1</td> + <td class="slice-diagram-not-selected">2</td> + <td><div class="circle-blue slice-diagram-selected">3</div></td> + <td><div class="circle-blue slice-diagram-selected">4</div></td> + <td><div class="circle-red slice-diagram-not-selected">5</div></td> + <td class="slice-diagram-not-selected">6</td> + </tr> + <tr> + <th>negative index</th> + <td></td> + <td></td> + <td class="slice-diagram-not-selected">−7</td> + <td class="slice-diagram-not-selected">−6</td> + <td class="slice-diagram-not-selected">−5</td> + <td><div class="circle-blue slice-diagram-selected">−4</div></td> + <td><div class="circle-blue slice-diagram-selected">−3</div></td> + <td><div class="circle-red slice-diagram-not-selected">−2</div></td> + <td class="slice-diagram-not-selected">−1</td> + </tr> + </table> + </div> +</div> + +```py +>>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] +>>> a[3:5] +['d', 'e'] +>>> a[-4:-2] +['d', 'e'] +>>> a[3:-2] +['d', 'e'] +>>> a[-4:5] +['d', 'e'] +``` + +If a negative `stop` indexes an element on or before a nonnegative `start`, the +slice is empty, akin to when `stop <= start` when both are nonnegative. + +```py +>>> a[3:-5] +[] +>>> a[3:2] +[] +``` + +As with [integer indices](integer-indices), negative indices `-i` in slices +can always be replaced with `len(a) - i` (replacing `len(a)` with the size of +the given axis for NumPy arrays). So they are primarily a syntactic +convenience. + +While negative indexing is convenient, it may introduce subtle bugs due to the +inherent discontinuity it creates. This is especially likely to happen if the +slice entries are arithmetical expressions. **One should always double check +if the `start` or `stop` values of a slice can be negative, and if they can, +if those values produce the correct results.** + +(negative-indices-example)= +For example, say you wanted to slice `n` values from the middle of `a`. +Something like the following would work: + +```py +>>> mid = len(a)//2 +>>> n = 4 +>>> a[mid - n//2: mid + n//2] +['b', 'c', 'd', 'e'] +``` + +From our [sanity check](sanity-check), `mid + n//2 - (mid - n//2)` does equal +`n` if `n` is even (we could find a similar expression for odd `n`, but for +now let us assume `n` is even for simplicity). + +However, let's look at what happens when `n` is larger than the size of `a`: + +```py +>>> n = 8 +>>> a[mid - n//2: mid + n//2] +['g'] +``` + +The result `['g']` is not the "middle eight elements of `a`." What we likely +really wanted here was full list `['a', 'b', 'c', 'd', 'e', 'f', 'g']`. + +What happened here? Let's look at the slice values: + +```py +>>> mid - n//2 +-1 +>>> mid + n//2 +7 +``` + +The `stop` slice value is out of bounds for `a`, but this just causes it to +[clip](clipping) to the end, which is what we want. + +But the `start` expression contains a subtraction, which causes it to become +negative. So rather than clipping to the start, it wraps around and indexes +from the end of `a`. The resulting slice `a[-1:7]` selects everything from +`'g'` to the end of the list, which in this case is just `['g']`. + +Unfortunately, the "correct" fix here depends on the desired behavior for each +individual slice. In some cases, the "slice from the end" behavior of negative +values is in fact what is desired. In others, you might prefer an error, so you +should add a value check or assertion. In others, you might want clipping, in +which case you could modify the expression to always be nonnegative. + +In this example, we do want clipping. Instead of using `mid - n//2`, we could +manually clip with `max(mid - n//2, 0)`: + +```py +>>> n = 4 +>>> a[max(mid - n//2, 0): mid + n//2] +['b', 'c', 'd', 'e'] +>>> n = 8 +>>> a[max(mid - n//2, 0): mid + n//2] +['a', 'b', 'c', 'd', 'e', 'f', 'g'] +``` + +Or we could replace the `start` with a value that is always negative. This +avoids the discontinuity problem because values that are too negative will +[clip](clipping) to the start of the array, just as they do for the `stop`. +But this does require thinking a bit about how to translate from 0-based +indexing to −1-based indexing. In this example, the `start` becomes `-n//2 - +mid - 1`: + +```py +>>> n = 4 +>>> a[-n//2 - mid - 1:mid + n//2] +['b', 'c', 'd', 'e'] +>>> n = 8 +>>> a[-n//2 - mid - 1:mid + n//2] +['a', 'b', 'c', 'd', 'e', 'f', 'g'] +``` + +It's a good idea to play around in an interpreter and check all the corner +cases when dealing with situations like this. + +And note that even this improved version can give unexpected results when `n` +is negative, for the exact same reasons. So value checking that the inputs are +in an expected range is not a bad idea. + +**Exercise:** Write a slice to index the middle `n` elements of `a` when `n` +is odd, clipping to all of `a` if `n` is larger than `len(a)`. + +~~~~{dropdown} Click here to show the solution + +Solution: `a[-n//2 - mid:mid + n//2 + 1]`. + +Note that this also works when `n` is even, although unlike above, `n=2` gives +`['d', 'e']` instead of `['c', 'd']`. + +```py +>>> n = 2 +>>> a[-n//2 - mid:mid + n//2 + 1] +['d', 'e'] +>>> n = 3 +>>> a[-n//2 - mid:mid + n//2 + 1] +['c', 'd', 'e'] +>>> n = 4 +>>> a[-n//2 - mid:mid + n//2 + 1] +['c', 'd', 'e', 'f'] +``` + +~~~~ +(clipping)= +### Clipping + +Slices can never result in an out-of-bounds `IndexError`. This differs from +[integer indices](integer-indices), which require the index to be in bounds. +Instead, slice values *clip* to the bounds of the array. + +The rule for clipping is this: + +> **If the `start` or `stop` extend before the beginning or after the end of + `a`, they will clip to the bounds of `a`.** + +For example: + +```py +>>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] +>>> a[-100:100] +['a', 'b', 'c', 'd', 'e', 'f', 'g'] +``` + +Here, `-100` is "clipped" to `-7`, the smallest possible negative start value +that actually selects something, and `100` is clipped down to `7`, the +smallest possible positive stop value that selects the last element. + +```py +>>> a[-7:7] +['a', 'b', 'c', 'd', 'e', 'f', 'g'] +``` + +Of course, if we actually wanted to just select everything, we could use +[omitted entries](omitted) (i.e., `a[:]`). The point with clipping is that the +same slice can be used on lists or arrays that are smaller than the bounds of +the slice, and it will just select "as much as it can." + +``` +>>> a[1:3] # The second and third element +['b', 'c'] +>>> ['a', 'b'][1:3] # There is no third element, so just the second +['b'] +``` + +This behavior can be useful, but it can also bite you. Usually you really do +want to select something like "the first $n$ elements, or everything if there +are fewer than $n$ elements," and a slice like `:n` will do exactly this for +any size input. But you have to be careful. Simply seeing a slice like `x = +a[:n]` does not mean that `x` now has `n` elements. Because of clipping +behavior, you can never rely on the length of a slice being `stop - start` +([for `step = 1` and `start`, `stop` nonnegative](sanity-check)), unless you +are sure that the length of the input is at least that. Rather, this is the +*maximum* length of the slice. It could end up slicing something +smaller.[^ndindex-calculations-footnote] + +[^ndindex-calculations-footnote]: ndindex can help in calculations here: +{meth}`len(ndindex.Slice(...)) <ndindex.Slice.__len__>` can be used to compute +the *maximum* length of a slice. If the shape or length of the input is known, +{meth}`len(ndindex.Slice(...).reduce(shape)) <ndindex.Slice.reduce>` will +compute the true length of the slice. Of course, if you already have a list or +a NumPy array, you can just slice it and check the shape. Slicing a NumPy +array always produces a [view on the array](views-vs-copies), so it is a very +inexpensive operation. Slicing a `list` does make a copy, but it's a shallow +copy so it isn't particularly expensive either. + +The clipping behavior of slices also means that you cannot rely on runtime +checks for out-of-bounds slices. Simply put, there is no such thing as an +"out-of-bounds slice." If you really want a bounds check, you have to do it +manually. + +There's a cute trick you can sometimes use that takes advantage of clipping. +By using a slice that selects a single element instead of an integer index, +you can avoid `IndexError` when the index is out-of-bounds. For example, +suppose you want to implement a quick script with a rudimentary optional +command line argument (without the hassle of +[argparse](https://docs.python.org/3/library/argparse.html)). This can be done +by manually parsing `sys.argv`, which is a list of the arguments of passed at +the command line, including the filename. For example, `python script.py arg1 +arg2` would have `sys.argv == ['script.py', 'arg1', 'arg2']`. Suppose you want +your script to do something special if it called as `myscript.py help`. You +can do something like + +```py +import sys +if sys.argv[1] == 'help': + print("Usage: myscript.py") +``` + +The problem with this code is that it fails if no command line arguments are +passed, because `sys.argv[1]` will give an `IndexError` for the out-of-bounds +index 1. The most obvious fix is to add a length check: + +```py +import sys +if len(sys.argv) > 1 and sys.argv[1] == 'help': + print("Usage: myscript.py") +``` + +But another way would be to use a slice that gets the second element if there +is one. + +```py +import sys +if sys.argv[1:2] == ['help']: + print("Usage: myscript.py") +``` + +Now if `sys.argv` has at least two elements, the slice `sys.argv[1:2]` will be +the sublist consisting of just the second element. But if it has only one +element, i.e., the script is just run as `python myscript.py` with no +arguments, then `sys.argv[1:2]` will be an empty list `[]`. This will fail the +`==` check without raising an exception. + +If instead we want to only support exactly `myscript.py help` with no further +arguments, we could modify the check just slightly: + +```py +import sys +if sys.argv[1:] == ['help']: + print("Usage: myscript.py") +``` + +Now `myscript.py help` would print the help message but `myscript.py help me` +would not. + +The point here is that we are embedding both the bounds check and the element +check into the same conditional. That's because `==` on a container type (like +a `list` or `str`) checks two things: if containers have the same length and +the elements are the same. When we modified the code to compare lists instead +of strings, the `if len(sys.argv) > 1` check became unnecessary because it's +already built-in to the `==` comparison. + +This trick works especially well when working with strings. Unlike with lists, +both [integer ](integer-indices) and slice indices on a string result in +[another string](strings-integer-indexing), so changing the code logic to work +in this way often only requires adding a `:` to the index so that it is a +slice that selects a single element instead of an integer index. For example, +consider a function like + +```py +# Wrong for a = '' +def ends_in_punctuation(a: str) -> bool: + return a[-1] in ['.', '!', '?'] +``` + +This function is wrong for an empty string input: `ends_in_punctuation('')` +will raise `IndexError`. This could be fixed by adding a length check. Or we +could simply change the `a[-1]` to `a[-1:]`. This will usually be a string +consisting of the last character, but if `a` is empty it will be `''`. The +`in` check will be correct either way.[^string-check-footnote] + +[^string-check-footnote]: As an aside, the `in` check would be *wrong* if we + instead wrote `a[-1:] in '.!?'`. The two forms of comparison are not + equivalent. + +```py +# Better +def ends_in_punctuation(a: str) -> bool: + return a[-1:] in ['.', '!', '?'] +``` + +This sort of trick may seem scary and magic, but once you have digested this +guide and become comfortable with slice semantics, it is a natural and clean +way to embed length checks into comparison logic and avoid out-of-bounds +corner cases. + +(steps)= +### Steps + +If a third integer, `k`, is provided in a slice, such as `i:j:k`, it specifies +the step size. If `k` is not provided, the step size defaults to `1`. + +Thus far, we have considered only slices with the default step size of 1. When +the `step` is greater than 1, the slice selects every `step`-th element within +the bounds defined by the `start` and `stop`. + +> **The proper way to think about the `step` is that the slice starts at the + `start` and successively adds `step` until it reaches an index that is at or + past the `stop`, and then stops without including that index.** + +The important thing to remember about the `step` is that its presence does not +change the fundamental [rules](rules) of slices that we have learned so far. +The `start` and `stop` still use [0-based indexing](0-based). The `stop` is +[never included](half-open) in the slice. [Negative](negative-indices) `start` +and `stop` index from the end of the list. Out-of-bounds `start` and `stop` +still [clip](clipping) to the beginning or end of the list. And (see below) an +[omitted](omitted) `start` or `stop` still extends to the beginning or end of +`a`. + +Let us consider an example where the step size is `3`. + +<div class="slice-diagram"> +<code style="font-size: 16pt;">a[<span class="slice-diagram-slice">0:6:3</span>] == ['a', 'd']</code> +<table> + <tr> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>[</pre></td> + <td class="underline-cell"><pre>'a',</pre></td> + <td></td> + <td><pre> 'b',</pre></td> + <td></td> + <td><pre> 'c',</pre></td> + <td></td> + <td class="underline-cell"><pre> 'd',</pre></td> + <td></td> + <td><pre> 'e',</pre></td> + <td></td> + <td><pre> 'f',</pre></td> + <td></td> + <td><pre>'g']</pre></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td></td> + <td><div class="slice-diagram-selected circle-blue">0</div></td> + <td></td> + <td class="slice-diagram-not-selected">1</td> + <td></td> + <td class="slice-diagram-not-selected">2</td> + <td></td> + <td><div class="circle-blue slice-diagram-selected">3</div></td> + <td></td> + <td class="slice-diagram-not-selected">4</td> + <td></td> + <td class="slice-diagram-not-selected">5</td> + <td></td> + <td><div class="circle-red slice-diagram-not-selected">6</div></td> + </tr> + <tr> + <td></td> + <td></td> + <td></td> + <td style="vertical-align: top; color: var(--color-slice-diagram-selected)">start</td> + <td colspan="5" class="right-arrow-curved-cell"></td> + <td></td> + <td colspan="5" class="right-arrow-curved-cell"></td> + <td class="slice-diagram-index-label-not-selected">≥ stop</td> + </tr> + <tr> + <td></td> + <td></td> + <td></td> + <td></td> + <td colspan="5" style="padding-top: 0; transform: translateY(-0.7em)">+3</td> + <td></td> + <td colspan="5" style="padding-top: 0; transform: translateY(-0.7em)">+3</td> + <td></td> +</table> +</div> + +```py +>>> a[0:6:3] +['a', 'd'] +``` + +Note that the `start` index, `0`, is included, but the `stop` index, `6` +(corresponding to `'g'`), is *not* included, even though it is a multiple of +`3` away from the start. This is because the `stop` is [never +included](half-open). + +It can be tempting to think about the `step` in terms of modular arithmetic. +In fact, it is often the case in practice that you require a `step` greater +than 1 because you are dealing with modular arithmetic in some way. However, +this requires care. + +Indeed, the resulting indices `0`, `3` from the slice `a[0:6:3]` are multiples +of 3. This is because the `start` index, `0`, is a multiple of 3. Choosing a +start index that is $1 \pmod{3}$ would result in all indices also being $1 +\pmod{3}$. + + +<div class="slice-diagram"> +<code style="font-size: 16pt;">a[<span class="slice-diagram-slice">1:6:3</span>] == ['b', 'e']</code> +<table> + <tr> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>['a',</pre></td> + <td></td> + <td class="underline-cell"><pre> 'b',</pre></td> + <td></td> + <td><pre> 'c',</pre></td> + <td></td> + <td><pre> 'd',</pre></td> + <td></td> + <td class="underline-cell"><pre> 'e',</pre></td> + <td></td> + <td><pre> 'f',</pre></td> + <td></td> + <td><pre>'g']</pre></td> + <td></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td class="slice-diagram-not-selected">0</td> + <td></td> + <td><div class="circle-blue slice-diagram-selected">1</div></td> + <td></td> + <td class="slice-diagram-not-selected">2</td> + <td></td> + <td class="slice-diagram-not-selected">3</td> + <td></td> + <td><div class="circle-blue slice-diagram-selected">4</div></td> + <td></td> + <td class="slice-diagram-not-selected">5</td> + <td></td> + <td class="slice-diagram-not-selected">6</td> + <td></td> + <td><div class="circle-red slice-diagram-not-selected"></div></td> + </tr> + <tr> + <td></td> + <td></td> + <td></td> + <td></td> + <td class="slice-diagram-index-label-selected">start</td> + <td colspan="5" class="right-arrow-curved-cell"></td> + <td></td> + <td colspan="5" class="right-arrow-curved-cell"></td> + <td class="slice-diagram-index-label-not-selected">≥ stop</td> + </tr> + <tr> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td colspan="5" style="padding-top: 0; transform: translateY(-0.7em)">+3</td> + <td></td> + <td colspan="5" style="padding-top: 0; transform: translateY(-0.7em)">+3</td> + <td></td> + </tr> +</table> +</div> + +```py +>>> a[1:6:3] +['b', 'e'] +``` + +However, be careful, as this rule is *only* true when the `start` is +nonnegative. If the `start` is negative, the value of `start % step` has no +bearing on the indices chosen for the slice: + +```py +>>> list(range(21))[-15::3] +[6, 9, 12, 15, 18] +>>> list(range(22))[-15::3] +[7, 10, 13, 16, 19] +``` + +In the first case, `-15` is divisible by 3 and all the indices chosen by the +slice `-15::3` were also divisible by 3 (remember that indices and values are +the same for simple ranges). But this is only because the length of the list, +`21`, also happened to be a multiple of 3. In the second example it is `22` +and the resulting indices are not multiples of `3`. This caveat also applies +when the [step is negative](negative-steps). + +Another thing to be aware of is that if the `start` is [clipped](clipping), +**the clipping occurs *before* the step**. Specifically, if the `start` is +less than `len(a)`, the `step`ed values are computed as if the `start` were +`0`. + +```py +>>> a[-100::2] +['a', 'c', 'e', 'g'] +>>> a[-101::2] +['a', 'c', 'e', 'g'] +``` + +Because of these two caveats, you must be careful when using negative `start` +values with a `step`, and it's better to avoid this if +possible.[^negative-steps-ndindex-footnote] If the `start` is nonnegative, +then it *will* be true that the sliced indices will be equal to `start % +step`. + +[^negative-steps-ndindex-footnote]: If you do need to use a negative start + with a step, [ndindex](ndindex.slice.Slice) can be used to help compute + things to avoid making mistakes. + +```py +>>> l = list(range(20)) +>>> l[::3] # All the multiples of 3 up to 19 +[0, 3, 6, 9, 12, 15, 18] +>>> l[1::3] # All the numbers that are 1 (mod 3) +[1, 4, 7, 10, 13, 16, 19] +>>> l[2::3] # All the numbers that are 2 (mod 3) +[2, 5, 8, 11, 14, 17] +``` + +(negative-steps)= +### Negative Steps + +Recall what we said [above](steps): + +> **The proper way to think about the `step` is that the slice starts at + `start` and successively adds `step` until it reaches an index that is at or + past the `stop`, and then stops without including that index.** + +The key thing to remember with negative `step` values is that this rule still +applies. That is, the index starts at the `start` then adds the `step` (which +makes the index smaller), and stops when it is at or past the `stop`. Here "at +or past" means "greater than or equal to" if the `step` is positive and "less +than or equal to" if the `step` is negative. + +Think of a slice as starting at the `start` and sliding along the list, +jumping along by `step`, and spitting out elements. Once you see that you are +at or have gone past the `stop` in the direction you are going (left for +negative `step` and right for positive `step`), you stop. + +Unlike all the above examples, when the `step` is negative, generally the +`start` will be an index *after* the `stop` (otherwise the slice will be +[empty](empty-slice)). + +One of the most obvious features of negative `step` values is that unlike +every other slice we have seen so far, a negative `step` selects elements in +reversed order relative to the original list. In fact, one of the most common +uses of a negative `step` is the slice `a[::-1]`, which reverses the list: + +```py +>>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] +>>> a[::-1] +['g', 'f', 'e', 'd', 'c', 'b', 'a'] +``` + +It is tempting therefore to think of a negative `step` as a "reversing" +operation. However, this is a bad way of thinking about negative steps. This +is because `a[i:j:-1]` is *not* equivalent to `reversed(a[j:i:1])`. The reason +for this is basically the same as was described in [wrong rule +1](wrong-rule-1) above. The issue is that for `a[start:stop:step]`, the `stop` +is *always* what is [not included](half-open), which means if we swap `i` and +`j`, we go from "`j` is not included" to "`i` is not included". For example, +as [before](wrong-rule-1): + +```py +>>> a[5:3:-1] +['f', 'e'] +>>> list(reversed(a[3:5:1])) # This is not the same thing +['e', 'd'] +``` + +In the first case, index `3` is not included. In the second case, index `5` is +not included. + +Worse, this way of thinking may even lead one to imagine the completely wrong +idea that `a[i:j:-1]` is the same as `reversed(a)[j:i]` (that is, that +`step=-1` somehow "reverses and then swaps the `start` and `stop`"): + +```py +>>> list(reversed(a))[3:5] # Not the same as a[5:3:-1] as shown above +['d', 'c'] +``` + +Once `a` is reversed, the indices `3` and `5` have nothing to do with the +original indices `3` and `5`. To see why, consider a much larger list: + +```py +>>> list(range(100))[5:3:-1] +[5, 4] +>>> list(reversed(range(100)))[3:5] +[96, 95] +``` + +Instead of thinking about "reversing", it is much more conceptually robust to +think about the slice as starting at the `start`, then moving across every +`step`-th element until reaching the `stop`, which is not included. + +Negative steps can of course be less than −1 as well, with similar +behavior to steps greater than 1, again, keeping in mind that the `stop` is +not included. + +```py +>>> a[6:0:-3] +['g', 'd'] +``` + +<div class="slice-diagram"> +<code style="font-size: 16pt;">a[<span class="slice-diagram-slice">6:0:-3</span>] == ['g', 'd']</code> +<table> + <tr> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>['a',</pre></td> + <td></td> + <td><pre> 'b',</pre></td> + <td></td> + <td><pre> 'c',</pre></td> + <td></td> + <td class="underline-cell"><pre> 'd',</pre></td> + <td></td> + <td><pre> 'e',</pre></td> + <td></td> + <td><pre> 'f',</pre></td> + <td></td> + <td class="underline-cell"><pre>'g'</pre></td> + <td><pre>]</pre></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td><div class="circle-red slice-diagram-not-selected">0</div></td> + <td></td> + <td class="slice-diagram-not-selected">1</td> + <td></td> + <td class="slice-diagram-not-selected">2</td> + <td></td> + <td><div class="circle-blue slice-diagram-selected">3</div></td> + <td></td> + <td class="slice-diagram-not-selected">4</td> + <td></td> + <td class="slice-diagram-not-selected">5</td> + <td></td> + <td><div class="circle-blue slice-diagram-selected">6</div></td> + <td></td> + </tr> + <tr> + <td></td> + <td></td> + <td class="slice-diagram-index-label-not-selected">≤ stop</td> + <td colspan="5" class="left-arrow-curved-cell"></td> + <td></td> + <td colspan="5" class="left-arrow-curved-cell"></td> + <td class="slice-diagram-index-label-selected">start</td> + <td></td> + </tr> + <tr> + <td></td> + <td></td> + <td></td> + <td colspan="5" style="padding-top: 0; transform: translateY(-0.7em)">−3</td> + <td></td> + <td colspan="5" style="padding-top: 0; transform: translateY(-0.7em)">−3</td> + <td></td> + <td></td> +</table> +</div> + +The `step` can never be equal to 0. This unconditionally produces an error: + +```py +>>> a[::0] +Traceback (most recent call last): +... +ValueError: slice step cannot be zero +``` + +(omitted)= +### Omitted Entries + +The final point of confusion is omitted entries.[^omitted-none-footnote] + +[^omitted-none-footnote]: `start`, `stop`, or `step` may also be `None`, which +is syntactically equivalent to them being omitted. That is to say, `a[::]` is +a syntactic shorthand for `a[None:None:None]`. It is rare to see `None` in a +slice. This is only relevant for code that consumes slices, such as a +`__getitem__` method on an object. The `slice()` object corresponding to +`a[::]` is `slice(None, None, None)`. [`ndindex.Slice()`](ndindex.slice.Slice) also uses +`None` to indicate omitted entries in the same way. + +**The best way to think about omitted entries is just that, as omitted +entries.** That is, for a slice like `a[:i]`, think of it as the `start` being +omitted, and the `stop` equal to `i`. Conversely, `a[i:]` has the `start` as `i` +and the `stop` omitted. The *wrong way* to think about these is as a colon +being before or after the index `i`. Thinking about it this way will only lead +to confusion, because you won't be thinking about `start` and `stop`, but +rather trying to remember some rule based on where a colon is. But the colons +in a slice are not *indicators*; they are *separators*. + +As to the semantic meaning of omitted entries, the easiest one is the `step`. + +> **If the `step` is omitted, it always defaults to `1`.** + +If the `step` is omitted the second colon can also be omitted. That is to say, +the following are all completely equivalent[^equivalent-slices-footnote]: + +[^equivalent-slices-footnote]: Strictly speaking `a[i:j:1]` creates `slice(i, + j, 1)` whereas `a[i:j:]` and `a[i:j]` produce `slice(i, j, None)`. This + only matters if you are implementing `a.__getitem__`. The ndindex + [`Slice.reduce()`](ndindex.Slice.reduce) method can be used to + normalize slices do you don't have to worry about these kinds of + distinctions. + +```py +a[i:j:1] +a[i:j:] +a[i:j] +``` + +<!-- TODO: Better wording for this rule? --> + +> **For the `start` and `stop`, the rule is that being omitted extends the + slice all the way to the beginning or end of `a` in the direction being + sliced.** + +If the `step` is positive, this means `start` extends to the beginning of `a` +and `stop` extends to the end. If the `step` is negative, this is reversed: +`start` extends to the end of `a` and `stop` extends to the beginning. + +Writing down the rule in this way makes it sound more confusing than it really +is. Simply put, omitting the `start` or `stop` of a slice will make it slice +"as much as possible" instead. + +<div class="slice-diagram"> + <code style="font-size: 16pt;">a[<span class="slice-diagram-slice">:3</span>] == a[<span class="slice-diagram-slice">:3:1</span>] == ['a', 'b', 'c']</code> + <table> + <tr> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>[</pre></td> + <td class="underline-cell"><pre>'a',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre> 'b',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre> 'c',</pre></td> + <td></td> + <td><pre> 'd',</pre></td> + <td></td> + <td><pre> 'e',</pre></td> + <td></td> + <td><pre> 'f',</pre></td> + <td></td> + <td><pre> 'g'</pre></td> + <td><pre>]</pre></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td></td> + <td><div class="circle-blue slice-diagram-selected">0</div></td> + <td class="right-arrow-cell"><div style="font-size: smaller; transform: translateY(-12px) translateX(-3px)">+1</div></td> + <td><div class="circle-blue slice-diagram-selected">1</div></td> + <td class="right-arrow-cell"><div style="font-size: smaller; transform: translateY(-12px) translateX(-3px)">+1</div></td> + <td><div class="circle-blue slice-diagram-selected">2</div></td> + <td class="right-arrow-cell"><div style="font-size: smaller; transform: translateY(-12px) translateX(-3px)">+1</div></td> + <td><div class="circle-red slice-diagram-not-selected">3</div></td> + <td></td> + <td class="slice-diagram-not-selected">4</td> + <td></td> + <td class="slice-diagram-not-selected">5</td> + <td></td> + <td class="slice-diagram-not-selected">6</td> + <td></td> + </tr> + <tr> + <th></th> + <td></td> + <td></td> + <td class="slice-diagram-index-label-selected"> + <div class="overflow-content">start (beginning)</div> + </td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td class="slice-diagram-index-label-not-selected">stop</td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + </tr> + </table> +</div> + +<div class="slice-diagram"> + <code style="font-size: 16pt;">a[<span class="slice-diagram-slice">3:</span>] == a[<span class="slice-diagram-slice">3::1</span>] == ['d', 'e', 'f', 'g']</code> + <table> + <tr> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>[</pre></td> + <td><pre>'a',</pre></td> + <td></td> + <td><pre> 'b',</pre></td> + <td></td> + <td><pre> 'c',</pre></td> + <td></td> + <td class="underline-cell"><pre> 'd',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre> 'e',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre> 'f',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre> 'g'</pre></td> + <td><pre>]</pre></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td></td> + <td class="slice-diagram-not-selected">0</td> + <td></td> + <td class="slice-diagram-not-selected">1</td> + <td></td> + <td class="slice-diagram-not-selected">2</td> + <td></td> + <td><div class="circle-blue slice-diagram-selected">3</div></td> + <td class="right-arrow-cell"><div style="font-size: smaller; transform: translateY(-12px) translateX(-3px)">+1</div></td> + <td><div class="circle-blue slice-diagram-selected">4</div></td> + <td class="right-arrow-cell"><div style="font-size: smaller; transform: translateY(-12px) translateX(-3px)">+1</div></td> + <td><div class="circle-blue slice-diagram-selected">5</div></td> + <td class="right-arrow-cell"><div style="font-size: smaller; transform: translateY(-12px) translateX(-3px)">+1</div></td> + <td><div class="circle-blue slice-diagram-selected">6</div></td> + <td></td> + </tr> + <tr> + <th></th> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td class="slice-diagram-index-label-selected">start</td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td class="slice-diagram-index-label-selected"> + <div class="overflow-content">stop (end)</div> + <td></td> + </td> + </tr> + </table> +</div> + +<div class="slice-diagram"> + <code style="font-size: 16pt;">a[<span class="slice-diagram-slice"><span class="slice-diagram-slice">:3:-1</span></span>] == ['g', 'f', 'e']</code> + <table> + <tr> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>[</pre></td> + <td><pre>'a',</pre></td> + <td></td> + <td><pre> 'b',</pre></td> + <td></td> + <td><pre> 'c',</pre></td> + <td></td> + <td><pre> 'd',</pre></td> + <td></td> + <td class="underline-cell"><pre> 'e',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre> 'f',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre> 'g'</pre></td> + <td><pre>]</pre></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td></td> + <td class="slice-diagram-not-selected">0</td> + <td></td> + <td class="slice-diagram-not-selected">1</td> + <td></td> + <td class="slice-diagram-not-selected">2</td> + <td></td> + <td><div class="circle-red slice-diagram-not-selected">3</div></td> + <td class="left-arrow-cell"><div style="font-size: smaller; transform: translateY(-12px) translateX(3px)">−1</div></td> + <td><div class="circle-blue slice-diagram-selected">4</div></td> + <td class="left-arrow-cell"><div style="font-size: smaller; transform: translateY(-12px) translateX(3px)">−1</div></td> + <td><div class="circle-blue slice-diagram-selected">5</div></td> + <td class="left-arrow-cell"><div style="font-size: smaller; transform: translateY(-12px) translateX(3px)">−1</div></td> + <td><div class="circle-blue slice-diagram-selected">6</div></td> + <td></td> + </tr> + <tr> + <th></th> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td class="slice-diagram-index-label-not-selected">stop</td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td class="slice-diagram-index-label-selected"> + <div class="overflow-content">start (end)</div> + </td> + <td></td> + </tr> + </table> +</div> + +<div class="slice-diagram"> + <code style="font-size: 16pt;">a[<span class="slice-diagram-slice">3::-1</span>] == ['d', 'c', 'b', 'a']</code> + <table> + <tr> + <td><pre>a</pre></td> + <td><pre>=</pre></td> + <td><pre>[</pre></td> + <td class="underline-cell"><pre>'a',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre> 'b',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre> 'c',</pre></td> + <td class="underline-cell"></td> + <td class="underline-cell"><pre> 'd',</pre></td> + <td></td> + <td><pre> 'e',</pre></td> + <td></td> + <td><pre> 'f',</pre></td> + <td></td> + <td><pre> 'g'</pre></td> + <td><pre>]</pre></td> + </tr> + <tr> + <th>index</th> + <td></td> + <td></td> + <td><div class="circle-blue slice-diagram-selected">0</div></td> + <td class="left-arrow-cell"><div style="font-size: smaller; transform: translateY(-12px) translateX(3px)">−1</div></td> + <td><div class="circle-blue slice-diagram-selected">1</div></td> + <td class="left-arrow-cell"><div style="font-size: smaller; transform: translateY(-12px) translateX(3px)">−1</div></td> + <td><div class="circle-blue slice-diagram-selected">2</div></td> + <td class="left-arrow-cell"><div style="font-size: smaller; transform: translateY(-12px) translateX(3px)">−1</div></td> + <td><div class="circle-blue slice-diagram-selected">3</div></td> + <td></td> + <td class="slice-diagram-not-selected">4</td> + <td></td> + <td class="slice-diagram-not-selected">5</td> + <td></td> + <td class="slice-diagram-not-selected">6</td> + <td></td> + </tr> + <tr> + <th></th> + <td></td> + <td></td> + <td class="slice-diagram-index-label-selected"> + <div class="overflow-content">stop (beginning)</div> + </td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td class="slice-diagram-index-label-selected">start</td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + </tr> + </table> +</div> + +```py +>>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] +>>> a[:3] # From the beginning to index 3 (but not including index 3) +['a', 'b', 'c'] +>>> a[3:] # From index 3 to the end +['d', 'e', 'f', 'g'] +>>> a[:3:-1] # From the end to index 3 (but not including index 3), reversed +['g', 'f', 'e'] +>>> a[3::-1] # From index 3 to the beginning, reversed +['d', 'c', 'b', 'a'] +``` + +A slice with both `start` and `stop` omitted, `a[:]`, therefore is just all of +`a`: + +```py +>>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] +>>> a[:] +['a', 'b', 'c', 'd', 'e', 'f', 'g'] +``` + +If `a` is a `list`, this is a convenient way of creating a (shallow) copy of +`a`.[^tuple-copy-footnote] On the other hand, if `a` is a NumPy array, this is +a convenient way of creating a [view](views-vs-copies) of all of `a` (which is +*not* a copy). Or more commonly, `:` is used to select an entire axis in a +[multidimensional index](multidimensional-indices/index.md). + +[^tuple-copy-footnote]: If `a` is a `tuple` or `str`, there is little point to + copying `a` since these are immutable types, meaning that a shallow copy + of `a` and the original `a` are effectively indistinguishable. + +## Soapbox + +While this guide is opinionated about the right and wrong ways to think about +slices in Python, I have tried to stay neutral regarding the merits of the +rules themselves. But I want to take a moment to give my views on them. I have +worked with slice objects quite a bit in building ndindex, as well as just +general usage with Python and NumPy. + +Without a doubt, Python's slice syntax is extremely expressive and +straightforward. However, simply put, the semantic rules for slices are +completely bonkers. They lend themselves to several invalid interpretations, +which I have outlined above, and which seem valid at first glance but fall +apart in corner cases. The "correct" ways to think about slices are very +particular. I have tried to [outline](rules) them carefully, but one gets the +impression that unless one works with slices regularly, it will be hard to +remember the "right" ways and not fallback to thinking about the "wrong" ways, +or, as most Python programmers probably do, simply "guessing and checking" and +probably not correctly handling corner cases. + +Furthermore, the discontinuous nature of the `start` and `stop` parameters not +only makes it hard to remember how slices work but it also makes it *extremely* +hard to write slice arithmetic (i.e., for anyone implementing ` __getitem__` +or `__setitem__` that accepts slices matching the standard semantics). The +arithmetic is already hard enough due to the modular nature of `step`, but the +discontinuous aspect of `start` and `stop` increases this tenfold. If you are +unconvinced of this, take a look at the [source +code](https://github.com/Quansight-labs/ndindex/blob/main/ndindex/slice.py) +for `ndindex.Slice()`. You will see lots of nested `if` +blocks.[^source-footnote] This is because slices have *fundamentally* +different definitions if the `start` or `stop` are `None`, negative, or +nonnegative. Furthermore, `None` is not an integer, so one must always be +careful to either check for it first or to be certain that it cannot happen, +before performing any arithmetical operation or numerical comparison. Under +each `if` block you will see some formula or other. Many of these formulas +were difficult to come up with. In many cases they are asymmetrical in +surprising ways. It is only through the rigorous [testing](testing) that +ndindex uses that I can have confidence the formulas are correct for all +corner cases. + +[^source-footnote]: To be sure, I make no claims that the source of any +function in ndindex cannot be simplified. In writing ndindex, I have primarily +focused on making the logic correct, and less on making it elegant. I welcome +any pull requests that simplifies the logic of a function. The extensive +[testing](testing) should ensure that any rewritten function remains correct. + +I believe that Python's slicing semantics could remain just as expressive +while being less confusing and easier to work with for both end-users and +developers writing slice arithmetic (a typical user of ndindex). The changes I +would make to improve the semantics would be + +1. Remove the special meaning of negative numbers. +2. Use 1-based indexing instead of 0-based indexing. +3. Make a slice always include both the start and the stop. + +<!-- This comment is here to force Markdown to reset the numbering --> + +1. **Negative numbers.** The special meaning of negative numbers, to index + from the end of the list, is by far the biggest problem with Python's slice + semantics. It introduces a fundamental discontinuity to the definition of + an index. This makes it completely impossible to write a formula for almost + anything relating to slices that will not end up having branching `if` + conditions. But the problem isn't just for code that manipulates slices. + The [example above](negative-indices-example) shows how negative indices + can easily lead to bugs in end-user code. Effectively, any time you have a + slice `a[i:j]`, if `i` and `j` are nontrivial expressions, they must be + checked to ensure they do not go negative. If they can be both negative and + nonnegative, it is virtually never the case that the slice will give you + what you want in both cases. This is because the discontinuity inherent in + the definition of [negative indexing](negative-indices) disagrees with the + concept of [clipping](clipping). `a[i:j]` will slice "as far as it can" if + `j` is "too big" (greater than `len(a)`), but it does something completely + different if `i` is "too small" as soon as "too small" means "negative". + Clipping is a good idea. It tends to lead to behavior that gives what you + would want for slices that go out-of-bounds. + + Negative indexing is, strictly speaking, a syntactic sugar only. + Slicing/indexing from the end of a list can always be done in terms of the + length of the list. `a[-x]` is the same as `a[len(a)-x]` (when using + 0-based indexing), but the problem is that it is tedious to write `a` + twice, and `a` may in fact be a larger expression, so writing `a[len(a)-x]` + would require assigning it to a variable. It also becomes more complicated + when `a` is a NumPy array and the slice appears as part of a larger + multidimensional (tuple) index. However, I think it would be possible to + introduce a special syntax to mean "reversed" or "from the end the list" + indexing, and leave negative numbers to simply extend beyond the left side + of a list with clipping. For example, in [Julia](https://julialang.org/), + one can use `a[end]` to index the last element of an array (Julia also uses + 1-based indexing). Since this is a moot point for Python---I don't expect + Python's indexing semantics to change; they are already baked into the + language---I won't suggest any syntax. Perhaps this can inspire people + writing new languages or DSLs to come up with better semantics backed by + good syntax (again, I think Python slicing has good *syntax*. I only take + issue with some of its *semantics*). + +2. **0-based vs. 1-based indexing.** The suggestion to switch from 0-based to + 1-based indexing is likely to be the most controversial. For many people + reading this, the notion that 0-based indexing is superior has been + preached as irreproachable gospel. I encourage you to open your mind and + try to unlearn what you have been taught and take a fresh view of the + matter. (Or don't. These are just my opinions after all, and none of it + changes the fact that Python is what it is and isn't going to change.) + + 0-based indexing certainly has its uses. In C, where an index is literally + a syntactic macro for adding two pointers, 0-based indexing makes sense, + since `a[i]` literally means `*(a + i)` under those semantics. However, for + higher level languages such as Python, people think of indexing as pointing + to specific numbered elements of a collection, not as pointer arithmetic. + Every human being is taught from an early age to count from 1. If you show + someone the list "a, b, c", they will tell you that "a" is the 1st, "b" is + the 2nd, and "c" is the 3rd. [Sentences](fourth-sentence) in this guide + like "`a[3]` selects the fourth element of `a`" sound very off, even for + those of us used to 0-based indexing. 0-based indexing requires a shift in + thinking from the way that you have been taught to count from early + childhood. Counting is a very fundamental thing for any human, but + especially so for a programmer. Forcing someone to learn a new way to do + such a foundational thing is a huge cognitive burden, and so it shouldn't + be done without a very good reason. In a language like C, one can argue + there is a good reason, just as one can argue that it is beneficial to + learn new base number systems like base-2 and base-16 when doing certain + kinds of programming. + + But for Python, what are the true benefits of counting starting at 0? The + main benefit is that the implementation is easier, because Python is itself + written in C, which uses 0-based indexing, so Python does not need to + handle shifting in the translation. But this has never been a valid + argument for Python semantics. The whole point of Python is to provide + higher level semantics than C, and leave those hard details of translating + them to the interpreter and library code. In fact, Python's slices + themselves are much more complicated than what is available in C, and the + interpreter code to handle them is more than just a trivial translation to + C. Adding shifts to this translation code would not be much additional + complexity. + + The other advantage of 0-based indexing is that it makes it easier for + people who know C to learn Python. This may have been a good reason when + Python was new, but [now more people know Python than + C](https://www.tiobe.com/tiobe-index/). A good programming language like + Python should strive to be better than its predecessors, not let itself be + dragged behind by them. + + Even experienced programmers of languages like Python that use 0-based + indexing must occasionally stop themselves from writing something like + `a[3]` instead of `a[2]` to get the third element of `a`. It is very + difficult to "unlearn" 1-based counting, which was not only the first way + that you learned to count, but is also the way that you and everyone else + around you continues to count outside of programming contexts. + + When you teach a child how to count things, you teach them to enumerate the + items starting at 1. For example, 🍎🍎🍎🍎 is "4 apples" because you count + them off, "1, 2, 3, 4." The number that is enumerated for the final object + is equal to the number of items (in technical terms, the final + [ordinal](https://en.wikipedia.org/wiki/Ordinal_number) is equal to the + [cardinal](https://en.wikipedia.org/wiki/Cardinal_number)). This only works + if you start at 1. If the child instead starts at 0 ("0, 1, 2, 3"), the + final ordinal (the last number spoken aloud) would not match the cardinal + (the number of items). The distinction between ordinals and cardinals is + not something most people think about often, because the convention of + counting starting at 1 makes it so that they are equal. But as programmers + in a language that rejects this elegant convention, we are forced to think + about such philosophical distinctions just to solve whatever problem we are + trying to solve. + + In most instances (outside of programming) where a reckoning starts at 0 + instead of 1, it is because it is measuring a distance. The distance from + your house to your friend's house may be "2 miles", but the distance from + your house to itself is "0 miles". On the other hand, when counting or + enumerating individual objects, counting always starts at 1. The notion of + a "zeroth" object doesn't make sense when counting, say, apples, because + you are counting the apples themselves, not some quantity relating them. + + So the question then becomes, should indexing work like a measurement of + distance, which would naturally start at 0, or like an enumeration of + distinct terms, which would naturally start at 1? If we think of an index + as a pointer offset, as C does, then it is indeed a measurement of a + distance. But if we instead think of an indexable list as a discrete + ordered collection of items, then the notion of a measurement of distance + is harder to justify. But enumeration is a natural concept for any ordered + discrete collection. + + What are the benefits of 0-based indexing? + + - It makes translation to lower level code (like C or machine code) easier. + But as I already argued, this is not a valid argument for Python, which + aims to be high-level and abstract away translation complexities that + make coding more difficult. The translation that necessarily takes place + in the interpreter itself can afford this complexity if it means making + the language itself simpler. + + - It makes translation from code written in other languages that use + 0-based indexing simpler. If Python used 1-based indexing, then to + translate a C algorithm to Python, for instance, one would have to adapt + all the places that use indexing, which would be a bug-prone task. But + Python's primary mantra is having syntax and semantics that make code + easy to read and easy to write. Being similar to other existing languages + is second to this, and should not take priority when it conflicts with + it. Translation of code from other languages to Python does happen, but + it is much rarer than novel code written in Python. Furthermore, + automated tooling could be used to avoid translation bugs. Such tooling + would help avoid other translation bugs unrelated to indexing as well. + + - It works nicely with half-open semantics. It is true that half-open + semantics and 0-based indexing, while technically distinct, are virtually + always implemented together because they play so nicely with each other. + However, as I argue below, half-open semantics are just as absurd as + 0-based indexing, and abandoning both for the more standard + closed-closed/1-based semantics is very reasonable. + + To me, the ideal indexing system defaults to 1-based, but allows starting + at any index. That way, if you are dealing with a use case where 0-based + indexing really does make more sense, you can easily use it. Indices should + also be able to start at any other number, including negative numbers + (which is another reason to remove the special meaning of negative + indices). An example of a use case where 0-based indexing truly is more + natural than 1-based indexing is polynomials. Say we have a polynomial <!-- + --> $a_0 + a_1x + a_2x^2 + \cdots$. Then we can represent the coefficients + $a_0, a_1, a_2, \ldots$ in a list `[a0, a1, a2, ...]`. Since a polynomial + naturally has a 0th coefficient, it makes sense to index the list starting + at 0 (though even then, one must still be careful about off-by-one errors, + e.g., a degree-$n$ polynomial has $n+1$ coefficients). + + If this seems like absurd idea, note that this is how Fortran works (see + <https://www.fortran90.org/src/faq.html#what-is-the-most-natural-starting-index-for-numbering>). + In Fortran, arrays index starting at 1 by default, but any integer can be + used as a starting index. Fortran predates Python by many decades, but is + still in use today, particularly in scientific applications, and many + Python libraries themselves such as SciPy are backed by Fortran code. These + codes tend to be very mathematical and may make heavy use of indexing (for + instance, linear algebra packages like BLAS and LAPACK). Many other popular + programming languages use 1-based indexing, such as Julia, MATLAB, + Mathematica, R, Lua, and + [others](https://en.wikipedia.org/wiki/Comparison_of_programming_languages_(array)#Array_system_cross-reference_list). + In fact, a majority of the popular programming languages that use 1-based + indexing are languages that are primarily used for scientific applications. + Scientific applications tend to make much heavier use of arrays than most + other programming tasks, and hence a heavy use of indexing. + +3. **Half-open semantics.** Finally, the idea of half-open semantics, where the + `stop` value of a slice is never included, is bad, for many of the same + reasons that 0-based indexing is bad. In most contexts outside of programming, + including virtually all mathematical contexts, when one sees a range of + values, it is implicitly assumed that both endpoints are included in the + range. For example, if you see a phrase like "ages 7 to 12", "the letters A to + Z", or "sum of the numbers from 1 to 10", without any further qualification + you assume that both endpoints are included in the range. Half-open semantics + also break down when considering non-numeric quantities. For example, one + cannot represent the set of letters "from A to Z" except by including both + endpoints, as there is no letter after Z to not include. + + It is simply more natural to think about a range as including both endpoints. + Half-open semantics are often tied to 0-based indexing, since it is a + convenient way to allow the range 0--N to contain N values, by not including + N.[^python-history-footnote] I see this as taking a bad decision (0-based + indexing) and putting a bad bandaid on it that makes it worse. But certainly + this argument goes away for 1-based indexing. The range 1--N contains N values + exactly when N *is* included in the range. + + [^python-history-footnote]: In fact, the original reason that Python uses + 0-based indexing is that Guido preferred the half-open semantics, which only + work out well when combined with 0-based indexing + ([reference](https://web.archive.org/web/20190321101606/https://plus.google.com/115212051037621986145/posts/YTUxbXYZyfi)). + + You might argue that there are instances in everyday life where half-open as + well as 0-based semantics are used. For example, in the West, the reckoning of + a person's age is typically done in a way that matches half-open 0-based + indexing semantics. If has been less than 1 year since a person's birthdate, + you might say they are "zero years old" (although typically you use a smaller + unit of measure such as months to avoid this). And if tomorrow is my 30th + birthday, then today I will say, "I am 29 years old", even though I am + actually 29.99 years old (I may continue to say "I am 29 years old" tomorrow, + but at least today no one could accuse me of lying). This matches the + "half-open" semantics used by slices. The end date of an age, the birthday, is + not accounted for until it has passed. This example shows that half-open + semantics do indeed go nicely with 0-based counting, and it's indeed typically + good to use one when using the other. But age is a distance. It is the + distance in time since a person's birthdate. So 0-based indexing makes sense + for it. Half-open semantics play nicely with age not just because it lets us + lie to ourselves about being younger than we really are, but because age is a + continuous quantity which is reckoned by integer values for convenience. Since + people rarely concern themselves with fractional ages, they must increment an + age counter at some point, and doing so on a birthday, which leads to a + "half-open" semantic, makes sense. But a collection of items like a list, + array, or string in Python usually does not represent a continuous quantity + which is discretized, but rather a quantity that is naturally discrete. So + while half-open 0-indexed semantics are perfectly reasonable for human ages, + the same argument doesn't make sense for collections in Python. + + When it comes to indexing, half-open semantics are problematic for a few + reasons: + + - A commonly touted benefit of half-open slicing semantics is that you can + "glue" half-open intervals together. For example, `a[0:N] + a[N:M]` is + the same as `a[0:M]`. But `a[1:N] + a[N+1:M]` is just as clear. People + are perfectly used to adding 1 to get to the next term in a sequence, and + it is easier to see that `[1:N]` and `[N+1:M]` are non-overlapping if + they do not share endpoint values. Ranges that include both endpoints are + standard in both everyday language and mathematics. $\sum_{i=1}^n$ means + a summation from $1$ to $n$ inclusive. Formulas like $\sum_{i=1}^n a_i = + \sum_{i=1}^k a_i + \sum_{i=k+1}^n a_i$ are natural to anyone who has + studied enough mathematics. If you were to say "the first $n$ numbers are + $1$ to $n+1$; the next $n$ numbers are $n+1$ to $2n+1$", or "'the 70s' + refers to the years 1970--1980", imagining "to" and "--" to mean + half-open semantics, anyone would tell you were wrong. + + - Another benefit of half-open intervals is that they allow the range `a[i:j]` + to contain $j - i$ elements (assuming $0 \leq i \leq j$ and `a` is large + enough). I tout this myself in the guide above, since it is a useful [sanity + check](sanity-check). However, as useful as it is, it isn't worth the more + general confusion caused by half-open semantics. I contend people are + perfectly used to the usual [fencepost](fencepost) offset that a range + $i\ldots j$ contains $j - i + 1$ numbers. Half-open semantics replace this + fencepost error with more subtle ones, which arise from forgetting that the + range doesn't include the endpoint, unlike most natural ranges that occur in + day-to-day life. See [wrong rule 3](wrong-rule-3) above for an example of how + half-open semantics can lead to subtle fencepost errors. + + It is true that including both endpoints in range can lead to [fencepost + errors](fencepost). But the fencepost problem is fundamentally unavoidable. A + 100 foot fence truly has one more fencepost than fence lengths. The best way + to deal with the fencepost problem is not to try to change the way we count + fenceposts, so that somehow 11 fenceposts is really only + 10.[^fencepost-footnote] It is rather to reuse the most natural and intuitive + way of thinking about the problem, which occurs both in programming and + non-programming contexts, which is that certain quantities, like the number of + elements in a range $1\ldots N$, will require an extra "$+\,1$" to be correct. + + [^fencepost-footnote]: [This + article](https://betterexplained.com/articles/learning-how-to-count-avoiding-the-fencepost-problem/) + has a nice writeup of why the fencepost problem exists. It's related to + the difference between measurement and enumeration that I touched on + earlier. + + - Half-open semantics become particularly confusing when the step is negative. + This is because one must remember that the end that is not included in the + half-open interval is the second index in the slice, *not* the larger index + (see wrong rules [1](wrong-rule-1) and [3](wrong-rule-3) above). Were both + endpoints included, this confusion would be impossible, because positive and + negative steps would be symmetric in this regard. + + - Half-open semantics are generally undesirable to apply to extensions to + slicing on non-integer labels. For example, the pandas + {external+pandas:attr}`~pandas.DataFrame.loc` attribute allows slicing + string labels (like `df.loc['a':'f']`), but this syntax always includes + both ends. This is because when you slice on labels, you probably aren't + thinking about which label comes before or after the one you want, and + you might not even know. But this same reasoning also applies to + integers. You're probably thinking about the index that you want to slice + up to, not the one before or after it. + + Furthermore, if label slicing used half-open semantics, to slice to the end + of the sequence, you'd have to use an [omitted](omitted) `end`, instead of + just using the last label. With integers you can get away with this because + [there is always a bigger + integer](https://en.wikipedia.org/wiki/Archimedean_property), but this + property doesn't apply to other types of label objects. + + In general, half-open semantics are naïvely superior because they have some + properties that appear to be nice (easy unions, no +1s in length formulas). + But the "niceness" of these properties ignores the fact that most people + are already used to closed-closed intervals from mathematics and from + everyday life, and so are used to accounting for them already. So while + these properties are nice, they also break the natural intuition of how + ranges work. Half-open semantics are also closely tied to 0-based indexing, + which as I argued above, is itself problematic for many of the same + reasons. + +Again, there is no way Python itself can change any of these things at this +point. It would be way too big of a change to the language, far bigger than +any change that was made as part of Python 3 (and the Python developers have +already stated that they will never do a big breaking change like Python 3 +again). But I hope I can inspire new languages and DSLs that include slicing +semantics to be written in clearer ways. And I also hope that I can break some +of the cognitive dissonance that leads people to believe that the Python +slicing rules are superior, despite the endless confusion that they provide. + +Finally, I believe that simply understanding that Python has made these +decisions, whether you agree with them or not, will help you to remember the +slicing [rules](rules), and that's my true goal here. + +```{rubric} Footnotes +``` +<!-- Footnotes are written inline above but markdown will put them here at the +end of the document. --> diff --git a/docs/requirements.txt b/docs/requirements.txt index dbec7740..f703a755 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,10 @@ furo linkify-it-py +matplotlib @ git+https://github.com/asmeurer/matplotlib.git@output-base-name myst-parser +scikit-image sphinx sphinx-copybutton +sphinx-reredirects +sphinx-design sphinx-autobuild diff --git a/docs/slices.md b/docs/slices.md deleted file mode 100644 index 0c76d625..00000000 --- a/docs/slices.md +++ /dev/null @@ -1,2011 +0,0 @@ -(slices-docs)= -Slices -====== - -Python's slice syntax is one of the more confusing parts of the language, even -to experienced developers. In this page, I carefully break down the rules for -slicing, and examine just what it is that makes it so confusing. - -There are two primary aspects of slices that make them difficult to -understand: confusing conventions, and discontinuous definitions. By confusing -conventions, we mean that slice semantics have definitions that are often -difficult to reason about mathematically. These conventions were chosen for -syntactic convenience, and one can easily see for most of them how they lead -to concise notation for very common operations, but it remains nonetheless -true that they can make figuring out the *right* slice to use in the first -place complicated. By discontinuous definitions, we mean that the definition -of a slice takes on fundamentally different meanings if the start, stop, or -step are negative, nonnegative, or omitted. This again is done for syntactic -convenience, but it means that as a user, you must switch your mode of -thinking about slices depending on value of the arguments. There are no -uniform formulas that apply to all slices. - -The [ndindex](index) library can help with much of this, especially for people -developing libraries that consume slices. But for end-users the challenge is -often just to write down a slice. Even if you rarely work with NumPy arrays, -you will most likely require slices to select parts of lists or strings as -part of the normal course of Python coding. - -ndindex focuses on NumPy array index semantics, but everything on this page -equally applies to sliceable Python builtin objects like lists, tuples, and -strings. This is because on a single dimension, NumPy slice semantics are -identical to the Python slice semantics (NumPy only begins to differ from -Python for multi-dimensional indices). - -## What is a slice? - -In Python, a slice is a special syntax that is allowed only in an index, that -is, inside of square brackets proceeding an expression. A slice consists of -one or two colons, with either an expression or nothing on either side of each -colon. For example, the following are all valid slices on the object `a`: - -```py -a[x:y] -a[x:y:z] -a[:] -a[x::] -a[x::z] -``` - -Furthermore, for a slice `a[x:y:z]`, if `a` is a Python built-in object or a -NumPy array, there is an additional semantic restriction, which is that the -expressions `x`, `y`, and `z` must be integers. - -The three arguments to a slice are traditionally called `start`, `stop`, and -`step`: - -```py -a[start:stop:step] -``` - -We will use these names throughout this guide. - -It is worth noting that the `x:y:z` syntax is not valid outside of square -brackets. However, slice objects can be created manually using the `slice()` -builtin (`a[x:y:z]` is the same as `a[slice(x, y, z)]`). If you want to -perform more advanced operations like arithmetic on slices, consider using -the [`ndindex.Slice()`](slice-api) object. - -(rules)= -## Rules - -These are the rules to keep in mind to understand how slices work. Each of -these is explained in detail below. Many of the detailed descriptions below -also outline several *wrong* rules, which are bad ways of thinking about -slices but which you may be tempted to think about as rules. The below 7 rules -are always correct. - -In this document, "*nonnegative*" means $\geq 0$ and "*negative*" means $< 0$. - -For a slice `a[start:stop:step]`: - -1. `start` and `stop` use **0-based indexing** from the **beginning** of `a` - when they are **nonnegative**, and **−1-based indexing** from **end** of - `a` when they are **negative**. (See sections {ref}`0-based` and - {ref}`negative-indices`) -2. `stop` is never included in the slice. (See section {ref}`half-open`) -3. `start` and `stop` are clipped to the bounds of `a`. (See section {ref}`clipping`) -4. The slice starts at `start` and successively adds `step` until it reaches - an index that is at or past `stop`, and then stops without including that - `stop` index. (See sections {ref}`steps` and {ref}`negative-steps`) -5. If `step` is omitted it defaults to `1`. (See section {ref}`omitted`) -6. If `start` or `stop` are omitted they extend to the beginning or end of `a` - in the direction being sliced. Slices like `a[:i]` or `a[i:]` should be - though of as the `start` or `stop` being omitted, not as a colon to the - left or right of an index. (See section {ref}`omitted`) -7. Slicing something never raises an `IndexError`, even if the slice is empty. - For a NumPy array, a slice always keeps the axis being sliced, even if that - means the resulting dimension will be 0 or 1. (See section {ref}`subarray`) - -(integer-indices)= -## Integer indices - -To understand slices, it is good to first review how integer indices work. -Throughout this guide, we will use as an example this prototype list: - -$$ -a = [\mathtt{\textsf{'}a\textsf{'}},\ \mathtt{\textsf{'}b\textsf{'}},\ \mathtt{\textsf{'}c\textsf{'}},\ -\mathtt{\textsf{'}d\textsf{'}},\ \mathtt{\textsf{'}e\textsf{'}},\ \mathtt{\textsf{'}f\textsf{'}},\ \mathtt{\textsf{'}g\textsf{'}}] -$$ - -The list `a` has 7 elements. - -The elements of `a` are strings, but the indices and slices on the list `a` -will always use integers. An index or slice is never based on the value of the -elements, but rather the position of the elements in the list.[^dict-footnote] - -[^dict-footnote]: If you are looking for something that allows non-integer -indices or that indexes by value, you may want a `dict`. Despite using similar -syntax, `dict`s do not allow slicing. - -An integer index picks a single element from the list `a`. For NumPy arrays, -integer indices pick a subarray corresponding to a particular element from a -given axis (and as a result, an integer index always reduces the -dimensionality of an array by one). - -(fourth-sentence)= -The key thing to remember about indexing in Python, both for integer and -slice indexing, is that it is 0-based. This means that the indices start -at 0. This is the case for all **nonnegative** indices. -For example, `a[3]` would pick the **fourth** element of `a`, in this case, `'d'`. - -<div style="text-align:center"> -<code style="font-size: 16pt;">a[3] == 'd'</code> -$$ -\begin{aligned} -\begin{array}{r c c c c c c c} -a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{#EE0000}{\text{index}} - & \color{#EE0000}{0\phantom{,}} - & \color{#EE0000}{1\phantom{,}} - & \color{#EE0000}{2\phantom{,}} - & \color{#5E5EFF}3{\phantom{,}} - & \color{#EE0000}{4\phantom{,}} - & \color{#EE0000}{5\phantom{,}} - & \color{#EE0000}{6\phantom{,}}\\ -\end{array} -\end{aligned} -$$ -</div> - -```py ->>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] ->>> a[3] -'d' -``` - -0-based indexing is different from how people typically count things, which is -1-based (1, 2, 3, ...). Thinking in terms of 0-based indexing requires some -practice. - -For **negative** integers, indices index from the end of the list. These -indices are necessarily 1-based (or rather, −1-based), since `0` already -refers to the first element of the list. `-1` chooses the last element, `-2` -the second-to-last, and so on. For example, `a[-3]` picks the -**third-to-last** element of `a`, in this case, `'e'`: - -<div style="text-align:center"> -<code style="font-size: 16pt;">a[-3] == 'e'</code> -$$ -\begin{aligned} -\begin{array}{r c c c c c c c} -a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{#EE0000}{\text{index}} - & \color{#EE0000}{-7\phantom{,}} - & \color{#EE0000}{-6\phantom{,}} - & \color{#EE0000}{-5\phantom{,}} - & \color{#EE0000}{-4\phantom{,}} - & \color{#5E5EFF}{-3\phantom{,}} - & \color{#EE0000}{-2\phantom{,}} - & \color{#EE0000}{-1\phantom{,}}\\ -\end{array} -\end{aligned} -$$ -</div> - -```py ->>> a[-3] -'e' -``` - -An equivalent way to think about negative indices is that an index -`a[-i]` picks `a[len(a) - i]`, that is, you can subtract the negative -index off of the size of `a` (for a NumPy array, replace `len(a)` -with the size of the axis being sliced). For example, `len(a)` is `7`, so -`a[-3]` is the same as `a[7 - 3]`: - -```py ->>> len(a) -7 ->>> a[7 - 3] -'e' -``` - -Therefore, negative indices are primarily a syntactic convenience that -allows one to specify parts of a list that would otherwise need to be -specified in terms of the size of the list. - -If an integer index is greater than or equal to the size of the list, or less -than negative the size of the list (`i >= len(a)` or `i < -len(a)`), then it -is out of bounds and will raise an `IndexError`. - -```py ->>> a[7] -Traceback (most recent call last): -... -IndexError: list index out of range ->>> a[-8] -Traceback (most recent call last): -... -IndexError: list index out of range -``` - -## Points of Confusion - -Now let us come back to slices. The full definition of a slice could be -written down in a couple of sentences, although the discontinuous definitions -would necessitate several "if" conditions. The [NumPy -docs](https://numpy.org/doc/stable/reference/arrays.indexing.html) on slices -say - -(numpy-definition)= - -> The basic slice syntax is `i:j:k` where *i* is the starting index, *j* is -> the stopping index, and *k* is the step ( $k\neq 0$ ). This selects the `m` -> elements (in the corresponding dimension) with index values *i, i + k, ..., -> i + (m - 1) k* where $m = q + (r\neq 0)$ and *q* and *r* are the quotient and -> remainder obtained by dividing *j - i* by *k*: *j - i = q k + r*, so that -> *i + (m - 1) k \< j*. - -While definitions like this may give a technically accurate description of -slices, they aren't especially helpful to someone who is trying to construct a -slice from a higher level of abstraction such as "I want to select this -particular subset of my array".[^numpy-definition-footnote] - -[^numpy-definition-footnote]: This formulation actually isn't particularly -helpful for formulating higher level slice formulas such as the ones used by -ndindex either. - -Instead, we shall examine slices by carefully going over all the various -aspects of the syntax and semantics that can lead to confusion, and attempting -to demystify them through simple [rules](rules). - -(subarray)= -### Subarray - -> **A slice always produces a subarray (or sub-list, sub-tuple, sub-string, -etc.). For NumPy arrays, this means that a slice will always *preserve* the -dimension that is sliced.** - -This is true even if the slice chooses only a single element, or even if it -chooses no elements. This is also true for lists, tuples, and strings, in the -sense that a slice on a list, tuple, or string will always produce a list, -tuple, or string. This behavior is different from integer indices, which -always remove the dimension that they index. - -For example - -```py ->>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] ->>> a[3] -'d' ->>> a[3:4] -['d'] ->>> a[5:2] # Empty slice -[] ->>> import numpy as np ->>> arr = np.array([[1, 2], [3, 4]]) ->>> arr.shape -(2, 2) ->>> arr[0].shape # Integer index removes the first dimension -(2,) ->>> arr[0:1].shape # Slice preserves the first dimension -(1, 2) -``` - -One consequence of this is that, unlike integer indices, **slices will never -raise `IndexError`, even if the slice is empty**. Therefore you cannot rely on -runtime errors to alert you to coding mistakes relating to slice bounds that -are too large. A slice cannot be "out of bounds." See the section on -[clipping](clipping) below. - -(0-based)= -### 0-based - -For the slice `a[start:stop]`, with `start` and `stop` nonnegative integers, -the indices `start` and `stop` are 0-based, just as with [integer -indexing](integer-indices) (although one should be careful that even though -`stop` is 0-based, it is not included in the slice, see [below](half-open)). - -For example: - -<div style="text-align:center"> -<code style="font-size: 16pt;">a[3:5] == ['d', 'e']</code> -$$ -\begin{aligned} -\begin{array}{r c c c c c c c} -a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{#EE0000}{\text{index}} - & \color{#EE0000}{0\phantom{,}} - & \color{#EE0000}{1\phantom{,}} - & \color{#EE0000}{2\phantom{,}} - & \color{#5E5EFF}{3\phantom{,}} - & \color{#5E5EFF}{4\phantom{,}} - & \color{#EE0000}{5\phantom{,}} - & \color{#EE0000}{6\phantom{,}}\\ -\end{array} -\end{aligned} -$$ -</div> - -```py ->>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] ->>> a[3:5] -['d', 'e'] -``` - -Do not be worried if you find 0-based indexing hard to get used to, or if you -find yourself forgetting about it. Even experienced Python developers (this -author included) still find themselves writing `a[3]` instead of `a[2]` from -time to time. The best way to learn to use 0-based indexing is to practice -using it enough that you use it automatically without thinking about it. - -(half-open)= -### Half-open - -Slices behave like half-open intervals. What this means is that the `stop` in -`a[start:stop]` is *never* included in the slice (the exception is if the -`stop` is omitted, see [below](omitted)). - -For example, `a[3:5]` slices the indices `3` and `4`, but not `5` -([0-based](0-based)). - -<div style="text-align:center"> -<code style="font-size: 16pt;">a[3:5] == ['d', 'e']</code> -$$ -\require{enclose} -\begin{aligned} -\begin{array}{r c c c c c c c} -a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{#EE0000}{\text{index}} - & \color{#EE0000}{0\phantom{,}} - & \color{#EE0000}{1\phantom{,}} - & \color{#EE0000}{2\phantom{,}} - & \color{#5E5EFF}{\enclose{circle}{3}} - & \color{#5E5EFF}{\enclose{circle}{4}} - & \color{#EE0000}{\enclose{circle}{5}} - & \color{#EE0000}{6\phantom{,}}\\ -\end{array} -\end{aligned} -$$ -</div> - -```py ->>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] ->>> a[3:5] -['d', 'e'] -``` - -The half-open nature of slices means that you must always remember that the -`stop` slice element is not included in the slice. However, it has a few -advantages: - -(sanity-check)= -- The maximum length of a slice `a[start:stop]`, when `start` and `stop` are - nonnegative, is always `stop - start`. For example, `a[i:i+n]` will slice - `n` elements from `a`. The caveat "maximum" is here because if `stop` - extends beyond the end of `a`, then `a[start:stop]` will only slice up - to `len(a) - start` (see {ref}`clipping` below). Also be careful that this - is only true when `start` and `stop` are nonnegative (see - {ref}`negative-indices` below). However, given those caveats, this is often - a very useful sanity check that a slice is correct. If you expect a slice to - have length `n` but `stop - start` is clearly different from `n`, then the - slice is likely wrong. Length calculations are more complicated when `step - != 1`; in those cases, {meth}`len(ndindex.Slice(...)) <ndindex.Slice.__len__>` can be useful. - -- `len(a)` can be used as a `stop` value to slice to the end of `a`. For - example, `a[1:len(a)]` slices from the second element to the end of `a` - (this is equivalent to `a[1:]`, see {ref}`omitted`) - - ```py - >>> a[1:len(a)] - ['b', 'c', 'd', 'e', 'f', 'g'] - >>> a[1:] - ['b', 'c', 'd', 'e', 'f', 'g'] - ``` - -- Consecutive slices can be appended to one another by making each successive - slice's `start` the same as the previous slice's `stop`. For example, for our - list `a`, `a[2:3] + a[3:5]` is the same as `a[2:5]`. - - ```py - >>> a[2:3] + a[3:5] - ['c', 'd', 'e'] - >>> a[2:5] - ['c', 'd', 'e'] - ``` - - A common usage of this is to split a slice into two slices. For example, the - slice `a[i:j]` can be split as `a[i:k]` and `a[k:j]`. - -#### Wrong Ways of Thinking about Half-open Semantics - -> **The proper rule to remember for half-open semantics is "the `stop` is not - included".** - -There are several alternative ways that one might think of slice semantics, -but they are all wrong in subtle ways. To be sure, for each of these, one -could "fix" the rule by adding some conditions, "it's this in the case where -such and such is nonnegative and that when such and such is negative, and so -on". But that's not the point. The goal here is to *understand* slices. -Remember that one of the reasons that slices are difficult to understand is -these branching rules. By trying to remember a rule that has branching -conditions, you open yourself up to confusion. The rule becomes much more -complicated than it appears at first glance, making it hard to remember. You -may forget the "uncommon" cases and get things wrong when they come up in -practice. You might as well think about slices using the [definition from the -NumPy docs](numpy-definition). - -Rather, it is best to remember the simplest rule possible that is *always* -correct. That rule is, "the `stop` is not included". This rule is extremely -simple, and is always right, regardless of what the values of `start`, `stop`, -or `step` are. The only exception is if `stop` is omitted. In this case, the -rule obviously doesn't apply as-is, and so you can fallback to the rule about -omitted `start`/`stop` (see {ref}`omitted` below). - -(wrong-rule-1)= -<strong style="font-size:120%;" style="font-size:120%;">Wrong Rule 1: "a slice `a[start:stop]` slices the half-open interval -$[\text{start}, \text{stop})$ (equivalently, a slice `a[start:stop]` picks the -elements `i` such that `start <= i < stop`)." <a class="headerlink" -href="#wrong-rule-1" title="Permalink to this headline">¶</a> </strong> - -This is *only* the case if the `step` is positive. It also isn't directly true -for negative `start` or `stop`. For example, with a `step` of `-1`, -`a[start:stop:-1]` slices starting at `start` going in reverse order to -`stop`, but not including `stop`. Mathematically, this creates a half open -interval $(\text{stop}, \text{start}]$ (except reversed). - -For example, say way believed that `a[5:3:-1]` sliced the half-open interval -$[3, 5)$ but in reverse order. - -<!-- TODO: improve this --> -<div style="text-align:center;" > -<code style="font-size: 16pt;">a[5:3:-1] "==" ['e', 'd']</code> -<div style="font-size: 16pt;color:#EE0000;">(WRONG)</div> -$$ -\begin{aligned} -\begin{array}{r c c c c c c c} -a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{#EE0000}{\text{index}} - & \color{#EE0000}{0\phantom{,}} - & \color{#EE0000}{1\phantom{,}} - & \color{#EE0000}{2\phantom{,}} - & \color{#5E5EFF}{3\phantom{,}} - & \color{#5E5EFF}{4\phantom{,}} - & \color{#EE0000}{5\phantom{,}} - & \color{#EE0000}{6\phantom{,}}\\ -\color{#EE0000}{\text{WRONG}}& - & - & - & [\phantom{3,} - & - & ) - & \\ -\end{array}\\ -\small{\text{(reversed)}}\hspace{3.5em} -\end{aligned} -$$ -</div> - -We might assume we would get - -```py ->> a[5:3:-1] -['e', 'd'] # WRONG -``` - -Actually, what we really get is - -```py ->>> a[5:3:-1] -['f', 'e'] -``` - -This is because the slice `5:3:-1` starts at index `5` and steps backwards to -index `3`, but not including `3` (see [](negative-steps) below). - -<div style="text-align:center"> -<code style="font-size: 16pt;">a[5:3:-1] == ['f', 'e']</code> -$$ -\require{enclose} -\begin{aligned} -\begin{array}{r r r r r r r r} -a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{#EE0000}{\text{index}} - & \color{#EE0000}{0\phantom{\textsf{'},}} - & \color{#EE0000}{1\phantom{\textsf{'},}} - & \color{#EE0000}{2\phantom{\textsf{'},}} - & \color{#EE0000}{\enclose{circle}{3}\phantom{,}} - & \leftarrow\color{#5E5EFF}{\enclose{circle}{4}\phantom{,}} - & \leftarrow\color{#5E5EFF}{\enclose{circle}{5}\phantom{,}} - & \color{#EE0000}{6\phantom{\textsf{'},}}\\ -\end{array} -\end{aligned} -$$ -</div> - -(wrong-rule-2)= -<strong style="font-size:120%;">Wrong Rule 2: "A slice works like `range()`." <a class="headerlink" -href="#wrong-rule-2" title="Permalink to this headline">¶</a> </strong> - -There are many similarities between the behaviors of slices and the behavior -of `range()`. However, they do not behave the same. A slice -`a[start:stop:step]` only acts like `range(start, stop, step)` if `start` and -`stop` are **nonnegative**. If either of them are negative, the slice wraps -around and slices from the end of the list (see {ref}`negative-indices` -below). `range()` on the other hand treats negative numbers as the actual -start and stop values for the range. For example: - -```py ->>> list(range(3, 5)) -[3, 4] ->>> b = list(range(7)) ->>> b[3:5] # b is range(7), and these are the same -[3, 4] ->>> list(range(3, -2)) # Empty, because -2 is less than 3 -[] ->>> b[3:-2] # Indexes from 3 to the second to last (5) -[3, 4] -``` - -This rule is tempting because `range()` makes some computations easy. For -example, you can index or take the `len()` of a range. If you want to perform -computations on slices, we recommend using [ndindex](slice-api). This is what -it was designed for. - -(wrong-rule-3)= -<strong style="font-size:120%;">Wrong Rule 3: "Slices index the spaces between the elements of the list."<a class="headerlink" -href="#wrong-rule-3" title="Permalink to this headline">¶</a> </strong> - -This is a very common rule that is taught for both slices and integer -indexing. The reasoning goes as follows: 0-based indexing is confusing, where -the first element of a list is indexed by 0, the second by 1, and so on. -Rather than thinking about that, consider the spaces between the elements: - -<div style="text-align:center"> -<code style="font-size: 16pt;">a[3:5] == ['d', 'e']</code> -$$ -\require{enclose} -\begin{aligned} -\begin{array}{r r r r r r r r r r r r r r r r r r} -a = & [&\phantom{|}&\mathtt{\textsf{'}a\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}b\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}c\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}d\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}e\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}f\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}g\textsf{'}}&\phantom{|}&]&\\ - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|}\\ -\color{#EE0000}{\text{index}} - & - & \color{#EE0000}{0} - & - & \color{#EE0000}{1} - & - & \color{#EE0000}{2} - & - & \color{#5E5EFF}{3} - & - & \color{#5E5EFF}{4} - & - & \color{#5E5EFF}{5} - & - & \color{#EE0000}{6} - & - & \color{#EE0000}{7}\\ -\end{array}\\ -\end{aligned} -$$ -<i>(not a great way of thinking about indices)</i> -</div> - -Using this way of thinking, the first element of `a` is to the left of -the "1-divider". An integer index `i` produces the element to the right of the -"`i`-divider", and a slice `a[i:j]` picks the elements between the `i` and `j` -dividers. - -At first glance, this seems like a rather clever way to think about the -half-open rule. For instance, between the `3` and `5` dividers is the subarray -`['d', 'e']`, which is indeed what we get for `a[3:5]`. However, there are several -reasons why this way of thinking creates more confusion than it removes. - -- As with [wrong rule 1](wrong-rule-1), it works well enough if the step is - positive, but falls apart when it is negative. - - Consider again the slice `a[5:3:-1]`. Looking at the above figure, we might - imagine it to give the same incorrect subarray that we imagined before. - - <div style="text-align:center"> - <code style="font-size: 16pt;">a[5:3:-1] "==" ['e', 'd']</code> - <div style="font-size: 16pt;color:#EE0000;">(WRONG)</div> - $$ - \require{enclose} - \begin{aligned} - \begin{array}{c} - \begin{array}{r r r r r r r r r r r r r r r r r r} - a = & [&\phantom{|}&\mathtt{\textsf{'}a\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}b\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}c\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}d\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}e\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}f\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}g\textsf{'}}&\phantom{|}&]&\\ - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|}\\ - \color{#EE0000}{\text{index}} - & - & \color{#EE0000}{0} - & - & \color{#EE0000}{1} - & - & \color{#EE0000}{2} - & - & \color{#5E5EFF}{3} - & - & \color{#5E5EFF}{4} - & - & \color{#5E5EFF}{5} - & - & \color{#EE0000}{6} - & - & \color{#EE0000}{7}\\ - \end{array}\\ - \small{\color{#EE0000}{\textbf{THIS IS WRONG!}}} - \end{array} - \end{aligned} - $$ - </div> - - As before, we might assume we would get - - ```py - >> a[5:3:-1] - ['e', 'd'] # WRONG - ``` - - but this is incorrect! What we really get is - - ```py - >>> a[5:3:-1] - ['f', 'e'] - ``` - - If you've ever espoused the "spaces between elements" way of thinking about - indices, this should give you serious pause. One can see from the above - diagram that there is clearly no way to salvage this rule to see `a[5:3:-1]` - as giving `['f', 'e']`. Contrast the same slice when simply thinking about - it as stepping backwards from index `5` to index `3`, but not including - index `3`. - - <div style="text-align:center"> - <code style="font-size: 16pt;">a[5:3:-1] == ['f', 'e']</code> - $$ - \require{enclose} - \begin{aligned} - \begin{array}{r r c c c c c c c c c c c c l} - a = & [\mathtt{\textsf{'}a\textsf{'}}, && \mathtt{\textsf{'}b\textsf{'}}, && \mathtt{\textsf{'}c\textsf{'}}, && \mathtt{\textsf{'}d\textsf{'}}, && \mathtt{\textsf{'}e\textsf{'}}, && \mathtt{\textsf{'}f\textsf{'}}, && \mathtt{\textsf{'}g\textsf{'}}]\\ - \color{#EE0000}{\text{index}} - & \color{#EE0000}{0\phantom{,}} - & - & \color{#EE0000}{1\phantom{,}} - & - & \color{#EE0000}{2\phantom{,}} - & - & \color{#EE0000}{\enclose{circle}{3}} - & - & \color{#5E5EFF}{\enclose{circle}{4}} - & - & \color{#5E5EFF}{\enclose{circle}{5}} - & - & \color{#EE0000}{6\phantom{,}}\\ - & - & \phantom{\leftarrow} - & - & \phantom{\leftarrow} - & - & \phantom{\leftarrow} - & \color{#EE0000}{-1} - & \leftarrow - & \color{#5E5EFF}{-1} - & \leftarrow - & \color{#5E5EFF}{\text{start}} - \end{array} - \end{aligned} - $$ - </div> - -- The rule does work for negative `start` and `stop`, but only if you think - about it correctly. The correct way to think about it is to reverse the - indices: - - <div style="text-align:center" > - <code style="font-size: 16pt;">a[-4:-2] == ['d', 'e']</code> - $$ - \require{enclose} - \begin{aligned} - \begin{array}{c} - \begin{array}{r r r r r r r r r r r r r r r r r r} - a = & [&\phantom{|}&\mathtt{\textsf{'}a\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}b\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}c\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}d\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}e\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}f\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}g\textsf{'}}&\phantom{|}&]&\\ - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|}\\ - \color{#EE0000}{\text{index}} - & - & \color{#EE0000}{-7} - & - & \color{#EE0000}{-6} - & - & \color{#EE0000}{-5} - & - & \color{#5E5EFF}{-4} - & - & \color{#5E5EFF}{-3} - & - & \color{#5E5EFF}{-2} - & - & \color{#EE0000}{-1} - & - & \color{#EE0000}{0}\\ - \end{array}\\ - \small{\text{(not a great way of thinking about negative indices)}} - \end{array} - \end{aligned} - $$ - </div> - - For example, `a[-4:-2]` will give `['d', 'e']` - - ```py - >>> a[-4:-2] - ['d', 'e'] - ``` - - However, it would be quite easy to get confused here, as the "other" way of - thinking about negative indices (the way we are recommending) is that the - end starts at -1. So you might mistakenly imagine something like this: - - <div style="text-align:center" > - <code style="font-size: 16pt;">a[-4:-2] "==" ['e', 'f']</code> - <div style="font-size: 16pt;color:#EE0000;">(WRONG)</div> - $$ - \require{enclose} - \begin{aligned} - \begin{array}{c} - \begin{array}{r r r r r r r r r r r r r r r r r r} - a = & [&\phantom{|}&\mathtt{\textsf{'}a\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}b\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}c\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}d\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}e\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}f\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}g\textsf{'}}&\phantom{|}&]&\\ - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#EE0000}{|}\\ - \color{#EE0000}{\text{index}} - & - & \color{#EE0000}{-8} - & - & \color{#EE0000}{-7} - & - & \color{#EE0000}{-6} - & - & \color{#EE0000}{-5} - & - & \color{#5E5EFF}{-4} - & - & \color{#5E5EFF}{-3} - & - & \color{#5E5EFF}{-2} - & - & \color{#EE0000}{-1}\\ - \end{array}\\ - \small{\color{#EE0000}{\textbf{THIS IS WRONG!}}} - \end{array} - \end{aligned} - $$ - </div> - - - But things are even worse than that. If we combine negative `start` and - `stop` and negative `step`, things get even more confusing. Consider the - slice `a[-2:-4:-1]`. This gives `['f', 'e']`: - - ```py - >>> a[-2:-4:-1] - ['f', 'e'] - ``` - - To get this with the "spacers" idea, we have to use the above "wrong" - diagram: - - <div style="text-align:center" > - <code style="font-size: 16pt;">a[-2:-4:-1] == ['f', 'e']</code> - <div style="font-size: 16pt;color:#5E5EFF;">NOW RIGHT!</div> - $$ - \require{enclose} - \begin{aligned} - \begin{array}{c} - \begin{array}{r r r r r r r r r r r r r r r r r r} - a = & [&\phantom{|}&\mathtt{\textsf{'}a\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}b\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}c\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}d\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}e\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}f\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}g\textsf{'}}&\phantom{|}&]&\\ - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#EE0000}{|}\\ - \color{#EE0000}{\text{index}} - & - & \color{#EE0000}{-8} - & - & \color{#EE0000}{-7} - & - & \color{#EE0000}{-6} - & - & \color{#EE0000}{-5} - & - & \color{#5E5EFF}{-4} - & - & \color{#5E5EFF}{-3} - & - & \color{#5E5EFF}{-2} - & - & \color{#EE0000}{-1}\\ - \end{array}\\ - \small{\text{(not a great way of thinking about negative indices)}} - \end{array} - \end{aligned} - $$ - </div> - - <div style="text-align:center" > - <code style="font-size: 16pt;">a[-2:-4:-1] "==" ['e', 'd']</code> - <div style="font-size: 16pt;color:#EE0000;">(WRONG)</div> - $$ - \require{enclose} - \begin{aligned} - \begin{array}{c} - \begin{array}{r r r r r r r r r r r r r r r r r r} - a = & [&\phantom{|}&\mathtt{\textsf{'}a\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}b\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}c\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}d\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}e\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}f\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}g\textsf{'}}&\phantom{|}&]&\\ - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#5E5EFF}{|} - & - & \color{#EE0000}{|} - & - & \color{#EE0000}{|}\\ - \color{#EE0000}{\text{index}} - & - & \color{#EE0000}{-7} - & - & \color{#EE0000}{-6} - & - & \color{#EE0000}{-5} - & - & \color{#5E5EFF}{-4} - & - & \color{#5E5EFF}{-3} - & - & \color{#5E5EFF}{-2} - & - & \color{#EE0000}{-1} - & - & \color{#EE0000}{0}\\ - \end{array}\\ - \small{\color{#EE0000}{\textbf{THIS IS WRONG!}}} - \end{array} - \end{aligned} - $$ - </div> - - In other words, the "right" way to think of spacers with negative `start` - and `stop` depends if the `step` is positive or negative. - - This is because the correct half-open rule is based on not including the - `stop`. It *isn't* based on not including the larger end of the interval. If - the `step` is positive, the `stop` will be larger, but if it is - [negative](negative-steps), the `start` will be larger. - -- The rule "works" for slices, but is harder to imagine for integer indices. - In the divider way of thinking, an integer index `n` corresponds to the - entry to the *right* of the `n` divider. Rules that involve remembering left - or right aren't great when it comes to memorability. - -(fencepost)= -- This rule leads to off-by-one errors due to "fencepost" errors. The - fencepost problem is this: say you want to build a fence that is 100 feet - long with posts spaced every 10 feet. How many fenceposts do you need? - - The naive answer is 10, but the correct answer is 11. The reason is the - fenceposts go in between the 10 feet divisions, including at the end points. - So there is an "extra" fencepost compared to the number of fence sections. - - - ```{figure} imgs/jeff-burak-lPO0VzF_4s8-unsplash.jpg - A section of a fence that has 6 segments and 7 fenceposts.[^fencepost-jeff-burbak-footnote] - - [^fencepost-jeff-burbak-footnote]: Image credit [Jeff Burak via - Unsplash](https://unsplash.com/photos/lPO0VzF_4s8). The image is of - Chautauqua Park in Boulder, Colorado. - ``` - - Fencepost problems are a leading cause of off-by-one errors. Thinking about - slices in this way is to think about lists as separated by fenceposts, and - is only begging for problems. This will especially be the case if you still - find yourself otherwise thinking about indices as pointing to list elements - themselves, rather than the divisions between them. And given the behavior - of negative slices and integer indices under this model, one can hardly - blame you for doing so (see the previous two bullet points). - -Rather than trying to think about dividers between elements, it's much simpler -to just think about the elements themselves, but being counted starting at 0. -To be sure, 0-based indexing itself leads to off-by-one errors, since it is -not the usually way humans are taught to count things, but this is nonetheless -the best way to think about things, especially as you gain practice in -counting that way. As long as you apply the rule "the `stop` is not included," -you will get the correct results. - -(wrong-rule-4)= -<strong style="font-size:120%;">Wrong Rule 4: "The `stop` of a slice `a[start:stop]` is 1-based."<a class="headerlink" -href="#wrong-rule-4" title="Permalink to this headline">¶</a> </strong> - -You might get clever and say `a[3:5]` indexes from the 3rd element with -0-based indexing to the 5th element with 1-based indexing. Don't do this. It -is confusing. Not only that, but the rule must necessarily be reversed for -negative indices. `a[-5:-3]` indexes from the (−5)th element with −1-based -indexing to the (−3)rd element with 0-based indexing (and of course, negative -and nonnegative starts and stops can be mixed, like `a[3:-3]`). Don't get cute -here. It isn't worth it. - -(negative-indices)= -### Negative Indices - -Negative indices in slices work the same way they do with [integer -indices](integer-indices). - -> **For `a[start:stop:step]`, negative `start` or `stop` use −1-based indexing - from the end of `a`.** - -However, negative `start` or `stop` does *not* change the order of the -slicing---only the [`step`](steps) does that. The other [rules](rules) of -slicing do not change when the `start` or `stop` is negative. [The `stop` is -still not included](half-open), values less than `-len(a)` still -[clip](clipping), and so on. - -Note that positive and negative indices can be mixed. The following slices of -`a` all produce `['d', 'e']`: - -<div style="text-align:center"> -<div style="font-size: 16pt;"><code>a[3:5] == a[-4:-2] == a[3:-2] == a[-4:5] -== ['d', 'e']</code></div> -$$ -\begin{aligned} -\begin{array}{r c c c c c c c} -a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{#EE0000}{\text{nonnegative index}} - & \color{#EE0000}{0\phantom{,}} - & \color{#EE0000}{1\phantom{,}} - & \color{#EE0000}{\enclose{circle}{2}\phantom{,}} - & \color{#5E5EFF}{3\phantom{,}} - & \color{#5E5EFF}{4\phantom{,}} - & \color{#EE0000}{\enclose{circle}{5}\phantom{,}} - & \color{#EE0000}{6\phantom{,}}\\ -\color{#EE0000}{\text{negative index}} - & \color{#EE0000}{-7\phantom{,}} - & \color{#EE0000}{-6\phantom{,}} - & \color{#EE0000}{\enclose{circle}{-5}\phantom{,}} - & \color{#5E5EFF}{-4\phantom{,}} - & \color{#5E5EFF}{-3\phantom{,}} - & \color{#EE0000}{\enclose{circle}{-2}\phantom{,}} - & \color{#EE0000}{-1\phantom{,}}\\ -\end{array} -\end{aligned} -$$ -</div> - -```py ->>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] ->>> a[3:5] -['d', 'e'] ->>> a[-4:-2] -['d', 'e'] ->>> a[3:-2] -['d', 'e'] ->>> a[-4:5] -['d', 'e'] -``` - -If a negative `stop` indexes an element on or before a nonnegative `start`, the -slice is empty, the same as if `stop <= start` when both are nonnegative. - -```py ->>> a[3:-5] -[] ->>> a[3:2] -[] -``` - -Similar to integer indices, negative indices `-i` in slices can always be -replaced by adding `len(a)` to `-i` until it is in the range $[0, -\operatorname{len}(a))$ (replacing `len(a)` with the size of the given axis -for NumPy arrays), so they are primarily a syntactic convenience. - -The negative indexing behavior is convenient, but it can also lead to subtle -bugs, due to the fundamental discontinuity it produces. This is especially -likely to happen if the slice entries are arithmetical expressions. **One -should always double check if the `start` or `stop` values of a slice can be -negative, and if they can, if those values produce the correct results.** - -(negative-indices-example)= -For example, say you wanted to slice `n` values from the middle of `a`. -Something like the following would work - -```py ->>> mid = len(a)//2 ->>> n = 4 ->>> a[mid - n//2: mid + n//2] -['b', 'c', 'd', 'e'] -``` - -From our [sanity check](sanity-check), `mid + n//2 - (mid - n//2)` does -equal `n` if `n` is even (we could find a similar expression for `n` odd, but -for now let us assume `n` is even). - -However, let's look at what happens when `n` is larger than the size of `a`: - -```py ->>> n = 8 ->>> a[mid - n//2: mid + n//2] -['g'] -``` - -This is mostly likely not what we would want. Depending on our use-case, we -would most likely want either an error or the full list `['a', 'b', 'c', 'd', -'e', 'f', 'g']`. - -What happened here? Let's look at the slice values: - -```py ->>> mid - n//2 --1 ->>> mid + n//2 -7 -``` - -The `stop` slice value is out of bounds for `a`, but this just causes it -to [clip](clipping) to the end. - -But `start` contains a subtraction, which causes it to become negative. Rather -than clipping to the start, it indexes from the end of `a`, producing the -slice `a[-1:7]`. This picks the elements from the last element (`'g'`) up to -but not including the 7th element (0-based). Index `7` is out of bounds for -`a`, so this picks all elements including and after `'g'`, which in this case -is just `['g']`. - -Unfortunately, the "correct" fix here depends on the desired behavior for each -individual slice. In some cases, the "slice from the end" behavior of negative -values is in fact what is desired. In others, you might prefer an error, so -should add a value check or assertion. In others, you might want clipping, in -which case you could modify the expression to always be nonnegative. For -example, instead of using `mid - n//2`, we could use `max(mid - n//2, 0)`. - -```py ->>> n = 8 ->>> a[max(mid - n//2, 0): mid + n//2] -['a', 'b', 'c', 'd', 'e', 'f', 'g'] -``` - -(clipping)= -### Clipping - -Slices can never give an out-of-bounds `IndexError`. This is different from -[integer indices](integer-indices) which require the index to be in bounds. - -> **If `start` or `stop` index before the beginning or after the end of the -`a`, they will clip to the bounds of `a`**: - -```py ->>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] ->>> a[-100:100] -['a', 'b', 'c', 'd', 'e', 'f', 'g'] -``` - -Furthermore, if the `start` is on or after the `stop`, the slice will be -empty. - -```py ->>> a[3:3] -[] ->>> a[5:2] -[] -``` - -For NumPy arrays, a consequence of this is that a slice will always keep the -axis being sliced, even if the size of the resulting axis is 0 or 1. - -```py ->>> import numpy as np ->>> arr = np.array([[1, 2], [3, 4]]) ->>> arr.shape -(2, 2) ->>> arr[0].shape # Integer index removes the first dimension -(2,) ->>> arr[0:1].shape # Slice preserves the first dimension -(1, 2) ->>> arr[0:0].shape # Slice preserves the first dimension as an empty dimension -(0, 2) -``` - -An important consequence of the clipping behavior of slices is that you cannot -rely on runtime checks for out-of-bounds slices. See the [example -above](negative-indices-example). Another consequence is that you can never -rely on the length of a slice being `stop - start` (for `step = 1` and -`start`, `stop` nonnegative). This is rather the *maximum* length of the -slice. It could end up slicing something smaller. For example, an empty list -will always slice to an empty list. ndindex can help in calculations here: -`len(ndindex.Slice(...))` can be used to compute the *maximum* length of a -slice. If the shape of the input is known, -`len(ndindex.Slice(...).reduce(shape))` will compute the true length of the -slice (see {meth}`ndindex.Slice.__len__` and {meth}`ndindex.Slice.reduce`). Of -course, if you already have a NumPy array, you can just slice the array and -check the shape (slicing a NumPy array always produces a view on the array, so -it is a very inexpensive operation). - -(steps)= -### Steps - -Thus far, we have only considered slices with the default step size of 1. When -the step is greater than 1, the slice picks every `step` element contained in -the bounds of `start` and `stop`. - -> **The proper way to think about `step` is that the slice starts at `start` - and successively adds `step` until it reaches an index that is at or past - the `stop`, and then stops without including that index.** - -The important thing to remember about the `step` is that it being non-1 does -not change the fundamental [rules](rules) of slices that we have learned so -far. `start` and `stop` still use [0-based indexing](0-based). The `stop` is -[never included](half-open) in the slice. [Negative](negative-indices) `start` -and `stop` index from the end of the list. Out-of-bounds `start` and `stop` -still [clip](clipping) to the beginning or end of the list. - -Let us consider an example where the step size is `3`. - -<div style="text-align:center"> -<code style="font-size: 16pt;">a[0:6:3] == ['a', 'd']</code> -$$ -\require{enclose} -\begin{aligned} -\begin{array}{r c c c c c c l} -a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{#EE0000}{\text{index}} - & \color{#5E5EFF}{\enclose{circle}{0}} - & \color{#EE0000}{1\phantom{,}} - & \color{#EE0000}{2\phantom{,}} - & \color{#5E5EFF}{\enclose{circle}{3}} - & \color{#EE0000}{4\phantom{,}} - & \color{#EE0000}{5\phantom{,}} - & \color{#EE0000}{\enclose{circle}{6}}\\ - & \color{#5E5EFF}{\text{start}} - & - & \rightarrow - & \color{#5E5EFF}{+3} - & - & \rightarrow - & \color{#EE0000}{+3\ (\geq \text{stop})} -\end{array} -\end{aligned} -$$ -</div> - -```py ->>> a[0:6:3] -['a', 'd'] -``` - -Note that the `start` index, `0`, is included, but the `stop` index, `6` -(corresponding to `'g'`), is *not* included, even though it is a multiple of -`3` away from the start. This is because the `stop` is [never -included](half-open). - -It can be tempting to think about the `step` in terms of modular arithmetic. -In fact, it is often the case in practice that you require a `step` greater -than 1 because you are dealing with modular arithmetic in some way. However, -this requires care. - -Indeed, we can note that resulting indices `0`, `3` of the above slice -`a[0:6:3]` are all multiples of 3. This is because the `start` index, `0`, is -a multiple of 3. If we instead choose a start index that is $1 \pmod{3}$ then -all the indices would also be $1 \pmod{3}$. - -<div style="text-align:center"> -<code style="font-size: 16pt;">a[1:6:3] == ['b', 'e']</code> -$$ -\require{enclose} -\begin{aligned} -\begin{array}{r c c c c c c c l} -a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{#EE0000}{\text{index}} - & \color{#EE0000}{0\phantom{,}} - & \color{#5E5EFF}{\enclose{circle}{1}} - & \color{#EE0000}{2\phantom{,}} - & \color{#EE0000}{3\phantom{,}} - & \color{#5E5EFF}{\enclose{circle}{4}} - & \color{#EE0000}{5\phantom{,}} - & \color{#EE0000}{\underline{6}\phantom{,}} - & \color{#EE0000}{\enclose{circle}{\phantom{7}}}\\ - & - & \color{#5E5EFF}{\text{start}} - & - & \rightarrow - & \color{#5E5EFF}{+3} - & - & \rightarrow - & \color{#EE0000}{+3\ (\geq \text{stop})} -\end{array} -\end{aligned} -$$ -</div> - -```py ->>> a[1:6:3] -['b', 'e'] -``` - -However, be careful as this rule is *only* true for nonnegative `start`. If -`start` is negative, the value of `start (mod step)` has no -bearing on the indices chosen for the slice: - -```py ->>> list(range(21))[-15::3] -[6, 9, 12, 15, 18] ->>> list(range(22))[-15::3] -[7, 10, 13, 16, 19] -``` - -In the first case, `-15` is divisible by 3 and all the indices chosen by the -slice `-15::3` were also divisible by 3 (remember that indices and values -are the same for simple ranges). But this is only because the length of the -list, `21`, also happened to be a multiple of 3. In the second example it is -`22` and the resulting indices are not multiples of `3`. - -Another thing to be aware of is that if the start is [clipped](clipping), the -clipping occurs *before* the step. That is, if the `start` is less than -`len(a)`, it is the same as `start = 0` regardless of the `step`. - -```py ->>> a[-100::2] -['a', 'c', 'e', 'g'] ->>> a[-101::2] -['a', 'c', 'e', 'g'] -``` - -If you need to think about steps in terms of modular arithmetic, -[`ndindex.Slice()`](slice-api) can be used to perform various slice -calculations so that you don't have to come up with modulo formulas yourself. -If you try to write such formulas yourself, chances are you will get them -wrong, as it is easy to fail to properly account for [negative vs. nonnegative -indices](negative-indices), [clipping](clipping), and [negative -steps](negative-steps). As noted before, any correct "formula" regarding -slices will necessarily have many piecewise conditions. - -(negative-steps)= -### Negative Steps - -Recall what we said [above](steps): - -> **The proper way to think about `step` is that the slice starts at `start` - and successively adds `step` until it reaches an index that is at or past - the `stop`, and then stops without including that index.** - -The key thing to remember with negative `step` is that this rule still -applies. That is, the index starts at `start` then adds the `step` (which -makes the index smaller), and stops when it is at or past the `stop`. Note the -phrase "at or past". If the `step` is positive this means "greater than or -equal to", but if the `step` is negative this means "less than or equal to". - -Think of a slice as starting at the `start` and sliding along the list, -jumping along by `step`, and spitting out elements. Once you see that you are -at or have gone past the `stop` in the direction you are going (left for -negative `step` and right for positive `step`), you stop. - -It's worth pointing out that unlike all other slices we have seen so far, a -negative `step` reverses the order that the elements are returned relative to -the original list. In fact, one of the most common uses of a negative `step` is -`a[::-1]`, which reverses the list: - -```py ->>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] ->>> a[::-1] -['g', 'f', 'e', 'd', 'c', 'b', 'a'] -``` - -It is tempting therefore to think of a negative `step` as a "reversing" -operation. However, this is a bad way of thinking about negative steps. The -reason is that `a[i:j:-1]` is *not* equivalent to `reversed(a[j:i:1])`. The -reason is basically the same as was described in [wrong rule 1](wrong-rule-1) -above. The issue is that for `a[start:stop:step]`, `stop` is *always* what is -[not included](half-open). Which means if we swap `i` and `j`, we go from "`j` -is not included" to "`i` is not included", producing a wrong result. For -example, as [before](wrong-rule-1): - -```py ->>> a[5:3:-1] -['f', 'e'] ->>> list(reversed(a[3:5:1])) # This is not the same thing -['e', 'd'] -``` - -In the first case, index `3` is not included. In the second case, index `5` is -not included. - -Worse, this way of thinking may even lead one to imagine the completely wrong -idea that `a[i:j:-1]` is the same as `reversed(a)[j:i]`: - -```py ->>> list(reversed(a))[3:5] -['d', 'c'] -``` - -Once `a` is reversed, the indices `3` and `5` have nothing to do with the -original indices `3` and `5`. To see why, consider a much larger list: - -```py ->>> list(range(100))[5:3:-1] -[5, 4] ->>> list(reversed(range(100)))[3:5] -[96, 95] -``` - -It is much more robust to think about the slice as starting at `start`, then -moving across the list by `step` until reaching `stop`, which is not included. - -Negative steps can of course be less than -1 as well, with similar behavior to -steps greater than 1, again, keeping in mind that the `stop` is not included. - -```py ->>> a[6:0:-3] -['g', 'd'] -``` - -<div style="text-align:center"> -<code style="font-size: 16pt;">a[6:0:-3] == ['g', 'd']</code> -$$ -\require{enclose} -\begin{aligned} -\begin{array}{r r c c c c c c l} -a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{#EE0000}{\text{index}} - & \color{#EE0000}{\enclose{circle}{0}\phantom{,}} - & \color{#EE0000}{1\phantom{,}} - & \color{#EE0000}{2\phantom{,}} - & \color{#5E5EFF}{\enclose{circle}{3}} - & \color{#EE0000}{4\phantom{,}} - & \color{#EE0000}{5\phantom{,}} - & \color{#5E5EFF}{\enclose{circle}{6}}\\ - & \color{#EE0000}{-3}\phantom{\mathtt{\textsf{'},}} - & \leftarrow - & - & \color{#5E5EFF}{-3}\phantom{\mathtt{\textsf{'},}} - & \leftarrow - & - & \color{#5E5EFF}{\text{start}}\\ - & \color{#EE0000}{(\leq \text{stop})} -\end{array} -\end{aligned} -$$ -</div> - -The `step` can never be equal to 0. This unconditionally produces an error: - -```py ->>> a[::0] -Traceback (most recent call last): -... -ValueError: slice step cannot be zero -``` - -(omitted)= -### Omitted Entries - -The final point of confusion is omitted entries.[^omitted-none-footnote] - -[^omitted-none-footnote]: `start`, `stop`, or `step` may also be `None`, which -is syntactically equivalent to them being omitted. That is to say, `a[::]` is -a syntactic shorthand for `a[None:None:None]`. It is rare to see `None` in a -slice. This is only relevant for code that consumes slices, such as a -`__getitem__` method on an object. The `slice()` object corresponding to -`a[::]` is `slice(None, None, None)`. [`ndindex.Slice()`](slice-api) also uses -`None` to indicate omitted entries in the same way. - -**The best way to think about omitted entries is just like that, as omitted -entries.** That is, for a slice like `a[:i]`, think of it as the `start` being -omitted, and `stop` equal to `i`. Conversely, `a[i:]` has the `start` as `i` -and the `stop` omitted. The wrong way to think about these is as a colon being -before or after the index `i`. Thinking about it this way will only lead to -confusion, because you won't be thinking about `start` and `stop`, but rather -trying to remember some rule based on where a colon is. But the colons in a -slice are not indicators, they are separators. - -As to the semantic meaning of omitted entries, the easiest one is the `step`. - -> **If the `step` is omitted, it always defaults to `1`.** - -If the `step` is omitted the second colon before the `step` can also be -omitted. That is to say, the following are completely equivalent: - -```py -a[i:j:1] -a[i:j:] -a[i:j] -``` - -<!-- TODO: Better wording for this rule? --> - -> **For the `start` and `stop`, the rule is that being omitted extends the - slice all the way to the beginning or end of `a` in the direction being - sliced.** - -If -the `step` is positive, this means `start` extends to the beginning of `a` and -`stop` extends to the end. If `step` is negative, it is reversed: `start` -extends to the end of `a` and `stop` extends to the beginning. - -<div style="text-align:center"> -<code style="font-size: 16pt;">a[:3] == a[:3:1] == ['a', 'b', 'c']</code> -$$ -\require{enclose} -\begin{aligned} -\begin{array}{r r c c c c c c c c c c c c l} -a = & [\mathtt{\textsf{'}a\textsf{'}}, && \mathtt{\textsf{'}b\textsf{'}}, && \mathtt{\textsf{'}c\textsf{'}}, && \mathtt{\textsf{'}d\textsf{'}}, && \mathtt{\textsf{'}e\textsf{'}}, && \mathtt{\textsf{'}f\textsf{'}}, && \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{#EE0000}{\text{index}} - & \color{#5E5EFF}{\enclose{circle}{0}} - & - & \color{#5E5EFF}{\enclose{circle}{1}} - & - & \color{#5E5EFF}{\enclose{circle}{2}} - & - & \color{#EE0000}{\enclose{circle}{3}} - & - & \color{#EE0000}{4\phantom{,}} - & - & \color{#EE0000}{5\phantom{,}} - & - & \color{#EE0000}{6\phantom{,}}\\ - \color{#5E5EFF}{\text{start}} - & \color{#5E5EFF}{\text{(beginning)}} - & \rightarrow - & \color{#5E5EFF}{+1} - & \rightarrow - & \color{#5E5EFF}{+1} - & \rightarrow - & \color{#EE0000}{\text{stop}} - & - & \phantom{\rightarrow} - & - & \phantom{\rightarrow} - & - & \phantom{\rightarrow} -\end{array} -\end{aligned} -$$ -</div> - -<div style="text-align:center"> -<code style="font-size: 16pt;">a[3:] == a[3::1] == ['d', 'e', 'f', 'g']</code> -$$ -\require{enclose} -\begin{aligned} -\begin{array}{r r c c c c c c c c c c c c l} -a = & [\mathtt{\textsf{'}a\textsf{'}}, && \mathtt{\textsf{'}b\textsf{'}}, && \mathtt{\textsf{'}c\textsf{'}}, && \mathtt{\textsf{'}d\textsf{'}}, && \mathtt{\textsf{'}e\textsf{'}}, && \mathtt{\textsf{'}f\textsf{'}}, && \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{#EE0000}{\text{index}} - & \color{#EE0000}{0\phantom{,}} - & - & \color{#EE0000}{1\phantom{,}} - & - & \color{#EE0000}{2\phantom{,}} - & - & \color{#5E5EFF}{\enclose{circle}{3}} - & - & \color{#5E5EFF}{\enclose{circle}{4}} - & - & \color{#5E5EFF}{\enclose{circle}{5}} - & - & \color{#5E5EFF}{\enclose{circle}{6}\phantom{,}}\\ - & - & \phantom{\rightarrow} - & - & \phantom{\rightarrow} - & - & \phantom{\rightarrow} - & \color{#5E5EFF}{\text{start}} - & \rightarrow - & \color{#5E5EFF}{+1} - & \rightarrow - & \color{#5E5EFF}{+1} - & \rightarrow - & \color{#5E5EFF}{\text{stop}} - & \color{#5E5EFF}{\text{(end)}} -\end{array} -\end{aligned} -$$ -</div> - -<div style="text-align:center"> -<code style="font-size: 16pt;">a[:3:-1] == ['g', 'f', 'e']</code> -$$ -\require{enclose} -\begin{aligned} -\begin{array}{r r c c c c c c c c c c c c l} -a = & [\mathtt{\textsf{'}a\textsf{'}}, && \mathtt{\textsf{'}b\textsf{'}}, && \mathtt{\textsf{'}c\textsf{'}}, && \mathtt{\textsf{'}d\textsf{'}}, && \mathtt{\textsf{'}e\textsf{'}}, && \mathtt{\textsf{'}f\textsf{'}}, && \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{#EE0000}{\text{index}} - & \color{#EE0000}{0\phantom{,}} - & - & \color{#EE0000}{1\phantom{,}} - & - & \color{#EE0000}{2\phantom{,}} - & - & \color{#EE0000}{\enclose{circle}{3}} - & - & \color{#5E5EFF}{\enclose{circle}{4}} - & - & \color{#5E5EFF}{\enclose{circle}{5}} - & - & \color{#5E5EFF}{\enclose{circle}{6}\phantom{,}}\\ - & - & \phantom{\leftarrow} - & - & \phantom{\leftarrow} - & - & \phantom{\leftarrow} - & \color{#EE0000}{\text{stop}} - & \leftarrow - & \color{#5E5EFF}{-1} - & \leftarrow - & \color{#5E5EFF}{-1} - & \leftarrow - & \color{#5E5EFF}{\text{start}} - & \color{#5E5EFF}{\text{(end)}} -\end{array} -\end{aligned} -$$ -</div> - -<div style="text-align:center"> -<code style="font-size: 16pt;">a[3::-1] == ['d', 'c', 'b', 'a']</code> -$$ -\require{enclose} -\begin{aligned} -\begin{array}{r r c c c c c c c c c c c c l} -a = & [\mathtt{\textsf{'}a\textsf{'}}, && \mathtt{\textsf{'}b\textsf{'}}, && \mathtt{\textsf{'}c\textsf{'}}, && \mathtt{\textsf{'}d\textsf{'}}, && \mathtt{\textsf{'}e\textsf{'}}, && \mathtt{\textsf{'}f\textsf{'}}, && \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{#EE0000}{\text{index}} - & \color{#5E5EFF}{\enclose{circle}{0}} - & - & \color{#5E5EFF}{\enclose{circle}{1}} - & - & \color{#5E5EFF}{\enclose{circle}{2}} - & - & \color{#5E5EFF}{\enclose{circle}{3}} - & - & \color{#EE0000}{4\phantom{,}} - & - & \color{#EE0000}{5\phantom{,}} - & - & \color{#EE0000}{6\phantom{,}}\\ - \color{#5E5EFF}{\text{stop}} - & \color{#5E5EFF}{\text{(beginning)}} - & \leftarrow - & \color{#5E5EFF}{-1} - & \leftarrow - & \color{#5E5EFF}{-1} - & \leftarrow - & \color{#5E5EFF}{\text{start}} - & - & \phantom{\leftarrow} - & - & \phantom{\leftarrow} - & - & \phantom{\leftarrow} -\end{array} -\end{aligned} -$$ -</div> - -```py ->>> a = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] ->>> a[:3] # From the beginning to index 3 (but not including index 3) -['a', 'b', 'c'] ->>> a[3:] # From index 3 to the end -['d', 'e', 'f', 'g'] ->>> a[:3:-1] # From the end to index 3 (but not including index 3), reversed -['g', 'f', 'e'] ->>> a[3::-1] # From index 3 to the beginning, reversed -['d', 'c', 'b', 'a'] -``` - -## Soapbox - -While this guide is opinionated about the right and wrong ways to think about -slices in Python, I have tried to stay neutral regarding the merits of the -rules themselves. But I want to take a moment to give my views on them. I have -worked with slice objects quite a bit in building ndindex, as well as just -general usage with Python and NumPy. - -Python's slice syntax is, without a doubt, extremely expressive, and has a -straightforward and simple syntax. However, simply put, the semantic rules for -slices are completely bonkers. They lend themselves to several invalid -interpretations, which I have outlined above, which seem valid at first glance -but fall apart in corner cases. The "correct" ways to think about slices are -very particular. I have tried to [outline](rules) them carefully, but one gets -the impression that unless one works with slices regularly, it will be hard to -remember the "right" ways and not fallback to thinking about the "wrong" ways, -or, as most Python programmers probably do, simply "guessing and checking". - -Furthermore, the discontinuous nature of the `start` and `stop` parameters not -only makes it hard to remember how slices work, but it makes it *extremely* -hard to write slice arithmetic. The arithmetic is already hard enough due to -the modular nature of `step`, but the discontinuous aspect of `start` and -`stop` increases this tenfold. If you are unconvinced of this, take a look at -the [source -code](https://github.com/Quansight-labs/ndindex/blob/main/ndindex/slice.py) for -`ndindex.Slice()`. You will see lots of nested `if` blocks.[^source-footnote] -This is because slices have *fundamentally* different definitions if the -`start` or `stop` are `None`, negative, or nonnegative. Furthermore, `None` is -not an integer, so one must always be careful to either check for it first or -to be certain that it cannot happen, before performing any arithmetical -operation or numerical comparison. Under each `if` block you will see some -formula or other. Many of these formulas were difficult to come up with. In -many cases they are asymmetrical in surprising ways. It is only through the -rigorous [testing](testing) that ndindex uses that I can have confidence the -formulas are correct for all corner cases. - -[^source-footnote]: To be sure, I make no claims that the source of any -function in ndindex cannot be simplified. In writing ndindex, I have primarily -focused on making the logic correct, and less on making it elegant. I welcome -any pull requests that simplifies the logic of a function. The extensive -[testing](testing) should ensure that any rewritten function remains correct. - -It is my opinion that Python's slicing semantics could be just as expressive, -but much less confusing and difficult to work with, both for end-users of -slices and people writing slice arithmetic (a typical user of ndindex). The -changes I would make to improve the semantics would be - -1. Remove the special meaning of negative numbers. -2. Use 1-based indexing instead of 0-based indexing. -3. Make a slice always include both the start and the stop. - -**Negative numbers.** The special meaning of negative numbers, to index from -the end of the list, is by far the biggest problem with Python's slice -semantics. It is a fundamental discontinuity in the definition of an index. -This makes it completely impossible to write a formula for almost anything -relating to slices that will not end up having branching `if` conditions. But -the problem isn't just for code that manipulates slices. The [example -above](negative-indices-example) shows how negative indices can easily lead to -bugs in end-user code. Effectively, any time you have a slice `a[i:j]`, if `i` -and `j` are nontrivial expressions, they must be checked to ensure they do not -go negative. If they can be both negative and nonnegative, it is virtually -never the case that the slice will give you what you want in both cases. This -is because the discontinuity inherent in the definition of [negative -indexing](negative-indices) disagrees with the concept of -[clipping](clipping). `a[i:j]` will slice "as far as it can" if `j` is "too -big" (greater than `len(a)`), but it does something completely different if -`i` is "too small" as soon as "too small" means "negative". Clipping is a good -idea. It tends to lead to behavior that gives what you would want slices that -go out of bounds. - -Negative indexing is, strictly speaking, a syntactic sugar only. -Slicing/indexing from the end of a list can always be done in terms of the -length of the list. `a[-x]` is the same as `a[len(a)-x]` (when using 0-based -indexing), but the problem is that it is tedious to write `a` twice, and `a` -may in fact be a larger expression, so writing `a[len(a)-x]` would require -assigning it to a variable. It also becomes more complicated when `a` is a -NumPy array and the slice appears as part of a larger multidimensional (tuple) -index. However, I think it would be possible to introduce a special syntax to -mean "reversed" or "from the end the list" indexing, and leave negative -numbers to simply extend beyond the left side of a list with clipping. For -example, in [Julia](https://julialang.org/), one can use `a[end]` to index the -last element of an array (Julia also uses 1-based indexing). Since this is a -moot point for Python---I don't expect Python's indexing semantics to change; -they are already baked into the language---I won't suggest any syntax. Perhaps -this can inspire people writing new languages or DSLs to come up with better -semantics backed by good syntax (again, I think Python slicing has good -*syntax*. I only take issue with some of its *semantics*.). - -**0-based vs. 1-based indexing.** The second point, on using 1-based indexing -instead of 0-based indexing, will likely be the most controversial. For many -people reading this, the notion that 0-based indexing is superior has been -preached as irreproachable gospel. I encourage you to open your mind and try -to unlearn what you have been taught and take a fresh view of the matter (or -don't. These are just my opinions after all, and none of it changes the fact -that Python is what it is and isn't going to change). - -0-based indexing certainly has its uses. In C, where an index is literally a -syntactic macro for adding two pointers, 0-based indexing makes sense, since -`a[i]` literally means `*(a + i)` under those semantics. However, for higher -level languages such as Python, people think of indexing as pointing to -specific numbered elements of a collection, not as pointer arithmetic. Every -human being is taught from an early age to count from 1. If you show someone -the list "a, b, c", they will tell you that "a" is the 1st, "b" is the 2nd, -and "c" is the 3rd. [Sentences](fourth-sentence) in the above guide like -"`a[3]` would pick the fourth element of `a`" sound very off, even for those -of us used to 0-based indexing. 0-based indexing requires a shift in thinking -from the way that you have been taught to count from early childhood. Counting -is a very fundamental thing for any human, but especially so for a programmer. -Forcing someone to learn a new way to do such a foundational thing is a huge -cognitive burden, and so it shouldn't be done without a very good reason. In a -language like C, one can argue there is a good reason, just as one can argue -that it is beneficial to learn new base number systems like base-2 and base-16 -when doing certain kinds of programming. - -But for Python, what truly is the benefit of counting starting at 0? The main -benefit is that the implementation is easier, because Python is itself written -in C, which uses 0-based indexing, so Python does not need to handle shifting -in the translation. But this has never been a valid argument for Python -semantics. The whole point of Python is to provide higher level semantics than -C, and leave those hard details of translating them to the interpreter and -library code. In fact, Python's slices themselves are much more complicated -than what is available in C, and the interpreter code to handle them is more -than just a trivial translation to C. Adding shifts to this translation code -would not be much additional complexity. - -Even experienced programmers of languages like Python that use 0-based -indexing must occasionally stop themselves from writing something like `a[3]` -instead of `a[2]` to get the third element of `a`. It is very difficult to -"unlearn" 1-based counting," which was not only the first way that you learned -to count, but is also the way that you and everyone else around you continues -to count outside of programming contexts. - -When you teach a child how to count things, you teach them to enumerate the -items starting at 1 ("1, 2, 3, ..."). The number that is enumerated for the -final object is equal to the number of items (the final ordinal is equal to -the cardinal). This only works if you start at 1. If the child instead starts -at 0 ("0, 1, 2, ...") the final ordinal (the last number spoken aloud) would -not match the cardinal (the number of items). The distinction between ordinals -and cardinals is not something most people think about often, because the -convention of counting starting at 1 makes it so that they are equal. But as -programmers in a language that rejects this elegant convention, we are forced -to think about such philosophical distinctions just to solve whatever problem -we are trying to solve. - -In most instances (outside of programming) where a reckoning starts at 0 -instead of 1, it is because it is measuring a distance. The distance from your -house to the nearest pub may be "2 miles", but the distance from your house to -itself is "0 miles". On the other hand, when counting or enumerating -individual objects, counting always starts at 1. The notion of a "zeroth" -object doesn't make sense when counting say apples, because you are counting -the apples themselves, not some quantity relating them. It doesn't make sense -to start at 0 with an "un-apple" because you are only counting apples, not -non-apples. - -So the question then becomes, should indexing work like a measurement of -distance, which would naturally start at 0, or like an enumeration of distinct -terms, which would naturally start at 0. If we think of an index as a pointer -offset, as C does, then it is indeed a measurement of a distance. But if we -instead think of an indexable list as a discrete ordered collection of items, -then the notion of a measurement of distance is harder to justify. But -enumeration is a natural concept for any ordered discrete collection. - -What are the benefits of 0-based indexing? - -- It makes translation to lower level code (like C or machine code) easier. - But as I already argued, this is not a valid argument for Python, which aims - to be high-level and abstract away translation complexities that make coding - more difficult. The translation that necessarily takes place in the - interpreter itself can afford this complexity if it means making the - language itself simpler. -- It makes translation of code written in other languages that use 0-based - indexing simpler. If Python used 1-based indexing, then to translate a C - algorithm to Python, for instance, one would have to adapt all the places - that use indexing, which would be a bug-prone task. But Python's primary - mantra is having syntax and semantics that make code easy to read and easy - to write. Being similar to other existing languages is second to this, and - should not take priority when it conflicts with it. Translation of code from - other languages to Python does happen, but it is much rarer than novel code - written in Python. Furthermore, automated tooling could be used to avoid - translation bugs. Such tooling would help avoid other translation bugs - unrelated to indexing as well. -- It works nicely with half-open semantics. It is true that half-open - semantics and 0-based indexing, while technically distinct, are virtually - always implemented together because they play so nicely with each other. - However, as I argue below, half-open semantics are just as absurd as 0-based - indexing, and abandoning both for the more standard closed-closed/1-based - semantics is very reasonable. - -To me, the ideal indexing system defaults to 1-based, but allows starting at -any index. That way, if you are dealing with a use-case where 0-based indexing -really does make more sense, you can easily use it. Indices should also be -able to start at any other number, including negative numbers (which is -another reason to remove the special meaning of negative indices). An example -of a use-case where 0-based indexing truly is more natural than 1-based -indexing is polynomials. Say we have a polynomial $a_0 + a_1x + a_2x^2 + -\cdots$. Then we can represent the coefficients $a_0, a_1, a_2, \ldots$ in a -list `[a0, a1, a2, ...]`. Since a polynomial naturally has a 0th coefficient, -it makes sense to index the list starting at 0 (and one must still be careful -about off-by-one errors; a degree-$n$ polynomial has $n+1$ coefficients). - -If this seems like absurd idea, note that this is how Fortran works (see -<https://www.fortran90.org/src/faq.html#what-is-the-most-natural-starting-index-for-numbering>). -In Fortran, arrays index starting at 1 by default, but any integer can be used -as a starting index. Fortran predates Python by many decades, but is still in -use today, particularly in scientific applications, and many Python libraries -themselves such as SciPy are backed by Fortran code. Many other popular -programming languages use 1-based indexing, such as Julia, MATLAB, -Mathematica, R, Lua, and -[others](https://en.wikipedia.org/wiki/Comparison_of_programming_languages_(array)#Array_system_cross-reference_list). -In fact, a majority of the popular programming languages that use 1-based -indexing are languages that are primarily used for scientific applications. -Scientific applications tend to make much heavier use of arrays than most -other programming tasks, and hence a heavy use of indexing. - -**Half-open semantics.** Finally, the idea of half-open semantics, where the -`stop` value of a slice is never included, is bad, for many of the same -reasons that 0-based indexing is bad. In most contexts outside of programming, -including virtually all mathematical contexts, when one sees a range of -values, it is implicitly assumed that both endpoints are included in the -range. For example, if you see a phrase like "ages 7 to 12", "the letters A to -Z", or "sum of the numbers from 1 to 10", without any further qualification -you assume that both endpoints are included in the range. Half-open semantics -also break down when considering non-numeric quantities. For example, one -cannot represent the set of letters "from A to Z" except by including both -endpoints, as there is no letter after Z to not include. - -It is simply more natural to think about a range as including both endpoints. -Half-open semantics are often tied to 0-based indexing, since it is a -convenient way to allow the range 0--N to contain N values, by not including -N.[^python-history-footnote] I see this as taking a bad decision (0-based -indexing) and putting a bad bandaid on it that makes it worse. But certainly -this argument goes away for 1-based indexing. The range 1--N contains N values -exactly when N *is* included in the range. - -[^python-history-footnote]: In fact, the original reason that Python uses -0-based indexing is that Guido preferred the half-open semantics, which only -work out well when combined with 0-based indexing -([reference](https://web.archive.org/web/20190321101606/https://plus.google.com/115212051037621986145/posts/YTUxbXYZyfi)). - -You might argue that there are instances in everyday life where half-open as -well as 0-based semantics are used. For example, in the West, the reckoning of -a person's age is typically done in a way that matches half-open 0-based -indexing semantics. If has been less than 1 year since a person's birthdate, -you might say they are "zero years old" (although typically you use a smaller -unit of measure such as months to avoid this). And if tomorrow is my 30th -birthday, then today I will say, "I am 29 years old", even though I am -actually 29.99 years old (I may continue to say "I am 29 years old" tomorrow, -but at least today no one would accuse me of lying). This matches the -"half-open" semantics used by slices. The end date of an age, the birthday, is -not accounted for until it has passed. This example shows that half-open -semantics do indeed go nicely with 0-based counting, and it's indeed typically -good to use one when using the other. But age is a distance. It is the -distance in time since a person's birthdate. So 0-based indexing makes sense -for it. Half-open semantics play nicely with age not just because it lets us -lie to ourselves about being younger than we really are, but because age is a -continuous quantity which is reckoned by integer values for convenience. Since -people rarely concern themselves with fractional ages, they must increment an -age counter at some point, and doing so on a birthday, which leads to a -"half-open" semantic, makes sense. But a collection of items like a list, -array, or string in Python usually does not represent a continuous quantity -which is discretized, but rather a quantity that is naturally discrete. So -while half-open 0-indexed semantics are perfectly reasonable for human ages, -the same argument doesn't make sense for collections in Python. - -When it comes to indexing, half-open semantics are problematic for a few -reasons: - -- A commonly touted benefit of half-open slicing semantics is that you can - "glue" half-open intervals together. For example, `a[0:N] + a[N:M]` is the - same as `a[0:M]`. But `a[1:N] + a[N+1:M]` is just as clear. People are - perfectly used to adding 1 to get to the next term in a sequence, and it is - more natural to see `[1:N]` and `[N+1:M]` as being non-overlapping if they - do not share endpoint values. Ranges that include both endpoints are - standard in both mathematics and everyday language. $\sum_{i=1}^n$ means a - summation from $1$ to $n$ inclusive. Formulas like $\sum_{i=1}^n a_i = - \sum_{i=1}^k a_i + \sum_{i=k+1}^n a_i$ are natural to anyone who has studied - enough mathematics. If you were to rewrite the sentences "the first $n$ - numbers are $1\ldots n$; the next $n$ numbers are $n+1\ldots 2n$", or "'the - 70s' refers to the years 1970--1979" using half-open semantics, anyone would - tell you they were phrased wrong. - -- Another benefit of half-open intervals is that they allow the range `a[i:j]` - to contain $j - i$ elements (assuming $0 \leq i \leq j$ and `a` is large - enough). I tout this myself in the guide above, since it is a useful [sanity - check](sanity-check). However, as useful as it is, it isn't worth the more - general confusion caused by half-open semantics. I contend people are - perfectly used to the usual [fencepost](fencepost) offset that a range - $i\ldots j$ contains $j - i + 1$ numbers. Half-open semantics replace this - fencepost error with more subtle ones, which arise from forgetting that the - range doesn't include the endpoint, unlike most natural ranges that occur in - day-to-day life. See [wrong rule 3](wrong-rule-3) above for an example of how - half-open semantics can lead to subtle fencepost errors. - - It is true that including both endpoints in range can lead to [fencepost - errors](fencepost). But the fencepost problem is fundamentally unavoidable. A - 100 foot fence truly has one more fencepost than fence lengths. The best way - to deal with the fencepost problem is not to try to change the way we count - fenceposts, so that somehow 11 fenceposts is really only - 10.[^fencepost-footnote] It is rather to reuse the most natural and intuitive - way of thinking about the problem, which occurs both in programming and - non-programming contexts, which is that certain quantities, like the number of - elements in a range $1\ldots N$ will require an extra "+ 1" to be correct. - -[^fencepost-footnote]: [This blog - post](https://betterexplained.com/articles/learning-how-to-count-avoiding-the-fencepost-problem/) - has a nice writeup of *why* the fencepost problem exists. It's related to the - difference between measurement and enumeration that I touched on earlier. - -- Half-open semantics become particularly confusing when the step is negative. - This is because one must remember that the end that is not included in the - half-open interval is the second index in the slice, *not* the larger index - (see wrong rules [1](wrong-rule-1) and [3](wrong-rule-3) above). Were both - endpoints included, this confusion would be impossible, because positive and - negative steps would be symmetric in this regard. - - -In general, half-open semantics are naively superior because they have some -properties that appear to be nice (easy unions, no +1s in length formulas). -But the "niceness" of these properties ignores the fact that most people are -already used to closed-closed intervals from mathematics and from everyday -life, and so are used to accounting for them already. So while these -properties are nice, they also break the natural intuition of how ranges work. -Half-open semantics are also closely tied to 0-based indexing, which as I -argued above, is itself problematic for many of the same reasons. - -Again, there is no way Python itself can change any of these things at this -point. It would be way too big of a change to the language, far bigger than -any change that was made as part of Python 3 (and the Python developers have -already stated that they will never do a big breaking change like Python 3 -again). But I hope I can inspire new languages and DSLs that include slicing -semantics to be written in clearer ways. And I also hope that I can break some -of the cognitive dissonance that leads people to believe that the Python -slicing rules are superior, despite the endless confusion that they provide. -Finally, I believe that simply understanding that Python has made these -decisions, whether you agree with them or not, will help you to remember the -slicing [rules](rules), and that's my true goal here. - -## Footnotes -<!-- Footnotes are written inline above but markdown will put them here at the -end of the document. --> diff --git a/docs/style-guide.md b/docs/style-guide.md index ccb5d26b..549fa749 100644 --- a/docs/style-guide.md +++ b/docs/style-guide.md @@ -1,7 +1,7 @@ # Documentation Style Guide ```{note} -This document is still a work in progress +This document is still a work in progress. ``` This is a style guide for the ndindex documentation. @@ -10,7 +10,7 @@ This is a style guide for the ndindex documentation. - Use American English spelling for all documentation. - The Oxford comma should be used. -- "ndindex" is always written in all lowercase, unless referring to the +- "ndindex" is always written in all lowercase unless referring to the `NDIndex` base class. - The plural of "index" is "indices". "Indexes" should only be used as a verb. For example, "in `a[i, j]`, the indices are `i` and `j`. They represent a @@ -18,17 +18,16 @@ This is a style guide for the ndindex documentation. - The arguments of a slice should be referred to as "start", "stop", and "step", respectively. This matches the argument names and attributes of the `Slice` and `slice` objects. -- A generic index variable should be called `idx`. -- A generic slice variable should be called `s`. -- Example array variables should be called `a`. -- The more concise Python notation for indices should be used where it is - allowed in doctests and code examples, unless not using an ndindex type - would lead to confusion or ambiguity. For example, `Tuple` always converts - its arguments to ndindex types, so `Tuple(slice(1, 2), ..., 3)` is preferred +- A generic index variable should be named `idx`. +- A generic slice variable should be named `s`. +- Example array variables should be named `a`. +- The more concise Python notation for indices should be used in doctests and + code examples where it is allowed unless not using an ndindex type would + lead to confusion or ambiguity. For example, `Tuple` always converts its + arguments to ndindex types, so `Tuple(slice(1, 2), ..., 3)` is preferred over `Tuple(Slice(1, 2), ellipsis(), Integer(3))`. -- `...` should be used in place of `Ellipsis` or `ellipsis()` wherever - possible (this and the previous rule also apply to the code, not just - documentation examples). +- Use `...` instead of `Ellipsis` or `ellipsis()` wherever possible (this and + the previous rule apply to the code as well as to documentation examples). - The above rules are only guides. If following them to the word would lead to ambiguity, they should be broken. @@ -38,7 +37,7 @@ This is a style guide for the ndindex documentation. use [MyST](https://myst-parser.readthedocs.io/en/latest/). Markdown is generally preferred over RST, although in some cases RST is required if something isn't supported by MyST. -- Both RST and MyST support cross references. In RST, a cross reference looks +- Both RST and MyST support cross-references. In RST, a cross-reference looks like ``` :ref:`reference` ```. In MyST, use either `[link text](reference)` or ``` {ref}`link text <reference>` ```. See the [MyST docs](https://myst-parser.readthedocs.io/en/latest/using/syntax.html) and @@ -49,14 +48,17 @@ This is a style guide for the ndindex documentation. like ``` `code` ```) whenever it refers to a variable or expression in code. Note that only single backticks are required even for RST as the default role is set to `'code'`. -- Inline mathematical formulas can be formatted with single dollar signs, like - `$x^2 + 1$`, which creates $x^2 + 1$. Display formulas, which appear on - their own line, should use the `.. math::` directive for RST or ```` - ```{math} ```` for Markdown. +- Format inline mathematical formulas with single dollar signs, e.g., `$x^2 + + 1$`, which displays as $x^2 + 1$. Display formulas, which appear on their + own line, should use the `.. math::` directive for RST or + ```` + ```{math} + ```` + for Markdown. - Docstrings are currently written in RST. We may move to Markdown at some point. Docstrings use the napoleon extension, meaning they can be written in numpydoc format. -- All public docstrings should include a doctest. Doctests can be run with the +- All public docstrings should include a doctest, which can be run with the `./run_doctests` script at the root of the repo. This also runs doctests in the RST and Markdown files. Doctests are configured so that the doctests for each function or method must import all names used in that function or diff --git a/docs/type-confusion.md b/docs/type-confusion.md index 7ff3a6ba..91d55f06 100644 --- a/docs/type-confusion.md +++ b/docs/type-confusion.md @@ -1,3 +1,4 @@ +(type-confusion)= Type Confusion ============== @@ -20,18 +21,18 @@ Some general types to help avoid type confusion: can use the `slice` built-in object to create slices. `slice(a, b, c)` is the same as `a:b:c`. - **Right:** - - ```py - idx.as_subindex((1,)) - ``` - **Wrong:** ``` idx.as_subindex(Tuple(Integer(1))) # More verbose than necessary ``` + **Right:** + + ```py + idx.as_subindex((1,)) + ``` + - **Keep all index objects as ndindex types until performing actual indexing.** If all you are doing with an index is indexing it, and not manipulating it or storing it somewhere else, there is no need to use @@ -44,7 +45,7 @@ Some general types to help avoid type confusion: this: - Raw types (such as `int`, `slice`, `tuple`, `array`, and so on), do not - have any of the same methods as ndindex, so your code may fail + have any of the same methods as ndindex, so your code may fail. - Some raw types, such as slices, arrays, and tuples containing slices or arrays, are not hashable, so if you try to use them as a dictionary key, @@ -111,6 +112,10 @@ Some general types to help avoid type confusion: ... ``` + Additionally, all ndindex types are immutable, including types + representing NumPy arrays, so it is impossible to accidentally mutate an + ndindex array index object. + - **Use `.raw` to convert an ndindex object to an indexable type.** With the exception of `Integer`, it is impossible for custom types to define themselves as indices to NumPy arrays, so it is necessary to use @@ -163,20 +168,22 @@ Some general types to help avoid type confusion: - **Try to use ndindex methods to manipulate indices.** The whole reason ndindex exists is that writing formulas for manipulating indices is hard, - and it's easy to get the corner cases wrong. If you find yourself - manipulating index args directly in complex ways, it's a sign you should - probably be using a higher level abstraction. If what you are trying to do - doesn't exist yet, [open an - issue](https://github.com/Quansight-labs/ndindex/issues) so we can implement it. + and it's easy to get the corner cases wrong. ndindex is [rigorously + tested](testing) so you can be highly confident of its correctness. If you + find yourself manipulating index args directly in complex ways, it's a sign + you should probably be using a higher level abstraction. If what you are + trying to do doesn't exist yet, [open an + issue](https://github.com/Quansight-labs/ndindex/issues) so we can implement + it. Additionally, some advice for specific types: ## Integer - **{class}`~.Integer` should not be thought of as an int type.** It - represents integers **as indices**. It is not usable in contexts where - integers are usable. For example, arithmetic will not work on it. If you - need to manipulate the integer index as an integer, use `idx.raw`. + represents integers **as indices**. It is not usable in other contexts where + ints are usable. For example, arithmetic will not work on it. If you need to + manipulate the `Integer` index as an `int`, use `idx.raw`. **Right:** @@ -431,14 +438,14 @@ for `None`. **Right:** ```py - idx = IntegerArray(array([0, 1])) + idx = IntegerArray(np.array([0, 1])) idx.array[0] ``` **Wrong:** ```py - idx = IntegerArray(array([0, 1])) + idx = IntegerArray(np.array([0, 1])) idx[0] # Gives an error ``` @@ -453,7 +460,7 @@ for `None`. **Right:** ```py - idx = IntegerArray(array([0, 1])) + idx = IntegerArray(np.array([0, 1])) arr = idx.array.copy() arr[0] = 1 idx2 = IntegerArray(arr) @@ -462,6 +469,6 @@ for `None`. **Wrong:** ```py - idx = IntegerArray(array([0, 1])) + idx = IntegerArray(np.array([0, 1])) idx.array[0] = 1 # Gives an error ``` diff --git a/ndindex/chunking.py b/ndindex/chunking.py index b1570a2b..634d0b12 100644 --- a/ndindex/chunking.py +++ b/ndindex/chunking.py @@ -323,9 +323,10 @@ def num_subchunks(self, idx, shape): if len(shape) != len(self): raise ValueError("chunks dimensions must equal the array dimensions") + idx = ndindex(idx).expand(shape) + if 0 in shape: return 0 - idx = ndindex(idx).expand(shape) if idx.isempty(shape): return 0 diff --git a/ndindex/ellipsis.py b/ndindex/ellipsis.py index 81cf2291..a1dc14c3 100644 --- a/ndindex/ellipsis.py +++ b/ndindex/ellipsis.py @@ -38,7 +38,8 @@ class ellipsis(NDIndex): ndindex contexts, `...` can be used instead of `ellipsis()`, for instance, when creating a `Tuple` object. Also unlike `Ellipsis`, `ellipsis()` is not singletonized, so you should not use `is` to - compare it. See the document on :any:`type-confusion` for more details. + compare it. See the document on :any:`type confusion + <type-confusion-ellipsis>` for more details. """ __slots__ = () diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index 6bb4cf13..71225e5a 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -222,18 +222,9 @@ class should redefine the following methods - `raw` (a **@property** method) should return the raw index that can be passed as an index to a numpy array. - In addition other methods should be defined as necessary. - - - `__len__` should return the largest possible shape of an axis sliced by - the index (for single-axis indices), or raise ValueError if no such - maximum exists. - - - `reduce(shape=None)` should reduce an index to an equivalent form for - arrays of shape `shape`, or raise an `IndexError`. The error messages - should match numpy as much as possible. The class of the equivalent - index may be different. If `shape` is `None`, it should return a - canonical form that is equivalent for all array shapes (assuming no - IndexErrors). + In addition other methods on this should be re-defined as necessary. Some + methods have a default implementation on this class, which is sufficient + for some subclasses. The methods `__init__` and `__eq__` should *not* be overridden. Equality (and hashability) on `NDIndex` subclasses is determined by equality of @@ -410,7 +401,8 @@ def expand(self, shape): combined into a single term (the same as with :meth:`.Tuple.reduce`). - Non-scalar :class:`~.BooleanArray`\ s are all converted into - equivalent :class:`~.IntegerArray`\ s via `nonzero()` and broadcast. + equivalent :class:`~.IntegerArray`\ s via `nonzero()` and + broadcasted. >>> from ndindex import Tuple, Slice >>> Slice(None).expand((2, 3)) @@ -570,10 +562,10 @@ def isempty(self, shape=None): `None`, isempty() will return `True` when `self` is always empty for any array shape. However, if it gives `False`, it could still give an empty array for some array shapes, but not all. If you know the shape - of the array that will be indexed, you can call `idx.isempty(shape)` - first and the result will be correct for arrays of shape `shape`. If - `shape` is given and `self` would raise an `IndexError` on an array of - shape `shape`, `isempty()` also raises `IndexError`. + of the array that will be indexed, use `idx.isempty(shape)` and the + result will be correct for arrays of shape `shape`. If `shape` is + given and `self` would raise an `IndexError` on an array of shape + `shape`, `isempty()` also raises `IndexError`. >>> from ndindex import Tuple, Slice >>> Tuple(0, slice(0, 1)).isempty() diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 061c47c5..bfba1ee4 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -45,7 +45,7 @@ def broadcast_shapes(*shapes, skip_axes=()): """ Broadcast the input shapes `shapes` to a single shape. - This is the same as :py:func:`np.broadcast_shapes() + This is the same as :external+numpy:py:func:`np.broadcast_shapes() <numpy.broadcast_shapes>`, except is also supports skipping axes in the shape with `skip_axes`. @@ -115,7 +115,7 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): tuple of indices, one for each shape, which would correspond to the same elements if the arrays of the given shapes were first broadcast together. - This is a generalization of the NumPy :py:class:`np.ndindex() + This is a generalization of the NumPy :external+numpy:py:class:`np.ndindex() <numpy.ndindex>` function (which otherwise has no relation). `np.ndindex()` only iterates indices for a single shape, whereas `iter_indices()` supports generating indices for multiple broadcast diff --git a/ndindex/slice.py b/ndindex/slice.py index 80b8491a..494c9fe4 100644 --- a/ndindex/slice.py +++ b/ndindex/slice.py @@ -32,8 +32,8 @@ class Slice(NDIndex): because Python itself does not make the distinction between `x:y` and `x:y:` syntactically. - See :ref:`slices-docs` for a description of the semantic meaning of slices - on arrays. + See :doc:`../indexing-guide/slices` for a description of the semantic + meaning of slices on arrays. Slice has attributes `start`, `stop`, and `step` to access the corresponding attributes. @@ -49,7 +49,7 @@ class Slice(NDIndex): >>> s.raw slice(None, 10, None) - For most use-cases, it's more convenient to create Slice objects using + For most use cases, it's more convenient to create Slice objects using `ndindex[slice]`, which allows using `a:b` slicing syntax: >>> from ndindex import ndindex @@ -215,11 +215,12 @@ def reduce(self, shape=None, *, axis=0, negative_int=False): given shape, or for any shape if `shape` is `None` (the default). `Slice.reduce` is a perfect canonicalization, meaning that two slices - are equal---for all array shapes if `shape=None` or for arrays of shape - `shape` otherwise---if and only if they `reduce` to the same Slice - object. Note that ndindex objects do not simplify automatically, and - `==` only does exact equality comparison, so to test that two slices - are equal, use `slice1.reduce(shape) == slice2.reduce(shape)`. + are equal---for all array shapes if `shape=None` or for arrays of + shape `shape` otherwise---if and only if they `reduce` to the same + `Slice` object. Note that ndindex objects do not simplify + automatically, and `==` only does exact equality comparison, so to + test that two slices are equal, use `slice1.reduce(shape) == + slice2.reduce(shape)`. - If `shape` is `None`, the following properties hold after calling `reduce()`: diff --git a/ndindex/tests/doctest.py b/ndindex/tests/doctest.py index 55a0a02e..f16ba9f9 100644 --- a/ndindex/tests/doctest.py +++ b/ndindex/tests/doctest.py @@ -24,7 +24,8 @@ import os from contextlib import contextmanager from doctest import (DocTestRunner, DocFileSuite, DocTestSuite, - NORMALIZE_WHITESPACE, register_optionflag) + NORMALIZE_WHITESPACE, ELLIPSIS, register_optionflag) +import doctest SKIPNP1 = register_optionflag("SKIPNP1") NP1 = numpy.__version__.startswith('1') @@ -41,6 +42,7 @@ def patch_doctest(): The doctester must be patched """ orig_run = DocTestRunner.run + orig_indent = doctest._indent def run(self, test, **kwargs): filtered_examples = [] @@ -56,11 +58,18 @@ def run(self, test, **kwargs): test.examples = filtered_examples return orig_run(self, test, **kwargs) + # Doctest indents the output, which is annoying for copy-paste, so disable + # it. + def _indent(s, **kwargs): + return s + try: DocTestRunner.run = run + doctest._indent = _indent yield finally: DocTestRunner.run = orig_run + doctest._indent = orig_indent DOCS = os.path.realpath(os.path.join(__file__, os.path.pardir, os.path.pardir, os.pardir, 'docs')) @@ -75,14 +84,15 @@ def load_tests(loader, tests, ignore): tests.addTests(DocTestSuite(sys.modules[mod], globs={}, optionflags=NORMALIZE_WHITESPACE)) tests.addTests(DocFileSuite(*MARKDOWN, *RST, README, - optionflags=NORMALIZE_WHITESPACE, + optionflags=NORMALIZE_WHITESPACE | ELLIPSIS, module_relative=False)) return tests -def doctest(): +def run_doctests(): + numpy.seterr(all='ignore') with patch_doctest(): return unittest.main(module='ndindex.tests.doctest', exit=False).result if __name__ == '__main__': # TODO: Allow specifying which doctests to run at the command line - doctest() + run_doctests() diff --git a/ndindex/tests/test_chunking.py b/ndindex/tests/test_chunking.py index 650d5375..ac718613 100644 --- a/ndindex/tests/test_chunking.py +++ b/ndindex/tests/test_chunking.py @@ -174,6 +174,8 @@ def test_as_subchunks(chunk_size, idx, shape): def test_as_subchunks_error(): raises(ValueError, lambda: next(ChunkSize((1, 2)).as_subchunks(..., (1, 2, 3)))) +@example(chunk_size=(1,), idx=(slice(None), slice(None)), shape=(0,)) +@example(chunk_size=(), idx=(), shape=()) @example(chunk_size=(1, 1), idx=[[False, True], [True, True]], shape=(2, 2)) @example(chunk_size=(1,), idx=None, shape=(1,)) diff --git a/ndindex/tuple.py b/ndindex/tuple.py index cdf6969e..fe8e1836 100644 --- a/ndindex/tuple.py +++ b/ndindex/tuple.py @@ -236,9 +236,9 @@ def reduce(self, shape=None, *, negative_int=False): ==== ndindex presently does not distinguish between scalar objects and - rank-0 arrays. It is possible for the original index to produce one + 0-D arrays. It is possible for the original index to produce one and the reduced index to produce the other. In particular, the - presence of a redundant ellipsis forces NumPy to return a rank-0 array + presence of a redundant ellipsis forces NumPy to return a 0-D array instead of a scalar. >>> import numpy as np diff --git a/requirements-dev.txt b/requirements-dev.txt index 25cd7f8c..3fea5925 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,5 @@ hypothesis +matplotlib numpy packaging pyflakes @@ -6,4 +7,5 @@ pytest pytest-cov pytest-doctestplus pytest-flakes +scikit-image slotscheck diff --git a/run_doctests b/run_doctests index 39160720..fa08400d 100755 --- a/run_doctests +++ b/run_doctests @@ -2,5 +2,5 @@ import sys import ndindex.tests.doctest -tests = ndindex.tests.doctest.doctest() +tests = ndindex.tests.doctest.run_doctests() sys.exit(bool(tests.errors or tests.failures))