Merge remote-tracking branch 'upstream/main' into sty/ruff/docstrings

.circleci/config.yml

@@ -18,6 +18,29 @@ jobs:
           PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH
           LD_PRELOAD=$HOME/miniconda3/envs/pandas-dev/lib/libgomp.so.1:$LD_PRELOAD
           ci/run_tests.sh
+  linux-musl:
+    docker:
+      - image: quay.io/pypa/musllinux_1_1_aarch64
+    resource_class: arm.large
+    steps:
+      # Install pkgs first to have git in the image
+      # (needed for checkout)
+      - run: |
+          apk update
+          apk add git
+          apk add musl-locales
+      - checkout
+      - run: |
+          /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
+          . ~/virtualenvs/pandas-dev/bin/activate
+          python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
+          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
+          python -m pip list --no-cache-dir
+      - run: |
+          . ~/virtualenvs/pandas-dev/bin/activate
+          export PANDAS_CI=1
+          python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml
   build-aarch64:
     parameters:
       cibw-build:
@@ -89,6 +112,13 @@ workflows:
         equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
     jobs:
       - test-arm
+  test-musl:
+    # Don't run trigger this one when scheduled pipeline runs
+    when:
+      not:
+        equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
+    jobs:
+      - linux-musl
   build-wheels:
     jobs:
       - build-aarch64:
@@ -97,4 +127,11 @@ workflows:
               only: /^v.*/
           matrix:
             parameters:
-              cibw-build: ["cp39-manylinux_aarch64", "cp310-manylinux_aarch64", "cp311-manylinux_aarch64", "cp312-manylinux_aarch64"]
+              cibw-build: ["cp39-manylinux_aarch64",
+                           "cp310-manylinux_aarch64",
+                           "cp311-manylinux_aarch64",
+                           "cp312-manylinux_aarch64",
+                           "cp39-musllinux_aarch64",
+                           "cp310-musllinux_aarch64",
+                           "cp311-musllinux_aarch64",
+                           "cp312-musllinux_aarch64",]

.github/actions/build_pandas/action.yml

@@ -4,6 +4,12 @@ inputs:
   editable:
     description: Whether to build pandas in editable mode (default true)
     default: true
+  meson_args:
+    description: Extra flags to pass to meson
+    required: false
+  cflags_adds:
+    description: Items to append to the CFLAGS variable
+    required: false
 runs:
   using: composite
   steps:
@@ -24,11 +30,12 @@ runs:
 
     - name: Build Pandas
       run: |
+        export CFLAGS="$CFLAGS ${{ inputs.cflags_adds }}"
         if [[ ${{ inputs.editable }} == "true" ]]; then
-          pip install -e . --no-build-isolation -v --no-deps \
+          pip install -e . --no-build-isolation -v --no-deps ${{ inputs.meson_args }} \
             --config-settings=setup-args="--werror"
         else
-          pip install . --no-build-isolation -v --no-deps \
+          pip install . --no-build-isolation -v --no-deps ${{ inputs.meson_args }} \
             --config-settings=setup-args="--werror"
         fi
       shell: bash -el {0}

.github/actions/run-tests/action.yml

@@ -1,9 +1,16 @@
 name: Run tests and report results
+inputs:
+  preload:
+    description: Preload arguments for sanitizer
+    required: false
+  asan_options:
+    description: Arguments for Address Sanitizer (ASAN)
+    required: false
 runs:
   using: composite
   steps:
     - name: Test
-      run: ci/run_tests.sh
+      run: ${{ inputs.asan_options }} ${{ inputs.preload }} ci/run_tests.sh
       shell: bash -el {0}
 
     - name: Publish test results

.github/workflows/deprecation-tracking-bot.yml

@@ -19,7 +19,7 @@ jobs:
       issues: write
     runs-on: ubuntu-22.04
     env:
-      DEPRECATION_TRACKER_ISSUE: 50578
+      DEPRECATION_TRACKER_ISSUE: 56596
     steps:
     - uses: actions/github-script@v7
       id: update-deprecation-issue

.github/workflows/unit-tests.yml

@@ -92,10 +92,18 @@ jobs:
           - name: "Numpy Dev"
             env_file: actions-311-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
-            test_args: "-W error::DeprecationWarning -W error::FutureWarning"
+            test_args: "-W error::FutureWarning"
           - name: "Pyarrow Nightly"
             env_file: actions-311-pyarrownightly.yaml
             pattern: "not slow and not network and not single_cpu"
+          - name: "ASAN / UBSAN"
+            env_file: actions-311-sanitizers.yaml
+            pattern: "not slow and not network and not single_cpu and not skip_ubsan"
+            asan_options: "ASAN_OPTIONS=detect_leaks=0"
+            preload: LD_PRELOAD=$(gcc -print-file-name=libasan.so)
+            meson_args: --config-settings=setup-args="-Db_sanitize=address,undefined"
+            cflags_adds: -fno-sanitize-recover=all
+            pytest_workers: -1  # disable pytest-xdist as it swallows stderr from ASAN
       fail-fast: false
     name: ${{ matrix.name || format('ubuntu-latest {0}', matrix.env_file) }}
     env:
@@ -105,8 +113,9 @@ jobs:
       PANDAS_COPY_ON_WRITE: ${{ matrix.pandas_copy_on_write || '0' }}
       PANDAS_CI: ${{ matrix.pandas_ci || '1' }}
       TEST_ARGS: ${{ matrix.test_args || '' }}
-      PYTEST_WORKERS: 'auto'
+      PYTEST_WORKERS: ${{ matrix.pytest_workers || 'auto' }}
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
+      NPY_PROMOTION_STATE: ${{ matrix.env_file == 'actions-311-numpydev.yaml' && 'weak' || 'legacy' }}
       # Clipboard tests
       QT_QPA_PLATFORM: offscreen
     concurrency:
@@ -174,16 +183,25 @@ jobs:
     - name: Build Pandas
       id: build
       uses: ./.github/actions/build_pandas
+      with:
+        meson_args: ${{ matrix.meson_args }}
+        cflags_adds: ${{ matrix.cflags_adds }}
 
     - name: Test (not single_cpu)
       uses: ./.github/actions/run-tests
       if: ${{ matrix.name != 'Pypy' }}
+      with:
+        preload: ${{ matrix.preload }}
+        asan_options: ${{ matrix.asan_options }}
       env:
         # Set pattern to not single_cpu if not already set
         PATTERN: ${{ env.PATTERN == '' && 'not single_cpu' || matrix.pattern }}
 
     - name: Test (single_cpu)
       uses: ./.github/actions/run-tests
+      with:
+        preload: ${{ matrix.preload }}
+        asan_options: ${{ matrix.asan_options }}
       env:
         PATTERN: 'single_cpu'
         PYTEST_WORKERS: 0

.pre-commit-config.yaml

@@ -27,6 +27,8 @@ repos:
         # TODO: remove autofixe-only rules when they are checked by ruff
         name: ruff-selected-autofixes
         alias: ruff-selected-autofixes
+        files: ^pandas
+        exclude: ^pandas/tests
         args: [--select, "ANN001,ANN2", --fix-only, --exit-non-zero-on-fix]
     -   id: ruff-format
 -   repo: https://github.com/jendrikseipp/vulture

CITATION.cff

@@ -3,12 +3,50 @@ title: 'pandas-dev/pandas: Pandas'
 message: 'If you use this software, please cite it as below.'
 authors:
   - name: "The pandas development team"
+    website: "https://pandas.pydata.org/about/team.html"
 abstract: "Pandas is a powerful data structures for data analysis, time series, and statistics."
+doi: 10.5281/zenodo.3509134
 license: BSD-3-Clause
 license-url: "https://github.com/pandas-dev/pandas/blob/main/LICENSE"
 repository-code: "https://github.com/pandas-dev/pandas"
 keywords:
   - python
   - data science
+  - flexible
+  - pandas
+  - alignment
+  - data analysis
 type: software
-url: "https://github.com/pandas-dev/pandas"
+url: "https://pandas.pydata.org/"
+references:
+  - type: article
+    authors:
+      - given-names: Wes
+        family-names: McKinney
+        affiliation: AQR Capital Management, LLC
+        email: [email protected]
+    title: Data Structures for Statistical Computing in Python
+    doi: 10.25080/Majora-92bf1922-00a
+    license: CC-BY-3.0
+    start: 56
+    end: 61
+    year: 2010
+    collection-title: Proceedings of the 9th Python in Science Conference
+    collection-doi: 10.25080/Majora-92bf1922-012
+    collection-type: proceedings
+    editors:
+      - given-names: Stéfan
+        name-particle: van der
+        family-names: Walt
+      - given-names: Jarrod
+        family-names: Millman
+    conference:
+      name: 9th Python in Science Conference (SciPy 2010)
+      city: Austin, TX
+      country: US
+      date-start: "2010-06-28"
+      date-end: "2010-07-03"
+    keywords:
+      - data structure
+      - statistics
+      - R

ci/deps/actions-311-sanitizers.yaml

@@ -0,0 +1,32 @@
+name: pandas-dev
+channels:
+  - conda-forge
+dependencies:
+  - python=3.11
+
+  # build dependencies
+  - versioneer[toml]
+  - cython>=0.29.33
+  - meson[ninja]=1.2.1
+  - meson-python=0.13.1
+
+  # test dependencies
+  - pytest>=7.3.2
+  - pytest-cov
+  - pytest-xdist>=2.2.0
+  - pytest-localserver>=0.7.1
+  - pytest-qt>=4.2.0
+  - boto3
+  - hypothesis>=6.46.1
+  - pyqt>=5.15.9
+
+  # required dependencies
+  - python-dateutil
+  - numpy<2
+  - pytz
+
+  # pandas dependencies
+  - pip
+
+  - pip:
+    - "tzdata>=2022.7"

ci/run_tests.sh

@@ -10,8 +10,7 @@ echo PYTHONHASHSEED=$PYTHONHASHSEED
 
 COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=pyproject.toml"
 
-# TODO: Support NEP 50 and remove NPY_PROMOTION_STATE
-PYTEST_CMD="NPY_PROMOTION_STATE=legacy MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
+PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
 
 if [[ "$PATTERN" ]]; then
   PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\""