From f767f66dd97605a04fff5e2f985fe412ce37cb54 Mon Sep 17 00:00:00 2001 From: Ronen Date: Thu, 12 Sep 2024 12:12:55 -0400 Subject: [PATCH] chore: Python 3.12 (SCE-16) (#1100) --- .github/workflows/e2e-tests.yml | 4 ++-- .github/workflows/push_tests.yml | 18 +++++++++--------- .github/workflows/scale-test.yml | 8 ++++---- .infra/rdev/values.yaml | 2 +- dev_docs/developer_scripts.md | 8 ++++---- server/common/diffexpdu/postingslist.py | 6 ++++-- server/common/fbs/matrix.py | 2 +- server/requirements.txt | 4 ++-- .../common/utils/test_type_conversion_utils.py | 2 +- server/tests/unit/dataset/test_cxg_dataset.py | 12 ++++++++++-- 10 files changed, 38 insertions(+), 28 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index fb0e10b39..00e51cd9a 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -23,10 +23,10 @@ jobs: with: # Chromatic needs full Git history graph fetch-depth: 0 - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@v1 with: - python-version: 3.11 + python-version: 3.12 - name: Python cache uses: actions/cache@v1 with: diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 1cf9e5485..d062d8872 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -16,15 +16,15 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: | git fetch --depth=1 origin +${{github.base_ref}} - - name: Set up Python 3.11 - uses: actions/setup-python@v1 + - name: Set up Python 3.12 + uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.12 - name: Node cache - uses: actions/cache@v1 + uses: actions/cache@v4 with: path: ~/.npm key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} @@ -57,12 +57,12 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v2 - - name: Set up Python 3.11 - uses: actions/setup-python@v1 + - name: Set up Python 3.12 + uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.12 - name: Python cache - uses: actions/cache@v1 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} diff --git a/.github/workflows/scale-test.yml b/.github/workflows/scale-test.yml index 1068ee484..eaf097adb 100644 --- a/.github/workflows/scale-test.yml +++ b/.github/workflows/scale-test.yml @@ -8,11 +8,11 @@ jobs: locust-build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.11 - uses: actions/setup-python@v1 + - uses: actions/checkout@v4 + - name: Set up Python 3.12 + uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.12 - name: Install dependencies run: | pip install -r server/tests/locust/requirements-locust.txt diff --git a/.infra/rdev/values.yaml b/.infra/rdev/values.yaml index 1ca2f7316..60a0f09c9 100644 --- a/.infra/rdev/values.yaml +++ b/.infra/rdev/values.yaml @@ -2,7 +2,7 @@ stack: services: explorer: image: - tag: sha-727744f5 + tag: sha-5ec059bf replicaCount: 1 env: # env vars common to all deployment stages diff --git a/dev_docs/developer_scripts.md b/dev_docs/developer_scripts.md index c57c037de..f8ace0992 100644 --- a/dev_docs/developer_scripts.md +++ b/dev_docs/developer_scripts.md @@ -55,8 +55,8 @@ Installs requirements files. **Usage:** from the `$PROJECT_ROOT/client` directory run `make start-frontend` -NB: the frontend server reads in the desired base_url and dataset name to form the complete url base for API calls. _In -order to use an arbitrary dataset successfully, the frontend server must be started **after** the backend server_, which +NB: the frontend server reads in the desired base_url and dataset name to form the complete url base for API calls. \_In +order to use an arbitrary dataset successfully, the frontend server must be started **after** the backend server\*, which writes out the given base_url and dataset anew each time. #### backend_dev @@ -67,7 +67,7 @@ the FE developer gets the current version of the backend with a single command and no knowledge of python necessary. It creates and activates a virtual environment and installs explorer requirements from the current branch. -**Requires** `Python3.6+`, `virtual-env`, `pip` +**Requires** `Python3.10+`, `virtual-env`, `pip` **Usage:** from the `$PROJECT_ROOT` directory run `./scripts/backend_dev` @@ -83,7 +83,7 @@ environment and installs explorer requirements from the current branch. | command | purpose | | ------------------------------------ | ------------------------------------------------- | -| python3.11 -m venv explorer | creates explorer virtual environment | +| python3.12 -m venv explorer | creates explorer virtual environment | | source explorer/bin/activate | activates virtual environment | | ./launch_dev_server.sh [cli options] | launches api server (can supply arbitrary config) | diff --git a/server/common/diffexpdu/postingslist.py b/server/common/diffexpdu/postingslist.py index 5e8e0e114..52d8847e8 100644 --- a/server/common/diffexpdu/postingslist.py +++ b/server/common/diffexpdu/postingslist.py @@ -340,7 +340,9 @@ def _un_byteshuffle(buf: Union[bytes, bytearray, memoryview], dtype) -> np.ndarr def _delta(arr: np.ndarray) -> np.ndarray: - return np.diff(arr, prepend=arr.dtype.type(0)) + delta_arr = arr.astype(np.int64) # Cast to higher precision + delta_arr[1:] -= delta_arr[:-1] # Subtract to compute delta + return delta_arr.astype(np.uint16) # Cast back to uint16 if needed def _un_delta(arr: np.ndarray) -> np.ndarray: @@ -365,7 +367,7 @@ def _interval_invert(arr: np.ndarray, interval=None) -> np.ndarray: assert arr.dtype.kind == "u" assert interval is not None or len(arr) > 0 if interval is None: - interval = (arr[0], arr[-1] + 1) + interval = (int(arr[0]), int(arr[-1]) + 1) # Cast to int to prevent overflow assert len(arr) == 0 or (arr[0] >= interval[0] and arr[0] < interval[1]) assert len(arr) == 0 or (arr[-1] >= interval[0] and arr[-1] < interval[1]) return _interval_invert_inner( diff --git a/server/common/fbs/matrix.py b/server/common/fbs/matrix.py index 8e97ec91a..6a4324c9b 100644 --- a/server/common/fbs/matrix.py +++ b/server/common/fbs/matrix.py @@ -72,7 +72,7 @@ def encode_matrix_fbs(matrix, row_idx=None, col_idx=None, num_bins=None): raise ValueError("FBS Matrix must be 2D") if sparse.issparse(matrix): - matrix = matrix.A + matrix = matrix.toarray() (n_rows, n_cols) = matrix.shape # estimate size needed, so we don't unnecessarily realloc. diff --git a/server/requirements.txt b/server/requirements.txt index ce7d36782..31ced36de 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -8,7 +8,7 @@ flask-compress==1.13 flask-cors==5.0.0 flask-restful==0.3.9 flask-talisman==1.0.0 -flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires migration +flatbuffers==2.0.7 # flatbuffers generation has not been tested on any version > 2.0.0 yet flatten_dict==0.4.2 fsspec>=0.4.4,<0.8.0 gunicorn[gevent]==20.0.4 @@ -17,7 +17,7 @@ numpy>=1.24.0,<2.1.0 pandas>=2.2.2 pydantic==1.10.13 requests==2.32.3 -scipy==1.11.1 +scipy==1.14.1 flask-server-timing==0.1.2 s3fs==0.4.2 tiledb # unpinned to match Portal's requirements diff --git a/server/tests/unit/common/utils/test_type_conversion_utils.py b/server/tests/unit/common/utils/test_type_conversion_utils.py index 10f0f323b..8ea10712b 100644 --- a/server/tests/unit/common/utils/test_type_conversion_utils.py +++ b/server/tests/unit/common/utils/test_type_conversion_utils.py @@ -133,7 +133,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): np.arange(-128, 1000, dtype=dtype), pd.Series(np.arange(-128, 1000, dtype=dtype)), pd.Index(np.arange(-129, 1000, dtype=dtype)), - np.array([-np.nan, np.NINF, -1, np.NZERO, 0, np.PZERO, 1, np.PINF, np.nan], dtype=dtype), + np.array([-np.nan, -np.inf, -1, -0.0, 0, 0.0, 1, np.inf, np.nan], dtype=dtype), np.array([np.finfo(dtype).min, 0, np.finfo(dtype).max], dtype=dtype), sparse.csr_matrix((10, 100), dtype=dtype), ] diff --git a/server/tests/unit/dataset/test_cxg_dataset.py b/server/tests/unit/dataset/test_cxg_dataset.py index fc6f32950..18592b809 100644 --- a/server/tests/unit/dataset/test_cxg_dataset.py +++ b/server/tests/unit/dataset/test_cxg_dataset.py @@ -51,7 +51,11 @@ def test_tdb_bug(self): ) dat = data.summarize_var("mean", filt, 0) summary = decode_fbs.decode_matrix_FBS(dat) - self.assertDictContainsSubset({"n_rows": 2638, "n_cols": 1, "row_idx": None}, summary) + + expected_subset = {"n_rows": 2638, "n_cols": 1, "row_idx": None} + actual_subset = {key: summary[key] for key in expected_subset if key in summary} + + self.assertEqual(expected_subset, actual_subset) self.assertIs(type(summary["columns"]), list) self.assertEqual(len(summary["columns"]), 1) self.assertEqual(len(summary["columns"][0]), 2638) @@ -74,7 +78,11 @@ def test_tdb_bug_lossy(self): ) dat = data.summarize_var("mean", filt, 0, 500) summary = decode_fbs.decode_matrix_FBS(dat) - self.assertDictContainsSubset({"n_rows": 2638, "n_cols": 1, "row_idx": None}, summary) + + expected_subset = {"n_rows": 2638, "n_cols": 1, "row_idx": None} + actual_subset = {key: summary[key] for key in expected_subset if key in summary} + + self.assertEqual(expected_subset, actual_subset) self.assertIs(type(summary["columns"]), list) self.assertEqual(len(summary["columns"]), 1) self.assertEqual(len(summary["columns"][0]), 2638)