chore: Python 3.12 (SCE-16) (#1100)

.github/workflows/e2e-tests.yml

@@ -23,10 +23,10 @@ jobs:
         with:
           # Chromatic needs full Git history graph
           fetch-depth: 0
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v1
         with:
-          python-version: 3.11
+          python-version: 3.12
       - name: Python cache
         uses: actions/cache@v1
         with:

.github/workflows/push_tests.yml

@@ -16,15 +16,15 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - run: |
           git fetch --depth=1 origin +${{github.base_ref}}
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v1
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: 3.12
       - name: Node cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -57,12 +57,12 @@ jobs:
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v1
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: 3.12
       - name: Python cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}

.github/workflows/scale-test.yml

@@ -8,11 +8,11 @@ jobs:
   locust-build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v1
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: 3.12
       - name: Install dependencies
         run: |
           pip install -r server/tests/locust/requirements-locust.txt

.infra/rdev/values.yaml

@@ -2,7 +2,7 @@ stack:
   services:
     explorer:
       image:
-        tag: sha-727744f5
+        tag: sha-5ec059bf
       replicaCount: 1
       env:
         # env vars common to all deployment stages

dev_docs/developer_scripts.md

@@ -55,8 +55,8 @@ Installs requirements files.
 
 **Usage:** from the `$PROJECT_ROOT/client` directory run `make start-frontend`
 
-NB: the frontend server reads in the desired base_url and dataset name to form the complete url base for API calls. _In
-order to use an arbitrary dataset successfully, the frontend server must be started **after** the backend server_, which
+NB: the frontend server reads in the desired base_url and dataset name to form the complete url base for API calls. \_In
+order to use an arbitrary dataset successfully, the frontend server must be started **after** the backend server\*, which
 writes out the given base_url and dataset anew each time.
 
 #### backend_dev
@@ -67,7 +67,7 @@ the FE developer gets the current version of the backend with a single command
 and no knowledge of python necessary. It creates and activates a virtual
 environment and installs explorer requirements from the current branch.
 
-**Requires** `Python3.6+`, `virtual-env`, `pip`
+**Requires** `Python3.10+`, `virtual-env`, `pip`
 
 **Usage:** from the `$PROJECT_ROOT` directory run `./scripts/backend_dev`
 
@@ -83,7 +83,7 @@ environment and installs explorer requirements from the current branch.
 
 | command                              | purpose                                           |
 | ------------------------------------ | ------------------------------------------------- |
-| python3.11 -m venv explorer          | creates explorer virtual environment              |
+| python3.12 -m venv explorer          | creates explorer virtual environment              |
 | source explorer/bin/activate         | activates virtual environment                     |
 | ./launch_dev_server.sh [cli options] | launches api server (can supply arbitrary config) |
 

server/common/diffexpdu/postingslist.py

@@ -340,7 +340,9 @@ def _un_byteshuffle(buf: Union[bytes, bytearray, memoryview], dtype) -> np.ndarr
 
 
 def _delta(arr: np.ndarray) -> np.ndarray:
-    return np.diff(arr, prepend=arr.dtype.type(0))
+    delta_arr = arr.astype(np.int64)  # Cast to higher precision
+    delta_arr[1:] -= delta_arr[:-1]  # Subtract to compute delta
+    return delta_arr.astype(np.uint16)  # Cast back to uint16 if needed
 
 
 def _un_delta(arr: np.ndarray) -> np.ndarray:
@@ -365,7 +367,7 @@ def _interval_invert(arr: np.ndarray, interval=None) -> np.ndarray:
     assert arr.dtype.kind == "u"
     assert interval is not None or len(arr) > 0
     if interval is None:
-        interval = (arr[0], arr[-1] + 1)
+        interval = (int(arr[0]), int(arr[-1]) + 1)  # Cast to int to prevent overflow
     assert len(arr) == 0 or (arr[0] >= interval[0] and arr[0] < interval[1])
     assert len(arr) == 0 or (arr[-1] >= interval[0] and arr[-1] < interval[1])
     return _interval_invert_inner(

server/common/fbs/matrix.py

@@ -72,7 +72,7 @@ def encode_matrix_fbs(matrix, row_idx=None, col_idx=None, num_bins=None):
         raise ValueError("FBS Matrix must be 2D")
 
     if sparse.issparse(matrix):
-        matrix = matrix.A
+        matrix = matrix.toarray()
 
     (n_rows, n_cols) = matrix.shape
     # estimate size needed, so we don't unnecessarily realloc.

server/requirements.txt

@@ -8,7 +8,7 @@ flask-compress==1.13
 flask-cors==5.0.0
 flask-restful==0.3.9
 flask-talisman==1.0.0
-flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires migration
+flatbuffers==2.0.7 # flatbuffers generation has not been tested on any version > 2.0.0 yet
 flatten_dict==0.4.2
 fsspec>=0.4.4,<0.8.0
 gunicorn[gevent]==20.0.4
@@ -17,7 +17,7 @@ numpy>=1.24.0,<2.1.0
 pandas>=2.2.2
 pydantic==1.10.13
 requests==2.32.3
-scipy==1.11.1
+scipy==1.14.1
 flask-server-timing==0.1.2
 s3fs==0.4.2
 tiledb # unpinned to match Portal's requirements

server/tests/unit/common/utils/test_type_conversion_utils.py

@@ -133,7 +133,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         np.arange(-128, 1000, dtype=dtype),
         pd.Series(np.arange(-128, 1000, dtype=dtype)),
         pd.Index(np.arange(-129, 1000, dtype=dtype)),
-        np.array([-np.nan, np.NINF, -1, np.NZERO, 0, np.PZERO, 1, np.PINF, np.nan], dtype=dtype),
+        np.array([-np.nan, -np.inf, -1, -0.0, 0, 0.0, 1, np.inf, np.nan], dtype=dtype),
         np.array([np.finfo(dtype).min, 0, np.finfo(dtype).max], dtype=dtype),
         sparse.csr_matrix((10, 100), dtype=dtype),
     ]

server/tests/unit/dataset/test_cxg_dataset.py

@@ -51,7 +51,11 @@ def test_tdb_bug(self):
                 )
                 dat = data.summarize_var("mean", filt, 0)
                 summary = decode_fbs.decode_matrix_FBS(dat)
-                self.assertDictContainsSubset({"n_rows": 2638, "n_cols": 1, "row_idx": None}, summary)
+
+                expected_subset = {"n_rows": 2638, "n_cols": 1, "row_idx": None}
+                actual_subset = {key: summary[key] for key in expected_subset if key in summary}
+
+                self.assertEqual(expected_subset, actual_subset)
                 self.assertIs(type(summary["columns"]), list)
                 self.assertEqual(len(summary["columns"]), 1)
                 self.assertEqual(len(summary["columns"][0]), 2638)
@@ -74,7 +78,11 @@ def test_tdb_bug_lossy(self):
         )
         dat = data.summarize_var("mean", filt, 0, 500)
         summary = decode_fbs.decode_matrix_FBS(dat)
-        self.assertDictContainsSubset({"n_rows": 2638, "n_cols": 1, "row_idx": None}, summary)
+
+        expected_subset = {"n_rows": 2638, "n_cols": 1, "row_idx": None}
+        actual_subset = {key: summary[key] for key in expected_subset if key in summary}
+
+        self.assertEqual(expected_subset, actual_subset)
         self.assertIs(type(summary["columns"]), list)
         self.assertEqual(len(summary["columns"]), 1)
         self.assertEqual(len(summary["columns"][0]), 2638)