Drop 3.10, improve/fix scheduled test (#584)

* add python matrix, drop 3.10

* add simbad check

* restrict action to upstream repo

* remove internet check. format output

* up to 3.11

---------

Co-authored-by: David Rodriguez <[email protected]>

Author: kelle

.github/workflows/gen-db.yml

@@ -21,10 +21,10 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Set up Python 3.11
+    - name: Set up Python 3.13
       uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.13'
 
     - name: Install dependencies
       run: |

.github/workflows/scheduled-tests.yml

@@ -1,25 +1,25 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
 name: Scheduled Checks
 
 on:
   schedule:
-    - cron: '30 1 1 * *'
+    - cron: '30 1 1 * *' # Runs at 1:30 AM on the first of every month
   workflow_dispatch:  # manual execution
 
 jobs:
   build:
-
+    if: github.repository == 'SIMPLE-AstroDB/SIMPLE-db'
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.11, 3.12, 3.13]
 
     steps:
     - uses: actions/checkout@v4
 
-    - name: Set up Python 3.11
+    - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: ${{ matrix.python-version }}
 
     - name: Install dependencies
       run: |

pyproject.toml

@@ -16,7 +16,7 @@ authors = [
     {name = "Kelle Cruz", email = "[email protected]"},
     {name = "David Rodriguez", email = "[email protected]"},
 ]
-requires-python = ">= 3.10"
+requires-python = ">= 3.11"
 dependencies = [
     "astrodbkit",
     "astrodb_utils",

simple/utils/astrometry.py

@@ -6,7 +6,9 @@
 from astropy.units import Quantity
 from astropy.table import Table
 from astrodbkit.astrodb import Database
-from astrodb_utils import AstroDBError, find_source_in_db, find_publication
+from astrodb_utils import AstroDBError
+from astrodb_utils.sources import find_source_in_db
+from astrodb_utils.publications import find_publication
 
 
 __all__ = [

simple/utils/companions.py

@@ -1,9 +1,7 @@
 import logging
 import sqlalchemy.exc
-from astrodb_utils import (
-    AstroDBError, find_source_in_db
-)
-
+from astrodb_utils import AstroDBError
+from astrodb_utils.sources import find_source_in_db
 
 __all__ = [
     "ingest_companion_relationships",

simple/utils/spectra.py

@@ -6,9 +6,9 @@
 import sqlalchemy.exc
 from astrodb_utils import (
     AstroDBError,
-    find_source_in_db,
     internet_connection,
 )
+from astrodb_utils.sources import find_source_in_db
 from astrodb_utils.spectra import check_spectrum_plottable
 from astrodbkit.astrodb import Database
 from astropy.io import fits

simple/utils/spectral_types.py

@@ -6,8 +6,8 @@
 from simple.schema import SpectralTypes
 from astrodb_utils import (
     AstroDBError,
-    find_source_in_db,
 )
+from astrodb_utils.sources import find_source_in_db
 
 
 __all__ = [

tests/scheduled_checks.py

@@ -3,67 +3,126 @@
 
 import pytest
 import requests
-from astrodb_utils.utils import internet_connection
-from astrodbkit.astrodb import Database, create_database
+from astrodb_utils import load_astrodb
 from tqdm import tqdm
+from astroquery.simbad import Simbad
+from astrodbkit.utils import _name_formatter
 
 sys.path.append(".")
-from simple.schema import *
 from simple.schema import REFERENCE_TABLES
 
-DB_NAME = "temp.sqlite"
+DB_NAME = "temp_SIMPLE.sqlite"
 DB_PATH = "data"
 
 
-# Load the database for use in individual tests
+# Load the SIMPLE database
 @pytest.fixture(scope="module")
 def db():
-    # Create a fresh temporary database and assert it exists
-    # Because we've imported simple.schema, we will be using that schema for the database
-
-    if os.path.exists(DB_NAME):
-        os.remove(DB_NAME)
-    connection_string = "sqlite:///" + DB_NAME
-    create_database(connection_string)
-    assert os.path.exists(DB_NAME)
 
     # Connect to the new database and confirm it has the Sources table
-    db = Database(connection_string, reference_tables=REFERENCE_TABLES)
-    assert db
-    assert "source" in [c.name for c in db.Sources.columns]
-
-    # Load data into an in-memory sqlite database first, for performance
-
-    # create and connects to a temporary in-memory database
-    temp_db = Database("sqlite://", reference_tables=REFERENCE_TABLES)
+    db = load_astrodb(DB_NAME, data_path=DB_PATH, reference_tables=REFERENCE_TABLES)
 
-    # load the data from the data files into the database
-    temp_db.load_database(DB_PATH, verbose=False)
+    return db
 
-    # dump in-memory database to file
-    temp_db.dump_sqlite(DB_NAME)
-    # replace database object with new file version
-    db = Database("sqlite:///" + DB_NAME, reference_tables=REFERENCE_TABLES)
 
-    return db
+def test_db(db):
+    assert os.path.exists(DB_NAME)
+    assert db
+    assert "source" in [c.name for c in db.Sources.columns]
 
 
 def test_spectra_urls(db):
     spectra_urls = db.query(db.Spectra.c.access_url).astropy()
     broken_urls = []
     codes = []
-    internet, _ = internet_connection()
-    if internet:
-        for spectrum_url in tqdm(spectra_urls["access_url"]):
-            request_response = requests.head(spectrum_url)
-            status_code = request_response.status_code
-            # The website is up if the status code is 200
-            # cuny academic commons links give 301 status code
-            if status_code != 200 and status_code != 301:
-                broken_urls.append(spectrum_url)
-                codes.append(status_code)
+
+    for spectrum_url in tqdm(spectra_urls["access_url"]):
+        request_response = requests.head(spectrum_url)
+        status_code = request_response.status_code
+        # The website is up if the status code is 200
+        # cuny academic commons links give 301 status code
+        if status_code not in (200, 301):
+            broken_urls.append(spectrum_url)
+            codes.append(status_code)
 
     # Display broken spectra regardless if it's the number we expect or not
     print(f"found {len(broken_urls)} broken spectra urls: {broken_urls}, {codes}")
 
-    assert 4 <= len(broken_urls) <= 4
+    assert 5 == len(broken_urls)
+
+# Expected fails:
+# 11123099-7653342.txt',
+# L1_OPT_2MASS_J10595138-2113082_Cruz2003.txt
+# 0000%252B2554_IRS_spectrum.fits'
+# 0415-0935.fits',
+# 2MASS+J22541892%2B3123498.fits'])
+
+
+def test_source_simbad(db):
+    # Query Simbad and confirm that there are no duplicates with different names
+
+    # Get list of all source names
+    results = db.query(db.Sources.c.source).all()
+    name_list = [s[0] for s in results]
+
+    # Add all IDS to the Simbad output as well as the user-provided id
+    Simbad.add_votable_fields("ids")
+
+    print("Querying SIMBAD for all SIMPLE sources")
+    simbad_results = Simbad.query_objects(name_list)
+
+    duplicate_count = 0
+    not_in_simbad = []
+    in_simbad = []
+
+    print("Checking all SIMPLE sources for Simbad names")
+    for row in tqdm(simbad_results[["main_id", "ids", "user_specified_id"]].iterrows()):
+        simple_name = row[2]
+        try:
+            simbad_ids = row[1].decode("utf-8")
+        except AttributeError:
+            # Catch decoding error
+            simbad_ids = row[1]
+
+        # Get a nicely formatted list of Simbad names for each input row
+        simbad_names = [
+            _name_formatter(s)
+            for s in simbad_ids.split("|")
+            if _name_formatter(s) != "" and _name_formatter(s) is not None
+        ]
+
+        if len(simbad_names) == 0:
+            not_in_simbad.append(simple_name)
+            continue
+        else:
+            in_simbad.append(simple_name)
+
+        # Examine DB for each input, displaying results when more than one source matches
+        t = db.search_object(
+            simbad_names,
+            output_table="Sources",
+            fmt="astropy",
+            fuzzy_search=False,
+            verbose=False,
+        )
+        if len(t) > 1:
+            print(f"Multiple matches for {simple_name}: {simbad_names}")
+            print(
+                db.query(db.Names).filter(db.Names.c.source.in_(t["source"])).astropy()
+            )
+            duplicate_count += 1
+
+    assert duplicate_count == 0, "Duplicate sources identified via Simbad queries"
+    assert (
+        len(not_in_simbad) == 425
+    ), f"Expecting {len(not_in_simbad)} sources not found in Simbad"
+
+    assert len(in_simbad) == 3012, "Sources found in Simbad"
+    print(f"Found {len(in_simbad)} SIMPLE sources in Simbad")
+
+    assert len(not_in_simbad) + len(in_simbad) == len(
+        name_list
+    ), "Not all sources checked"
+
+    print(f"Found {len(not_in_simbad)} SIMPLE sources not in Simbad")
+    print("\n".join(not_in_simbad))

tests/test_integrity.py

@@ -1,11 +1,8 @@
 # Test to verify database integrity
 # database object 'db' defined in conftest.py
-import pytest
 from astrodbkit.astrodb import or_
-from astrodbkit.utils import _name_formatter
 from astropy import units as u
 from astropy.table import unique
-from astroquery.simbad import Simbad
 from sqlalchemy import and_, func
 
 from simple.schema import ParallaxView  # , PhotometryView
@@ -247,52 +244,6 @@ def test_source_uniqueness2(db):
     assert len(duplicate_names) == 0
 
 
-@pytest.mark.skip(reason="SIMBAD unreliable")
-def test_source_simbad(db):
-    # Query Simbad and confirm that there are no duplicates with different names
-
-    # Get list of all source names
-    results = db.query(db.Sources.c.source).all()
-    name_list = [s[0] for s in results]
-
-    # Add all IDS to the Simbad output as well as the user-provided id
-    Simbad.add_votable_fields("ids")
-    Simbad.add_votable_fields("typed_id")
-
-    simbad_results = Simbad.query_objects(name_list)
-    # Get a nicely formatted list of Simbad names for each input row
-    duplicate_count = 0
-    for row in simbad_results[["TYPED_ID", "IDS"]].iterrows():
-        try:
-            name, ids = row[0].decode("utf-8"), row[1].decode("utf-8")
-        except AttributeError:
-            # Catch decoding error
-            name, ids = row[0], row[1]
-
-        simbad_names = [
-            _name_formatter(s)
-            for s in ids.split("|")
-            if _name_formatter(s) != "" and _name_formatter(s) is not None
-        ]
-
-        if len(simbad_names) == 0:
-            print(f"No Simbad names for {name}")
-            continue
-
-        # Examine DB for each input, displaying results when more than one source matches
-        t = db.search_object(
-            simbad_names, output_table="Sources", fmt="astropy", fuzzy_search=False
-        )
-        if len(t) > 1:
-            print(f"Multiple matches for {name}: {simbad_names}")
-            print(
-                db.query(db.Names).filter(db.Names.c.source.in_(t["source"])).astropy()
-            )
-            duplicate_count += 1
-
-    assert duplicate_count == 0, "Duplicate sources identified via Simbad queries"
-
-
 def test_photometry(db):
     # Tests for Photometry table