Ingest CATWISE

catwise_matches.csv

@@ -0,0 +1,83 @@
+source_name,ra,dec,PMRA,sigPMRA,PMDec,sigPMDec,w1mpro,w1sigmpro,w2mpro,w2sigmpro
+J035519.99+143902.2,-0.01002,0.0859,0.03007,0.0882,17.282,0.074,17.464,0.256,58.8333068,14.6506374
+J114925.01-014406.6,-0.02256,0.0314,-0.01881,0.036,16.259,0.033,15.765,0.059,177.3542473,-1.7351709
+J175656.87-480525.9,-0.02016,0.0214,0.04206,0.0212,15.498,0.023,15.794,0.07,269.2369725,-48.0905452
+J013743.65+090749.5,-0.14238,0.1127,0.07491,0.118,17.801,0.104,17.901,0.357,24.4318933,9.130431
+J161420.41+004642.6,-0.08177,0.0099,-0.08356,0.0112,14.591,0.016,14.393,0.018,243.5850476,0.7785209
+J094901.85+264923.2,0.16402,0.1659,0.119,0.1902,18.171,0.143,17.586,0.295,147.2577424,26.8231252
+J083716.33-000045.0,0.03692,0.1085,-0.317,0.1234,17.865,0.113,17.214,0.189,129.318069,-0.0125017
+J060802.62-294520.3,-0.01462,0.0346,0.0289,0.0374,16.511,0.034,16.553,0.091,92.010938,-29.7556654
+J093733.73+293120.3,-0.05163,0.0313,0.00444,0.0346,16.249,0.032,15.818,0.062,144.3905662,29.522312
+J133943.75+010436.1,-0.28452,0.0653,-0.06417,0.0715,17.694,0.09,15.982,0.067,204.9323213,1.0766982
+J214812.49-632353.9,0.01494,0.008,-0.02262,0.0072,13.682,0.014,13.826,0.013,327.0520636,-63.398312
+J230154.75-645911.7,0.02045,0.0502,0.02025,0.0455,16.902,0.041,16.807,0.113,345.4781277,-64.9866103
+J200403.12-263813.5,0.0172,0.0317,0.00251,0.0306,16.096,0.033,16.113,0.076,301.0130299,-26.6370907
+J024956.47-055706.9,-0.16663,0.1656,0.46381,0.169,18.251,0.145,17.354,--,42.4853075,-5.9519319
+J142950.34+333000.4,-0.06414,0.1385,0.02983,0.1487,18.248,0.126,18.072,0.345,217.4597737,33.5001176
+J232748.84+045126.7,0.45892,0.0046,0.17634,0.004,7.531,0.016,7.307,0.008,351.9535171,4.8574215
+J155320.46+140022.6,-0.00282,0.0211,0.02118,0.0249,15.955,0.029,15.494,0.043,238.3352764,14.0062964
+J112601.05+481904.6,-0.00958,0.0057,-0.02108,0.0064,13.459,0.013,13.484,0.011,171.5043749,48.317969
+J074106.62+173839.0,-0.27835,0.0067,-0.61937,0.0067,11.989,0.019,11.752,0.013,115.2776034,17.6441716
+J095045.71+011715.0,-0.01797,0.0083,-0.00123,0.0098,14.314,0.014,14.388,0.019,147.6904958,1.2875253
+J075117.38-253104.0,0.00122,0.0082,0.02828,0.0086,14.201,0.022,14.227,0.022,117.8224368,-25.517799
+J213915.19-353004.7,0.29838,0.1494,0.30813,0.1574,18.278,0.162,17.369,0.243,324.8133078,-35.5013187
+J102922.44+162626.7,-0.03686,0.0995,0.148,0.112,17.811,0.105,16.937,0.16,157.3435034,16.4407529
+J161730.27+401846.2,0.03623,0.0945,0.00901,0.1094,17.951,0.079,17.683,0.204,244.3761429,40.3128576
+J141115.12+393623.3,0.05778,0.0671,0.18232,0.0759,17.34,0.051,17.32,0.165,212.8130122,39.6064955
+J193428.80-214222.8,-0.02143,0.0119,0.0118,0.0125,14.53,0.032,14.548,0.036,293.6200202,-21.7063371
+J105036.47-451722.0,-0.02635,0.0136,0.00767,0.0152,15.202,0.019,15.27,0.033,162.6519581,-45.2894698
+J115230.03+035911.1,-0.02163,0.0974,-0.18357,0.1157,17.641,0.093,16.833,0.138,178.1251603,3.9864288
+J121952.27+312825.6,-0.01688,0.2828,0.0834,0.3406,18.65,0.191,17.52,--,184.9677957,31.4738039
+J012836.37-280313.4,-0.0911,0.0918,0.03724,0.0859,17.543,0.075,16.941,0.14,22.1515753,-28.0537379
+J064722.27+394446.2,-0.00823,0.0153,-0.0581,0.0169,15.242,0.019,14.987,0.028,101.8428285,39.7461819
+J014103.10+131806.3,-0.01868,0.1089,-0.04196,0.1033,17.706,0.091,17.758,0.341,25.2629449,13.3017617
+J235402.94+024012.5,0.4634,0.0381,-0.48928,0.0389,18.118,0.145,15.107,0.032,358.5122544,2.6701485
+J155214.09+065028.2,0.14994,0.1536,-0.17218,0.1591,18.005,0.112,17.692,--,238.0587342,6.8411796
+J083052.00+361248.5,0.35315,0.0151,-0.878,0.0174,15.323,0.024,14.985,0.03,127.7166905,36.2134968
+J001449.45-083845.0,0.04485,0.122,-0.00294,0.1262,17.747,0.106,17.239,0.2,3.7060563,-8.6458447
+J014304.25-703418.0,-0.02493,0.0595,-0.04771,0.0498,17.304,0.047,16.76,0.082,25.7677156,-70.571692
+J105546.99+080840.7,-0.34253,0.004,-0.13661,0.004,11.133,0.013,10.879,0.009,163.9458234,8.1446501
+J142058.61+213127.9,0.00386,0.0279,-0.04877,0.0326,16.361,0.031,16.15,0.067,215.2442177,21.5244303
+J164034.96+292155.7,0.04574,0.0623,0.06009,0.066,17.307,0.054,17.697,0.22,250.1456805,29.3654965
+J050647.90+523618.6,0.01547,0.0436,0.0184,0.0471,16.677,0.04,16.39,0.085,76.6995846,52.6051854
+J064203.49+410141.3,0.01884,0.1297,-0.212,0.1426,17.983,0.12,17.252,0.203,100.5145417,41.0281539
+J074201.91+205512.2,0.0027,0.0622,0.01168,0.07,17.182,0.071,16.249,0.091,115.5079626,20.9200635
+J115433.80+623417.9,-0.03617,0.0272,0.01224,0.032,16.359,0.028,16.024,0.053,178.6408472,62.5716408
+J195520.62-030333.3,0.00407,0.0042,-0.0239,0.0038,8.0,0.013,7.543,0.008,298.8359486,-3.0592753
+J123322.60+213803.4,-0.03744,0.0111,-0.22727,0.0128,14.799,0.016,14.558,0.021,188.3441647,21.6342991
+J171146.43+233115.9,-0.0323,0.0609,0.0485,0.069,17.27,0.059,16.782,0.108,257.9434767,23.5210885
+J101950.66+044926.3,-0.06442,0.0071,-0.00355,0.0073,13.711,0.013,13.547,0.013,154.961111,4.8239863
+J114341.68+061337.4,-0.0099,0.104,-0.05705,0.1208,17.534,0.087,17.11,0.175,175.9236715,6.2270592
+J023941.71-173607.4,0.09441,0.1649,0.06979,0.1613,18.348,0.163,18.152,0.46,39.9237999,-17.6020805
+J151914.26-000046.0,0.13004,0.1446,-0.06592,0.1672,18.08,0.126,17.994,--,229.8094543,-0.0127921
+J091309.94-003150.1,-0.01962,0.0119,-0.00973,0.0142,14.914,0.018,14.974,0.028,138.291431,-0.5306092
+J111934.29-113752.6,-0.08579,0.0051,-0.02876,0.0057,12.654,0.013,12.517,0.009,169.8928897,-11.631294
+J081857.72+233346.3,-0.26432,0.0036,-0.345,0.0042,10.941,0.012,10.739,0.009,124.7405247,23.5628882
+J124640.89+032301.6,0.17286,0.085,-0.09544,0.0919,17.362,0.074,16.969,0.167,191.6703861,3.3837852
+J042212.65+153041.6,0.01305,0.0057,-0.0234,0.0055,12.696,0.013,12.627,0.011,65.5527185,15.5115631
+J000112.39+153511.6,0.03234,0.115,-0.29187,0.1077,17.603,0.08,18.058,--,0.3016605,15.586563
+J050214.66+144212.6,0.02744,0.0125,-0.00598,0.0128,14.815,0.017,14.68,0.024,75.5610828,14.7035006
+J030234.47-011640.8,0.02792,0.0654,0.05253,0.0669,17.151,0.056,17.221,0.186,45.6436558,-1.2780047
+J063953.34-741912.5,-0.0304,0.0274,0.0452,0.0258,16.491,0.024,16.618,0.062,99.9722739,-74.3201467
+J102845.73-283037.7,-0.08567,0.0041,-0.01777,0.0049,9.684,0.012,9.224,0.008,157.1905509,-28.5104861
+J043005.44+260803.8,-0.00845,0.099,-0.284,0.1059,17.669,0.097,16.716,0.136,67.522686,26.1344117
+J130730.42-055829.6,-0.05093,0.0915,0.20553,0.1109,17.692,0.094,16.754,0.147,196.876761,-5.9748992
+J185305.89-501051.1,-0.02717,0.0052,-0.06332,0.0055,6.478,0.025,6.387,0.012,283.2745682,-50.180862
+J055905.54-132321.8,-0.12487,0.1122,0.212,0.1221,18.012,0.116,17.184,0.176,89.7731011,-13.3894152
+J201155.37-620139.1,0.07183,0.0568,-0.0667,0.0658,17.085,0.054,17.564,0.263,302.9807283,-62.0275296
+J015144.13+134627.8,0.03347,0.0124,-0.008,0.0121,14.967,0.019,14.539,0.021,27.9338928,13.7743983
+J104617.66+351237.7,-0.05178,0.0245,0.00737,0.032,16.118,0.028,15.858,0.057,161.5735903,35.2104815
+J211846.83-001044.1,0.05762,0.0119,0.05031,0.0121,14.845,0.017,14.708,0.024,319.6951275,-0.1789393
+J023801.32+054520.4,0.03123,0.0117,-0.02323,0.011,14.63,0.042,14.489,0.047,39.5054993,5.7556813
+J042922.87+153518.6,-0.00346,0.0182,0.00459,0.0193,15.387,0.023,15.604,0.055,67.3452951,15.5885255
+J131142.11+362908.1,-0.05494,0.0299,-0.0624,0.0323,16.346,0.029,16.135,0.062,197.925476,36.485593
+J114146.07+411651.3,-0.05532,0.0318,-0.09924,0.0359,16.355,0.036,16.248,0.082,175.4419968,41.2809443
+J115500.31+230638.1,0.01439,0.059,-0.05872,0.0688,17.064,0.049,17.7,--,178.7512986,23.11061
+J020013.17+090813.6,-0.01942,0.063,0.03492,0.064,17.043,0.056,16.602,0.123,30.0549021,9.1371375
+J053859.95-022206.8,-0.06388,0.1176,-0.0453,0.1274,17.704,0.093,17.243,0.207,84.7498075,-2.3685626
+J013836.61-032222.8,0.13414,0.0081,-0.34495,0.0078,14.467,0.015,13.436,0.013,24.6525763,-3.3730227
+J095321.78-101438.7,-0.02793,0.0689,-0.0415,0.0798,17.265,0.065,16.802,0.14,148.3407498,-10.2440956
+J032842.66+230204.1,0.0484,0.0088,-0.09346,0.0078,14.169,0.015,13.669,0.014,52.1777779,23.0344927
+J014115.49-241756.0,-0.08355,0.1811,0.05349,0.1854,18.468,0.178,17.774,0.303,25.3145769,-24.2988951
+J051531.15+591104.3,0.01538,0.006,-0.865,0.0045,11.133,0.018,10.855,0.013,78.8797903,59.1845473
+J204731.73-080824.0,-0.03303,0.0049,-0.246,0.0045,12.333,0.012,12.124,0.009,311.8822237,-8.1400253

scripts/ingests/catwise/create_matches_csv.py

@@ -0,0 +1,77 @@
+from astrodb_utils import load_astrodb
+import sys
+sys.path.append(".")
+from simple import *
+from simple import REFERENCE_TABLES
+from astrodb_utils.publications import (
+    logger
+)
+from sqlalchemy import select
+from astroquery.ipac.irsa import Irsa #we will use astroquery 
+from astropy.coordinates import SkyCoord
+import astropy.units as u
+import os
+import csv
+
+
+SAVE_DB = False  # save the data files in addition to modifying the .db file
+RECREATE_DB = True  # recreates the .db file from the data files
+SCHEMA_PATH = "simple/schema.yaml" 
+# LOAD THE DATABASE
+db = load_astrodb("SIMPLE.sqlite", recreatedb=RECREATE_DB, reference_tables=REFERENCE_TABLES, felis_schema=SCHEMA_PATH)
+
+# generates all the sources in the db
+sources_table = select(db.Sources)
+with db.engine.connect() as conn:
+    sources = conn.execute(sources_table).mappings().all()
+
+# should be 3598
+logger.info(f"Found {len(sources)} sources to process.")
+
+
+
+csv_file = "catwise_matches.csv"
+write_header = not os.path.exists(csv_file)  # Only write header if the file doesn't exist
+
+# Open CSV file in append mode
+with open(csv_file, mode="a", newline="") as f:
+    writer = csv.writer(f)
+
+    # Write the header once if file doesn't exist
+    if write_header:
+        writer.writerow(["source_name", "ra", "dec", "PMRA", "sigPMRA", "PMDec", "sigPMDec", "w1mpro", "w1sigmpro", "w2mpro", "w2sigmpro"])
+
+    source_num = 0
+    for source in sources:
+        #create skycoord object because one of the parameters for query region for position
+        coord = SkyCoord(ra = source["ra"], dec = source["dec"], unit = "deg", frame = "icrs")
+
+        # generates a list of objects from the catwise2020 catalogs that are within this radius of a certain position/coordinate
+        results = Irsa.query_region(coordinates=coord, spatial='Cone', catalog='catwise_2020', radius=0.5 * u.arcmin, columns="source_name,PMRA,sigPMRA,PMDec,sigPMDec,ab_flags,cc_flags,w1mpro,w1sigmpro,w2mpro,w2sigmpro,ra,dec")
+
+        try:
+            filtered_results = results[(results["ab_flags"] == '00') & (results["cc_flags"] == '0000')][0]
+            writer.writerow([
+                filtered_results[0],
+                filtered_results[1],
+                filtered_results[2],
+                filtered_results[3],
+                filtered_results[4],
+                filtered_results[7],
+                filtered_results[8],
+                filtered_results[9],
+                filtered_results[10],
+                filtered_results[11],
+                filtered_results[12]
+            ])
+            logger.info("source match found and added to csv file")
+            source_num +=1
+            print(source_num)
+        except IndexError:
+            source_num+=1
+            print(source_num)
+            logger.warning("no source match found")
+    logger.info("done")
+
+
+

scripts/ingests/catwise/ingest_CATWISE.py

@@ -0,0 +1,143 @@
+from astrodb_utils import load_astrodb
+import sys
+sys.path.append(".")
+from simple import *
+from simple import REFERENCE_TABLES
+from astropy.io import ascii
+from astrodb_utils.publications import (
+    logger,
+    find_publication,
+    ingest_publication
+)
+from astrodb_utils.sources import (
+    find_source_in_db,
+    ingest_source,
+    AstroDBError
+)
+from sqlalchemy import select
+from astroquery.ipac.irsa import Irsa #we will use astroquery 
+from astropy.coordinates import SkyCoord
+import astropy.units as u
+
+
+SAVE_DB = False  # save the data files in addition to modifying the .db file
+RECREATE_DB = True  # recreates the .db file from the data files
+SCHEMA_PATH = "simple/schema.yaml" 
+# LOAD THE DATABASE
+db = load_astrodb("SIMPLE.sqlite", recreatedb=RECREATE_DB, reference_tables=REFERENCE_TABLES, felis_schema=SCHEMA_PATH)
+
+# generates all the sources in the db
+sources_table = select(db.Sources)
+with db.engine.connect() as conn:
+    sources = conn.execute(sources_table).mappings().all()
+
+# should be 3598
+logger.info(f"Found {len(sources)} sources to process.")
+
+# manually ingest the publication that the catalog uses
+ingest_publication(db = db, doi = "10.3847/1538-4365/abd805")
+
+pub = find_publication(db = db, doi = "10.3847/1538-4365/abd805")[1]
+
+
+for row in sources:
+    #create skycoord object because one of the parameters for query region for position
+    coord = SkyCoord(ra = row["ra"], dec = row["dec"], unit = "deg", frame = "icrs")
+
+    # generates a list of objects from the catwise2020 catalogs that are within this radius of a certain position/coordinate
+    results = Irsa.query_region(coordinates=coord, spatial='Cone', catalog='catwise_2020', radius=2 * u.arcmin, columns="source_name,PMRA,sigPMRA,PMDec,sigPMDec,ab_flags,cc_flags,w1mpro,w1sigmpro,w2mpro,w2sigmpro,ra,dec")
+
+    # generates a list of filtered results
+    filtered_results = []
+    for source in results:
+        if (source["ab_flags"] == '00') & (source["cc_flags"] == '0000'):
+            filtered_results.append(source["source_name"])
+
+    #use closest/best match
+        try:
+            match = filtered_results[0]
+            logger.info("source match found")
+
+            source_row = results[results["source_name"] == match]
+
+
+            ingested_source = find_source_in_db(
+                db = db,
+                source = source_row[0][0],
+                ra = source_row[0][11],
+                dec = source_row[0][12],
+                ra_col_name="ra",
+                dec_col_name="dec"
+            )
+
+            if(len(ingested_source)==0):
+                ingest_source(
+                    db = db,
+                    source = source_row[0][0],
+                    reference = pub,
+                    ra = source_row[0][11],
+                    dec = source_row[0][12],
+                    ra_col_name="ra",
+                    dec_col_name="dec",
+                    raise_error=True,
+                    search_db=True,
+                )  # ingest new sources
+
+            ingested_source = find_source_in_db(
+                db = db,
+                source = source_row[0][0],
+                ra = source_row[0][11],
+                dec = source_row[0][12],
+                ra_col_name="ra",
+                dec_col_name="dec"
+            )[0]
+
+            with db.engine.connect() as conn:
+                #ingest proper motions
+                conn.execute(
+                    db.ProperMotions.insert().values(
+                        {
+                            "source": ingested_source,
+                            "mu_ra": source_row[0][1],
+                            "mu_ra_error": source_row[0][2],
+                            "mu_dec": source_row[0][3],
+                            "mu_dec_error": source_row[0][4],
+                            "reference": pub
+                        }
+                    )
+                )
+                logger.info(f"{[ingested_source]} propermotions ingested")
+                #ingest WISE W1 photometry
+                conn.execute(
+                    db.Photometry.insert().values(
+                        {
+                            "source": ingested_source,
+                            "band": "WISE.W1",
+                            "magnitude": source_row[0][7],
+                            "magnitude_error": source_row[0][8],
+                            "reference": pub
+                        }
+                    )
+                )
+                logger.info(f"{[ingested_source]} photometry band 1 ingested")
+                #ingest WISE W2 photometry
+                conn.execute(
+                    db.Photometry.insert().values(
+                        {
+                            "source": ingested_source,
+                            "band": "WISE.W2",
+                            "magnitude": source_row[0][9],
+                            "magnitude_error": source_row[0][10],
+                            "reference": pub
+                        }
+                    )
+                )
+                logger.info(f"{[ingested_source]} photometry band 2 ingested")
+            conn.commit()
+        except IndexError:
+            logger.warning("no source match found")
+
+
+logger.info("done")
+
+