diff --git a/catwise_matches.csv b/catwise_matches.csv new file mode 100644 index 000000000..43897449a --- /dev/null +++ b/catwise_matches.csv @@ -0,0 +1,83 @@ +source_name,ra,dec,PMRA,sigPMRA,PMDec,sigPMDec,w1mpro,w1sigmpro,w2mpro,w2sigmpro +J035519.99+143902.2,-0.01002,0.0859,0.03007,0.0882,17.282,0.074,17.464,0.256,58.8333068,14.6506374 +J114925.01-014406.6,-0.02256,0.0314,-0.01881,0.036,16.259,0.033,15.765,0.059,177.3542473,-1.7351709 +J175656.87-480525.9,-0.02016,0.0214,0.04206,0.0212,15.498,0.023,15.794,0.07,269.2369725,-48.0905452 +J013743.65+090749.5,-0.14238,0.1127,0.07491,0.118,17.801,0.104,17.901,0.357,24.4318933,9.130431 +J161420.41+004642.6,-0.08177,0.0099,-0.08356,0.0112,14.591,0.016,14.393,0.018,243.5850476,0.7785209 +J094901.85+264923.2,0.16402,0.1659,0.119,0.1902,18.171,0.143,17.586,0.295,147.2577424,26.8231252 +J083716.33-000045.0,0.03692,0.1085,-0.317,0.1234,17.865,0.113,17.214,0.189,129.318069,-0.0125017 +J060802.62-294520.3,-0.01462,0.0346,0.0289,0.0374,16.511,0.034,16.553,0.091,92.010938,-29.7556654 +J093733.73+293120.3,-0.05163,0.0313,0.00444,0.0346,16.249,0.032,15.818,0.062,144.3905662,29.522312 +J133943.75+010436.1,-0.28452,0.0653,-0.06417,0.0715,17.694,0.09,15.982,0.067,204.9323213,1.0766982 +J214812.49-632353.9,0.01494,0.008,-0.02262,0.0072,13.682,0.014,13.826,0.013,327.0520636,-63.398312 +J230154.75-645911.7,0.02045,0.0502,0.02025,0.0455,16.902,0.041,16.807,0.113,345.4781277,-64.9866103 +J200403.12-263813.5,0.0172,0.0317,0.00251,0.0306,16.096,0.033,16.113,0.076,301.0130299,-26.6370907 +J024956.47-055706.9,-0.16663,0.1656,0.46381,0.169,18.251,0.145,17.354,--,42.4853075,-5.9519319 +J142950.34+333000.4,-0.06414,0.1385,0.02983,0.1487,18.248,0.126,18.072,0.345,217.4597737,33.5001176 +J232748.84+045126.7,0.45892,0.0046,0.17634,0.004,7.531,0.016,7.307,0.008,351.9535171,4.8574215 +J155320.46+140022.6,-0.00282,0.0211,0.02118,0.0249,15.955,0.029,15.494,0.043,238.3352764,14.0062964 +J112601.05+481904.6,-0.00958,0.0057,-0.02108,0.0064,13.459,0.013,13.484,0.011,171.5043749,48.317969 +J074106.62+173839.0,-0.27835,0.0067,-0.61937,0.0067,11.989,0.019,11.752,0.013,115.2776034,17.6441716 +J095045.71+011715.0,-0.01797,0.0083,-0.00123,0.0098,14.314,0.014,14.388,0.019,147.6904958,1.2875253 +J075117.38-253104.0,0.00122,0.0082,0.02828,0.0086,14.201,0.022,14.227,0.022,117.8224368,-25.517799 +J213915.19-353004.7,0.29838,0.1494,0.30813,0.1574,18.278,0.162,17.369,0.243,324.8133078,-35.5013187 +J102922.44+162626.7,-0.03686,0.0995,0.148,0.112,17.811,0.105,16.937,0.16,157.3435034,16.4407529 +J161730.27+401846.2,0.03623,0.0945,0.00901,0.1094,17.951,0.079,17.683,0.204,244.3761429,40.3128576 +J141115.12+393623.3,0.05778,0.0671,0.18232,0.0759,17.34,0.051,17.32,0.165,212.8130122,39.6064955 +J193428.80-214222.8,-0.02143,0.0119,0.0118,0.0125,14.53,0.032,14.548,0.036,293.6200202,-21.7063371 +J105036.47-451722.0,-0.02635,0.0136,0.00767,0.0152,15.202,0.019,15.27,0.033,162.6519581,-45.2894698 +J115230.03+035911.1,-0.02163,0.0974,-0.18357,0.1157,17.641,0.093,16.833,0.138,178.1251603,3.9864288 +J121952.27+312825.6,-0.01688,0.2828,0.0834,0.3406,18.65,0.191,17.52,--,184.9677957,31.4738039 +J012836.37-280313.4,-0.0911,0.0918,0.03724,0.0859,17.543,0.075,16.941,0.14,22.1515753,-28.0537379 +J064722.27+394446.2,-0.00823,0.0153,-0.0581,0.0169,15.242,0.019,14.987,0.028,101.8428285,39.7461819 +J014103.10+131806.3,-0.01868,0.1089,-0.04196,0.1033,17.706,0.091,17.758,0.341,25.2629449,13.3017617 +J235402.94+024012.5,0.4634,0.0381,-0.48928,0.0389,18.118,0.145,15.107,0.032,358.5122544,2.6701485 +J155214.09+065028.2,0.14994,0.1536,-0.17218,0.1591,18.005,0.112,17.692,--,238.0587342,6.8411796 +J083052.00+361248.5,0.35315,0.0151,-0.878,0.0174,15.323,0.024,14.985,0.03,127.7166905,36.2134968 +J001449.45-083845.0,0.04485,0.122,-0.00294,0.1262,17.747,0.106,17.239,0.2,3.7060563,-8.6458447 +J014304.25-703418.0,-0.02493,0.0595,-0.04771,0.0498,17.304,0.047,16.76,0.082,25.7677156,-70.571692 +J105546.99+080840.7,-0.34253,0.004,-0.13661,0.004,11.133,0.013,10.879,0.009,163.9458234,8.1446501 +J142058.61+213127.9,0.00386,0.0279,-0.04877,0.0326,16.361,0.031,16.15,0.067,215.2442177,21.5244303 +J164034.96+292155.7,0.04574,0.0623,0.06009,0.066,17.307,0.054,17.697,0.22,250.1456805,29.3654965 +J050647.90+523618.6,0.01547,0.0436,0.0184,0.0471,16.677,0.04,16.39,0.085,76.6995846,52.6051854 +J064203.49+410141.3,0.01884,0.1297,-0.212,0.1426,17.983,0.12,17.252,0.203,100.5145417,41.0281539 +J074201.91+205512.2,0.0027,0.0622,0.01168,0.07,17.182,0.071,16.249,0.091,115.5079626,20.9200635 +J115433.80+623417.9,-0.03617,0.0272,0.01224,0.032,16.359,0.028,16.024,0.053,178.6408472,62.5716408 +J195520.62-030333.3,0.00407,0.0042,-0.0239,0.0038,8.0,0.013,7.543,0.008,298.8359486,-3.0592753 +J123322.60+213803.4,-0.03744,0.0111,-0.22727,0.0128,14.799,0.016,14.558,0.021,188.3441647,21.6342991 +J171146.43+233115.9,-0.0323,0.0609,0.0485,0.069,17.27,0.059,16.782,0.108,257.9434767,23.5210885 +J101950.66+044926.3,-0.06442,0.0071,-0.00355,0.0073,13.711,0.013,13.547,0.013,154.961111,4.8239863 +J114341.68+061337.4,-0.0099,0.104,-0.05705,0.1208,17.534,0.087,17.11,0.175,175.9236715,6.2270592 +J023941.71-173607.4,0.09441,0.1649,0.06979,0.1613,18.348,0.163,18.152,0.46,39.9237999,-17.6020805 +J151914.26-000046.0,0.13004,0.1446,-0.06592,0.1672,18.08,0.126,17.994,--,229.8094543,-0.0127921 +J091309.94-003150.1,-0.01962,0.0119,-0.00973,0.0142,14.914,0.018,14.974,0.028,138.291431,-0.5306092 +J111934.29-113752.6,-0.08579,0.0051,-0.02876,0.0057,12.654,0.013,12.517,0.009,169.8928897,-11.631294 +J081857.72+233346.3,-0.26432,0.0036,-0.345,0.0042,10.941,0.012,10.739,0.009,124.7405247,23.5628882 +J124640.89+032301.6,0.17286,0.085,-0.09544,0.0919,17.362,0.074,16.969,0.167,191.6703861,3.3837852 +J042212.65+153041.6,0.01305,0.0057,-0.0234,0.0055,12.696,0.013,12.627,0.011,65.5527185,15.5115631 +J000112.39+153511.6,0.03234,0.115,-0.29187,0.1077,17.603,0.08,18.058,--,0.3016605,15.586563 +J050214.66+144212.6,0.02744,0.0125,-0.00598,0.0128,14.815,0.017,14.68,0.024,75.5610828,14.7035006 +J030234.47-011640.8,0.02792,0.0654,0.05253,0.0669,17.151,0.056,17.221,0.186,45.6436558,-1.2780047 +J063953.34-741912.5,-0.0304,0.0274,0.0452,0.0258,16.491,0.024,16.618,0.062,99.9722739,-74.3201467 +J102845.73-283037.7,-0.08567,0.0041,-0.01777,0.0049,9.684,0.012,9.224,0.008,157.1905509,-28.5104861 +J043005.44+260803.8,-0.00845,0.099,-0.284,0.1059,17.669,0.097,16.716,0.136,67.522686,26.1344117 +J130730.42-055829.6,-0.05093,0.0915,0.20553,0.1109,17.692,0.094,16.754,0.147,196.876761,-5.9748992 +J185305.89-501051.1,-0.02717,0.0052,-0.06332,0.0055,6.478,0.025,6.387,0.012,283.2745682,-50.180862 +J055905.54-132321.8,-0.12487,0.1122,0.212,0.1221,18.012,0.116,17.184,0.176,89.7731011,-13.3894152 +J201155.37-620139.1,0.07183,0.0568,-0.0667,0.0658,17.085,0.054,17.564,0.263,302.9807283,-62.0275296 +J015144.13+134627.8,0.03347,0.0124,-0.008,0.0121,14.967,0.019,14.539,0.021,27.9338928,13.7743983 +J104617.66+351237.7,-0.05178,0.0245,0.00737,0.032,16.118,0.028,15.858,0.057,161.5735903,35.2104815 +J211846.83-001044.1,0.05762,0.0119,0.05031,0.0121,14.845,0.017,14.708,0.024,319.6951275,-0.1789393 +J023801.32+054520.4,0.03123,0.0117,-0.02323,0.011,14.63,0.042,14.489,0.047,39.5054993,5.7556813 +J042922.87+153518.6,-0.00346,0.0182,0.00459,0.0193,15.387,0.023,15.604,0.055,67.3452951,15.5885255 +J131142.11+362908.1,-0.05494,0.0299,-0.0624,0.0323,16.346,0.029,16.135,0.062,197.925476,36.485593 +J114146.07+411651.3,-0.05532,0.0318,-0.09924,0.0359,16.355,0.036,16.248,0.082,175.4419968,41.2809443 +J115500.31+230638.1,0.01439,0.059,-0.05872,0.0688,17.064,0.049,17.7,--,178.7512986,23.11061 +J020013.17+090813.6,-0.01942,0.063,0.03492,0.064,17.043,0.056,16.602,0.123,30.0549021,9.1371375 +J053859.95-022206.8,-0.06388,0.1176,-0.0453,0.1274,17.704,0.093,17.243,0.207,84.7498075,-2.3685626 +J013836.61-032222.8,0.13414,0.0081,-0.34495,0.0078,14.467,0.015,13.436,0.013,24.6525763,-3.3730227 +J095321.78-101438.7,-0.02793,0.0689,-0.0415,0.0798,17.265,0.065,16.802,0.14,148.3407498,-10.2440956 +J032842.66+230204.1,0.0484,0.0088,-0.09346,0.0078,14.169,0.015,13.669,0.014,52.1777779,23.0344927 +J014115.49-241756.0,-0.08355,0.1811,0.05349,0.1854,18.468,0.178,17.774,0.303,25.3145769,-24.2988951 +J051531.15+591104.3,0.01538,0.006,-0.865,0.0045,11.133,0.018,10.855,0.013,78.8797903,59.1845473 +J204731.73-080824.0,-0.03303,0.0049,-0.246,0.0045,12.333,0.012,12.124,0.009,311.8822237,-8.1400253 diff --git a/scripts/ingests/catwise/create_matches_csv.py b/scripts/ingests/catwise/create_matches_csv.py new file mode 100644 index 000000000..5e31e8978 --- /dev/null +++ b/scripts/ingests/catwise/create_matches_csv.py @@ -0,0 +1,82 @@ +from astrodb_utils import load_astrodb +import sys +sys.path.append(".") +from simple import * +from simple import REFERENCE_TABLES +from astrodb_utils.publications import ( + logger +) +from sqlalchemy import select +from astroquery.ipac.irsa import Irsa #we will use astroquery +from astropy.coordinates import SkyCoord +import astropy.units as u +import os +import csv +import numpy as np +import pandas as pd + +# Define the indices you want to include +indices = [0, 1, 2, 3, 4, 7, 8, 9, 10, 11, 12] + + + + +SAVE_DB = False # save the data files in addition to modifying the .db file +RECREATE_DB = True # recreates the .db file from the data files +SCHEMA_PATH = "simple/schema.yaml" +# LOAD THE DATABASE +db = load_astrodb("SIMPLE.sqlite", recreatedb=RECREATE_DB, reference_tables=REFERENCE_TABLES, felis_schema=SCHEMA_PATH) + +# generates all the sources in the db +sources_table = db.query(db.Sources).table() +print(sources_table) +coord_vector = SkyCoord(ra = sources_table["ra"], dec = sources_table["dec"], unit = "deg", frame = "icrs") +results = Irsa.query_region(coordinates=coord_vector, spatial='Cone', catalog='catwise_2020', radius=0.5 * u.arcmin, +columns="source_name,PMRA,sigPMRA,PMDec,sigPMDec,ab_flags,cc_flags,w1mpro,w1sigmpro,w2mpro,w2sigmpro,ra,dec", verbose=True) + +# should be 3598 +logger.info(f"Found {len(sources_table["source"])} sources to process.") + +source_num_counter = 0 +multiple_sources_counter = 0 +no_sources_counter = 0 +one_source = 0 + +one_source_names, multiple_source_names, no_source_names = [], [], [] +one_source_PMRA, multiple_source_PMRA, no_source_PMRA = [], [], [] +one_source_sigPMRA, multiple_source_sigPMRA, no_source_sigPMRA = [], [], [] +one_source_PMDec, multiple_source_PMDec, no_source_PMDec = [], [], [] +one_source_sigPMDec, multiple_source_sigPMDec, no_source_sigPMDec = [], [], [] +one_source_w1mpro, multiple_source_w1mpro, no_source_w1mpro = [], [], [] +one_source_w1sigmpro, multiple_source_w1sigmpro, no_source_w1sigmpro = [], [], [] +one_source_w2mpro, multiple_w2mpro, no_source_w2mpro = [], [], [] +one_source_w2sigmpro, multiple_source_w2sigmpro, no_source_w2sigmpro = [], [], [] +one_source_ra, multiple_source_ra, no_source_ra = [], [], [] +one_source_dec, multiple_source_dec, no_source_dec = [], [], [] + + +for result in results: + try: + filtered_results = results[(results["ab_flags"] == '00') & (results["cc_flags"] == '0000')] + if(len(filtered_results)>1): + multiple_sources_counter += 1 + # Convert selected values to strings and join with commas + row_string = ", ".join(str(filtered_results[i]) for i in indices if i < len(filtered_results)) + logger.info("source match found and added to csv file") + else: + one_source += 1 + row_string = ", ".join(str(filtered_results[i]) for i in indices if i < len(filtered_results)) + logger.info("source match found and added to csv file") + source_num_counter +=1 + print(source_num_counter) + except IndexError: + source_num_counter+=1 + no_sources_counter += 1 + row_string = ", ".join(str(filtered_results[i]) for i in indices if i < len(filtered_results)) + print(source_num_counter) + logger.warning("no source match found") + +logger.info("done") + + + diff --git a/scripts/ingests/catwise/ingest_CATWISE.py b/scripts/ingests/catwise/ingest_CATWISE.py new file mode 100644 index 000000000..402bfc6bd --- /dev/null +++ b/scripts/ingests/catwise/ingest_CATWISE.py @@ -0,0 +1,141 @@ +from astrodb_utils import load_astrodb +import sys +sys.path.append(".") +from simple import * +from simple import REFERENCE_TABLES +from astropy.io import ascii +from astrodb_utils.publications import ( + logger, + find_publication, + ingest_publication +) +from astrodb_utils.sources import ( + find_source_in_db, + ingest_source, + AstroDBError +) +from sqlalchemy import select +from astroquery.ipac.irsa import Irsa #we will use astroquery +from astropy.coordinates import SkyCoord +import astropy.units as u + + +SAVE_DB = False # save the data files in addition to modifying the .db file +RECREATE_DB = True # recreates the .db file from the data files +SCHEMA_PATH = "simple/schema.yaml" +# LOAD THE DATABASE +db = load_astrodb("SIMPLE.sqlite", recreatedb=RECREATE_DB, reference_tables=REFERENCE_TABLES, felis_schema=SCHEMA_PATH) + +# generates all the sources in the db +sources_table = db.query(db.Sources).table() + +# should be 3598 +logger.info(f"Found {len(sources)} sources to process.") + +# manually ingest the publication that the catalog uses +ingest_publication(db = db, doi = "10.3847/1538-4365/abd805") + +pub = find_publication(db = db, doi = "10.3847/1538-4365/abd805")[1] + + +for row in sources: + #create skycoord object because one of the parameters for query region for position + coord = SkyCoord(ra = row["ra"], dec = row["dec"], unit = "deg", frame = "icrs") + + # generates a list of objects from the catwise2020 catalogs that are within this radius of a certain position/coordinate + results = Irsa.query_region(coordinates=coord, spatial='Cone', catalog='catwise_2020', radius=2 * u.arcmin, columns="source_name,PMRA,sigPMRA,PMDec,sigPMDec,ab_flags,cc_flags,w1mpro,w1sigmpro,w2mpro,w2sigmpro,ra,dec") + + # generates a list of filtered results + filtered_results = [] + for source in results: + if (source["ab_flags"] == '00') & (source["cc_flags"] == '0000'): + filtered_results.append(source["source_name"]) + + #use closest/best match + try: + match = filtered_results[0] + logger.info("source match found") + + source_row = results[results["source_name"] == match] + + + ingested_source = find_source_in_db( + db = db, + source = source_row[0][0], + ra = source_row[0][11], + dec = source_row[0][12], + ra_col_name="ra", + dec_col_name="dec" + ) + + if(len(ingested_source)==0): + ingest_source( + db = db, + source = source_row[0][0], + reference = pub, + ra = source_row[0][11], + dec = source_row[0][12], + ra_col_name="ra", + dec_col_name="dec", + raise_error=True, + search_db=True, + ) # ingest new sources + + ingested_source = find_source_in_db( + db = db, + source = source_row[0][0], + ra = source_row[0][11], + dec = source_row[0][12], + ra_col_name="ra", + dec_col_name="dec" + )[0] + + with db.engine.connect() as conn: + #ingest proper motions + conn.execute( + db.ProperMotions.insert().values( + { + "source": ingested_source, + "mu_ra": source_row[0][1], + "mu_ra_error": source_row[0][2], + "mu_dec": source_row[0][3], + "mu_dec_error": source_row[0][4], + "reference": pub + } + ) + ) + logger.info(f"{[ingested_source]} propermotions ingested") + #ingest WISE W1 photometry + conn.execute( + db.Photometry.insert().values( + { + "source": ingested_source, + "band": "WISE.W1", + "magnitude": source_row[0][7], + "magnitude_error": source_row[0][8], + "reference": pub + } + ) + ) + logger.info(f"{[ingested_source]} photometry band 1 ingested") + #ingest WISE W2 photometry + conn.execute( + db.Photometry.insert().values( + { + "source": ingested_source, + "band": "WISE.W2", + "magnitude": source_row[0][9], + "magnitude_error": source_row[0][10], + "reference": pub + } + ) + ) + logger.info(f"{[ingested_source]} photometry band 2 ingested") + conn.commit() + except IndexError: + logger.warning("no source match found") + + +logger.info("done") + +