diff --git a/scripts/download/nsidc_sources.txt b/scripts/download/nsidc_sources.txt index 06888a9..195524f 100644 --- a/scripts/download/nsidc_sources.txt +++ b/scripts/download/nsidc_sources.txt @@ -3,8 +3,8 @@ https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_BSU_GPR.001/ https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_GM_SP.001/ https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_SMP.001/ https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_SD.001/ -https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_GM_CSU_GPR.001/2020.02.06/SNEX20_GM_CSU_GPR_1GHz_v01.csv -https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_UNM_GPR.001/2020.01.28/SNEX20_UNM_GPR.csv +https://data.nsidc.earthdatacloud.nasa.gov/nsidc-cumulus-prod-protected/SNOWEX/SNEX20_GM_CSU_GPR/1/2020/02/06/SNEX20_GM_CSU_GPR_1GHz_v01.csv +https://data.nsidc.earthdatacloud.nasa.gov/nsidc-cumulus-prod-protected/SNOWEX/SNEX20_UNM_GPR/1/2020/01/28/SNEX20_UNM_GPR.csv https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_SD_TLI.001/2019.09.29/SNEX20_SD_TLI_clean.csv https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_TS_SP.002/ https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX21_TS_SP.001/ diff --git a/scripts/upload/add_csu_gpr.py b/scripts/upload/add_csu_gpr.py index b2c30da..4bd267b 100644 --- a/scripts/upload/add_csu_gpr.py +++ b/scripts/upload/add_csu_gpr.py @@ -8,49 +8,43 @@ """ -import time -from os.path import abspath, expanduser, join +# Issue #70 -import pandas as pd +from os.path import abspath, expanduser -from snowexsql.db import get_db -from snowex_db.upload import * +from snowexsql.db import db_session_with_credentials +from snowex_db.upload.points import PointDataCSV def main(): - file = '../download/data/SNOWEX/SNEX20_GM_CSU_GPR.001/2020.02.06/SNEX20_GM_CSU_GPR_1GHz_v01.csv' + file = ( + '../download/data/nsidc-cumulus-prod-protected/SNOWEX/' + 'SNEX20_GM_CSU_GPR/1/2020/02/06/SNEX20_GM_CSU_GPR_1GHz_v01.csv' + ) kwargs = { - # Keyword argument to upload depth measurements - 'depth_is_metadata': False, - # Constant Metadata for the GPR data - 'site_name': 'Grand Mesa', - 'observers': 'Randall Bonnell', - 'instrument': 'pulse EKKO Pro multi-polarization 1 GHz GPR', - 'in_timezone': 'UTC', - 'out_timezone': 'UTC', + 'campaign_name': 'Grand Mesa', + 'observer': 'Randall Bonnell', + 'instrument': 'gpr', + 'instrument_model': 'pulse EKKO Pro multi-polarization 1 GHz GPR', + 'timezone': 'UTC', 'doi': 'https://doi.org/10.5067/S5EGFLCIAB18', - 'epsg': 26912 + 'name': 'CSU GPR Data', } # Break out the path and make it an absolute path file = abspath(expanduser(file)) - # Grab a db connection to a local db named snowex - db_name = 'localhost/snowex' - engine, session = get_db(db_name, credentials='./credentials.json') - - # Instantiate the point uploader - csv = PointDataCSV(file, **kwargs) - # Push it to the database - csv.submit(session) - - # Close out the session with the DB - session.close() + # Grab a db connection + with db_session_with_credentials() as (_engine, session): + # Instantiate the point uploader + uploader = PointDataCSV(session, file, **kwargs) + # Push it to the database + uploader.submit() # return the number of errors for run.py can report it - return len(csv.errors) + # return len(csv.errors) if __name__ == '__main__': diff --git a/scripts/upload/add_csu_gpr_AK.py b/scripts/upload/add_csu_gpr_AK.py index d56bb7b..42e4d30 100644 --- a/scripts/upload/add_csu_gpr_AK.py +++ b/scripts/upload/add_csu_gpr_AK.py @@ -7,59 +7,41 @@ """ +from os.path import abspath, expanduser from pathlib import Path -from snowexsql.db import get_db -from snowex_db.upload import PointDataCSV +from snowexsql.db import db_session_with_credentials +from snowex_db.upload.points import PointDataCSV import pandas as pd +# Issue #71 + def main(): file = Path('../download/data/SnowEx223_FLCF_1GHz_GPR_CSU.csv') - # Fix quirks - df = pd.read_csv(file, dtype=str) - - # Upload is not able to handle the Notes col. So just remove it for now - modified = file.parent.joinpath(file.stem + f'_mod{file.suffix}') - print(f"Removing Notes Column prior to upload. Writing to {modified}") - - coi = [c for c in df.columns if c != 'Notes'] - - # No time is a problem. Use 12 AKST == 9pm (21:00) UTC - df['Time[HHMM]'] = '21:00' - - # Write out the modified version - df[coi].to_csv(modified, index=False) - - kwargs = { - # Keyword argument to upload depth measurements - 'depth_is_metadata': False, - # Constant Metadata for the GPR data - 'site_name': 'farmers-creamers', - 'observers': 'Randall Bonnell', - 'instrument': 'pulseEkko pro 1 GHz GPR', - 'in_timezone': 'UTC', - 'out_timezone': 'UTC', - 'doi': None, # Data is preliminary - 'epsg': 26906 + 'campaign_name': 'farmers-creamers', # TODO: should this be AK-something? + 'observer': 'Randall Bonnell', + 'instrument': 'gpr', + 'instrument_model': 'pulseEkko pro 1 GHz GPR', + 'timezone': 'UTC', + 'doi': None, # TODO: presumably this exists now? + 'name': 'CSU GPR Data', } - # Grab a db connection to a local db named snowex - db_name = 'localhost/snowex' - engine, session = get_db(db_name, credentials='./credentials.json') - - # Instantiate the point uploader - csv = PointDataCSV(modified, **kwargs) - # Push it to the database - csv.submit(session) + # Break out the path and make it an absolute path + file = abspath(expanduser(file)) - # Close out the session with the DB - session.close() + # Grab a db connection + with db_session_with_credentials() as (_engine, session): + # Instantiate the point uploader + uploader = PointDataCSV(session, file, **kwargs) + # Push it to the database + uploader.submit() # return the number of errors for run.py can report it - return len(csv.errors) + # return len(csv.errors) if __name__ == '__main__': diff --git a/scripts/upload/add_unm_gpr.py b/scripts/upload/add_unm_gpr.py index 2d2b32e..8a80c79 100644 --- a/scripts/upload/add_unm_gpr.py +++ b/scripts/upload/add_unm_gpr.py @@ -8,67 +8,59 @@ """ -import time -from os.path import abspath, expanduser, join +from pathlib import Path import pandas as pd -from snowexsql.db import get_db -from snowex_db.upload import * +from snowexsql.db import db_session_with_credentials +from snowex_db.upload.points import PointDataCSV +# Issue #72 + def main(): - filename = '../download/data/SNOWEX/SNEX20_UNM_GPR.001/2020.01.28/SNEX20_UNM_GPR.csv' + filename = ('../download/data/nsidc-cumulus-prod-protected/SNOWEX/' + 'SNEX20_UNM_GPR/1/2020/01/28/SNEX20_UNM_GPR.csv') kwargs = { - # Keyword argument to upload depth measurements - 'depth_is_metadata': False, - # Constant Metadata for the GPR data - 'site_name': 'Grand Mesa', - 'observers': 'Ryan Webb', - 'instrument': None, # See loop below - 'in_timezone': 'UTC', - 'out_timezone': 'UTC', + 'observer': 'Ryan Webb', 'doi': 'https://doi.org/10.5067/WE9GI1GVMQF6', - 'epsg': 26912 + 'campaign_name': 'Grand Mesa', + 'instrument': 'gpr', + # 'instrument_model': 'pulse EKKO Pro multi-polarization 1 GHz GPR', + 'timezone': 'UTC', + 'name': 'UNM GPR Data', } # Break out the path and make it an absolute path - filename = abspath(expanduser(filename)) - - # Grab a db connection to a local db named snowex - db_name = 'localhost/snowex' - engine, session = get_db(db_name, credentials='./credentials.json') + file = Path(filename).absolute().resolve() + # Make two files, filter by frequency # Read in for management of instruments df_raw = pd.read_csv(filename) - low_freq = df_raw['FREQ_MHz'] == 800 - hi_freq = df_raw['FREQ_MHz'] == 1600 - - # Instantiate the point uploader - csv = PointDataCSV(filename, **kwargs) - - # Convert depth to centimeters - csv.log.info('Converting depth to centimeters...') - csv.df['depth'] = csv.df['depth'].mul(100) - df_original = csv.df.copy() - - # Loop over the two insturments in the file and separate them for two submissions - for hz, ind in [(800, low_freq), (1600, hi_freq)]: - instrument = f'Mala {hz} MHz GPR' - csv.log.info(f'Isolating {instrument} data for upload.') - csv.df = df_original[ind].copy() - # Change the instrument. - csv.df['instrument'] = instrument - # Push it to the database - csv.submit(session) - - # Close out the session with the DB - session.close() - # return the number of errors for run.py can report it - return len(csv.errors) + # Grab a db connection + with db_session_with_credentials() as (_engine, session): + for freq in [800, 1600]: + new_file_name = file.parent.joinpath(f'{file.stem}_{freq}{file.suffix}') + # Filter the data by frequency + df_filtered = df_raw[df_raw['FREQ_MHz'] == freq] + # convert depth to cm + df_filtered.loc[:, ['DEPTH']] = df_filtered['DEPTH_m'].mul(100) + # Drop the original depth column + df_filtered = df_filtered.drop(columns=['DEPTH_m']) + # Save the filtered data to a new file + df_filtered.to_csv(new_file_name, index=False) + specific_kwargs = dict( + instrument_model=f'Mala {freq} MHz GPR' + ) + # Instantiate the point uploader + uploader = PointDataCSV( + session, str(new_file_name), **{**kwargs, **specific_kwargs} + ) + # Push it to the database + uploader.submit() if __name__ == '__main__': diff --git a/snowex_db/point_data.py b/snowex_db/point_data.py index bf29726..2e34b6c 100644 --- a/snowex_db/point_data.py +++ b/snowex_db/point_data.py @@ -245,6 +245,7 @@ def _read_csv( meta_parser.primary_variables.entries["UTCYEAR"], meta_parser.primary_variables.entries["UTM_ZONE"], meta_parser.primary_variables.entries["VERSION_NUMBER"], + meta_parser.primary_variables.entries["FREQUENCY"], ] shared_columns = [ diff --git a/snowex_db/point_primary_variable_overrides.yaml b/snowex_db/point_primary_variable_overrides.yaml index 8c21c19..6f6bbdb 100644 --- a/snowex_db/point_primary_variable_overrides.yaml +++ b/snowex_db/point_primary_variable_overrides.yaml @@ -13,7 +13,7 @@ BP: - bp_kpa_avg match_on_code: true DATE: - auto_remap: false + auto_remap: true code: date description: Measurement Date (only date column) map_from: @@ -167,7 +167,7 @@ SW_OUT: - sdn_avg match_on_code: true TIME: - auto_remap: false + auto_remap: true code: time description: Measurement time map_from: @@ -235,6 +235,8 @@ CAMPAIGN: COMMENTS: code: comments description: Comments + map_from: + - notes match_on_code: true FLAGS: code: flags @@ -255,3 +257,9 @@ PIT_ID: - pit_id - pitid match_on_code: true +FREQUENCY: + auto_remap: true + code: frequency + description: Frequency of measurement + map_from: + - freq_mhz \ No newline at end of file diff --git a/tests/data/csu_ak_gpr.csv b/tests/data/csu_ak_gpr.csv new file mode 100644 index 0000000..ddf9089 --- /dev/null +++ b/tests/data/csu_ak_gpr.csv @@ -0,0 +1,6 @@ +Date[mmddyy],Time[HHMM],Longitude[DD],Latitude[DD],ElevationWGS84[mae],Easting[m],Northing[m],UTM_Zone,TWT[ns],Depth[cm],SWE[mm],Density[kg m-3],Notes +030723,NaN,-147.737230798003,64.8638112503524,151.806903,465058.37445,7193480.256156,6,1.45,18.6201608827132,36.8679185477722,198,DN013 +030723,NaN,-147.737231429528,64.8638103942091,151.81797,465058.343408,7193480.161096,6,1.487265,19.098699017399,37.81542405445,198,DN013 +030723,NaN,-147.737232060693,64.8638095380227,151.829036,465058.312383,7193480.066031,6,1.524917,19.5822068088169,38.7727694814574,198,DN013 +030723,NaN,-147.73723268845,64.8638086814314,151.839986,465058.281519,7193479.970919,6,1.55,19.9043099091073,39.4105336200324,198,DN013 +030723,NaN,-147.737233145465,64.863807805451,151.844639,465058.258722,7193479.873552,6,1.562569,20.0657146002347,39.7301149084648,198,DN013 \ No newline at end of file diff --git a/tests/data/csu_gpr.csv b/tests/data/csu_gpr.csv new file mode 100644 index 0000000..3511d70 --- /dev/null +++ b/tests/data/csu_gpr.csv @@ -0,0 +1,6 @@ +Date [mmddyy],Time [HHMM],Longitude [DD],Latitude [DD],ElevationWGS84 [mae],Easting [m],Northing [m],UTM_Zone,TWT [ns],Depth [cm],SWE [mm] +020620,NaN,-108.176474337253,39.0192013494439,3047.4021717252,744448.66495925,4322701.1564092,12,7.348628,89.5675335280758,244.519366531647 +020620,NaN,-108.176477731632,39.0192008340717,3047.4004489707,744448.37281516,4322701.0900766,12,7.225466,88.0663939188338,240.421255398416 +020620,NaN,-108.176481162331,39.0192004664121,3047.3760773238,744448.07701698,4322701.0400424,12,7.102305,86.5652664979259,236.323177539338 +020620,NaN,-108.176484593025,39.0192000987253,3047.3517073054,744447.78121926,4322700.9900052,12,6.979144,85.0641390770181,232.225099680259 +020620,NaN,-108.17648792684,39.01919998347,3047.3289465061,744447.49294034,4322700.9682483,12,6.814929,83.0626317863629,226.760984776771 \ No newline at end of file diff --git a/tests/data/unm_gpr.csv b/tests/data/unm_gpr.csv index 0be8121..2763001 100644 --- a/tests/data/unm_gpr.csv +++ b/tests/data/unm_gpr.csv @@ -3,5 +3,3 @@ DATE_dd_mmm_yy,TIME_GMT,FREQ_MHz,LONG,LAT,ELEV_m,NORTHING,EASTING,UTMzone,TWT_ns 29-Jan-20,20:35,800,-108.139506,39.014194,3106.94,4322247.041,747667.2868,12,7.34,0.9,244 30-Jan-20,22:20,800,-108.1622217,39.03095167,3110.8,4324045.489,745642.1231,12,9.55,1.17,318 31-Jan-20,20:04,800,-108.1762593,39.02676284,3090.19,4323542.671,744441.2436,12,8.73,1.07,291 -4-Feb-20,16:00,1600,-108.1391578,39.03117433,3141.17,4324132.814,747638.1351,12,7.27,0.89,242 -5-Feb-20,19:04,1600,-108.1715773,39.02401467,3103.3,4323250.231,744856.1064,12,6.15,0.75,205 diff --git a/tests/points/test_gpr_csu.py b/tests/points/test_gpr_csu.py new file mode 100644 index 0000000..1446304 --- /dev/null +++ b/tests/points/test_gpr_csu.py @@ -0,0 +1,112 @@ +from datetime import datetime, timezone, date + +import pytest +from geoalchemy2 import WKTElement +from snowexsql.tables import PointData, DOI, Campaign, Instrument, \ + MeasurementType, PointObservation +from snowexsql.tables.campaign_observation import CampaignObservation + +from snowex_db.upload.points import PointDataCSV + +from _base import PointBaseTesting + + +class TestCSUGPR(PointBaseTesting): + """ + Test that a density file is uploaded correctly including sample + averaging for the main value. + """ + kwargs = { + # Constant Metadata for the GPR data + 'campaign_name': 'Grand Mesa', + 'observer': 'Randall Bonnell', + 'instrument': 'gpr', + 'instrument_model': 'pulse EKKO Pro multi-polarization 1 GHz GPR', + 'timezone': 'UTC', + 'doi': 'https://doi.org/10.5067/S5EGFLCIAB18', + 'name': 'CSU GPR Data', + } + UploaderClass = PointDataCSV + TableClass = PointData + + @pytest.fixture(scope="class") + def uploaded_file(self, session, data_dir): + self.upload_file(session, str(data_dir.joinpath("csu_gpr.csv"))) + + def filter_measurement_type(self, session, measurement_type, query=None): + if query is None: + query = session.query(self.TableClass) + + query = query.join( + self.TableClass.observation + ).join( + PointObservation.measurement_type + ).filter(MeasurementType.name == measurement_type) + return query + + @pytest.mark.parametrize( + "table, attribute, expected_value", [ + (Campaign, "name", "Grand Mesa"), + (Instrument, "name", "gpr"), + (Instrument, "model", "pulse EKKO Pro multi-polarization 1 GHz GPR"), + (MeasurementType, "name", ['two_way_travel', 'depth', "swe"]), + (MeasurementType, "units", ['ns', 'cm', 'mm']), + (MeasurementType, "derived", [False, False, False]), + (DOI, "doi", "https://doi.org/10.5067/S5EGFLCIAB18"), + (CampaignObservation, "name", "CSU GPR Data_gpr_pulse EKKO Pro multi-polarization 1 GHz GPR_two_way_travel"), + (PointData, "geom", + WKTElement('POINT (-108.176474337253 39.0192013494439)', srid=4326) + ), + (PointObservation, "date", date(2020, 2, 6)), + ] + ) + def test_metadata(self, table, attribute, expected_value, uploaded_file): + self._check_metadata(table, attribute, expected_value) + + @pytest.mark.parametrize( + "data_name, attribute_to_check, filter_attribute, filter_value, expected", [ + ('two_way_travel', 'value', 'date', date(2020, 2, 6), [ + 7.348628, 7.225466, 7.102305, 6.979144, 6.814929 + ]), + ('depth', 'value', 'date', date(2020, 2, 6), [ + 89.5675335280758, 88.0663939188338, 86.5652664979259, + 85.0641390770181, 83.0626317863629 + ],), + ('swe', 'value', 'date', date(2020, 2, 6), [ + 244.519366531647, 240.421255398416, 236.323177539338, + 232.225099680259, 226.760984776771 + ]), + ] + ) + def test_value( + self, data_name, attribute_to_check, + filter_attribute, filter_value, expected, uploaded_file + ): + self.check_value( + data_name, attribute_to_check, + filter_attribute, filter_value, expected, + ) + + @pytest.mark.parametrize( + "data_name, expected", [ + ("depth", 5), + ("swe", 5), + ("two_way_travel", 5), + ("density", 0), # no measurements + ] + ) + def test_count(self, data_name, expected, uploaded_file): + n = self.check_count(data_name) + assert n == expected + + @pytest.mark.parametrize( + "data_name, attribute_to_count, expected", [ + ("depth", "value", 5), + ("swe", "value", 5), + ("swe", "units", 1) + ] + ) + def test_unique_count(self, data_name, attribute_to_count, expected, uploaded_file): + self.check_unique_count( + data_name, attribute_to_count, expected + ) diff --git a/tests/points/test_gpr_csu_ak.py b/tests/points/test_gpr_csu_ak.py new file mode 100644 index 0000000..6ee4148 --- /dev/null +++ b/tests/points/test_gpr_csu_ak.py @@ -0,0 +1,112 @@ +from datetime import datetime, timezone, date + +import pytest +from geoalchemy2 import WKTElement +from snowexsql.tables import PointData, DOI, Campaign, Instrument, \ + MeasurementType, PointObservation +from snowexsql.tables.campaign_observation import CampaignObservation + +from snowex_db.upload.points import PointDataCSV + +from _base import PointBaseTesting + + +class TestCSUAKGPR(PointBaseTesting): + """ + Test that a density file is uploaded correctly including sample + averaging for the main value. + """ + kwargs = { + # Constant Metadata for the GPR data + 'campaign_name': 'farmers-creamers', # TODO: should this be AK-something? + 'observer': 'Randall Bonnell', + 'instrument': 'gpr', + 'instrument_model': 'pulseEkko pro 1 GHz GPR', + 'timezone': 'UTC', + 'doi': "preliminary_gpr_ak_farmers-creamers", # TODO: presumably this exists now? + 'name': 'CSU GPR Data', + } + UploaderClass = PointDataCSV + TableClass = PointData + + @pytest.fixture(scope="class") + def uploaded_file(self, session, data_dir): + self.upload_file(session, str(data_dir.joinpath("csu_ak_gpr.csv"))) + + def filter_measurement_type(self, session, measurement_type, query=None): + if query is None: + query = session.query(self.TableClass) + + query = query.join( + self.TableClass.observation + ).join( + PointObservation.measurement_type + ).filter(MeasurementType.name == measurement_type) + return query + + @pytest.mark.parametrize( + "table, attribute, expected_value", [ + (Campaign, "name", "farmers-creamers"), + (Instrument, "name", "gpr"), + (Instrument, "model", "pulseEkko pro 1 GHz GPR"), + (MeasurementType, "name", ['two_way_travel', 'depth', "swe", "density"]), + (MeasurementType, "units", ['ns', 'cm', 'mm', "kg/m^3"]), + (MeasurementType, "derived", [False, False, False, False]), + (DOI, "doi", "preliminary_gpr_ak_farmers-creamers"), + (CampaignObservation, "name", "CSU GPR Data_gpr_pulseEkko pro 1 GHz GPR_two_way_travel"), + (PointData, "geom", + WKTElement('POINT (-147.737230798003 64.8638112503524)', srid=4326) + ), + (PointObservation, "date", date(2023, 3, 7)), + ] + ) + def test_metadata(self, table, attribute, expected_value, uploaded_file): + self._check_metadata(table, attribute, expected_value) + + @pytest.mark.parametrize( + "data_name, attribute_to_check, filter_attribute, filter_value, expected", [ + ('two_way_travel', 'value', 'date', date(2023, 3, 7), [ + 1.45, 1.487265, 1.524917, 1.55, 1.562569 + ]), + ('depth', 'value', 'date', date(2023, 3, 7), [ + 18.6201608827132, 19.098699017399, 19.5822068088169, + 19.9043099091073, 20.0657146002347 + ],), + ('swe', 'value', 'date', date(2023, 3, 7), [ + 36.8679185477722, 37.81542405445, 38.7727694814574, + 39.4105336200324, 39.7301149084648 + ]), + ] + ) + def test_value( + self, data_name, attribute_to_check, + filter_attribute, filter_value, expected, uploaded_file + ): + self.check_value( + data_name, attribute_to_check, + filter_attribute, filter_value, expected, + ) + + @pytest.mark.parametrize( + "data_name, expected", [ + ("depth", 5), + ("swe", 5), + ("two_way_travel", 5), + ("density", 5) + ] + ) + def test_count(self, data_name, expected, uploaded_file): + n = self.check_count(data_name) + assert n == expected + + @pytest.mark.parametrize( + "data_name, attribute_to_count, expected", [ + ("depth", "value", 5), + ("swe", "value", 5), + ("swe", "units", 1) + ] + ) + def test_unique_count(self, data_name, attribute_to_count, expected, uploaded_file): + self.check_unique_count( + data_name, attribute_to_count, expected + ) diff --git a/tests/points/test_gpr_unm.py b/tests/points/test_gpr_unm.py new file mode 100644 index 0000000..1a663fb --- /dev/null +++ b/tests/points/test_gpr_unm.py @@ -0,0 +1,104 @@ +from datetime import datetime, timezone, date + +import pytest +from geoalchemy2 import WKTElement +from snowexsql.tables import PointData, DOI, Campaign, Instrument, \ + MeasurementType, PointObservation +from snowexsql.tables.campaign_observation import CampaignObservation + +from snowex_db.upload.points import PointDataCSV + +from _base import PointBaseTesting + + +class TestUNMGPR(PointBaseTesting): + """ + Test that a density file is uploaded correctly including sample + averaging for the main value. + """ + kwargs = { + # Constant Metadata for the GPR data + 'observer': 'Ryan Webb', + 'doi': 'https://doi.org/10.5067/WE9GI1GVMQF6', + 'campaign_name': 'Grand Mesa', + 'instrument': 'gpr', + 'instrument_model': f'Mala 800 MHz GPR', + 'timezone': 'UTC', + 'name': 'UNM GPR Data', + } + UploaderClass = PointDataCSV + TableClass = PointData + + @pytest.fixture(scope="class") + def uploaded_file(self, session, data_dir): + self.upload_file(session, str(data_dir.joinpath("unm_gpr.csv"))) + + def filter_measurement_type(self, session, measurement_type, query=None): + if query is None: + query = session.query(self.TableClass) + + query = query.join( + self.TableClass.observation + ).join( + PointObservation.measurement_type + ).filter(MeasurementType.name == measurement_type) + return query + + @pytest.mark.parametrize( + "table, attribute, expected_value", [ + (Campaign, "name", "Grand Mesa"), + (Instrument, "name", "gpr"), + (Instrument, "model", "Mala 800 MHz GPR"), + (MeasurementType, "name", ['two_way_travel', 'depth', "swe"]), + (MeasurementType, "units", ['ns', 'cm', 'mm']), + (MeasurementType, "derived", [False, False, False]), + (DOI, "doi", "https://doi.org/10.5067/WE9GI1GVMQF6"), + (CampaignObservation, "name", "UNM GPR Data_gpr_Mala 800 MHz GPR_two_way_travel"), + (PointData, "geom", + WKTElement('POINT (-108.1340183 39.0296597)', srid=4326) + ), + (PointObservation, "date", date(2020, 1, 28)), + ] + ) + def test_metadata(self, table, attribute, expected_value, uploaded_file): + self._check_metadata(table, attribute, expected_value) + + @pytest.mark.parametrize( + "data_name, attribute_to_check, filter_attribute, filter_value, expected", [ + ('two_way_travel', 'value', 'date', date(2020, 1, 28), [8.97]), + ('depth', 'value', 'date', date(2020, 1, 29), [0.9],), + ('swe', 'value', 'date', date(2020, 1, 31), [291]), + ] + ) + def test_value( + self, data_name, attribute_to_check, + filter_attribute, filter_value, expected, uploaded_file + ): + self.check_value( + data_name, attribute_to_check, + filter_attribute, filter_value, expected, + ) + + @pytest.mark.parametrize( + "data_name, expected", [ + ("depth", 4), + ("swe", 4), + ("two_way_travel", 4), + ("density", 0), # no measurements + ] + ) + def test_count(self, data_name, expected, uploaded_file): + n = self.check_count(data_name) + assert n == expected + + @pytest.mark.parametrize( + "data_name, attribute_to_count, expected", [ + ("depth", "value", 4), + ("swe", "value", 4), + ("swe", "units", 1) + ] + ) + def test_unique_count(self, data_name, attribute_to_count, expected, uploaded_file): + self.check_unique_count( + data_name, attribute_to_count, expected + )