SnowEx · jomey · Aug 1, 2025 · Jul 28, 2025 · Jul 28, 2025 · Jul 28, 2025
diff --git a/scripts/download/nsidc_sources.txt b/scripts/download/nsidc_sources.txt
@@ -3,8 +3,8 @@ https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_BSU_GPR.001/
 https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_GM_SP.001/
 https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_SMP.001/
 https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_SD.001/
-https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_GM_CSU_GPR.001/2020.02.06/SNEX20_GM_CSU_GPR_1GHz_v01.csv
-https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_UNM_GPR.001/2020.01.28/SNEX20_UNM_GPR.csv
+https://data.nsidc.earthdatacloud.nasa.gov/nsidc-cumulus-prod-protected/SNOWEX/SNEX20_GM_CSU_GPR/1/2020/02/06/SNEX20_GM_CSU_GPR_1GHz_v01.csv
+https://data.nsidc.earthdatacloud.nasa.gov/nsidc-cumulus-prod-protected/SNOWEX/SNEX20_UNM_GPR/1/2020/01/28/SNEX20_UNM_GPR.csv
 https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_SD_TLI.001/2019.09.29/SNEX20_SD_TLI_clean.csv
 https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_TS_SP.002/
 https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX21_TS_SP.001/

diff --git a/scripts/upload/add_csu_gpr.py b/scripts/upload/add_csu_gpr.py
@@ -8,49 +8,43 @@
 
 """
 
-import time
-from os.path import abspath, expanduser, join
+# Issue #70
 
-import pandas as pd
+from os.path import abspath, expanduser
 
-from snowexsql.db import get_db
-from snowex_db.upload import *
+from snowexsql.db import db_session_with_credentials
+from snowex_db.upload.points import PointDataCSV
 
 
 def main():
-    file = '../download/data/SNOWEX/SNEX20_GM_CSU_GPR.001/2020.02.06/SNEX20_GM_CSU_GPR_1GHz_v01.csv'
+    file = (
+        '../download/data/nsidc-cumulus-prod-protected/SNOWEX/'
+        'SNEX20_GM_CSU_GPR/1/2020/02/06/SNEX20_GM_CSU_GPR_1GHz_v01.csv'
+    )
 
     kwargs = {
-        # Keyword argument to upload depth measurements
-        'depth_is_metadata': False,
-
         # Constant Metadata for the GPR data
-        'site_name': 'Grand Mesa',
-        'observers': 'Randall Bonnell',
-        'instrument': 'pulse EKKO Pro multi-polarization 1 GHz GPR',
-        'in_timezone': 'UTC',
-        'out_timezone': 'UTC',
+        'campaign_name': 'Grand Mesa',
+        'observer': 'Randall Bonnell',
+        'instrument': 'gpr',
+        'instrument_model': 'pulse EKKO Pro multi-polarization 1 GHz GPR',
+        'timezone': 'UTC',
         'doi': 'https://doi.org/10.5067/S5EGFLCIAB18',
-        'epsg': 26912
+        'name': 'CSU GPR Data',
     }
 
     # Break out the path and make it an absolute path
     file = abspath(expanduser(file))
 
-    # Grab a db connection to a local db named snowex
-    db_name = 'localhost/snowex'
-    engine, session = get_db(db_name, credentials='./credentials.json')
-
-    # Instantiate the point uploader
-    csv = PointDataCSV(file, **kwargs)
-    # Push it to the database
-    csv.submit(session)
-
-    # Close out the session with the DB
-    session.close()
+    # Grab a db connection
+    with db_session_with_credentials() as (_engine, session):
+        # Instantiate the point uploader
+        uploader = PointDataCSV(session, file, **kwargs)
+        # Push it to the database
+        uploader.submit()
 
     # return the number of errors for run.py can report it
-    return len(csv.errors)
+    # return len(csv.errors)
 
 
 if __name__ == '__main__':

diff --git a/scripts/upload/add_csu_gpr_AK.py b/scripts/upload/add_csu_gpr_AK.py
@@ -7,59 +7,41 @@
 
 """
 
+from os.path import abspath, expanduser
 from pathlib import Path
-from snowexsql.db import get_db
-from snowex_db.upload import PointDataCSV
+from snowexsql.db import db_session_with_credentials
+from snowex_db.upload.points import PointDataCSV
 import pandas as pd
 
 
+# Issue #71
+
 def main():
     file = Path('../download/data/SnowEx223_FLCF_1GHz_GPR_CSU.csv')
 
-    # Fix quirks
-    df = pd.read_csv(file, dtype=str)
-
-    # Upload is not able to handle the Notes col. So just remove it for now
-    modified = file.parent.joinpath(file.stem + f'_mod{file.suffix}')
-    print(f"Removing Notes Column prior to upload. Writing to {modified}")
-
-    coi = [c for c in df.columns if c != 'Notes']
-
-    # No time is a problem. Use 12 AKST == 9pm (21:00) UTC
-    df['Time[HHMM]'] = '21:00'
-
-    # Write out the modified version
-    df[coi].to_csv(modified, index=False)
-
-
     kwargs = {
-        # Keyword argument to upload depth measurements
-        'depth_is_metadata': False,
-
         # Constant Metadata for the GPR data
-        'site_name': 'farmers-creamers',
-        'observers': 'Randall Bonnell',
-        'instrument': 'pulseEkko pro 1 GHz GPR',
-        'in_timezone': 'UTC',
-        'out_timezone': 'UTC',
-        'doi': None, # Data is preliminary
-        'epsg': 26906
+        'campaign_name': 'farmers-creamers',  # TODO: should this be AK-something?
+        'observer': 'Randall Bonnell',
+        'instrument': 'gpr',
+        'instrument_model': 'pulseEkko pro 1 GHz GPR',
+        'timezone': 'UTC',
+        'doi': None,  # TODO: presumably this exists now?
+        'name': 'CSU GPR Data',
     }
 
-    # Grab a db connection to a local db named snowex
-    db_name = 'localhost/snowex'
-    engine, session = get_db(db_name, credentials='./credentials.json')
-
-    # Instantiate the point uploader
-    csv = PointDataCSV(modified, **kwargs)
-    # Push it to the database
-    csv.submit(session)
+    # Break out the path and make it an absolute path
+    file = abspath(expanduser(file))
 
-    # Close out the session with the DB
-    session.close()
+    # Grab a db connection
+    with db_session_with_credentials() as (_engine, session):
+        # Instantiate the point uploader
+        uploader = PointDataCSV(session, file, **kwargs)
+        # Push it to the database
+        uploader.submit()
 
     # return the number of errors for run.py can report it
-    return len(csv.errors)
+    # return len(csv.errors)
 
 
 if __name__ == '__main__':

diff --git a/scripts/upload/add_unm_gpr.py b/scripts/upload/add_unm_gpr.py
@@ -8,67 +8,59 @@
 
 """
 
-import time
-from os.path import abspath, expanduser, join
+from pathlib import Path
 
 import pandas as pd
 
-from snowexsql.db import get_db
-from snowex_db.upload import *
+from snowexsql.db import db_session_with_credentials
+from snowex_db.upload.points import PointDataCSV
 
 
+# Issue #72
+
 def main():
-    filename = '../download/data/SNOWEX/SNEX20_UNM_GPR.001/2020.01.28/SNEX20_UNM_GPR.csv'
+    filename = ('../download/data/nsidc-cumulus-prod-protected/SNOWEX/'
+                'SNEX20_UNM_GPR/1/2020/01/28/SNEX20_UNM_GPR.csv')
 
     kwargs = {
-        # Keyword argument to upload depth measurements
-        'depth_is_metadata': False,
-
         # Constant Metadata for the GPR data
-        'site_name': 'Grand Mesa',
-        'observers': 'Ryan Webb',
-        'instrument': None, # See loop below
-        'in_timezone': 'UTC',
-        'out_timezone': 'UTC',
+        'observer': 'Ryan Webb',
         'doi': 'https://doi.org/10.5067/WE9GI1GVMQF6',
-        'epsg': 26912
+        'campaign_name': 'Grand Mesa',
+        'instrument': 'gpr',
+        # 'instrument_model': 'pulse EKKO Pro multi-polarization 1 GHz GPR',
+        'timezone': 'UTC',
+        'name': 'UNM GPR Data',
     }
 
     # Break out the path and make it an absolute path
-    filename = abspath(expanduser(filename))
-
-    # Grab a db connection to a local db named snowex
-    db_name = 'localhost/snowex'
-    engine, session = get_db(db_name, credentials='./credentials.json')
+    file = Path(filename).absolute().resolve()
 
+    # Make two files, filter by frequency
     # Read in for management of instruments
     df_raw = pd.read_csv(filename)
-    low_freq = df_raw['FREQ_MHz'] == 800
-    hi_freq = df_raw['FREQ_MHz'] == 1600
-
-    # Instantiate the point uploader
-    csv = PointDataCSV(filename, **kwargs)
-
-    # Convert depth to centimeters
-    csv.log.info('Converting depth to centimeters...')
-    csv.df['depth'] = csv.df['depth'].mul(100)
-    df_original = csv.df.copy()
-
-    # Loop over the two insturments in the file and separate them for two submissions
-    for hz, ind in [(800, low_freq), (1600, hi_freq)]:
-        instrument = f'Mala {hz} MHz GPR'
-        csv.log.info(f'Isolating {instrument} data for upload.')
-        csv.df = df_original[ind].copy()
-        # Change the instrument.
-        csv.df['instrument'] = instrument
-        # Push it to the database
-        csv.submit(session)
-
-    # Close out the session with the DB
-    session.close()
 
-    # return the number of errors for run.py can report it
-    return len(csv.errors)
+    # Grab a db connection
+    with db_session_with_credentials() as (_engine, session):
+        for freq in [800, 1600]:
+            new_file_name = file.parent.joinpath(f'{file.stem}_{freq}{file.suffix}')
+            # Filter the data by frequency
+            df_filtered = df_raw[df_raw['FREQ_MHz'] == freq]
+            # convert depth to cm
+            df_filtered.loc[:, ['DEPTH']] = df_filtered['DEPTH_m'].mul(100)
+            # Drop the original depth column
+            df_filtered = df_filtered.drop(columns=['DEPTH_m'])
+            # Save the filtered data to a new file
+            df_filtered.to_csv(new_file_name, index=False)
+            specific_kwargs = dict(
+                instrument_model=f'Mala {freq} MHz GPR'
+            )
+            # Instantiate the point uploader
+            uploader = PointDataCSV(
+                session, str(new_file_name), **{**kwargs, **specific_kwargs}
+            )
+            # Push it to the database
+            uploader.submit()
 
 
 if __name__ == '__main__':

diff --git a/snowex_db/point_data.py b/snowex_db/point_data.py
@@ -245,6 +245,7 @@ def _read_csv(
             meta_parser.primary_variables.entries["UTCYEAR"],
             meta_parser.primary_variables.entries["UTM_ZONE"],
             meta_parser.primary_variables.entries["VERSION_NUMBER"],
+            meta_parser.primary_variables.entries["FREQUENCY"],
         ]
 
         shared_columns = [

diff --git a/snowex_db/point_primary_variable_overrides.yaml b/snowex_db/point_primary_variable_overrides.yaml
@@ -13,7 +13,7 @@ BP:
   - bp_kpa_avg
   match_on_code: true
 DATE:
-  auto_remap: false
+  auto_remap: true
   code: date
   description: Measurement Date (only date column)
   map_from:
@@ -167,7 +167,7 @@ SW_OUT:
   - sdn_avg
   match_on_code: true
 TIME:
-  auto_remap: false
+  auto_remap: true
   code: time
   description: Measurement time
   map_from:
@@ -235,6 +235,8 @@ CAMPAIGN:
 COMMENTS:
     code: comments
     description: Comments
+    map_from:
+        - notes
     match_on_code: true
 FLAGS:
   code: flags
@@ -255,3 +257,9 @@ PIT_ID:
     - pit_id
     - pitid
   match_on_code: true
+FREQUENCY:
+    auto_remap: true
+    code: frequency
+    description: Frequency of measurement
+    map_from:
+        - freq_mhz
diff --git a/tests/data/csu_ak_gpr.csv b/tests/data/csu_ak_gpr.csv
@@ -0,0 +1,6 @@
+Date[mmddyy],Time[HHMM],Longitude[DD],Latitude[DD],ElevationWGS84[mae],Easting[m],Northing[m],UTM_Zone,TWT[ns],Depth[cm],SWE[mm],Density[kg m-3],Notes
+030723,NaN,-147.737230798003,64.8638112503524,151.806903,465058.37445,7193480.256156,6,1.45,18.6201608827132,36.8679185477722,198,DN013
+030723,NaN,-147.737231429528,64.8638103942091,151.81797,465058.343408,7193480.161096,6,1.487265,19.098699017399,37.81542405445,198,DN013
+030723,NaN,-147.737232060693,64.8638095380227,151.829036,465058.312383,7193480.066031,6,1.524917,19.5822068088169,38.7727694814574,198,DN013
+030723,NaN,-147.73723268845,64.8638086814314,151.839986,465058.281519,7193479.970919,6,1.55,19.9043099091073,39.4105336200324,198,DN013
+030723,NaN,-147.737233145465,64.863807805451,151.844639,465058.258722,7193479.873552,6,1.562569,20.0657146002347,39.7301149084648,198,DN013
diff --git a/tests/data/csu_gpr.csv b/tests/data/csu_gpr.csv
@@ -0,0 +1,6 @@
+Date [mmddyy],Time [HHMM],Longitude [DD],Latitude [DD],ElevationWGS84 [mae],Easting [m],Northing [m],UTM_Zone,TWT [ns],Depth [cm],SWE [mm]
+020620,NaN,-108.176474337253,39.0192013494439,3047.4021717252,744448.66495925,4322701.1564092,12,7.348628,89.5675335280758,244.519366531647
+020620,NaN,-108.176477731632,39.0192008340717,3047.4004489707,744448.37281516,4322701.0900766,12,7.225466,88.0663939188338,240.421255398416
+020620,NaN,-108.176481162331,39.0192004664121,3047.3760773238,744448.07701698,4322701.0400424,12,7.102305,86.5652664979259,236.323177539338
+020620,NaN,-108.176484593025,39.0192000987253,3047.3517073054,744447.78121926,4322700.9900052,12,6.979144,85.0641390770181,232.225099680259
+020620,NaN,-108.17648792684,39.01919998347,3047.3289465061,744447.49294034,4322700.9682483,12,6.814929,83.0626317863629,226.760984776771
diff --git a/tests/data/unm_gpr.csv b/tests/data/unm_gpr.csv
@@ -3,5 +3,3 @@ DATE_dd_mmm_yy,TIME_GMT,FREQ_MHz,LONG,LAT,ELEV_m,NORTHING,EASTING,UTMzone,TWT_ns
 29-Jan-20,20:35,800,-108.139506,39.014194,3106.94,4322247.041,747667.2868,12,7.34,0.9,244
 30-Jan-20,22:20,800,-108.1622217,39.03095167,3110.8,4324045.489,745642.1231,12,9.55,1.17,318
 31-Jan-20,20:04,800,-108.1762593,39.02676284,3090.19,4323542.671,744441.2436,12,8.73,1.07,291
-4-Feb-20,16:00,1600,-108.1391578,39.03117433,3141.17,4324132.814,747638.1351,12,7.27,0.89,242
-5-Feb-20,19:04,1600,-108.1715773,39.02401467,3103.3,4323250.231,744856.1064,12,6.15,0.75,205