Merge pull request #318 from sot/limit-stat-update-range

Limit stat update range to end of AOPCADMD data

Author: jeanconn

mica/stats/update_acq_stats.py

@@ -469,9 +469,23 @@ def calc_acq_stats(manvr, vals, times):
 
 
 def _get_obsids_to_update(check_missing=False):
+    # Use the end of AOPCADMD data as the end of the time range for kadi obsids
+    _, aopcadmd_end_time = fetch.get_time_range("AOPCADMD")
+
+    # There's some duplication below in how stop time is applied, but in both
+    # cases the the kadi obsids are filtered on the stop time of the obsid as less than or equal to
+    # the end of AOPCADMD data as available from the CXC archive.
+    #
+    # Note that the obsid event interval is the full interval of constant obsid.
+    # Obsid commanding is normally in the maneuver between observations.
+    # And note that the normal stop kwarg is inclusive and includes intervals that intersect with
+    # that stop time.
+
     if check_missing:
         last_tstart = "2007:271:12:00:00"
-        kadi_obsids = events.obsids.filter(start=last_tstart)
+        kadi_obsids = events.obsids.filter(
+            start=last_tstart, stop__lte=aopcadmd_end_time
+        )
         with tables.open_file(table_file, "r") as h5:
             tbl_data = h5.root.data[:]
         # get all obsids that aren't already in tbl
@@ -483,7 +497,9 @@ def _get_obsids_to_update(check_missing=False):
                 last_tstart = tbl.cols.guide_tstart[tbl.colindexes["guide_tstart"][-1]]
         except Exception:
             last_tstart = "2002:012:12:00:00"
-        kadi_obsids = events.obsids.filter(start=last_tstart)
+        kadi_obsids = events.obsids.filter(
+            start=last_tstart, stop__lte=aopcadmd_end_time
+        )
         # Skip the first obsid (as we already have it in the table)
         obsids = [o.obsid for o in kadi_obsids][1:]
     return obsids

mica/stats/update_guide_stats.py

@@ -400,9 +400,24 @@ def calc_gui_stats(data):
 
 
 def _get_obsids_to_update(check_missing=False, table_file=None, start=None, stop=None):
+    # Use the end of AOPCADMD data or a supplied stop time.
+    _, aopcadmd_end_time = fetch.get_time_range("AOPCADMD")
+    stop = stop if stop is not None else aopcadmd_end_time
+
+    # There's some duplication below in how stop time is applied, but in both
+    # cases the the kadi obsids are filtered on the stop time of the obsid as less than or equal to
+    # the stop time defined above (the supplied stop time or the end of cxc archive
+    # AOPCADMD data). The kadi obsid events may be complete past the end of
+    # the AOPCADMD data, especially if kadi events are updated using MAUDE data.
+    #
+    # Note that the obsid event interval is the full interval of constant obsid.
+    # Obsid commanding is normally in the maneuver between observations.
+    # And note that the normal stop kwarg is inclusive and includes intervals that intersect with
+    # that stop time.
+
     if check_missing:
         last_tstart = start if start is not None else "2007:271:12:00:00"
-        kadi_obsids = events.obsids.filter(start=last_tstart)
+        kadi_obsids = events.obsids.filter(start=last_tstart, stop__lte=stop)
         with tables.open_file(table_file, "r") as h5:
             tbl_data = h5.root.data[:]
         # get all obsids that aren't already in tbl
@@ -416,7 +431,7 @@ def _get_obsids_to_update(check_missing=False, table_file=None, start=None, stop
                 ]
         except Exception:
             last_tstart = start if start is not None else "2002:012:12:00:00"
-        kadi_obsids = events.obsids.filter(start=last_tstart, stop=stop)
+        kadi_obsids = events.obsids.filter(start=last_tstart, stop__lte=stop)
         # Skip the first obsid (as we already have it in the table)
         obsids = [o.obsid for o in kadi_obsids][1:]
     return obsids