Found an ANTENNAS_IQ file missing fields

[RemingtonRohel]

I found an antennas_iq file at INV 20250628.2200.00.inv.0.antennas_iq.h5 which is missing fields like agc_status_word, antennas_iq_data, and possibly more. Need to investigate how this occurred, if it showed up in the logs for data_write, etc.

[RemingtonRohel]

This did show up in the data_write log, seems that lag_numbers was attempting to be written for a second time, which h5py didn't allow.

{
  "error": "ValueError('Unable to synchronously create dataset (name already exists)')",
  "event": "error when saving to file",
  "logger": "data_write",
  "level": "critical",
  "timestamp": "2025-06-28T22:00:17.106933Z",
  "module": "data_write",
  "process": 28041,
  "func_name": "_write_file",
  "unix_timestamp": 1751148017.1088233
}
{
  "exception": [
    {
      "exc_type": "ValueError",
      "exc_value": "Unable to synchronously create dataset (name already exists)",
      "syntax_error": null,
      "is_cause": false,
      "frames": [
        {
          "filename": "/home/radar/borealis/src/data_write.py",
          "lineno": 131,
          "name": "_write_file",
          "line": "",
          "locals": {
            "self": "<__main__.DataWrite object at 0x7fb3af8b3fd0>",
            "aveperiod_data": "\"{'agc_status_word': 2048, 'antenna_locations': array([[-114.3 ,    0.  ,    0.  \"+22154",
            "two_hr_file_with_type": "20250628.2200.00.inv.0.antennas_iq",
            "data_type": "antennas_iq",
            "writer": "<class 'utils.writers.HDF5Writer'>",
            "full_two_hr_file": "/data/borealis_data/20250628/20250628.2200.00.inv.0.antennas_iq.h5",
            "e": "ValueError('Unable to synchronously create dataset (name already exists)')"
          }
        },
        {
          "filename": "/home/radar/borealis/src/utils/writers.py",
          "lineno": 66,
          "name": "write_record",
          "line": "",
          "locals": {
            "filename": "/data/borealis_data/20250628/20250628.2200.00.inv.0.antennas_iq.h5",
            "slice_data": "\"{'agc_status_word': 2048, 'antenna_locations': array([[-114.3 ,    0.  ,    0.  \"+22154",
            "dt_str": "20250628-2200-00.529000",
            "file_type": "antennas_iq",
            "f": "<Closed HDF5 file>",
            "group": "<Closed HDF5 group>",
            "metadata": "<Closed HDF5 group>"
          }
        },
        {
          "filename": "/home/radar/borealis/src/utils/file_formats.py",
          "lineno": 897,
          "name": "to_hdf5",
          "line": "",
          "locals": {
            "self": "\"{'agc_status_word': 2048, 'antenna_locations': array([[-114.3 ,    0.  ,    0.  \"+22154",
            "group": "<Closed HDF5 group>",
            "metadata_group": "<Closed HDF5 group>",
            "data_type": "antennas_iq",
            "dim_scale_fields": "\"['antennas', 'cfs_freqs', 'local_coord', 'pulses', 'rx_antennas', 'lag_pulse_des\"+148",
            "f": "lag_numbers",
            "written": "True",
            "remaining_fields": "\"['pulse_phase_offset', 'scheduling_mode', 'rx_intf_excitations', 'borealis_git_h\"+624",
            "dataclass_fields": "\"{'agc_status_word': Field(name='agc_status_word',type=<class 'int'>,default=<dat\"+32630"
          }
        },
        {
          "filename": "/home/radar/borealis/src/utils/file_formats.py",
          "lineno": 796,
          "name": "_dispatch_to_write_method",
          "line": "",
          "locals": {
            "self": "\"{'agc_status_word': 2048, 'antenna_locations': array([[-114.3 ,    0.  ,    0.  \"+22154",
            "name": "lag_numbers",
            "fields_map": "\"{'agc_status_word': Field(name='agc_status_word',type=<class 'int'>,default=<dat\"+32630",
            "group": "<Closed HDF5 group>",
            "metadata_group": "<Closed HDF5 group>",
            "data_type": "antennas_iq",
            "data": "'array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 17,\\n     '+40",
            "formatted_data": "'array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 17,\\n     '+40",
            "metadata": "\"mappingproxy({'groups': ['antennas_iq', 'bfiq', 'rawacf'], 'level': 'file', 'nic\"+164"
          }
        },
        {
          "filename": "/home/radar/borealis/src/utils/file_formats.py",
          "lineno": 868,
          "name": "_write_metadata_field",
          "line": "",
          "locals": {
            "name": "lag_numbers",
            "data": "'array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 17,\\n     '+40",
            "metadata": "\"mappingproxy({'groups': ['antennas_iq', 'bfiq', 'rawacf'], 'level': 'file', 'nic\"+164",
            "group": "<Closed HDF5 group>",
            "ftype": "antennas_iq"
          }
        },
        {
          "filename": "/home/radar/borealis/src/utils/file_formats.py",
          "lineno": 821,
          "name": "_write_hdf5_field",
          "line": "",
          "locals": {
            "name": "lag_numbers",
            "data": "'array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 17,\\n     '+40",
            "metadata": "\"mappingproxy({'groups': ['antennas_iq', 'bfiq', 'rawacf'], 'level': 'file', 'nic\"+164",
            "group": "<Closed HDF5 group>",
            "ftype": "antennas_iq",
            "kw": "{'compression': 'gzip', 'compression_opts': 9}"
          }
        },
        {
          "filename": "/home/radar/borealis/borealis_env3.11/lib64/python3.11/site-packages/h5py/_hl/group.py",
          "lineno": 186,
          "name": "create_dataset",
          "line": "",
          "locals": {
            "self": "<Closed HDF5 group>",
            "name": "b'lag_numbers'",
            "shape": "None",
            "dtype": "None",
            "data": "'array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 17,\\n     '+40",
            "kwds": "{'compression': 'gzip', 'compression_opts': 9, 'track_order': False}",
            "group": "<Closed HDF5 group>"
          }
        },
        {
          "filename": "/home/radar/borealis/borealis_env3.11/lib64/python3.11/site-packages/h5py/_hl/dataset.py",
          "lineno": 175,
          "name": "make_new_dset",
          "line": "",
          "locals": {
            "parent": "<Closed HDF5 group>",
            "shape": "(23,)",
            "dtype": "dtype('uint32')",
            "data": "'array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 17,\\n     '+40",
            "name": "b'lag_numbers'",
            "chunks": "None",
            "compression": "gzip",
            "shuffle": "None",
            "fletcher32": "None",
            "maxshape": "None",
            "compression_opts": "9",
            "fillvalue": "None",
            "scaleoffset": "None",
            "track_times": "False",
            "external": "None",
            "track_order": "False",
            "dcpl": "<h5py.h5p.PropDCID object at 0x7fb3af93aca0>",
            "dapl": "None",
            "efile_prefix": "None",
            "virtual_prefix": "None",
            "allow_unknown_filter": "False",
            "rdcc_nslots": "None",
            "rdcc_nbytes": "None",
            "rdcc_w0": "None",
            "fill_time": "None",
            "tmp_shape": "(23,)",
            "tid": "<h5py.h5t.TypeIntegerID object at 0x7fb3af93b0b0>",
            "sid": "<h5py.h5s.SpaceID object at 0x7fb3af93ac50>"
          }
        },
        {
          "filename": "/home/radar/borealis/h5py/_objects.pyx",
          "lineno": 56,
          "name": "h5py._objects.with_phil.wrapper",
          "line": "",
          "locals": {}
        },
        {
          "filename": "/home/radar/borealis/h5py/_objects.pyx",
          "lineno": 57,
          "name": "h5py._objects.with_phil.wrapper",
          "line": "",
          "locals": {}
        },
        {
          "filename": "/home/radar/borealis/h5py/h5d.pyx",
          "lineno": 136,
          "name": "h5py.h5d.create",
          "line": "",
          "locals": {}
        }
      ]
    }
  ],
  "event": "error when saving to file",
  "logger": "data_write",
  "level": "error",
  "timestamp": "2025-06-28T22:00:17.110102Z",
  "module": "data_write",
  "process": 28041,
  "func_name": "_write_file",
  "unix_timestamp": 1751148018.2591774
}

[RemingtonRohel]

It also seems to have only affected the first record. Not quite the calamitous error that I was worried about.