Merge release/1.2.0

Author: shahramn

.gitignore

@@ -77,6 +77,4 @@ venv.bak/
 
 # local ignores
 .docker-tox/
-tests/sample-data/cds*.grib
-tests/sample-data/cds*.nc
 *.idx

CHANGELOG.rst

@@ -2,6 +2,14 @@
 Changelog for eccodes-python
 ============================
 
+1.2.0 (2021-03-23)
+--------------------
+
+- Added test for multi-field GRIBs
+- Fix deprecation warning: `np.float` is a deprecated alias for the builtin `float`
+- Experimental feature: grib_nearest_find
+
+
 1.1.0 (2021-01-20)
 --------------------
 

README.rst

@@ -47,7 +47,7 @@ https://software.ecmwf.int/wiki/display/ECC/ecCodes+installation
 You may run a simple selfcheck command to ensure that your system is set up correctly::
 
     $ python -m eccodes selfcheck
-    Found: ecCodes v2.20.0.
+    Found: ecCodes v2.21.0.
     Your system is ready.
 
 

eccodes/eccodes.py

@@ -21,6 +21,9 @@
 from gribapi import CODES_PRODUCT_ANY
 from gribapi import GRIB_MISSING_DOUBLE as CODES_MISSING_DOUBLE
 from gribapi import GRIB_MISSING_LONG as CODES_MISSING_LONG
+from gribapi import GRIB_NEAREST_SAME_GRID as CODES_GRIB_NEAREST_SAME_GRID
+from gribapi import GRIB_NEAREST_SAME_DATA as CODES_GRIB_NEAREST_SAME_DATA
+from gribapi import GRIB_NEAREST_SAME_POINT as CODES_GRIB_NEAREST_SAME_POINT
 
 from gribapi import gts_new_from_file as codes_gts_new_from_file
 from gribapi import metar_new_from_file as codes_metar_new_from_file
@@ -102,6 +105,11 @@
 from gribapi import grib_is_defined as codes_is_defined
 from gribapi import grib_find_nearest as codes_grib_find_nearest
 from gribapi import grib_find_nearest_multiple as codes_grib_find_nearest_multiple
+
+from gribapi import grib_nearest_new as codes_grib_nearest_new
+from gribapi import grib_nearest_delete as codes_grib_nearest_delete
+from gribapi import grib_nearest_find as codes_grib_nearest_find
+
 from gribapi import grib_get_native_type as codes_get_native_type
 from gribapi import grib_get as codes_get
 from gribapi import grib_get_array as codes_get_array

gribapi/__init__.py

@@ -15,8 +15,8 @@
 from .gribapi import bindings_version
 
 # The minimum required version for the ecCodes package
-min_reqd_version_str = "2.20.0"
-min_reqd_version_int = 22000
+min_reqd_version_str = "2.21.0"
+min_reqd_version_int = 22100
 
 if lib.grib_get_api_version() < min_reqd_version_int:
     print(

gribapi/bindings.py

@@ -21,7 +21,7 @@
 
 import cffi
 
-__version__ = "1.1.0"
+__version__ = "1.2.0"
 
 LOG = logging.getLogger(__name__)
 

gribapi/gribapi.py

@@ -64,6 +64,10 @@
 GRIB_MISSING_DOUBLE = -1e100
 GRIB_MISSING_LONG = 2147483647
 
+# Constants for GRIB nearest neighbour
+GRIB_NEAREST_SAME_GRID = 1 << 0
+GRIB_NEAREST_SAME_DATA = 1 << 1
+GRIB_NEAREST_SAME_POINT = 1 << 2
 
 # ECC-1029: Disable function-arguments type-checking unless
 # environment variable is defined and equal to 1
@@ -813,7 +817,6 @@ def grib_keys_iterator_delete(iterid):
     @param iterid      keys iterator id created with @ref grib_keys_iterator_new
     @exception CodesInternalError
     """
-    # aa: THIS LEAKS MEMORY as it doesn't free all the connected iterators
     kih = get_grib_keys_iterator(iterid)
     lib.grib_keys_iterator_delete(kih)
 
@@ -829,7 +832,6 @@ def grib_keys_iterator_get_name(iterid):
     @return          key name to be retrieved
     @exception CodesInternalError
     """
-    # aa: missing call to grib_keys_iterator_get_accessor
     kih = get_grib_keys_iterator(iterid)
     name = lib.grib_keys_iterator_get_name(kih)
     return ffi.string(name).decode(ENC)
@@ -1582,7 +1584,7 @@ def grib_get_double_element(msgid, key, index):
     """
     h = get_handle(msgid)
     value_p = ffi.new("double*")
-    err = lib.grib_get_double_element(h, key.encode(ENC), index, value_p)  # TODO
+    err = lib.grib_get_double_element(h, key.encode(ENC), index, value_p)
     GRIB_CHECK(err)
     return value_p[0]
 
@@ -2356,7 +2358,7 @@ def _convert_struct_to_dict(s):
 
 def codes_bufr_extract_headers(filepath, is_strict=True):
     """
-    @brief BUFR header extraction (EXPERIMENTAL FEATURE)
+    @brief BUFR header extraction
 
     @param filepath       path of input BUFR file
     @param is_strict      fail as soon as any invalid BUFR message is encountered
@@ -2407,7 +2409,7 @@ def codes_bufr_key_is_header(msgid, key):
 
 def codes_extract_offsets(filepath, product_kind, is_strict=True):
     """
-    @brief Message offset extraction (EXPERIMENTAL FEATURE)
+    @brief Message offset extraction
 
     @param filepath       path of input file
     @param is_strict      fail as soon as any invalid message is encountered
@@ -2430,3 +2432,80 @@ def codes_extract_offsets(filepath, product_kind, is_strict=True):
     while i < num_messages:
         yield offsets[i]
         i += 1
+
+
+# -------------------------------
+# EXPERIMENTAL FEATURES
+# -------------------------------
+@require(msgid=int)
+def grib_nearest_new(msgid):
+    h = get_handle(msgid)
+    err, nid = err_last(lib.grib_nearest_new)(h)
+    GRIB_CHECK(err)
+    return put_grib_nearest(nid)
+
+
+def put_grib_nearest(nid):
+    return int(ffi.cast("size_t", nid))
+
+
+def get_grib_nearest(nid):
+    return ffi.cast("grib_nearest*", nid)
+
+
+@require(nid=int)
+def grib_nearest_delete(nid):
+    nh = get_grib_nearest(nid)
+    lib.grib_nearest_delete(nh)
+
+
+@require(nid=int, gribid=int)
+def grib_nearest_find(nid, gribid, inlat, inlon, flags, is_lsm=False, npoints=4):
+    # flags has to be one of:
+    #  GRIB_NEAREST_SAME_GRID
+    #  GRIB_NEAREST_SAME_DATA
+    #  GRIB_NEAREST_SAME_POINT
+    if npoints != 4:
+        raise errors.FunctionNotImplementedError(
+            "grib_nearest_find npoints argument: Only 4 points supported"
+        )
+    if is_lsm:
+        raise errors.FunctionNotImplementedError(
+            "grib_nearest_find is_lsm argument: Land sea mask not supported"
+        )
+
+    h = get_handle(gribid)
+    outlats_p = ffi.new("double[]", npoints)
+    outlons_p = ffi.new("double[]", npoints)
+    values_p = ffi.new("double[]", npoints)
+    distances_p = ffi.new("double[]", npoints)
+    indexes_p = ffi.new("int[]", npoints)
+    size = ffi.new("size_t *")
+    nh = get_grib_nearest(nid)
+    err = lib.grib_nearest_find(
+        nh,
+        h,
+        inlat,
+        inlon,
+        flags,
+        outlats_p,
+        outlons_p,
+        values_p,
+        distances_p,
+        indexes_p,
+        size,
+    )
+    GRIB_CHECK(err)
+    result = []
+    for i in range(npoints):
+        result.append(
+            Bunch(
+                lat=outlats_p[i],
+                lon=outlons_p[i],
+                value=values_p[i],
+                distance=distances_p[i],
+                index=indexes_p[i],
+            )
+        )
+
+    return tuple(result)

tests/environment.yml

@@ -4,5 +4,5 @@ channels:
 dependencies:
   - attrs>=19.2
   - cffi
-  - eccodes=>2.20.0
+  - eccodes>=2.20.0
   - numpy

tests/test_eccodes.py

@@ -21,7 +21,7 @@
 from eccodes import *
 
 SAMPLE_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "sample-data")
-TEST_DATA = os.path.join(SAMPLE_DATA_FOLDER, "era5-levels-members.grib")
+TEST_DATA = os.path.join(SAMPLE_DATA_FOLDER, "multi_field.grib2")
 
 
 def get_sample_fullpath(sample):
@@ -148,6 +148,17 @@ def test_gts_header():
     codes_gts_header(False)
 
 
+def test_extract_offsets():
+    fpath = get_sample_fullpath("BUFR4.tmpl")
+    if fpath is None:
+        return
+    is_strict = True
+    offsets = codes_extract_offsets(fpath, CODES_PRODUCT_ANY, is_strict)
+    offsets_list = list(offsets)
+    assert len(offsets_list) == 1
+    assert offsets_list[0] == 0
+
+
 # ---------------------------------------------
 # GRIB
 # ---------------------------------------------
@@ -320,6 +331,7 @@ def test_grib_get_double_element():
     gid = codes_grib_new_from_samples("gg_sfc_grib2")
     elem = codes_get_double_element(gid, "values", 1)
     assert math.isclose(elem, 259.9865, abs_tol=0.001)
+    codes_release(gid)
 
 
 def test_grib_get_double_elements():
@@ -407,7 +419,7 @@ def test_grib_ecc_1042():
             2,
             3,
         ],
-        dtype=np.float,
+        dtype=float,
     )
     codes_set_values(gid, write_vals)
     read_vals = codes_get_values(gid)
@@ -492,14 +504,35 @@ def test_grib_index_new_from_file(tmpdir):
 
 
 def test_grib_multi_support_reset_file():
-    # TODO: read an actual multi-field GRIB here
-    fpath = get_sample_fullpath("GRIB2.tmpl")
-    if fpath is None:
-        return
-    codes_grib_multi_support_on()
-    with open(fpath, "rb") as f:
-        codes_grib_multi_support_reset_file(f)
-    codes_grib_multi_support_off()
+    try:
+        # TODO: read an actual multi-field GRIB here
+        fpath = get_sample_fullpath("GRIB2.tmpl")
+        if fpath is None:
+            return
+        codes_grib_multi_support_on()
+        with open(fpath, "rb") as f:
+            codes_grib_multi_support_reset_file(f)
+    finally:
+        codes_grib_multi_support_off()
+
+
+def test_grib_multi_field_write(tmpdir):
+    # Note: codes_grib_multi_new() calls codes_grib_multi_support_on()
+    #       hence the 'finally' block
+    try:
+        gid = codes_grib_new_from_samples("GRIB2")
+        mgid = codes_grib_multi_new()
+        section_num = 4
+        for step in range(12, 132, 12):
+            codes_set(gid, "step", step)
+            codes_grib_multi_append(gid, section_num, mgid)
+        output = tmpdir.join("test_grib_multi_field_write.grib2")
+        with open(str(output), "wb") as fout:
+            codes_grib_multi_write(mgid, fout)
+        codes_grib_multi_release(mgid)
+        codes_release(gid)
+    finally:
+        codes_grib_multi_support_off()
 
 
 def test_grib_uuid_get_set():
@@ -654,7 +687,6 @@ def test_bufr_get_message_offset():
     assert codes_get_message_offset(gid) == 0
 
 
-# Experimental features
 def test_codes_bufr_key_is_header():
     bid = codes_bufr_new_from_samples("BUFR4_local_satellite")
     assert codes_bufr_key_is_header(bid, "edition")
@@ -670,17 +702,6 @@ def test_codes_bufr_key_is_header():
     assert not codes_bufr_key_is_header(bid, "#6#brightnessTemperature")
 
 
-def test_extract_offsets():
-    fpath = get_sample_fullpath("BUFR4.tmpl")
-    if fpath is None:
-        return
-    is_strict = True
-    offsets = codes_extract_offsets(fpath, CODES_PRODUCT_ANY, is_strict)
-    offsets_list = list(offsets)
-    assert len(offsets_list) == 1
-    assert offsets_list[0] == 0
-
-
 def test_bufr_extract_headers():
     fpath = get_sample_fullpath("BUFR4_local.tmpl")
     if fpath is None:
@@ -695,3 +716,30 @@ def test_bufr_extract_headers():
     assert header["ident"].strip() == "91334"
     assert header["rdbtimeSecond"] == 19
     assert math.isclose(header["localLongitude"], 151.83)
+
+
+# ---------------------------------------------
+# Experimental features
+# ---------------------------------------------
+def test_grib_nearest2():
+    if "codes_grib_nearest_new" not in dir(eccodes):
+        return
+    gid = codes_grib_new_from_samples("gg_sfc_grib2")
+    lat, lon = 40, 20
+    flags = CODES_GRIB_NEAREST_SAME_GRID | CODES_GRIB_NEAREST_SAME_POINT
+    nid = codes_grib_nearest_new(gid)
+    assert nid > 0
+    nearest = codes_grib_nearest_find(nid, gid, lat, lon, flags)
+    assert len(nearest) == 4
+    expected_indexes = (2679, 2678, 2517, 2516)
+    returned_indexes = (
+        nearest[0].index,
+        nearest[1].index,
+        nearest[2].index,
+        nearest[3].index,
+    )
+    assert sorted(expected_indexes) == sorted(returned_indexes)
+    assert math.isclose(nearest[0].value, 295.22085, abs_tol=0.0001)
+    assert math.isclose(nearest[2].distance, 24.16520, abs_tol=0.0001)
+    codes_release(gid)
+    codes_grib_nearest_delete(nid)