Release 1.1.0

Author: shahramn

.github/workflows/on-push.yml

@@ -0,0 +1,61 @@
+name: on-push
+
+on: [push]
+
+jobs:
+  unit-tests:
+    runs-on: ${{ matrix.os }}-latest
+    strategy:
+      max-parallel: 5
+      matrix:
+        os: [ubuntu]
+        python: [3.6, 3.7, 3.8]
+        include:
+          - os: macos
+            python: 3.8
+          - os: windows
+            python: 3.8
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        python-version: ${{ matrix.python }}
+        activate-environment: ${{ matrix.os }}-${{ matrix.python }}
+#        environment-file: tests/environment-${{ matrix.os }}-${{ matrix.python }}.yml
+        environment-file: tests/environment.yml
+    - name: Test with pytest
+      shell: bash -l {0}
+      run: |
+        conda install pytest pytest-cov
+        python -m pip install --no-deps -e .
+        pytest -v --cov=. --cov-report=xml --cov-branch .
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v1
+
+  code-quality:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Lint with flake8
+      run: |
+        $CONDA/bin/conda install flake8
+        # stop the build if there are Python syntax errors or undefined names
+        $CONDA/bin/flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        $CONDA/bin/flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+  code-style:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Check code style with black
+      run: |
+        $CONDA/bin/conda install -c conda-forge black
+        $CONDA/bin/black --check .
+    - name: Check code style with isort
+      run: |
+        $CONDA/bin/conda install isort
+        $CONDA/bin/isort .

CHANGELOG.rst

@@ -2,6 +2,17 @@
 Changelog for eccodes-python
 ============================
 
+1.1.0 (2021-01-20)
+--------------------
+
+- ECC-1171: Performance: Python bindings: remove assert statements
+- ECC-1161: Python3 bindings: Do not raise exception on first failed attempt
+- ECC-1176: Python3 bindings: float32 recognised as int instead of float
+- GitHub pull request #41: Remove the apparent support for Python 2
+- GitHub pull request #44: Fix CFFI crash on windows
+- GitHub pull request #42: Add unit testing with GitHub actions (linux, macos and windows)
+
+
 1.0.0 (2020-10-14)
 --------------------
 
@@ -24,7 +35,7 @@ Changelog for eccodes-python
 - Fix codes_set_definitions_path() typo
 - Fix grib_get_double_element(). Missing last argument
 - Add more tests to increase coverage
-- Add .__next__() method to eccodes.CodesFile class (pull request #15)
+- GitHub pull request #15: Add .__next__() method to eccodes.CodesFile class
 - ECC-1113: Python3 bindings under Windows: codes_get_long_array returns incorrect values
 - ECC-1108: Python3 bindings under Windows: use of handle causes crash
 - ECC-1121: Segfault when closing GribFile if messages are closed manually

README.rst

@@ -47,7 +47,7 @@ https://software.ecmwf.int/wiki/display/ECC/ecCodes+installation
 You may run a simple selfcheck command to ensure that your system is set up correctly::
 
     $ python -m eccodes selfcheck
-    Found: ecCodes v2.19.0.
+    Found: ecCodes v2.20.0.
     Your system is ready.
 
 

eccodes/eccodes.py

@@ -129,5 +129,8 @@
 from gribapi import codes_bufr_multi_element_constant_arrays_off
 from gribapi import codes_bufr_extract_headers
 
+from gribapi import codes_bufr_key_is_header
+from gribapi import codes_extract_offsets
+
 from gribapi.errors import GribInternalError as CodesInternalError
 from gribapi.errors import *

gribapi/__init__.py

@@ -15,8 +15,8 @@
 from .gribapi import bindings_version
 
 # The minimum required version for the ecCodes package
-min_reqd_version_str = "2.19.0"
-min_reqd_version_int = 21900
+min_reqd_version_str = "2.20.0"
+min_reqd_version_int = 22000
 
 if lib.grib_get_api_version() < min_reqd_version_int:
     print(

gribapi/bindings.py

@@ -21,18 +21,17 @@
 
 import cffi
 
-__version__ = "1.0.0"
+__version__ = "1.1.0"
 
 LOG = logging.getLogger(__name__)
 
 try:
     from ._bindings import ffi, lib
 except ModuleNotFoundError:
     ffi = cffi.FFI()
-    ffi.cdef(
-        pkgutil.get_data(__name__, "grib_api.h").decode("utf-8")
-        + pkgutil.get_data(__name__, "eccodes.h").decode("utf-8")
-    )
+    CDEF = pkgutil.get_data(__name__, "grib_api.h")
+    CDEF += pkgutil.get_data(__name__, "eccodes.h")
+    ffi.cdef(CDEF.decode("utf-8").replace("\r", "\n"))
 
     LIBNAMES = ["eccodes", "libeccodes.so", "libeccodes"]
 
@@ -46,18 +45,19 @@
     if os.environ.get("ECCODES_DIR"):
         eccdir = os.environ["ECCODES_DIR"]
         LIBNAMES.insert(0, os.path.join(eccdir, "lib/libeccodes.so"))
-        LIBNAMES.insert(0, os.path.join(eccdir, "lib64/libeccodes.so"))
+        LIBNAMES.insert(1, os.path.join(eccdir, "lib64/libeccodes.so"))
 
+    lib = None
     for libname in LIBNAMES:
         try:
             lib = ffi.dlopen(libname)
             LOG.info("ecCodes library found using name '%s'.", libname)
             break
         except OSError:
-            # lazy exception
-            lib = None
             LOG.info("ecCodes library not found using name '%s'.", libname)
-            raise RuntimeError(f"ecCodes library not found using {LIBNAMES}")
+            pass
+    if lib is None:
+        raise RuntimeError(f"ecCodes library not found using {LIBNAMES}")
 
 # default encoding for ecCodes strings
 ENC = "ascii"

gribapi/eccodes.h

@@ -20,5 +20,8 @@ int codes_bufr_copy_data(grib_handle* hin, grib_handle* hout);
 void codes_bufr_multi_element_constant_arrays_on(codes_context* c);
 void codes_bufr_multi_element_constant_arrays_off(codes_context* c);
 int codes_bufr_extract_headers_malloc(codes_context* c, const char* filename, codes_bufr_header** result, int* num_messages, int strict_mode);
+int codes_extract_offsets_malloc(grib_context* c, const char* filename, ProductKind product, long int** offsets, int* num_messages, int strict_mode);
+int codes_bufr_key_is_header(const codes_handle* h, const char* key, int* err);
+
 char* codes_samples_path(const codes_context *c);
 char* codes_definition_path(const codes_context *c);

gribapi/grib_api.h

@@ -220,6 +220,7 @@ typedef struct codes_bufr_header {
     long rectimeHour;
     long rectimeMinute;
     long rectimeSecond;
+    long restricted;
 
     long   isSatellite;
     double localLongitude1;

gribapi/gribapi.py

@@ -155,7 +155,6 @@ def wrapper(*args):
 
 
 def get_handle(msgid):
-    assert isinstance(msgid, int)
     h = ffi.cast("grib_handle*", msgid)
     if h == ffi.NULL:
         raise errors.InvalidGribError(f"get_handle: Bad message ID {msgid}")
@@ -169,7 +168,6 @@ def put_handle(handle):
 
 
 def get_multi_handle(msgid):
-    assert isinstance(msgid, int)
     return ffi.cast("grib_multi_handle*", msgid)
 
 
@@ -178,7 +176,6 @@ def put_multi_handle(handle):
 
 
 def get_index(indexid):
-    assert isinstance(indexid, int)
     return ffi.cast("grib_index*", indexid)
 
 
@@ -187,7 +184,6 @@ def put_index(indexh):
 
 
 def get_iterator(iterid):
-    assert isinstance(iterid, int)
     return ffi.cast("grib_iterator*", iterid)
 
 
@@ -196,7 +192,6 @@ def put_iterator(iterh):
 
 
 def get_grib_keys_iterator(iterid):
-    assert isinstance(iterid, int)
     return ffi.cast("grib_keys_iterator*", iterid)
 
 
@@ -205,7 +200,6 @@ def put_grib_keys_iterator(iterh):
 
 
 def get_bufr_keys_iterator(iterid):
-    assert isinstance(iterid, int)
     return ffi.cast("bufr_keys_iterator*", iterid)
 
 
@@ -314,7 +308,7 @@ def codes_new_from_file(fileobj, product_kind, headers_only=False):
         return gts_new_from_file(fileobj, headers_only)
     if product_kind == CODES_PRODUCT_ANY:
         return any_new_from_file(fileobj, headers_only)
-    raise Exception("Invalid product kind: " + product_kind)
+    raise ValueError("Invalid product kind %d" % product_kind)
 
 
 @require(fileobj=file)
@@ -456,7 +450,7 @@ def grib_multi_support_reset_file(fileobj):
     @brief Reset file handle in multiple field support mode
     """
     context = lib.grib_context_get_default()
-    GRIB_CHECK(lib.grib_multi_support_reset_file(context, fileobj))
+    lib.grib_multi_support_reset_file(context, fileobj)
 
 
 @require(msgid=int)
@@ -971,7 +965,7 @@ def grib_get_double(msgid, key):
     return value_p[0]
 
 
-@require(msgid=int, key=str, value=(int, float, str))
+@require(msgid=int, key=str, value=(int, float, np.float16, np.float32, str))
 def grib_set_long(msgid, key, value):
     """
     @brief Set the integer value for a key in a message.
@@ -990,13 +984,13 @@ def grib_set_long(msgid, key, value):
         raise TypeError("Invalid type")
 
     if value > sys.maxsize:
-        raise TypeError("Invalid type")
+        raise ValueError("Value too large")
 
     h = get_handle(msgid)
     GRIB_CHECK(lib.grib_set_long(h, key.encode(ENC), value))
 
 
-@require(msgid=int, key=str, value=(int, float, str))
+@require(msgid=int, key=str, value=(int, float, np.float16, np.float32, str))
 def grib_set_double(msgid, key, value):
     """
     @brief Set the double value for a key in a message.
@@ -1037,7 +1031,7 @@ def codes_new_from_samples(samplename, product_kind):
         return grib_new_from_samples(samplename)
     if product_kind == CODES_PRODUCT_BUFR:
         return codes_bufr_new_from_samples(samplename)
-    raise Exception("Invalid product kind: " + product_kind)
+    raise ValueError("Invalid product kind %d" % product_kind)
 
 
 @require(samplename=str)
@@ -2039,7 +2033,7 @@ def grib_set(msgid, key, value):
     """
     if isinstance(value, int):
         grib_set_long(msgid, key, value)
-    elif isinstance(value, float):
+    elif isinstance(value, (float, np.float16, np.float32, np.float64)):
         grib_set_double(msgid, key, value)
     elif isinstance(value, str):
         grib_set_string(msgid, key, value)
@@ -2075,12 +2069,11 @@ def grib_set_array(msgid, key, value):
     except TypeError:
         pass
 
-    if isinstance(val0, float):
+    if isinstance(val0, (float, np.float16, np.float32, np.float64)):
         grib_set_double_array(msgid, key, value)
     elif isinstance(val0, str):
         grib_set_string_array(msgid, key, value)
     else:
-        # Note: Cannot do isinstance(val0,int) for numpy.int64
         try:
             int(val0)
         except (ValueError, TypeError):
@@ -2392,3 +2385,48 @@ def codes_bufr_extract_headers(filepath, is_strict=True):
     while i < num_messages:
         yield _convert_struct_to_dict(headers[i])
         i += 1
+
+
+def codes_bufr_key_is_header(msgid, key):
+    """
+    @brief Check if the BUFR key is in the header or in the data section.
+
+    If the data section has not been unpacked, then passing in a key from
+    the data section will throw KeyValueNotFoundError.
+
+    @param msgid      id of the BUFR message loaded in memory
+    @param key        key name
+    @return           1->header, 0->data section
+    @exception CodesInternalError
+    """
+    h = get_handle(msgid)
+    err, value = err_last(lib.codes_bufr_key_is_header)(h, key.encode(ENC))
+    GRIB_CHECK(err)
+    return value
+
+
+def codes_extract_offsets(filepath, product_kind, is_strict=True):
+    """
+    @brief Message offset extraction (EXPERIMENTAL FEATURE)
+
+    @param filepath       path of input file
+    @param is_strict      fail as soon as any invalid message is encountered
+    @return               a list of offsets
+    @exception CodesInternalError
+    """
+    context = lib.grib_context_get_default()
+    offsets_p = ffi.new("long int**")
+    num_message_p = ffi.new("int*")
+
+    err = lib.codes_extract_offsets_malloc(
+        context, filepath.encode(ENC), product_kind, offsets_p, num_message_p, is_strict
+    )
+    GRIB_CHECK(err)
+
+    num_messages = num_message_p[0]
+    offsets = offsets_p[0]
+
+    i = 0
+    while i < num_messages:
+        yield offsets[i]
+        i += 1

setup.cfg

@@ -1,6 +1,3 @@
-[bdist_wheel]
-universal = 1
-
 [aliases]
 test = pytest