diff --git a/.gitignore b/.gitignore index f654e1ad..f92e3433 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,14 @@ doc/latex .*.sw* .*un~ *.ccls-cache +*.py[cod] +*.swp +**/__pycache__ +.vscode +*.egg-info +.pytest_cache +*.prof +*.idx +.mypy_cache +build/ +Testing/ \ No newline at end of file diff --git a/pymetkit/README.md b/pymetkit/README.md new file mode 100644 index 00000000..c0554dc0 --- /dev/null +++ b/pymetkit/README.md @@ -0,0 +1,28 @@ +# pymetkit + +This repository contains an Python interface to the MetKit library for parsing MARS requests. + +## Example + +The function for parsing a MARS request is `metkit.parse_mars_request` which accepts a string or file-like object +as inputs. A list of `metkit.mars.Request` instances are returned, which is a dictionary containing the keys and +values in the MARS request and the attribute `verb` for the verb in the MARS request. + +### From String +``` +from metkit import parse_mars_request + +request_str = "retrieve,class=od,date=20240124,time=12,param=129,step=12,target=test.grib" +requests = parse_mars_request(requests) + +print(requests[0]) +# verb: retrieve, request: {'class': ['od'], 'date': ['20240124'], 'time': ['1200'], 'param': ['129'], 'step': ['12'], 'target': ['test.grib'], 'domain': ['g'], 'expver': ['0001'], 'levelist': ['1000', '850', '700', '500', '400', '300'], 'levtype': ['pl'], 'stream': ['oper'], 'type': ['an']} +``` + +### From File +If the MARS request is contained inside a file, e.g. test_requests.txt: +``` +from metkit import parse_mars_request + +requests = parse_mars_request(open("test_requests.txt", "r")) +``` diff --git a/pymetkit/src/pymetkit/__init__.py b/pymetkit/src/pymetkit/__init__.py new file mode 100644 index 00000000..f6497777 --- /dev/null +++ b/pymetkit/src/pymetkit/__init__.py @@ -0,0 +1 @@ +from .pymetkit import * diff --git a/pymetkit/src/pymetkit/_version.py b/pymetkit/src/pymetkit/_version.py new file mode 100644 index 00000000..07549de4 --- /dev/null +++ b/pymetkit/src/pymetkit/_version.py @@ -0,0 +1,5 @@ +from pathlib import Path +from .pymetkit import * +import importlib.metadata + +__version__ = importlib.metadata.version("pymetkit") diff --git a/pymetkit/src/pymetkit/metkit_c.h b/pymetkit/src/pymetkit/metkit_c.h new file mode 100644 index 00000000..fbb9073a --- /dev/null +++ b/pymetkit/src/pymetkit/metkit_c.h @@ -0,0 +1,51 @@ + +struct metkit_marsrequest_t; +typedef struct metkit_marsrequest_t metkit_marsrequest_t; +struct metkit_requestiterator_t; +typedef struct metkit_requestiterator_t metkit_requestiterator_t; +struct metkit_paramiterator_t; +typedef struct metkit_paramiterator_t metkit_paramiterator_t; + +typedef enum metkit_error_values_t +{ + METKIT_SUCCESS = 0, /* Operation succeded. */ + METKIT_ERROR = 1, /* Operation failed. */ + METKIT_ERROR_UNKNOWN = 2, /* Failed with an unknown error. */ + METKIT_ERROR_USER = 3, /* Failed with an user error. */ + METKIT_ERROR_ASSERT = 4 /* Failed with an assert() */ +} metkit_error_t; + +typedef enum metkit_iterator_status_t +{ + METKIT_ITERATOR_SUCCESS = 0, /* Operation succeded. */ + METKIT_ITERATOR_COMPLETE = 1, /* All elements have been returned */ + METKIT_ITERATOR_ERROR = 2 /* Operation failed. */ +} metkit_iterator_status_t; + + +const char* metkit_get_error_string(enum metkit_error_values_t err); +const char* metkit_version(); +const char* metkit_git_sha1(); +metkit_error_t metkit_initialise(); + +metkit_error_t metkit_parse_marsrequests(const char* str, metkit_requestiterator_t** requests, bool strict); +metkit_error_t metkit_marsrequest_new(metkit_marsrequest_t** request); +metkit_error_t metkit_marsrequest_delete(const metkit_marsrequest_t* request); +metkit_error_t metkit_marsrequest_set(metkit_marsrequest_t* request, const char* param, const char* values[], int numValues); +metkit_error_t metkit_marsrequest_set_one(metkit_marsrequest_t* request, const char* param, const char* value); +metkit_error_t metkit_marsrequest_set_verb(metkit_marsrequest_t* request, const char* verb); +metkit_error_t metkit_marsrequest_verb(const metkit_marsrequest_t* request, const char** verb); +metkit_error_t metkit_marsrequest_has_param(const metkit_marsrequest_t* request, const char* param, bool* has); +metkit_error_t metkit_marsrequest_params(const metkit_marsrequest_t* request, metkit_paramiterator_t** params); +metkit_error_t metkit_marsrequest_count_values(const metkit_marsrequest_t* request, const char* param, size_t* count); +metkit_error_t metkit_marsrequest_value(const metkit_marsrequest_t* request, const char* param, int index, const char** value); +metkit_error_t metkit_marsrequest_expand(const metkit_marsrequest_t* request, bool inherit, bool strict, metkit_marsrequest_t* expandedRequest); +metkit_error_t metkit_marsrequest_merge(metkit_marsrequest_t* request, const metkit_marsrequest_t* otherRequest); + +metkit_error_t metkit_requestiterator_delete(const metkit_requestiterator_t* it); +metkit_iterator_status_t metkit_requestiterator_next(metkit_requestiterator_t* it); +metkit_iterator_status_t metkit_requestiterator_current(metkit_requestiterator_t* it, metkit_marsrequest_t* request); + +metkit_error_t metkit_paramiterator_delete(const metkit_paramiterator_t* it); +metkit_iterator_status_t metkit_paramiterator_next(metkit_paramiterator_t* it); +metkit_iterator_status_t metkit_paramiterator_current(const metkit_paramiterator_t* it, const char** param); diff --git a/pymetkit/src/pymetkit/pymetkit.py b/pymetkit/src/pymetkit/pymetkit.py new file mode 100644 index 00000000..7c0594fa --- /dev/null +++ b/pymetkit/src/pymetkit/pymetkit.py @@ -0,0 +1,312 @@ +import os +from cffi import FFI +import findlibs +from typing import IO, Iterator +import warnings +from ._version import __version__ + +ffi = FFI() + + +def ffi_encode(data) -> bytes: + if isinstance(data, bytes): + return data + + if not isinstance(data, str): + data = str(data) + + return data.encode(encoding="utf-8", errors="surrogateescape") + + +def ffi_decode(data: FFI.CData) -> str: + buf = ffi.string(data) + if isinstance(buf, str): + return buf + else: + return buf.decode(encoding="utf-8", errors="surrogateescape") + + +class MarsRequest: + def __init__(self, verb: str | None = None, **kwargs): + """ + Create MetKit MarsRequest object. Parameters and values in + the request can be specified through kwargs, noting that + reserved words in Python must be suffixed with "_" e.g. "class_" + """ + crequest = ffi.new("metkit_marsrequest_t **") + lib.metkit_marsrequest_new(crequest) + self.__request = ffi.gc(crequest[0], lib.metkit_marsrequest_delete) + if verb is not None: + lib.metkit_marsrequest_set_verb(self.__request, ffi_encode(verb)) + for param, values in kwargs.items(): + self[param.rstrip("_")] = values + + def ctype(self) -> FFI.CData: + return self.__request + + def verb(self) -> str: + cverb = ffi.new("const char **") + lib.metkit_marsrequest_verb(self.__request, cverb) + return ffi_decode(cverb[0]) + + def expand(self, inherit: bool = True, strict: bool = False) -> "MarsRequest": + """ + Return expanded request + + Params + ------ + inherit: bool, if True, populates expanded request with default values + strict: bool, if True, raise error instead of warning for invalid values + + Returns + ------- + Request, resulting from expansion + """ + expanded_request = MarsRequest() + lib.metkit_marsrequest_expand( + self.__request, inherit, strict, expanded_request.ctype() + ) + return expanded_request + + def validate(self): + """ + Check if request is valid against MARS language definition. Does not + inherit missing parameters. + + Raises + ------ + Exception if request is incompatible with MARS language definition + """ + self.expand(False, True) + + def keys(self) -> Iterator[str]: + """ + Get iterator over parameters in request + + Returns + ------- + Iterator over parameter names + """ + it_c = ffi.new("metkit_paramiterator_t **") + lib.metkit_marsrequest_params(self.__request, it_c) + it = ffi.gc(it_c[0], lib.metkit_paramiterator_delete) + + while lib.metkit_paramiterator_next(it) == lib.METKIT_ITERATOR_SUCCESS: + cparam = ffi.new("const char **") + lib.metkit_paramiterator_current(it, cparam) + param = ffi_decode(cparam[0]) + yield param + + def num_values(self, param: str) -> int: + """ + Number of values for parameter + + Params + ------ + param: parameter name + + Returns + ------- + int + """ + cparam = ffi_encode(param) + count = ffi.new("size_t *", 0) + lib.metkit_marsrequest_count_values(self.__request, cparam, count) + return count[0] + + def merge(self, other: "MarsRequest") -> "MarsRequest": + """ + Merge the values in another request with existing request and returns result as a + new Request object. Does not modify inputs to merge. Both input requests must contain + the same values and the resulting request object must be compatible with MARS language + definition + + Params + ------ + other: Request, request to merge with self + + Returns + ------- + Request, containing the result of the merge + + Raises + ------ + ValueError if parameters in the two requests do not match + MetKitException if resulting request is not compatible with MARS language definition + """ + if set(self.keys()) != set(other.keys()): + raise ValueError("Can not merge requests with different parameters.") + res = MarsRequest(self.verb(), **{k: v for k, v in self}) + lib.metkit_marsrequest_merge(res.ctype(), other.ctype()) + res.validate() + return res + + def __iter__(self) -> Iterator[tuple[str, list[str]]]: + for param in self.keys(): + yield param, self[param] + + def __getitem__(self, param: str) -> str | list[str]: + nvalues = self.num_values(param) + values = [] + for index in range(nvalues): + cvalue = ffi.new("const char **") + lib.metkit_marsrequest_value(self.__request, ffi_encode(param), index, cvalue) + value = ffi_decode(cvalue[0]) + if nvalues == 1: + return value + values.append(value) + return values + + def __contains__(self, param: str) -> bool: + has = ffi.new("bool *", False) + lib.metkit_marsrequest_has_param(self.__request, ffi_encode(param), has) + return has[0] + + def __setitem__(self, param: str, values: int | str | list[str]): + if isinstance(values, (str, int)): + values = [values] + cvals = [] + for value in values: + if isinstance(value, int): + value = str(value) + cvals.append(ffi.new("const char[]", value.encode("ascii"))) + lib.metkit_marsrequest_set( + self.__request, + ffi_encode(param), + ffi.new("const char*[]", cvals), + len(values), + ) + + def __eq__(self, other: "MarsRequest") -> bool: + if self.verb() != other.verb(): + return False + expanded = self.expand() + other_expanded = other.expand() + return dict(expanded) == dict(other_expanded) + + +def parse_mars_request(file_or_str: IO | str, strict: bool = False) -> list[MarsRequest]: + """ + Function for parsing mars request from file object or string. + + Params + ------ + file_or_str: string or file-like object, containing mars request + strict: bool, whether to raise error or warning when request is not compatible with + MARS language definition. In the case of warning, when False, the incompatible + parameters are unset from the request. + + Returns + ------- + list of Request + """ + crequest_iter = ffi.new("metkit_requestiterator_t **") + + if isinstance(file_or_str, str): + lib.metkit_parse_marsrequests(ffi_encode(file_or_str), crequest_iter, strict) + else: + lib.metkit_parse_marsrequests( + ffi_encode(file_or_str.read()), crequest_iter, strict + ) + request_iter = ffi.gc(crequest_iter[0], lib.metkit_requestiterator_delete) + + requests = [] + while lib.metkit_requestiterator_next(request_iter) == lib.METKIT_ITERATOR_SUCCESS: + new_request = MarsRequest() + lib.metkit_requestiterator_current(request_iter, new_request.ctype()) + requests.append(new_request) + + return requests + + +class MetKitException(RuntimeError): + """Raised when MetKit library throws exception""" + + pass + + +class CFFIModuleLoadFailed(ImportError): + """Raised when the shared library fails to load""" + + pass + + +class PatchedLib: + """ + Patch a CFFI library with error handling + + Finds the header file associated with the MetKit C API and parses it, + loads the shared library, and patches the accessors with + automatic python-C error handling. + """ + + def __init__(self): + libName = findlibs.find("metkit") + + if libName is None: + raise RuntimeError("MetKit library not found") + + ffi.cdef(self.__read_header()) + self.__lib = ffi.dlopen(libName) + + # All of the executable members of the CFFI-loaded library are functions in the MetKit + # C API. These should be wrapped with the correct error handling. Otherwise forward + # these on directly. + + for f in dir(self.__lib): + try: + attr = getattr(self.__lib, f) + setattr( + self, f, self.__check_error(attr, f) if callable(attr) else attr + ) + except Exception as e: + print(e) + print("Error retrieving attribute", f, "from library") + + # Initialise the library, and set it up for python-appropriate behaviour + + self.metkit_initialise() + + # Check the library version + + versionstr = ffi.string(self.metkit_version()).decode("utf-8") + if versionstr != __version__: + warnings.warn(f"Metkit library version {versionstr} does not match python version {__version__}") + + def __read_header(self): + with open(os.path.join(os.path.dirname(__file__), "metkit_c.h"), "r") as f: + return f.read() + + def __check_error(self, fn, name: str): + """ + If calls into the MetKit library return errors, ensure that they get + detected and reported by throwing an appropriate python exception. + """ + + def wrapped_fn(*args, **kwargs): + + # debug + retval = fn(*args, **kwargs) + + # Some functions dont return error codes. Ignore these. + if name in ["metkit_version", "metkit_git_sha1"]: + return retval + + # error codes: + if retval not in ( + self.__lib.METKIT_SUCCESS, + self.__lib.METKIT_ITERATOR_SUCCESS, + self.__lib.METKIT_ITERATOR_COMPLETE, + ): + err = ffi_decode(self.__lib.metkit_get_error_string(retval)) + msg = "Error in function '{}': {}".format(name, err) + raise MetKitException(msg) + return retval + + return wrapped_fn + + +try: + lib = PatchedLib() +except CFFIModuleLoadFailed as e: + raise ImportError() from e diff --git a/pymetkit/tests/test_marsrequest.py b/pymetkit/tests/test_marsrequest.py new file mode 100644 index 00000000..9d4ce6f4 --- /dev/null +++ b/pymetkit/tests/test_marsrequest.py @@ -0,0 +1,176 @@ +from datetime import datetime, timedelta +from contextlib import nullcontext as does_not_raise +import pytest + +from pymetkit import parse_mars_request, MarsRequest, MetKitException + +request = """ +retrieve, + class=od, + domain=g, + expver=0001, + levtype=sfc, + stream=enfo, + date=-1, + time=12, + param=151.128, + grid=O640, + step=0/to/24/by/6, + target=test.grib, + type=em +retrieve, + class=od, + domain=g, + expver=0001, + levtype=pl, + stream=enfo, + date=-1, + time=12, + param=129, + levelist=500, + grid=O640, + step=0/to/24/by/6, + target=test.grib, + type=em +""" + +yesterday = (datetime.today() - timedelta(days=1)).strftime("%Y%m%d") + + +def test_parse_file(tmpdir): + request_file = f"{tmpdir}/requests" + with open(request_file, "w") as f: + f.write(request) + requests = parse_mars_request(open(request_file, "r")) + assert len(requests) == 2 + for req in requests: + assert req.verb() == "retrieve" + assert len(req["step"]) == 5 + assert req["date"] == yesterday + assert "class" in requests[0] + assert requests[1]["levelist"] == "500" + +# @todo: [1] no longer raises an exception. Disable until METK-126 is resolved. +@pytest.mark.parametrize( + "req_str, length, steps, strict, expectation", + [ + [request, 2, 5, False, does_not_raise()], + # [request, 2, 5, True, pytest.raises(MetKitException)], + [ + "retrieve,class=od,date=-1,time=12,param=129,step=12,target=test.grib", + 1, + 1, + False, + does_not_raise(), + ], + ], +) +def test_parse_string(req_str, length, steps, strict, expectation): + with expectation: + requests = parse_mars_request(req_str, strict) + assert len(requests) == length + for req in requests: + assert req.num_values("step") == steps + + +def test_empty_request(tmpdir): + request_file = f"{tmpdir}/requests" + with open(request_file, "w") as f: + f.write("") + requests = parse_mars_request(open(request_file, "r")) + assert len(requests) == 0 + + +def test_new_request(): + req = MarsRequest("retrieve") + assert req.verb() == "retrieve" + + req = MarsRequest("request", class_="od", type="pf", date=["20200101", "20200102"]) + assert req["class"] == "od" + assert req["type"] == "pf" + assert req["date"] == ["20200101", "20200102"] + + +def test_request_from_expand(): + req = MarsRequest( + "retrieve", + **{ + "class": "od", + "domain": "g", + "date": "-1", + "expver": "0001", + "step": range(0, 13, 6), + }, + ) + expanded = req.expand() + assert expanded.verb() == req.verb() + assert expanded["date"] == yesterday + assert "param" in expanded + expanded.validate() + assert req == expanded + +# @todo: [0] and [1] no longer raise an exception. Disable until METK-126 is resolved. +@pytest.mark.parametrize( + "extra_kv", + [ + # {"levelist": [500]}, + # {"type": "cf", "number": [1, 2]}, + {"class": "invalid"} + ], +) +def test_request_validate(extra_kv): + request = { + "class": "od", + "domain": "g", + "date": "-1", + "expver": "0001", + "step": range(0, 13, 6), + "levtype": "sfc", + } + request.update(extra_kv) + req = MarsRequest("retrieve", **request) + with pytest.raises(MetKitException): + req.validate() + + +@pytest.mark.parametrize( + "extra_kv, expectation", + [ + [{"levtype": "pl", "date": "-1"}, pytest.raises(MetKitException)], + [{"levtype": "sfc", "date": "-1", "type": "em"}, pytest.raises(ValueError)], + [{"levtype": "sfc", "date": "20230101"}, does_not_raise()], + ], +) +def test_request_merge(extra_kv, expectation): + request = { + "class": "od", + "domain": "g", + "expver": "0001", + "step": range(0, 13, 6), + } + req = MarsRequest("retrieve", **request, date="-1", levtype="sfc") + other_req = MarsRequest("retrieve", **request, **extra_kv) + with expectation: + req.merge(other_req) + + +@pytest.mark.parametrize( + "verb, updates, expected", + [["retrieve", {"date": 20230101, "param": 130}, True], ["compute", {}, False]], +) +def test_request_equality(verb, updates, expected): + init_request = { + "class": "od", + "domain": "g", + "date": "20230101", + "param": "130", + "expver": "0001", + "step": range(0, 13, 6), + } + req = MarsRequest( + "retrieve", + **init_request, + ) + second_request = {**init_request, **updates} + req2 = MarsRequest(verb, **second_request) + assert (req == req2) == expected diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..bdb32794 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,46 @@ +# pytest +[tool.pytest.ini_options] +minversion = "6.0" +addopts = "-vv -s" +testpaths = [ + "pymetkit/tests" +] + +# pyproject.toml + +[build-system] +requires = ["setuptools", "wheel", "cffi"] +build-backend = "setuptools.build_meta" + +[project] +name = "pymetkit" +description = "Python interface for metkit" +dynamic = ["version"] +authors = [ + { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" }, +] +license = { text = "Apache License Version 2.0" } +requires-python = ">=3.10" +dependencies = [ + "cffi", + "findlibs" +] + +[tool.setuptools.dynamic] +version = { file = ["VERSION"] } + +[tool.setuptools] +packages = ["pymetkit"] +package-dir = { "pymetkit" = "./pymetkit/src/pymetkit" } +include-package-data = true +zip-safe = false + +[tool.setuptools.package-data] +"pymetkit" = [ + "VERSION", + "metkit_c.h" +] + +[project.optional-dependencies] +tests = ["pytest"] + diff --git a/src/metkit/api/metkit_c.cc b/src/metkit/api/metkit_c.cc index 32701f40..5d1ece68 100644 --- a/src/metkit/api/metkit_c.cc +++ b/src/metkit/api/metkit_c.cc @@ -287,11 +287,15 @@ metkit_error_t metkit_marsrequest_merge(metkit_marsrequest_t* request, const met }); } +void metkit_string_delete(const char* str) { + delete[] str; +} + // ----------------------------------------------------------------------------- // REQUEST ITERATOR // ----------------------------------------------------------------------------- -metkit_error_t metkit_delete_requestiterator(const metkit_requestiterator_t* it) { +metkit_error_t metkit_requestiterator_delete(const metkit_requestiterator_t* it) { return tryCatch([it] { delete it; }); diff --git a/src/metkit/api/metkit_c.h b/src/metkit/api/metkit_c.h index 014bdaa6..427ad2ec 100644 --- a/src/metkit/api/metkit_c.h +++ b/src/metkit/api/metkit_c.h @@ -201,7 +201,7 @@ metkit_error_t metkit_marsrequest_merge(metkit_marsrequest_t* request, const met * @param it RequestIterator instance * @return metkit_error_t Error code */ -metkit_error_t metkit_delete_requestiterator(const metkit_requestiterator_t* it); +metkit_error_t metkit_requestiterator_delete(const metkit_requestiterator_t* it); /** Moves to the next Request element in RequestIterator * @param it RequestIterator instance diff --git a/tests/test_c_api.cc b/tests/test_c_api.cc index d2f47821..32d98d04 100644 --- a/tests/test_c_api.cc +++ b/tests/test_c_api.cc @@ -159,7 +159,7 @@ CASE( "metkit_requestiterator_t parsing" ) { } // cleanup - metkit_delete_requestiterator(it); // NB: requests have been moved out of the iterator + metkit_requestiterator_delete(it); // NB: requests have been moved out of the iterator for (auto req : requests) { metkit_marsrequest_delete(req); } @@ -213,7 +213,7 @@ CASE( "metkit_requestiterator_t 1 item" ) { EXPECT_EQUAL(requests.size(), 1); // cleanup - metkit_delete_requestiterator(it); + metkit_requestiterator_delete(it); for (auto req : requests) { metkit_marsrequest_delete(req); }