VER: Release 0.7.0

See release notes.

Author: cjdsellers

.github/workflows/release.yml

@@ -33,8 +33,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel
-          pip install -r requirements.txt
-          pip install -r requirements_dev.txt
+          scripts/build.sh
 
       # Tag the commit with the library version
       - name: Create git tag

.github/workflows/test.yml

@@ -25,17 +25,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      # Install dependencies
-      - name: Install dependencies
+      - name: Install
         run: |
           python -m pip install --upgrade pip setuptools wheel
-          pip install -r requirements.txt
-          pip install -r requirements_dev.txt
-
-      # Test pip installation
-      - name: Test pip installation
-        run: pip install .
+          scripts/build.sh
 
       # Run tests
       - name: Run tests
-        run: pytest tests .
+        run: scripts/test.sh

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.7.0 - 2023-01-10
+- Added support for `definition` schema
+- Updated `Flags` enum
+- Upgraded `dbz-python` to `0.2.1`
+- Upgraded `zstandard` to `0.19.0`
+
 ## 0.6.0 - 2022-12-02
 - Added `metadata.get_dataset_condition` method to `Historical` client
 - Upgraded `dbz-python` to `0.2.0`

README.md

@@ -10,13 +10,13 @@ The official Python client library for [Databento](https://databento.com).
 
 Key features include:
 - Fast, lightweight access to both live and historical data from [multiple markets](https://docs.databento.com/knowledge-base/new-users/venues-and-publishers?historical=python&live=python).
-- [Multiple schemas](https://docs.databento.com/knowledge-base/new-users/list-of-supported-market-data-schemas?historical=python&live=python) such as MBO, MBP, top of book, OHLCV, last sale, and more.
+- [Multiple schemas](https://docs.databento.com/knowledge-base/new-users/market-data-schemas?historical=python&live=python) such as MBO, MBP, top of book, OHLCV, last sale, and more.
 - [Fully normalized](https://docs.databento.com/knowledge-base/new-users/normalization?historical=python&live=python), i.e. identical message schemas for both live and historical data, across multiple asset classes.
 - Provides mappings between different symbology systems, including [smart symbology](https://docs.databento.com/reference-historical/basics/symbology?historical=python&live=python) for futures rollovers.
 - [Point-in-time]() instrument definitions, free of look-ahead bias and retroactive adjustments.
 - Reads and stores market data in an extremely efficient file format using [Databento Binary Encoding](https://docs.databento.com/knowledge-base/new-users/dbz-format?historical=python&live=python).
 - Event-driven [market replay](https://docs.databento.com/reference-historical/helpers/bento-replay?historical=python&live=python), including at high-frequency order book granularity.
-- Support for [batch download](https://docs.databento.com/knowledge-base/new-users/historical-data-streaming-vs-batch-download?historical=python&live=python) of flat files.
+- Support for [batch download](https://docs.databento.com/knowledge-base/new-users/stream-vs-batch?historical=python&live=python) of flat files.
 - Support for [pandas](https://pandas.pydata.org/docs/), CSV, and JSON.
 
 ## Documentation
@@ -31,11 +31,11 @@ The library is fully compatible with the latest distribution of Anaconda 3.7 and
 The minimum dependencies as found in the `requirements.txt` are also listed below:
 - Python (>=3.7)
 - aiohttp (>=3.7.2)
-- dbz-python (>=0.2.0)
+- dbz-python (>=0.2.1)
 - numpy (>=1.17.0)
 - pandas (>=1.1.3)
 - requests (>=2.24.0)
-- zstandard (>=0.18.0)
+- zstandard (>=0.19.0)
 
 ## Installation
 To install the latest stable version of the package from PyPI:
@@ -56,6 +56,8 @@ import databento as db
 client = db.Historical('YOUR_API_KEY')
 data = client.timeseries.stream(
     dataset='GLBX.MDP3',
+    symbols='ES.FUT',
+    stype_in='smart',
     start='2022-06-10T14:30',
     end='2022-06-10T14:40',
 )
@@ -75,14 +77,17 @@ array = data.to_ndarray()  # to ndarray
 ```
 
 Note that the API key was also passed as a parameter, which is
-[not recommended for production applications](https://docs.databento.com/knowledge-base/kb-new-users/kb-new-security-managing-api-keys?historical=python&live=python).
+[not recommended for production applications](https://docs.databento.com/knowledge-base/new-users/security-managing-api-keys?historical=python&live=python).
 Instead, you can leave out this parameter to pass your API key via the `DATABENTO_API_KEY` environment variable:
 
 ```python
 import databento as db
 
-client = db.Historical('YOUR_API_KEY')  # pass as parameter
-client = db.Historical()  # pass as `DATABENTO_API_KEY` environment variable
+# Pass as parameter
+client = db.Historical('YOUR_API_KEY')
+
+# Or, pass as `DATABENTO_API_KEY` environment variable
+client = db.Historical()
 ```
 
 ## License

databento/common/bento.py

@@ -6,7 +6,14 @@
 import numpy as np
 import pandas as pd
 import zstandard
-from databento.common.data import COLUMNS, DERIV_SCHEMAS, STRUCT_MAP
+from databento.common.data import (
+    COLUMNS,
+    DEFINITION_CHARARRAY_COLUMNS,
+    DEFINITION_PRICE_COLUMNS,
+    DEFINITION_TYPE_MAX_MAP,
+    DERIV_SCHEMAS,
+    STRUCT_MAP,
+)
 from databento.common.enums import Compression, Encoding, Schema, SType
 from databento.common.logging import log_debug
 from databento.common.metadata import MetadataDecoder
@@ -52,6 +59,8 @@ def _get_index_column(self) -> str:
                 Schema.OHLCV_1M,
                 Schema.OHLCV_1H,
                 Schema.OHLCV_1D,
+                Schema.GATEWAY_ERROR,
+                Schema.SYMBOL_MAPPING,
             )
             else "ts_recv"
         )
@@ -435,43 +444,75 @@ def to_df(
         """
         df = pd.DataFrame(self.to_ndarray())
         df.set_index(self._get_index_column(), inplace=True)
+        df = self._cleanup_dataframe(df)
 
-        # Cleanup dataframe
+        if pretty_ts:
+            df = self._apply_pretty_ts(df)
+
+        if pretty_px:
+            df = self._apply_pretty_px(df)
+
+        if map_symbols and self.schema != Schema.DEFINITION:
+            df = self._map_symbols(df, pretty_ts)
+
+        return df
+
+    def _cleanup_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
         df.drop(["length", "rtype"], axis=1, inplace=True)
         if self.schema == Schema.MBO or self.schema in DERIV_SCHEMAS:
             df = df.reindex(columns=COLUMNS[self.schema])
             df["flags"] = df["flags"] & 0xFF  # Apply bitmask
             df["side"] = df["side"].str.decode("utf-8")
             df["action"] = df["action"].str.decode("utf-8")
+        elif self.schema == Schema.DEFINITION:
+            for column in DEFINITION_CHARARRAY_COLUMNS:
+                df[column] = df[column].str.decode("utf-8")
+            for column, type_max in DEFINITION_TYPE_MAX_MAP.items():
+                if column in df.columns:
+                    df[column] = df[column].where(df[column] != type_max, np.nan)
 
-        if pretty_ts:
-            df.index = pd.to_datetime(df.index, utc=True)
-            for column in df.columns:
-                if column.startswith("ts_") and "delta" not in column:
-                    df[column] = pd.to_datetime(df[column], utc=True)
+        return df
 
-        if pretty_px:
-            for column in list(df.columns):
-                if (
-                    column in ("price", "open", "high", "low", "close")
-                    or column.startswith("bid_px")  # MBP
-                    or column.startswith("ask_px")  # MBP
-                ):
-                    df[column] = df[column] * 1e-9
-
-        if map_symbols:
-            # Build product ID index
-            if not self._product_id_index:
-                self._product_id_index = self._build_product_id_index()
-
-            # Map product IDs to native symbols
-            if self._product_id_index:
-                df_index = df.index if pretty_ts else pd.to_datetime(df.index, utc=True)
-                dates = [ts.date() for ts in df_index]
-                df["symbol"] = [
-                    self._product_id_index[dates[i]][p]
-                    for i, p in enumerate(df["product_id"])
-                ]
+    def _apply_pretty_ts(self, df: pd.DataFrame) -> pd.DataFrame:
+        df.index = pd.to_datetime(df.index, utc=True)
+        for column in df.columns:
+            if column.startswith("ts_") and "delta" not in column:
+                df[column] = pd.to_datetime(df[column], utc=True)
+
+        if self.schema == Schema.DEFINITION:
+            df["expiration"] = pd.to_datetime(df["expiration"], utc=True)
+            df["activation"] = pd.to_datetime(df["activation"], utc=True)
+
+        return df
+
+    def _apply_pretty_px(self, df: pd.DataFrame) -> pd.DataFrame:
+        for column in list(df.columns):
+            if (
+                column in ("price", "open", "high", "low", "close")
+                or column.startswith("bid_px")  # MBP
+                or column.startswith("ask_px")  # MBP
+            ):
+                df[column] = df[column] * 1e-9
+
+        if self.schema == Schema.DEFINITION:
+            for column in DEFINITION_PRICE_COLUMNS:
+                df[column] = df[column] * 1e-9
+
+        return df
+
+    def _map_symbols(self, df: pd.DataFrame, pretty_ts: bool) -> pd.DataFrame:
+        # Build product ID index
+        if not self._product_id_index:
+            self._product_id_index = self._build_product_id_index()
+
+        # Map product IDs to native symbols
+        if self._product_id_index:
+            df_index = df.index if pretty_ts else pd.to_datetime(df.index, utc=True)
+            dates = [ts.date() for ts in df_index]
+            df["symbol"] = [
+                self._product_id_index[dates[i]][p]
+                for i, p in enumerate(df["product_id"])
+            ]
 
         return df
 

databento/common/data.py

@@ -140,21 +140,21 @@ def get_deriv_ba_types(level: int) -> List[Tuple[str, Union[type, str]]]:
         ("related_security_id", np.uint32),
         ("trading_reference_date", np.uint16),
         ("appl_id", np.int16),
-        ("maturity_month_year", np.uint16),
+        ("maturity_year", np.uint16),
         ("decay_start_date", np.uint16),
-        ("chan", np.uint16),
-        ("currency", "S1"),  # 1 byte chararray
-        ("settl_currency", "S1"),  # 1 byte chararray
-        ("secsubtype", "S1"),  # 1 byte chararray
-        ("symbol", "S1"),  # 1 byte chararray
-        ("group", "S1"),  # 1 byte chararray
-        ("exchange", "S1"),  # 1 byte chararray
-        ("asset", "S1"),  # 1 byte chararray
-        ("cfi", "S1"),  # 1 byte chararray
-        ("security_type", "S1"),  # 1 byte chararray
-        ("unit_of_measure", "S1"),  # 1 byte chararray
-        ("underlying", "S1"),  # 1 byte chararray
-        ("related", "S1"),  # 1 byte chararray
+        ("channel_id", np.uint16),
+        ("currency", "S4"),  # 4 byte chararray
+        ("settl_currency", "S4"),  # 4 byte chararray
+        ("secsubtype", "S6"),  # 6 byte chararray
+        ("symbol", "S22"),  # 22 byte chararray
+        ("group", "S21"),  # 21 byte chararray
+        ("exchange", "S5"),  # 5 byte chararray
+        ("asset", "S7"),  # 7 byte chararray
+        ("cfi", "S7"),  # 7 byte chararray
+        ("security_type", "S7"),  # 7 byte chararray
+        ("unit_of_measure", "S31"),  # 31 byte chararray
+        ("underlying", "S21"),  # 21 byte chararray
+        ("related", "S21"),  # 21 byte chararray
         ("match_algorithm", "S1"),  # 1 byte chararray
         ("md_security_trading_status", np.uint8),
         ("main_fraction", np.uint8),
@@ -163,17 +163,62 @@ def get_deriv_ba_types(level: int) -> List[Tuple[str, Union[type, str]]]:
         ("sub_fraction", np.uint8),
         ("underlying_product", np.uint8),
         ("security_update_action", "S1"),  # 1 byte chararray
-        ("maturity_month_month", np.uint8),
-        ("maturity_month_day", np.uint8),
-        ("maturity_month_week", np.uint8),
+        ("maturity_month", np.uint8),
+        ("maturity_day", np.uint8),
+        ("maturity_week", np.uint8),
         ("user_defined_instrument", "S1"),  # 1 byte chararray
         ("contract_multiplier_unit", np.int8),
         ("flow_schedule_type", np.int8),
         ("tick_rule", np.uint8),
-        ("dummy", "S1"),  # 1 byte chararray
+        ("dummy", "S3"),  # 3 byte chararray (Adjustment filler for 8-bytes alignment)
+    ],
+    Schema.GATEWAY_ERROR: RECORD_HEADER
+    + [
+        ("error", "S64"),
+    ],
+    Schema.SYMBOL_MAPPING: RECORD_HEADER
+    + [
+        ("stype_in_symbol", "S22"),
+        ("stype_out_symbol", "S22"),
+        ("dummy", "S4"),
+        ("start_ts", np.uint64),
+        ("end_ts", np.uint64),
     ],
 }
 
+DEFINITION_CHARARRAY_COLUMNS = [
+    "currency",
+    "settl_currency",
+    "secsubtype",
+    "symbol",
+    "group",
+    "exchange",
+    "asset",
+    "cfi",
+    "security_type",
+    "unit_of_measure",
+    "underlying",
+    "related",
+    "match_algorithm",
+    "security_update_action",
+    "user_defined_instrument",
+]
+
+DEFINITION_PRICE_COLUMNS = [
+    "min_price_increment",
+    "display_factor",
+    "high_limit_price",
+    "low_limit_price",
+    "max_price_variation",
+    "trading_reference_price",
+    "min_price_increment_amount",
+]
+
+DEFINITION_TYPE_MAX_MAP = {
+    x[0]: np.iinfo(x[1]).max
+    for x in STRUCT_MAP[Schema.DEFINITION]
+    if not isinstance(x[1], str)
+}
 
 ################################################################################
 # DBZ fields

databento/common/enums.py

@@ -50,6 +50,8 @@ class Schema(Enum):
     DEFINITION = "definition"
     STATISTICS = "statistics"
     STATUS = "status"
+    GATEWAY_ERROR = "gateway_error"
+    SYMBOL_MAPPING = "symbol_mapping"
 
 
 @unique
@@ -134,11 +136,11 @@ class SymbologyResolution(Enum):
 class Flags(Enum):
     """Represents record flags."""
 
-    F_LAST = 1 << 7  # 128  Last msg in packet (flags < 0)
-    F_HALT = 1 << 6  # 64  Exchange-independent HALT signal
-    F_RESET = 1 << 5  # 32  Drop book, reset symbol for this exchange
-    F_DUPID = 1 << 4  # 16  This OrderID has valid fresh duplicate (Iceberg, etc)
-    F_MBP = 1 << 3  # 8  This is SIP/MBP ADD message, single per price level
-    F_RESERVED2 = 1 << 2  # 4 Reserved for future use
-    F_RESERVED1 = 1 << 1  # 2 Reserved for future use
-    F_RESERVED0 = 1  # Reserved for future use
+    # Last message in the packet from the venue for a given `product_id`
+    F_LAST = 1 << 7
+    # Message sourced from a replay, such as a snapshot server
+    F_SNAPSHOT = 1 << 5
+    # Aggregated price level message, not an individual order
+    F_MBP = 1 << 4
+    # The `ts_recv` value is inaccurate (clock issues or reordering)
+    F_BAD_TS_RECV = 1 << 3