Add 2021 data with some updates to infra (#64)

Took 25 minutes

Author: droher

.env

@@ -1,7 +1,7 @@
-CHADWICK_VERSION=v0.9.0
-BASEBALLDATABANK_VERSION=36b88bb6abebddc5b1f584fab40b5655adc1ba70
-RETROSHEET_VERSION=c76e2b0aef6bfe821c48f440ba94c9cad68202b8
+CHADWICK_VERSION=v0.9.3
+BASEBALLDATABANK_VERSION=dd1a4503b9d6ec2bdda5e345ba06c867e368dd13
+RETROSHEET_VERSION=e540755f22b65d2f85f4da9180d1a31754c331f9
 
 EXTRACT_DIR=extract
 REPO=doublewick/boxball
-VERSION=2021.0.0
+VERSION=2022.0.0

extract/Dockerfile

@@ -2,7 +2,7 @@ ARG BUILD_ENV
 ARG RETROSHEET_IMAGE=get-retrosheet-${BUILD_ENV}
 ARG BASEBALLDATABANK_IMAGE=get-baseballdatabank-${BUILD_ENV}
 
-FROM python:3.7.3-alpine3.9 AS build-common
+FROM python:3.10.4-alpine3.15 AS build-common
 RUN apk add --no-cache \
     parallel \
     libtool \
@@ -29,7 +29,7 @@ COPY fixtures/raw/retrosheet.zip .
 
 FROM build-common as get-baseballdatabank-prod
 ARG BASEBALLDATABANK_VERSION
-RUN wget https://github.com/droher/baseballdatabank/archive/${BASEBALLDATABANK_VERSION}.zip -O baseballdatabank.zip
+RUN wget https://github.com/chadwickbureau/baseballdatabank/archive/${BASEBALLDATABANK_VERSION}.zip -O baseballdatabank.zip
 
 FROM build-common as get-baseballdatabank-test
 COPY fixtures/raw/baseballdatabank.zip .

extract/parsers/baseballdatabank.py

@@ -5,11 +5,14 @@
 from parsers.util import compress, OUTPUT_PATH
 
 DOS_EOF = chr(26)
-BASEBALLDATABANK_PATH = Path("baseballdatabank/core")
+BASEBALLDATABANK_PATHS = Path("baseballdatabank/core"), Path("baseballdatabank/contrib")
 
 
 def get_baseballdatabank_files():
-    for file in BASEBALLDATABANK_PATH.glob("*.csv"):
+    files = [f for path in BASEBALLDATABANK_PATHS
+             for f in path.glob("*.csv")]
+    print("Processing Baseball Databank files:", files)
+    for file in files:
         # Just need to change from PascalCase to snake_case to match table names
         # Editing OF fielding files to get PascalCasev conformity for all databank filenames
         file_name = file.name.replace("OFs", "OfS").replace("OF", "Of")

load/mysql/Dockerfile

@@ -2,7 +2,7 @@ ARG VERSION
 FROM doublewick/boxball:ddl-${VERSION} as ddl
 FROM doublewick/boxball:csv-${VERSION} as csv
 
-FROM mysql:8.0.23 as mysql-build
+FROM mysql:8.0.28-debian as mysql-build
 ENV MYSQL_ALLOW_EMPTY_PASSWORD=yes
 COPY my.cnf /etc/mysql/conf.d/
 COPY A_unzip_csvs.sh z_remove_csvs.sh /docker-entrypoint-initdb.d/

transform/ddl.Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7.3-slim-stretch AS build-common
+FROM python:3.10-slim-bullseye AS build-common
 COPY requirements.txt .
 RUN pip install -r requirements.txt
 ENV PYTHONPATH="/"

transform/parquet.Dockerfile

@@ -1,7 +1,7 @@
 ARG VERSION
 FROM doublewick/boxball:extract-${VERSION} as extract
 
-FROM python:3.7.3-slim-stretch AS build-common
+FROM python:3.10-slim-bullseye AS build-common
 COPY requirements.txt .
 RUN pip install -r requirements.txt
 ENV PYTHONPATH="/"

transform/requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.23
 sqlalchemy-fdw==0.3.0
 clickhouse-sqlalchemy==0.1.5
-pyarrow==3.0.0
-zstandard==0.15.2
+pyarrow==7.0.0
+zstandard==0.17.0

transform/src/schemas/retrosheet.py

@@ -863,7 +863,7 @@ class Event(Base):
     base1_run_id = Column(CHAR(8), doc="ID of runner on first")
     base2_run_id = Column(CHAR(8), doc="ID of runner on second")
     base3_run_id = Column(CHAR(8), doc="ID of runner on third")
-    event_tx = Column(String(58), doc="Event text (in scoring shorthand")
+    event_tx = Column(String(128), doc="Event text (in scoring shorthand")
     leadoff_fl = Column(Boolean, doc="Batter is leading off the inning")
     ph_fl = Column(Boolean, doc="Batter is pinch-hitting")
     bat_fld_cd = Column(SmallInteger, doc="Defensive position of batter (10 for DH, 11 for PH, 12 for PR")