Skip to content

Commit c25298c

Browse files
zkwentzzkwentz
authored
Merge pull request #72 from meta-pytorch/hotfix/openspiel_env-1
Precompile OpenSpiel, fix openspiel_env HF deployment
2 parents 5eb0e22 + 84eac1b commit c25298c

File tree

5 files changed

+233
-117
lines changed

5 files changed

+233
-117
lines changed

‎.github/workflows/deploy-hf-env.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ jobs:
167167
# Cleanup
168168
cd ..
169169
rm -rf hf-space
170-
rm -rf hf-staging_$ENV_NAME_$ENV_NAME
170+
rm -rf hf-staging_$ENV_NAME
171171
172172
# Job to deploy single environment
173173
deploy-single:

‎.github/workflows/openspiel_base_build.yml‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
name: Build OpenSpiel Base Image
2+
3+
on:
4+
workflow_dispatch: # Manual trigger only
5+
inputs:
6+
force_rebuild:
7+
description: 'Force rebuild even if base image exists'
8+
required: false
9+
default: 'false'
10+
type: boolean
11+
12+
env:
13+
REGISTRY: ghcr.io
14+
IMAGE_PREFIX: ${{ github.repository_owner }}/openenv
15+
16+
jobs:
17+
# Job 1: Build base image first
18+
build-base:
19+
runs-on: ubuntu-latest
20+
permissions:
21+
contents: read
22+
packages: write
23+
24+
steps:
25+
- name: Checkout code
26+
uses: actions/checkout@v4
27+
28+
- name: Set up Docker Buildx
29+
uses: docker/setup-buildx-action@v3
30+
31+
- name: Log in to GHCR
32+
uses: docker/login-action@v3
33+
with:
34+
registry: ${{ env.REGISTRY }}
35+
username: ${{ github.actor }}
36+
password: ${{ secrets.GITHUB_TOKEN }}
37+
38+
- name: Extract metadata for base image
39+
id: meta
40+
uses: docker/metadata-action@v5
41+
with:
42+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-base
43+
tags: |
44+
type=raw,value=latest,enable={{is_default_branch}}
45+
type=sha
46+
47+
- name: Build and push base image
48+
uses: docker/build-push-action@v5
49+
with:
50+
context: .
51+
file: src/core/containers/images/Dockerfile
52+
push: true
53+
platforms: linux/amd64,linux/arm64
54+
tags: ${{ steps.meta.outputs.tags }}
55+
labels: ${{ steps.meta.outputs.labels }}
56+
cache-from: type=gha,scope=base
57+
cache-to: type=gha,mode=max,scope=base
58+
59+
# Job 2: Build OpenSpiel base image (depends on base)
60+
build-openspiel-base:
61+
runs-on: 8-core-ubuntu
62+
needs: build-base # Wait for base image to be built
63+
permissions:
64+
contents: read
65+
packages: write
66+
67+
steps:
68+
- name: Checkout code
69+
uses: actions/checkout@v4
70+
71+
- name: Set up Docker Buildx
72+
uses: docker/setup-buildx-action@v3
73+
74+
- name: Log in to GHCR
75+
uses: docker/login-action@v3
76+
with:
77+
registry: ${{ env.REGISTRY }}
78+
username: ${{ github.actor }}
79+
password: ${{ secrets.GITHUB_TOKEN }}
80+
81+
- name: Extract metadata for OpenSpiel base image
82+
id: meta-openspiel-base
83+
uses: docker/metadata-action@v5
84+
with:
85+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-openspiel-base
86+
tags: |
87+
type=raw,value=latest,enable={{is_default_branch}}
88+
type=sha
89+
90+
- name: Build and push OpenSpiel base image
91+
uses: docker/build-push-action@v5
92+
with:
93+
context: .
94+
file: src/envs/openspiel_env/server/Dockerfile.openspiel-base
95+
push: true
96+
platforms: linux/amd64,linux/arm64
97+
tags: ${{ steps.meta-openspiel-base.outputs.tags }}
98+
labels: ${{ steps.meta-openspiel-base.outputs.labels }}
99+
cache-from: type=gha,scope=openspiel-base
100+
cache-to: type=gha,mode=max,scope=openspiel-base
101+
102+
- name: Build summary
103+
run: |
104+
echo "✅ OpenSpiel base image built and pushed successfully!"
105+
echo "📦 Image: ${{ steps.meta-openspiel-base.outputs.tags }}"
106+
echo "🚀 Next regular build will use this new base image"

‎scripts/prepare_hf_deployment.sh‎

Lines changed: 53 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -94,63 +94,20 @@ DOCKERFILE_EOF
9494
"openspiel_env")
9595
# OpenSpiel requires special C++ build process - replace entire Dockerfile
9696
cat > $CURRENT_STAGING_DIR/Dockerfile << DOCKERFILE_EOF
97-
# OpenSpiel requires complex C++ build - using special multi-stage approach
98-
# Stage 1: Build OpenSpiel C++ bindings
99-
FROM python:3.11 AS openspiel-builder
100-
101-
# Avoid interactive prompts during build
102-
ENV DEBIAN_FRONTEND=noninteractive
103-
ENV TZ=UTC
104-
105-
# Install build dependencies
106-
RUN apt-get update && apt-get install -y --no-install-recommends \
107-
build-essential \
108-
clang \
109-
cmake \
110-
curl \
111-
git \
112-
sudo \
113-
&& rm -rf /var/lib/apt/lists/*
114-
115-
# Set up OpenSpiel build directory
116-
RUN mkdir /repo
117-
WORKDIR /repo
118-
119-
# Clone OpenSpiel
120-
RUN git clone https://github.com/google-deepmind/open_spiel.git .
121-
122-
# Run OpenSpiel's installation script (downloads C++ dependencies)
123-
RUN ./install.sh
124-
125-
# Install Python dependencies
126-
RUN pip3 install --no-cache-dir --upgrade setuptools testresources importlib_metadata
127-
RUN pip3 install --no-cache-dir --upgrade -r requirements.txt cmake
128-
129-
# Build OpenSpiel with Python 3.11
130-
RUN mkdir -p build
131-
WORKDIR /repo/build
132-
RUN cmake -DPython3_EXECUTABLE=$(which python3) -DCMAKE_CXX_COMPILER=$(which clang++) ../open_spiel
133-
RUN make -j$(nproc) pyspiel
134-
135-
# Stage 2: Use the specified openenv-base image
136-
FROM $BASE_IMAGE_REF
137-
138-
# Copy OpenSpiel build artifacts from builder
139-
RUN mkdir -p /repo
140-
COPY --from=openspiel-builder /repo /repo
141-
142-
# Install OpenSpiel Python requirements in runtime
143-
WORKDIR /repo
144-
RUN pip3 install --no-cache-dir --upgrade -r requirements.txt
97+
# OpenSpiel environment using pre-built OpenSpiel base image
98+
ARG OPENSPIEL_BASE_IMAGE=ghcr.io/meta-pytorch/openenv-openspiel-base:sha-e622c7e
99+
FROM \${OPENSPIEL_BASE_IMAGE}
145100
146-
# Set Python path for OpenSpiel
147-
ENV PYTHONPATH=/repo:/repo/build/python:${PYTHONPATH}
148-
149-
# Copy OpenEnv core
101+
# Copy OpenEnv core (base image already set WORKDIR=/app)
150102
WORKDIR /app
151103
COPY src/core/ /app/src/core/
104+
105+
# Copy OpenSpiel environment
152106
COPY src/envs/openspiel_env/ /app/src/envs/openspiel_env/
153107
108+
# Copy README for web interface documentation
109+
COPY src/envs/openspiel_env/README.md /app/README.md
110+
154111
# Extend Python path for OpenEnv (base image set PYTHONPATH=/app/src)
155112
# We prepend OpenSpiel paths
156113
ENV PYTHONPATH=/repo:/repo/build/python:/app/src
@@ -160,11 +117,13 @@ ENV OPENSPIEL_GAME=catch
160117
ENV OPENSPIEL_AGENT_PLAYER=0
161118
ENV OPENSPIEL_OPPONENT_POLICY=random
162119
163-
# Health check
120+
# Health check (curl is provided by openenv-base)
164121
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
165122
CMD curl -f http://localhost:8000/health || exit 1
166123
167-
# Run the FastAPI server
124+
# Note: EXPOSE 8000 already set by openenv-base
125+
126+
# Run the FastAPI server (uvicorn installed by openenv-base)
168127
CMD ["uvicorn", "envs.openspiel_env.server.app:app", "--host", "0.0.0.0", "--port", "8000"]
169128
DOCKERFILE_EOF
170129
echo "Created special OpenSpiel Dockerfile with C++ build process"
@@ -205,12 +164,46 @@ create_readme() {
205164
# Capitalize first letter of environment name
206165
env_title=$(echo "$env_name" | awk '{print toupper(substr($0,1,1)) substr($0,2)}')
207166

167+
# Set environment-specific colors and emoji
168+
case $env_name in
169+
"atari_env")
170+
EMOJI="🕹️"
171+
COLOR_FROM="#FF6200"
172+
COLOR_TO="#D4151B"
173+
;;
174+
"coding_env")
175+
EMOJI="💻"
176+
COLOR_FROM="#007ACC"
177+
COLOR_TO="#1E1E1E"
178+
;;
179+
"openspiel_env")
180+
EMOJI="🎮"
181+
COLOR_FROM="#9146FF"
182+
COLOR_TO="#00FFA3"
183+
;;
184+
"echo_env")
185+
EMOJI="🔊"
186+
COLOR_FROM="#00C9FF"
187+
COLOR_TO="#1B2845"
188+
;;
189+
"chat_env")
190+
EMOJI="💬"
191+
COLOR_FROM="#0084FF"
192+
COLOR_TO="#25D366"
193+
;;
194+
*)
195+
EMOJI="🐳"
196+
COLOR_FROM="blue"
197+
COLOR_TO="green"
198+
;;
199+
esac
200+
208201
cat > $CURRENT_STAGING_DIR/README.md << README_EOF
209202
---
210203
title: ${env_title} Environment Server
211-
emoji: 🐳
212-
colorFrom: blue
213-
colorTo: green
204+
emoji: ${EMOJI}
205+
colorFrom: ${COLOR_FROM}
206+
colorTo: ${COLOR_TO}
214207
sdk: docker
215208
pinned: false
216209
app_port: 8000
@@ -310,9 +303,10 @@ Provides access to OpenSpiel games for multi-agent reinforcement learning.
310303
Send a POST request to `/step` with:
311304
```json
312305
{
313-
"action": 0
306+
"action": {
307+
"action_id": 1
308+
}
314309
}
315-
```
316310
README_EOF
317311
;;
318312
esac

‎src/envs/openspiel_env/server/Dockerfile‎

Lines changed: 8 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -4,60 +4,11 @@
44
# This source code is licensed under the BSD-style license found in the
55
# LICENSE file in the root directory of this source tree.
66

7-
# Multi-stage build for OpenSpiel + OpenEnv
8-
# Stage 1: Build OpenSpiel C++ bindings
9-
# Using Python 3.11 to match envtorch-base
10-
FROM python:3.11 AS openspiel-builder
11-
12-
# Avoid interactive prompts during build
13-
ENV DEBIAN_FRONTEND=noninteractive
14-
ENV TZ=UTC
15-
16-
# Install build dependencies
17-
RUN apt-get update && apt-get install -y --no-install-recommends \
18-
build-essential \
19-
clang \
20-
cmake \
21-
curl \
22-
git \
23-
sudo \
24-
&& rm -rf /var/lib/apt/lists/*
25-
26-
# Set up OpenSpiel build directory
27-
RUN mkdir /repo
28-
WORKDIR /repo
29-
30-
# Clone OpenSpiel
31-
RUN git clone https://github.com/google-deepmind/open_spiel.git .
32-
33-
# Run OpenSpiel's installation script (downloads C++ dependencies)
34-
RUN ./install.sh
35-
36-
# Install Python dependencies
37-
RUN pip3 install --no-cache-dir --upgrade setuptools testresources importlib_metadata
38-
RUN pip3 install --no-cache-dir --upgrade -r requirements.txt cmake
39-
40-
# Build OpenSpiel with Python 3.11
41-
RUN mkdir -p build
42-
WORKDIR /repo/build
43-
RUN cmake -DPython3_EXECUTABLE=$(which python3) -DCMAKE_CXX_COMPILER=$(which clang++) ../open_spiel
44-
RUN make -j$(nproc) pyspiel
45-
46-
# Stage 2: Runtime image using published openenv-base
47-
# Uses the standardized base image from GitHub Container Registry
48-
# See: https://github.com/meta-pytorch/OpenEnv/pkgs/container/openenv-base
49-
FROM ghcr.io/meta-pytorch/openenv-base:latest
50-
51-
# Copy OpenSpiel build artifacts from builder
52-
RUN mkdir -p /repo
53-
COPY --from=openspiel-builder /repo /repo
54-
55-
# Install OpenSpiel Python requirements in runtime
56-
WORKDIR /repo
57-
RUN pip3 install --no-cache-dir --upgrade -r requirements.txt
58-
59-
# Set Python path for OpenSpiel
60-
ENV PYTHONPATH=/repo:/repo/build/python:${PYTHONPATH}
7+
# Use the pre-built OpenSpiel base image
8+
# Built from: docker build -t openspiel-base:latest -f src/envs/openspiel_env/server/Dockerfile.openspiel-base .
9+
# In GitHub Actions, this is overridden to use the GHCR base image
10+
ARG OPENSPIEL_BASE_IMAGE=openspiel-base:latest
11+
FROM ${OPENSPIEL_BASE_IMAGE}
6112

6213
# Copy OpenEnv core (base image already set WORKDIR=/app)
6314
WORKDIR /app
@@ -78,11 +29,11 @@ ENV OPENSPIEL_GAME=catch
7829
ENV OPENSPIEL_AGENT_PLAYER=0
7930
ENV OPENSPIEL_OPPONENT_POLICY=random
8031

81-
# Health check (curl is provided by envtorch-base)
32+
# Health check (curl is provided by openenv-base)
8233
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
8334
CMD curl -f http://localhost:8000/health || exit 1
8435

85-
# Note: EXPOSE 8000 already set by envtorch-base
36+
# Note: EXPOSE 8000 already set by openenv-base
8637

87-
# Run the FastAPI server (uvicorn installed by envtorch-base)
38+
# Run the FastAPI server (uvicorn installed by openenv-base)
8839
CMD ["uvicorn", "envs.openspiel_env.server.app:app", "--host", "0.0.0.0", "--port", "8000"]

0 commit comments

Comments
 (0)