update etl configuration input to yml instead of environment variables

buildSrc/src/main/groovy/DockerExec.groovy

@@ -0,0 +1,32 @@
+import org.gradle.api.provider.ProviderFactory
+import org.gradle.api.tasks.Exec
+import org.gradle.api.tasks.Internal
+
+import javax.inject.Inject
+
+/**
+ * An {@link Exec} task that is automatically skipped (with a lifecycle message) when
+ * Docker is not installed or the Docker daemon is unavailable, instead of failing the build.
+ */
+abstract class DockerExec extends Exec {
+
+    @Inject
+    abstract ProviderFactory getProviders()
+
+    DockerExec() {
+        onlyIf { isDockerAvailable() }
+    }
+
+    @Internal
+    boolean isDockerAvailable() {
+        def version = providers.of(DockerVersionValueSource) {}.get()
+
+        if (!version) {
+            logger.lifecycle("Skipping ${name} because Docker is not installed or the Docker daemon is unavailable.")
+            return false
+        }
+
+        logger.info("Docker is available: ${version}")
+        return true
+    }
+}

buildSrc/src/main/groovy/DockerVersionValueSource.groovy

@@ -0,0 +1,50 @@
+import org.gradle.api.logging.Logger
+import org.gradle.api.logging.Logging
+import org.gradle.api.provider.ValueSource
+import org.gradle.api.provider.ValueSourceParameters
+import org.gradle.process.ExecOperations
+import org.gradle.process.ExecSpec
+import org.gradle.process.internal.ExecException
+
+import javax.inject.Inject
+
+/**
+ * Resolves the Docker daemon's server version, or an empty string if Docker is not
+ * installed or the daemon is unreachable.
+ *
+ * Gradle guarantees a {@link ValueSource}'s {@code obtain()} is invoked at most once
+ * per build, so this replaces hand-rolled caching of the "is Docker available" check.
+ */
+abstract class DockerVersionValueSource implements ValueSource<String, ValueSourceParameters.None> {
+
+    private static final Logger LOGGER = Logging.getLogger(DockerVersionValueSource)
+
+    @Inject
+    abstract ExecOperations getExecOperations()
+
+    @Override
+    String obtain() {
+        def stdout = new ByteArrayOutputStream()
+        def stderr = new ByteArrayOutputStream()
+
+        try {
+            execOperations.exec { ExecSpec spec ->
+                spec.commandLine 'docker', 'info', '--format', '{{.ServerVersion}}'
+                spec.standardOutput = stdout
+                spec.errorOutput = stderr
+                spec.ignoreExitValue = true
+            }
+        } catch (ExecException e) {
+            return ''
+        }
+
+        def version = stdout.toString().trim()
+
+        if (!version) {
+            def error = stderr.toString().trim()
+            LOGGER.info("'docker info' produced no version output.{}", error ? " stderr: ${error}" : '')
+        }
+
+        return version
+    }
+}

cda-etl/Dockerfile

@@ -7,4 +7,4 @@ RUN pip install --no-cache-dir -r requirements.txt
 
 COPY src/ /app/src/
 
-ENTRYPOINT ["python", "src/cda_etl/main.py"]
+CMD ["python", "src/cda_etl/main.py"]

cda-etl/README.md

@@ -0,0 +1,171 @@
+# CDA ETL
+
+This project downloads CWMS data from a source CDA REST API, stages the retrieved JSON on the local filesystem, and then uploads the staged records to a destination CDA REST API.
+
+The workflow is intentionally split into two phases:
+
+1. Stage data from the source API onto disk.
+2. Publish the staged files to the destination API.
+
+When `SOURCE_CDA_URL` is configured, the stage phase always re-downloads source data and overwrites staged files for projects, locations, and timeseries.
+
+If `SOURCE_CDA_URL` is not configured, the pipeline skips the download phase and publishes whatever is already staged on disk.
+
+## What It Does
+
+The ETL process currently handles three CWMS resource types:
+
+- Locations
+- Projects
+- Timeseries data
+
+The data is organized by office, project, and resource type, then written to a filesystem staging area before being posted to the destination CDA API.
+
+## Configuration Overview
+
+The main runtime configuration is stored in a YAML file, defaulting to `sample-app.yml` in the working directory.
+
+The application reads the YAML path from the `ETL_CONFIG_PATH` environment variable. If the variable is not set, it looks for `sample-app.yml` next to where the process starts.
+
+### Example Structure
+
+```yaml
+version: 1
+settings:
+  startTime: "2026-01-01"
+  endTime: "now"
+  maxThreads: 10
+  logLevel: INFO
+  path: "./data"
+offices:
+  - id: SWT
+    enabled: true
+    projects:
+      - id: EUFA
+        enabled: true
+        locations:
+          - id: EUFA-Dam
+            enabled: true
+        timeseries:
+          - id: EUFA.Elev.Inst.1Hour.0.Ccp-Rev
+            enabled: true
+```
+
+### YAML Fields
+
+- `version`: Config version. Must be `1`.
+- `settings.startTime`: Default start time used for timeseries downloads when a timeseries does not define its own download window.
+- `settings.endTime`: Default end time used for timeseries downloads when a timeseries does not define its own download window.
+- `settings.maxThreads`: Maximum number of worker threads used for staging and publishing.
+- `settings.logLevel`: Logging level for the application.
+- `settings.path`: Filesystem root used for staged JSON files.
+- `offices`: List of office definitions.
+- `projects`: Projects under each office.
+- `locations`: Locations under each project.
+- `timeseries`: Timeseries under each project.
+
+### Enabled Flags
+
+The `enabled` field is optional everywhere. If it is omitted, the item is treated as enabled.
+
+### Filesystem Staging
+
+Staged data is written under the directory configured by `settings.path`.
+
+For timeseries data, the stored file name does not include the time window. During staging with `SOURCE_CDA_URL` configured, each run overwrites the staged file with a fresh source download.
+
+## Runtime Parameters
+
+### Required for Destination Upload
+
+- `DEST_CDA_URL`: Destination CDA REST API root.
+
+Environment variable values are trimmed. Empty or whitespace-only values are treated as unset.
+
+### Optional Source Download
+
+- `SOURCE_CDA_URL`: Source CDA REST API root. If set, source data is always re-downloaded and staged files are overwritten each run. If omitted (or set to an empty value), the download phase is skipped.
+- `SOURCE_CDA_API_KEY`: API key for the source CDA REST API.
+- `DEST_CDA_API_KEY`: API key for the destination CDA REST API.
+
+### Other Runtime Settings
+
+- `ETL_CONFIG_PATH`: Path to the YAML config file. Defaults to `sample-app.yml`.
+- `LOG_LEVEL`: Console log level for the application process. Defaults to `INFO`.
+
+## Docker Usage
+
+### docker run
+
+Mount the YAML file into the container and point `ETL_CONFIG_PATH` at it.
+
+```powershell
+docker run --rm `
+  -v ${PWD}\data\sample-data\sample-app.yml:/app/sample-app.yml `
+  -e ETL_CONFIG_PATH=/app/sample-app.yml `
+  -e SOURCE_CDA_URL=https://source.example/cwms-data `
+  -e SOURCE_CDA_API_KEY=your-source-key `
+  -e DEST_CDA_URL=https://dest.example/cwms-data `
+  -e DEST_CDA_API_KEY=your-dest-key `
+  cwms-data-api/etl
+```
+
+If you do not want to download from the source API, omit `SOURCE_CDA_URL` and the pipeline will publish only staged files.
+
+### docker-compose
+
+The included `docker-compose.yml` mounts `ETL_CONFIG_PATH` for the yml config file path.
+
+You still need to supply the API endpoint environment variables when running Compose.
+
+## Gradle Commands
+
+The Gradle build file provides Docker-based convenience tasks.
+
+### Build the image
+
+```bash
+./gradlew dockerBuild
+```
+
+### Run the ETL container
+
+```bash
+./gradlew runEtl
+```
+
+Optional Gradle property:
+
+- `-PetlEnvFile=<path>`: Override the environment file passed to `docker run`. Default: `etl.env`
+
+Example:
+
+```bash
+./gradlew runEtl -PetlEnvFile=etl.env.example
+```
+
+### Run the unit tests in Docker
+
+```bash
+./gradlew runEtlUnitTests
+```
+
+This uses Docker, mounts the local `src` and `tests` directories, and runs `pytest` inside the container.
+
+### Run the full verification task
+
+```bash
+./gradlew check
+```
+
+`check` depends on `runEtlUnitTests` in this project.
+
+## Local Development
+
+For local Python execution, ensure the environment variables for source and destination CDA endpoints are set, then run:
+
+```bash
+python src/cda_etl/main.py
+```
+
+The process will load the YAML config, stage files under `settings.path`, and publish to the destination CDA API.

cda-etl/build.gradle

@@ -22,56 +22,63 @@ plugins {
     id 'base'
 }
 
-def isWindows = {
-    System.getProperty('os.name').toLowerCase().contains('win')
-}
+final def imageName = 'cwms-data-api/etl'
+final def etlEnvFile = providers.gradleProperty('etlEnvFile').orElse('etl.env')
+
+tasks.register('dockerBuild', DockerExec) {
+    group = 'docker'
+    description = 'Builds the CDA ETL Docker image.'
 
-final def pythonCmd = isWindows() ? 'python' : 'python3'
-final def mainScript = 'src/cda_etl/main.py'
-final def envFile = 'etl.env'
-final def reqFile = 'requirements.txt'
-
-tasks.register('installRequirements', Exec) {
-    commandLine pythonCmd, '-m', 'pip', 'install', '-r', reqFile
-    workingDir projectDir
-    inputs.file(new File(projectDir, reqFile))
-    outputs.file(new File(buildDir, 'pip-install.marker'))
-    doLast {
-        mkdir buildDir
-        new File(buildDir, 'pip-install.marker').text = "installed at ${new Date()}\n"
+    doFirst {
+        commandLine 'docker', 'build', '--pull', '-t', imageName, '.'
     }
+
+    inputs.file('Dockerfile')
+    inputs.file('requirements.txt')
+    inputs.dir('src')
 }
 
-tasks.register('runEtl', Exec) {
-    dependsOn 'installRequirements'
-    group 'application'
-    executable pythonCmd
-    args mainScript
-    workingDir projectDir
-
-    // Load environment variables from etl.env
+tasks.register('runEtl', DockerExec) {
+    dependsOn 'dockerBuild'
+
+    group = 'application'
+    description = 'Runs the CDA ETL Docker container using the configured environment file.'
+
     doFirst {
-        def envFileObj = new File(projectDir, envFile)
-        if (envFileObj.exists()) {
-            envFileObj.eachLine { line ->
-                if (line.trim() && !line.startsWith('#')) {
-                    def parts = line.split('=', 2)
-                    if (parts.length == 2) {
-                        environment parts[0].trim(), parts[1].trim()
-                    }
-                }
-            }
-        } else {
-            logger.warn("Environment file ${envFile} not found.")
+        def envFile = file(etlEnvFile.get())
+
+        if (!envFile.exists()) {
+            throw new GradleException("ETL environment file not found: ${envFile.absolutePath}")
         }
+
+        commandLine 'docker', 'run', '--rm', '--env-file', envFile.absolutePath, imageName
     }
 }
 
-tasks.register('runEtlUnitTests', Exec) {
-    dependsOn 'installRequirements'
-    group 'verification'
-    executable pythonCmd
-    args '-m', 'pytest'
-    workingDir projectDir
-    environment 'PYTHONPATH', 'src/cda_etl'
+tasks.register('runEtlUnitTests', DockerExec) {
+    dependsOn 'dockerBuild'
+
+    group = 'verification'
+    description = 'Runs ETL unit tests in Docker with local source and tests mounted for faster iteration.'
+
+    doFirst {
+        def args = [
+            'docker', 'run', '--rm',
+            '-e', 'PYTHONPATH=/app/src/cda_etl',
+            '-v', "${projectDir}/src:/app/src"
+        ]
+
+        def testsDir = file('tests')
+        args += ['-v', "${testsDir.absolutePath}:/app/tests"]
+        args += [imageName, 'python', '-m', 'pytest']
+
+        commandLine args
+    }
+
+    inputs.dir('src')
+    inputs.dir('tests').optional()
 }
+
+tasks.named('check') {
+    dependsOn 'runEtlUnitTests'
+}