Add terraform

.github/workflows/terraform.yml

@@ -0,0 +1,95 @@
+name: "Terraform"
+
+on:
+  # push:
+  #   branches:
+  #     - main
+  release:
+    types: [ published ]
+  pull_request_target:
+    types: [opened, synchronize]
+
+permissions:
+  pull-requests: write
+  issues: write
+  id-token: write
+  contents: read
+
+jobs:
+  terraform:
+    name: "Terraform"
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: infra
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::266796786618:role/github-pipeline-role
+          aws-region: eu-north-1
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.14.0
+          terraform_wrapper: false
+
+      - name: Terraform Format
+        id: fmt
+        run: terraform fmt -check
+
+      - name: Terraform Init
+        id: init
+        run: terraform init
+
+      - name: Terraform Validate
+        id: validate
+        run: terraform validate -no-color
+
+      - name: Terraform Plan
+        id: plan
+        if: github.event_name == 'pull_request'
+        run: terraform plan -no-color -input=false
+        continue-on-error: true
+
+      - uses: actions/github-script@v8
+        if: github.event_name == 'pull_request'
+        env:
+          PLAN: "terraform\n${{ steps.plan.outputs.stdout }}"
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const output = `#### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\`
+            #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\`
+            #### Terraform Plan 📖\`${{ steps.plan.outcome }}\`
+
+            <details><summary>Show Plan</summary>
+
+            \`\`\`${process.env.PLAN}\`\`\`
+
+            </details>
+
+            *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`;
+
+
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: output
+            })
+
+      - name: Terraform Plan Status
+        if: steps.plan.outcome == 'failure'
+        run: exit 1
+
+      - name: Terraform Apply
+        if: github.event_name == 'release'
+        working-directory: infra
+        run: |
+          terraform init
+          terraform apply -auto-approve

.gitignore

@@ -52,3 +52,4 @@ data/processed/
 .env
 
 mlruns/
+/infra/.terraform/

infra/main.tf

@@ -0,0 +1,146 @@
+provider "aws" {
+  region = var.aws_region
+}
+
+data "aws_caller_identity" "current" {}
+
+data "aws_ecr_authorization_token" "token" {}
+
+provider "docker" {
+  registry_auth {
+    address  = "${data.aws_caller_identity.current.account_id}.dkr.ecr.${var.aws_region}.amazonaws.com"
+    username = data.aws_ecr_authorization_token.token.user_name
+    password = data.aws_ecr_authorization_token.token.password
+  }
+}
+
+# Create S3 bucket
+
+resource "aws_s3_bucket" "app_s3" {
+  bucket = "climate-analysis-bucket"
+
+  tags = {
+    Name        = "Climate Analysis Bucket"
+    Environment = "Dev"
+  }
+}
+
+resource "aws_s3_bucket_public_access_block" "s3_access" {
+  bucket = aws_s3_bucket.app_s3.id
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+###---------------------------------------------
+
+# Create policy to access the bucket
+
+data "aws_iam_policy_document" "aws_policy_doc" {
+  statement {
+    actions = [
+      "s3:ListBucket",
+    ]
+    effect    = "Allow"
+    resources = [aws_s3_bucket.app_s3.arn]
+  }
+  statement {
+    actions = [
+      "s3:GetObject",
+      "s3:PutObject"
+    ]
+    effect    = "Allow"
+    resources = ["${aws_s3_bucket.app_s3.arn}/*"]
+  }
+}
+
+resource "aws_iam_policy" "policy" {
+  name        = "climate_analysis_resources_policy"
+  path        = "/"
+  description = "Climate Analysis Resources access"
+
+  policy = data.aws_iam_policy_document.aws_policy_doc.json
+}
+
+###---------------------------------------------
+
+# Create local user for s3 access (optional)
+
+resource "aws_iam_user" "local_dev" {
+  count = var.create_local_user ? 1 : 0
+  name  = var.local_user
+}
+
+resource "aws_iam_user_policy_attachment" "local_dev_s3" {
+  count      = var.create_local_user ? 1 : 0
+  user       = aws_iam_user.local_dev[0].name
+  policy_arn = aws_iam_policy.policy.arn
+}
+
+resource "aws_iam_access_key" "local_dev_key" {
+  count = var.create_local_user ? 1 : 0
+  user  = aws_iam_user.local_dev[0].name
+}
+
+###---------------------------------------------
+
+# Create docker image, app lambda, it's role and attach policy
+
+module "aws_lambda_function" {
+  source = "terraform-aws-modules/lambda/aws"
+
+  function_name  = "climate_analysis_lambda"
+  create_package = false
+
+  image_uri    = module.docker_image.image_uri
+  package_type = "Image"
+}
+
+module "docker_image" {
+  source = "terraform-aws-modules/lambda/aws//modules/docker-build"
+
+  create_ecr_repo = true
+  ecr_repo        = "climate-analysis"
+
+  use_image_tag = true
+  image_tag     = "1.0"
+
+  source_path = "../"
+}
+
+# resource "aws_api_gateway_integration" "app_api_gateway" {
+#   rest_api_id = ""
+#   resource_id = aws_lambda_function.example.id
+#   http_method = aws_api_gateway_method.example.http_method
+#
+#   integration_http_method = "POST"
+#   type                    = "AWS_PROXY"
+#   uri                     = aws_lambda_function.app_lambda.invoke_arn
+# }
+
+resource "aws_iam_role" "app_role" {
+  name = "climate_app_role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [{
+      Effect    = "Allow",
+      Principal = { Service = "lambda.amazonaws.com" },
+      Action    = "sts:AssumeRole"
+    }]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "app_role_policy_attach" {
+  role       = aws_iam_role.app_role.name
+  policy_arn = aws_iam_policy.policy.arn
+}
+
+###---------------------------------------------
+
+# Create api gateway
+
+
+
+

infra/terraform.tf

@@ -0,0 +1,21 @@
+terraform {
+  backend "s3" {
+    bucket       = "terraform-state-bucket-4848"
+    key          = "climate_analysis/terraform.tfstate"
+    region       = "eu-north-1"
+    encrypt      = true
+    use_lockfile = true
+  }
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 6.22"
+    }
+    docker = {
+      source  = "kreuzwerker/docker"
+      version = "3.6.1"
+    }
+  }
+
+  required_version = ">= 1.2"
+}

infra/variables.tf

@@ -0,0 +1,15 @@
+variable "aws_region" {
+  description = "AWS region"
+  default     = "eu-north-1"
+}
+
+variable "create_local_user" {
+  description = "Create user with app required policies"
+  type        = bool
+  default     = false
+}
+
+variable "local_user" {
+  description = "Local user name to create in aws and add policies to"
+  default     = "local-dev"
+}

readme.md

@@ -79,7 +79,28 @@ Prerequisites:
 - Create environment from file: `conda env create -f environment.yml`
 - Activate environment: `conda activate climate_analysis`
 
-### Run the CLI entrypoint (main.py)
+### Run API locally
+
+- From project root:
+  - `fastapi run src/api.py`
+  - server should start at `http://127.0.0.1:8000`
+
+This project allows local storage option or creating AWS infrastructure with S3 bucket through Terraform.
+This would allow much greater performance and possibility to run model more efficently.
+
+### Run Terraform
+
+You should have your `aws cli` credentials available locally (aws profile or key/secret pair as environment variables, see AWS docs for more details).
+`terraform` and `docker` should be installed as well. 
+*Alternatively you could fork repository and simply provide different
+`role-to-assume: arn:aws:iam::<AWS_ACCOUNT_ID>:role/<ROLE_NAME>` in `terraform.yml`, that way GitHub Actions will create infrastructure for you.*
+
+- From `infra` directory:
+  - `terraform init`
+  - `terraform plan` - ensure all planned services are ok for you
+  - `terraform apply` - docker image in AWS ECR will be created, S3, lambda and required permissions.
+
+### Run CLI entrypoint (main.py)
 
 Typical usage:
 

src/api.py

@@ -1,6 +1,11 @@
+from datetime import datetime, timezone
+
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
+from src.data import read_raw_co2
+from src.utils.location import get_location
+
 app = FastAPI(
     title="Climate Analysis API",
     openapi_url="/api",
@@ -17,7 +22,16 @@
 )
 
 
-@app.get("/refresh-data")
-async def refresh_data():
+@app.get("/refresh-data/")
+async def refresh_data(year_range: int = 4, loc_range: int = 5):
+    utc_now = datetime.now(timezone.utc)
+    year_from = utc_now.year - year_range
+
+    lon, lat = get_location()
+
+    lat_min = lat - loc_range
+    lat_max = lat + loc_range
+    lon_min = lon - loc_range
+    lon_max = lon + loc_range
     # For now starting with co2 only
     co2_df = read_raw_co2(year_from, lat_min, lat_max, lon_min, lon_max)

src/config.py

@@ -1,4 +1,6 @@
 from pathlib import Path
+from typing import Self
+
 from pydantic import model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
@@ -9,23 +11,16 @@
 CO2 = 'co2'
 TEMPANOMALIES = 'tempanomalies'
 
+
 class Settings(BaseSettings):
     model_config = SettingsConfigDict(
         env_file="./.env",
         env_ignore_empty=True,
         extra="ignore",
     )
-    AWS_KEY: str | None = None
-    AWS_SECRET: str | None = None
     S3_BUCKET: str | None = None
     EARTHDATA_USERNAME: str = ""
     EARTHDATA_PASSWORD: str = ""
 
-    @model_validator(mode="after")
-    def _enforce_non_default_secrets(self) -> Self:
-        self._check_default_secret("EARTHDATA_USERNAME", self.EARTHDATA_USERNAME)
-        self._check_default_secret("EARTHDATA_PASSWORD", self.EARTHDATA_PASSWORD)
-
-        return self
 
-settings = Settings()
+settings = Settings()

src/data/load.py

@@ -8,7 +8,7 @@
 import h5py
 import earthaccess
 
-from config import DATA_DIR, TEMPANOMALIES, CO2
+from src.config import DATA_DIR, TEMPANOMALIES, CO2
 
 
 def load_co2() -> pd.DataFrame:

src/data/write.py

@@ -1,5 +1,7 @@
+from typing import Literal
+
 import pandas as pd
-from config import DATA_DIR, TEMPANOMALIES, CO2, settings
+from src.config import DATA_DIR, TEMPANOMALIES, CO2, settings
 
 
 def _write_csv(df: pd.DataFrame, filename: str, locally: bool = True):