diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml new file mode 100644 index 0000000..69a7fd9 --- /dev/null +++ b/.github/workflows/terraform.yml @@ -0,0 +1,95 @@ +name: "Terraform" + +on: + # push: + # branches: + # - main + release: + types: [ published ] + pull_request_target: + types: [opened, synchronize] + +permissions: + pull-requests: write + issues: write + id-token: write + contents: read + +jobs: + terraform: + name: "Terraform" + runs-on: ubuntu-latest + + defaults: + run: + working-directory: infra + steps: + - name: Checkout + uses: actions/checkout@v4 + + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::266796786618:role/github-pipeline-role + aws-region: eu-north-1 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.14.0 + terraform_wrapper: false + + - name: Terraform Format + id: fmt + run: terraform fmt -check + + - name: Terraform Init + id: init + run: terraform init + + - name: Terraform Validate + id: validate + run: terraform validate -no-color + + - name: Terraform Plan + id: plan + if: github.event_name == 'pull_request' + run: terraform plan -no-color -input=false + continue-on-error: true + + - uses: actions/github-script@v8 + if: github.event_name == 'pull_request' + env: + PLAN: "terraform\n${{ steps.plan.outputs.stdout }}" + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const output = `#### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\` + #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\` + #### Terraform Plan 📖\`${{ steps.plan.outcome }}\` + +
Show Plan + + \`\`\`${process.env.PLAN}\`\`\` + +
+ + *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`; + + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }) + + - name: Terraform Plan Status + if: steps.plan.outcome == 'failure' + run: exit 1 + + - name: Terraform Apply + if: github.event_name == 'release' + working-directory: infra + run: | + terraform init + terraform apply -auto-approve \ No newline at end of file diff --git a/.gitignore b/.gitignore index d738448..ef8f8d0 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,4 @@ data/processed/ .env mlruns/ +/infra/.terraform/ diff --git a/environment.yml b/environment.yml index 5e5cbb8..37b1088 100644 Binary files a/environment.yml and b/environment.yml differ diff --git a/infra/main.tf b/infra/main.tf new file mode 100644 index 0000000..e81fc77 --- /dev/null +++ b/infra/main.tf @@ -0,0 +1,146 @@ +provider "aws" { + region = var.aws_region +} + +data "aws_caller_identity" "current" {} + +data "aws_ecr_authorization_token" "token" {} + +provider "docker" { + registry_auth { + address = "${data.aws_caller_identity.current.account_id}.dkr.ecr.${var.aws_region}.amazonaws.com" + username = data.aws_ecr_authorization_token.token.user_name + password = data.aws_ecr_authorization_token.token.password + } +} + +# Create S3 bucket + +resource "aws_s3_bucket" "app_s3" { + bucket = "climate-analysis-bucket" + + tags = { + Name = "Climate Analysis Bucket" + Environment = "Dev" + } +} + +resource "aws_s3_bucket_public_access_block" "s3_access" { + bucket = aws_s3_bucket.app_s3.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} +###--------------------------------------------- + +# Create policy to access the bucket + +data "aws_iam_policy_document" "aws_policy_doc" { + statement { + actions = [ + "s3:ListBucket", + ] + effect = "Allow" + resources = [aws_s3_bucket.app_s3.arn] + } + statement { + actions = [ + "s3:GetObject", + "s3:PutObject" + ] + effect = "Allow" + resources = ["${aws_s3_bucket.app_s3.arn}/*"] + } +} + +resource "aws_iam_policy" "policy" { + name = "climate_analysis_resources_policy" + path = "/" + description = "Climate Analysis Resources access" + + policy = data.aws_iam_policy_document.aws_policy_doc.json +} + +###--------------------------------------------- + +# Create local user for s3 access (optional) + +resource "aws_iam_user" "local_dev" { + count = var.create_local_user ? 1 : 0 + name = var.local_user +} + +resource "aws_iam_user_policy_attachment" "local_dev_s3" { + count = var.create_local_user ? 1 : 0 + user = aws_iam_user.local_dev[0].name + policy_arn = aws_iam_policy.policy.arn +} + +resource "aws_iam_access_key" "local_dev_key" { + count = var.create_local_user ? 1 : 0 + user = aws_iam_user.local_dev[0].name +} + +###--------------------------------------------- + +# Create docker image, app lambda, it's role and attach policy + +module "aws_lambda_function" { + source = "terraform-aws-modules/lambda/aws" + + function_name = "climate_analysis_lambda" + create_package = false + + image_uri = module.docker_image.image_uri + package_type = "Image" +} + +module "docker_image" { + source = "terraform-aws-modules/lambda/aws//modules/docker-build" + + create_ecr_repo = true + ecr_repo = "climate-analysis" + + use_image_tag = true + image_tag = "1.0" + + source_path = "../" +} + +# resource "aws_api_gateway_integration" "app_api_gateway" { +# rest_api_id = "" +# resource_id = aws_lambda_function.example.id +# http_method = aws_api_gateway_method.example.http_method +# +# integration_http_method = "POST" +# type = "AWS_PROXY" +# uri = aws_lambda_function.app_lambda.invoke_arn +# } + +resource "aws_iam_role" "app_role" { + name = "climate_app_role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [{ + Effect = "Allow", + Principal = { Service = "lambda.amazonaws.com" }, + Action = "sts:AssumeRole" + }] + }) +} + +resource "aws_iam_role_policy_attachment" "app_role_policy_attach" { + role = aws_iam_role.app_role.name + policy_arn = aws_iam_policy.policy.arn +} + +###--------------------------------------------- + +# Create api gateway + + + + diff --git a/infra/terraform.tf b/infra/terraform.tf new file mode 100644 index 0000000..fa17c8e --- /dev/null +++ b/infra/terraform.tf @@ -0,0 +1,21 @@ +terraform { + backend "s3" { + bucket = "terraform-state-bucket-4848" + key = "climate_analysis/terraform.tfstate" + region = "eu-north-1" + encrypt = true + use_lockfile = true + } + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.22" + } + docker = { + source = "kreuzwerker/docker" + version = "3.6.1" + } + } + + required_version = ">= 1.2" +} diff --git a/infra/variables.tf b/infra/variables.tf new file mode 100644 index 0000000..8aa003e --- /dev/null +++ b/infra/variables.tf @@ -0,0 +1,15 @@ +variable "aws_region" { + description = "AWS region" + default = "eu-north-1" +} + +variable "create_local_user" { + description = "Create user with app required policies" + type = bool + default = false +} + +variable "local_user" { + description = "Local user name to create in aws and add policies to" + default = "local-dev" +} diff --git a/readme.md b/readme.md index 52a30f6..2cebc62 100644 --- a/readme.md +++ b/readme.md @@ -79,7 +79,28 @@ Prerequisites: - Create environment from file: `conda env create -f environment.yml` - Activate environment: `conda activate climate_analysis` -### Run the CLI entrypoint (main.py) +### Run API locally + +- From project root: + - `fastapi run src/api.py` + - server should start at `http://127.0.0.1:8000` + +This project allows local storage option or creating AWS infrastructure with S3 bucket through Terraform. +This would allow much greater performance and possibility to run model more efficently. + +### Run Terraform + +You should have your `aws cli` credentials available locally (aws profile or key/secret pair as environment variables, see AWS docs for more details). +`terraform` and `docker` should be installed as well. +*Alternatively you could fork repository and simply provide different +`role-to-assume: arn:aws:iam:::role/` in `terraform.yml`, that way GitHub Actions will create infrastructure for you.* + +- From `infra` directory: + - `terraform init` + - `terraform plan` - ensure all planned services are ok for you + - `terraform apply` - docker image in AWS ECR will be created, S3, lambda and required permissions. + +### Run CLI entrypoint (main.py) Typical usage: diff --git a/src/api.py b/src/api.py index ae1fdaf..791a956 100644 --- a/src/api.py +++ b/src/api.py @@ -1,6 +1,11 @@ +from datetime import datetime, timezone + from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from src.data import read_raw_co2 +from src.utils.location import get_location + app = FastAPI( title="Climate Analysis API", openapi_url="/api", @@ -17,7 +22,16 @@ ) -@app.get("/refresh-data") -async def refresh_data(): +@app.get("/refresh-data/") +async def refresh_data(year_range: int = 4, loc_range: int = 5): + utc_now = datetime.now(timezone.utc) + year_from = utc_now.year - year_range + + lon, lat = get_location() + + lat_min = lat - loc_range + lat_max = lat + loc_range + lon_min = lon - loc_range + lon_max = lon + loc_range # For now starting with co2 only co2_df = read_raw_co2(year_from, lat_min, lat_max, lon_min, lon_max) diff --git a/src/config.py b/src/config.py index d531ab0..e352b2f 100644 --- a/src/config.py +++ b/src/config.py @@ -1,4 +1,6 @@ from pathlib import Path +from typing import Self + from pydantic import model_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -9,23 +11,16 @@ CO2 = 'co2' TEMPANOMALIES = 'tempanomalies' + class Settings(BaseSettings): model_config = SettingsConfigDict( env_file="./.env", env_ignore_empty=True, extra="ignore", ) - AWS_KEY: str | None = None - AWS_SECRET: str | None = None S3_BUCKET: str | None = None EARTHDATA_USERNAME: str = "" EARTHDATA_PASSWORD: str = "" - @model_validator(mode="after") - def _enforce_non_default_secrets(self) -> Self: - self._check_default_secret("EARTHDATA_USERNAME", self.EARTHDATA_USERNAME) - self._check_default_secret("EARTHDATA_PASSWORD", self.EARTHDATA_PASSWORD) - - return self -settings = Settings() \ No newline at end of file +settings = Settings() diff --git a/src/data/load.py b/src/data/load.py index 68d873b..57287ab 100644 --- a/src/data/load.py +++ b/src/data/load.py @@ -8,7 +8,7 @@ import h5py import earthaccess -from config import DATA_DIR, TEMPANOMALIES, CO2 +from src.config import DATA_DIR, TEMPANOMALIES, CO2 def load_co2() -> pd.DataFrame: diff --git a/src/data/write.py b/src/data/write.py index 2059f20..e267b50 100644 --- a/src/data/write.py +++ b/src/data/write.py @@ -1,5 +1,7 @@ +from typing import Literal + import pandas as pd -from config import DATA_DIR, TEMPANOMALIES, CO2, settings +from src.config import DATA_DIR, TEMPANOMALIES, CO2, settings def _write_csv(df: pd.DataFrame, filename: str, locally: bool = True): diff --git a/src/utils/location.py b/src/utils/location.py index 54a6e5f..716ce55 100644 --- a/src/utils/location.py +++ b/src/utils/location.py @@ -4,4 +4,4 @@ def get_location(): res = requests.get("https://ipinfo.io/json") data = res.json() lat, lon = map(float, data["loc"].split(",")) - return (lon, lat) \ No newline at end of file + return lon, lat \ No newline at end of file diff --git a/src/visualization/plotting.py b/src/visualization/plotting.py index 277921a..c287a04 100644 --- a/src/visualization/plotting.py +++ b/src/visualization/plotting.py @@ -11,7 +11,7 @@ from cartopy.feature.nightshade import Nightshade import matplotlib.colors as mcolors -from config import PLOTS_DIR +from src.config import PLOTS_DIR def _save_plot(fig: Figure, name: str):