diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e4cb683 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +.git +__pycache__ +*.pyc +infra/ +data/ +notebooks/ +mlruns/ +outputs/ +.jupyter +.github \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index bf880eb..dc26847 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,8 @@ -FROM continuumio/miniconda3 +FROM public.ecr.aws/lambda/python:3.11 -COPY environment.yml /tmp/environment.yml -RUN conda env create -f /tmp/environment.yml +COPY requirements-lambda.txt . +RUN pip install --prefer-binary -r requirements-lambda.txt -SHELL ["conda", "run", "-n", "climate_analysis", "/bin/bash", "-c"] - -COPY . /app -WORKDIR /app - -CMD ["fastapi", "run", "src/api.py", "--port", "80"] \ No newline at end of file +COPY src /var/task/src +ENV PYTHONPATH="/var/task/src" +CMD ["api.handler"] \ No newline at end of file diff --git a/environment.yml b/environment.yml index 37b1088..376f873 100644 Binary files a/environment.yml and b/environment.yml differ diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl new file mode 100644 index 0000000..63b71b4 --- /dev/null +++ b/infra/.terraform.lock.hcl @@ -0,0 +1,126 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/archive" { + version = "2.7.1" + constraints = "2.7.1" + hashes = [ + "h1:RzToQiFwVaxcV0QmgbyaKgNOhqc6oLKiFyZTrQSGcog=", + "zh:19881bb356a4a656a865f48aee70c0b8a03c35951b7799b6113883f67f196e8e", + "zh:2fcfbf6318dd514863268b09bbe19bfc958339c636bcbcc3664b45f2b8bf5cc6", + "zh:3323ab9a504ce0a115c28e64d0739369fe85151291a2ce480d51ccbb0c381ac5", + "zh:362674746fb3da3ab9bd4e70c75a3cdd9801a6cf258991102e2c46669cf68e19", + "zh:7140a46d748fdd12212161445c46bbbf30a3f4586c6ac97dd497f0c2565fe949", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:875e6ce78b10f73b1efc849bfcc7af3a28c83a52f878f503bb22776f71d79521", + "zh:b872c6ed24e38428d817ebfb214da69ea7eefc2c38e5a774db2ccd58e54d3a22", + "zh:cd6a44f731c1633ae5d37662af86e7b01ae4c96eb8b04144255824c3f350392d", + "zh:e0600f5e8da12710b0c52d6df0ba147a5486427c1a2cc78f31eea37a47ee1b07", + "zh:f21b2e2563bbb1e44e73557bcd6cdbc1ceb369d471049c40eb56cb84b6317a60", + "zh:f752829eba1cc04a479cf7ae7271526b402e206d5bcf1fcce9f535de5ff9e4e6", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.23.0" + constraints = ">= 6.0.0, ~> 6.22" + hashes = [ + "h1:rue7iU6J6MznSVz3EOAy/0Ko26Gh0oH1QYxqAxUmqWg=", + "zh:254eab8da4534fe937cb48992e4843d4165a6455eb6e989ef6cae7ebeb6fc107", + "zh:3c3913e4853c491ae3b65560fd5911c21959b57f2d1f890c252a12d4ac201852", + "zh:5baf2900b4b5fd7be40b0ce219359a6ec9a8896f267b932a249cee6cf0eccf09", + "zh:6c0784a51dfc31390a4d6a93f61c91c307fe8625872227236ba4dac5a59d33f6", + "zh:6c6af8b96d14893edb4e23c877a406ed85b970546c3b318edb20da05e505cd43", + "zh:732724c69ec82582804052ebe1f01d470e4af9a6c42a2a5d17df5da3d7e6e5af", + "zh:869c3c71510d9079ca7147a1660c233cffc872840995c4a3745ae26a59431414", + "zh:99db418c8b2473366a944164b7ce25c8fbcfd8ba6d1146154d885ce898351be0", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a54c2a3672d50a4c17ca04230af7a236b3d7f13418cc4b1982c2c4523744d5de", + "zh:acfd7014b33af2f36da63ebda72c500dc3e9bf1f2497b5423480bfce7db22fe8", + "zh:af0a81fcdf53064521ff9fb3d4e3e98098d5de04ed2938704807b2ef454456b5", + "zh:b042d5546151256a08f58eb7eaa1f65911b77be4ae2988c993d7ee94de1520a8", + "zh:cd033e0e52a2f13b59ab2e9cb84870306f3143d0793e0161a7ec12765e949a28", + "zh:d04a9e1861a08bdbe3a2be4c77680933cbbd8b87e6879d8079c3e5cd8d0f1ee7", + ] +} + +provider "registry.terraform.io/hashicorp/external" { + version = "2.3.5" + constraints = ">= 1.0.0" + hashes = [ + "h1:ORX3G4aEHO9sFP3dyv821JJTF9qa8J9XVp441SvvjvA=", + "zh:6e89509d056091266532fa64de8c06950010498adf9070bf6ff85bc485a82562", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:86868aec05b58dc0aa1904646a2c26b9367d69b890c9ad70c33c0d3aa7b1485a", + "zh:a2ce38fda83a62fa5fb5a70e6ca8453b168575feb3459fa39803f6f40bd42154", + "zh:a6c72798f4a9a36d1d1433c0372006cc9b904e8cfd60a2ae03ac5b7d2abd2398", + "zh:a8a3141d2fc71c86bf7f3c13b0b3be8a1b0f0144a47572a15af4dfafc051e28a", + "zh:aa20a1242eb97445ad26ebcfb9babf2cd675bdb81cac5f989268ebefa4ef278c", + "zh:b58a22445fb8804e933dcf835ab06c29a0f33148dce61316814783ee7f4e4332", + "zh:cb5626a661ee761e0576defb2a2d75230a3244799d380864f3089c66e99d0dcc", + "zh:d1acb00d20445f682c4e705c965e5220530209c95609194c2dc39324f3d4fcce", + "zh:d91a254ba77b69a29d8eae8ed0e9367cbf0ea6ac1a85b58e190f8cb096a40871", + "zh:f6592327673c9f85cdb6f20336faef240abae7621b834f189c4a62276ea5db41", + ] +} + +provider "registry.terraform.io/hashicorp/local" { + version = "2.6.1" + constraints = ">= 1.0.0" + hashes = [ + "h1:Ey2jPUGfuahWUs8w82yeZKBMYqh9SM3e6J+EQt7QTKk=", + "zh:10050d08f416de42a857e4b6f76809aae63ea4ec6f5c852a126a915dede814b4", + "zh:2df2a3ebe9830d4759c59b51702e209fe053f47453cb4688f43c063bac8746b7", + "zh:2e759568bcc38c86ca0e43701d34cf29945736fdc8e429c5b287ddc2703c7b18", + "zh:6a62a34e48500ab4aea778e355e162ebde03260b7a9eb9edc7e534c84fbca4c6", + "zh:74373728ba32a1d5450a3a88ac45624579e32755b086cd4e51e88d9aca240ef6", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8dddae588971a996f622e7589cd8b9da7834c744ac12bfb59c97fa77ded95255", + "zh:946f82f66353bb97aefa8d95c4ca86db227f9b7c50b82415289ac47e4e74d08d", + "zh:e9a5c09e6f35e510acf15b666fd0b34a30164cecdcd81ce7cda0f4b2dade8d91", + "zh:eafe5b873ef42b32feb2f969c38ff8652507e695620cbaf03b9db714bee52249", + "zh:ec146289fa27650c9d433bb5c7847379180c0b7a323b1b94e6e7ad5d2a7dbe71", + "zh:fc882c35ce05631d76c0973b35adde26980778fc81d9da81a2fade2b9d73423b", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.4" + constraints = ">= 2.0.0" + hashes = [ + "h1:+Ag4hSb4qQjNtAS6gj2+gsGl7v0iB/Bif6zZZU8lXsw=", + "zh:59f6b52ab4ff35739647f9509ee6d93d7c032985d9f8c6237d1f8a59471bbbe2", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:795c897119ff082133150121d39ff26cb5f89a730a2c8c26f3a9c1abf81a9c43", + "zh:7b9c7b16f118fbc2b05a983817b8ce2f86df125857966ad356353baf4bff5c0a", + "zh:85e33ab43e0e1726e5f97a874b8e24820b6565ff8076523cc2922ba671492991", + "zh:9d32ac3619cfc93eb3c4f423492a8e0f79db05fec58e449dee9b2d5873d5f69f", + "zh:9e15c3c9dd8e0d1e3731841d44c34571b6c97f5b95e8296a45318b94e5287a6e", + "zh:b4c2ab35d1b7696c30b64bf2c0f3a62329107bd1a9121ce70683dec58af19615", + "zh:c43723e8cc65bcdf5e0c92581dcbbdcbdcf18b8d2037406a5f2033b1e22de442", + "zh:ceb5495d9c31bfb299d246ab333f08c7fb0d67a4f82681fbf47f2a21c3e11ab5", + "zh:e171026b3659305c558d9804062762d168f50ba02b88b231d20ec99578a6233f", + "zh:ed0fe2acdb61330b01841fa790be00ec6beaac91d41f311fb8254f74eb6a711f", + ] +} + +provider "registry.terraform.io/kreuzwerker/docker" { + version = "3.6.1" + constraints = ">= 3.5.0, 3.6.1" + hashes = [ + "h1:Lg7pewlYc/pb+R5jVlIYa1c5A3sGGdH2rtKOlYKZy2Y=", + "zh:0856751d4c84f93832325bb8f4f1748f8b1239661aa389fb16e17c3de08d0544", + "zh:2888637e41f77a790053369e9df0d60242185dc5f1671503d19d1de0e5e9ab0e", + "zh:3082b0cd2faece730e6dc15750572f7893c1b07a63feaf971b2c9c8dae6fa998", + "zh:49932a8a036ed5058419a6ed860efbdc4054da74cc376d8385999a5d09b7beb8", + "zh:5aa9dff22d06fa2d3ab1d8798cb596fbd0ecbc49ae6f122b4a894a96c2885ed8", + "zh:83d34ef7b8896a8107559b96e673e252fa72d409e1034870ad46d04e79dba0fe", + "zh:862915b6a924c4f7957e908c658554452fc0d0ae10d284d2c3e0997ab06081e2", + "zh:98560cf20da973b69b3528877dbc03b4bafcbc820b2caf3060e33d6320041b56", + "zh:99f4bf53760ae13b6228c4053ef4800f87bfae4e20906aac555a757ed3df39ec", + "zh:c55e29f7561e53ab0e6a0d0b10231a4dba3538533e27082577a2cf2f09f2093f", + "zh:dfaeef2504137fb53027d8889cc85e76905d671ad41626b93a097ad113719990", + "zh:e749e2ef9c1841de69f21fa4f0cee3a1aae6ec7b624af8f9a5a07c42642f1e5d", + "zh:e88f7ee04a5b56548fa3460418a8072bd49628d238e5498e0d031cf59dab4bf3", + ] +} diff --git a/infra/main.tf b/infra/main.tf index aafe7b3..46bca79 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -95,6 +95,34 @@ module "aws_lambda_function" { image_uri = module.docker_image.image_uri package_type = "Image" + + create_role = false + lambda_role = aws_iam_role.app_role.arn + + policies = [ + aws_iam_role_policy_attachment.ecs_read.policy_arn, + aws_iam_role_policy_attachment.lambda_basic.policy_arn + ] +} + +data "archive_file" "docker_context" { + type = "zip" + source_dir = "../src/" + output_path = "${path.module}/context.zip" +} + +locals { + docker_tag = substr(data.archive_file.docker_context.output_base64sha256, 0, 12) +} + +resource "null_resource" "cleanup_zip" { + triggers = { + docker_tag = local.docker_tag + } + + provisioner "local-exec" { + command = "rm -f ${path.module}/context.zip" + } } module "docker_image" { @@ -104,20 +132,27 @@ module "docker_image" { ecr_repo = "climate-analysis" use_image_tag = true - image_tag = "1.0" + image_tag = local.docker_tag source_path = "../" -} -# resource "aws_api_gateway_integration" "app_api_gateway" { -# rest_api_id = "" -# resource_id = aws_lambda_function.example.id -# http_method = aws_api_gateway_method.example.http_method -# -# integration_http_method = "POST" -# type = "AWS_PROXY" -# uri = aws_lambda_function.app_lambda.invoke_arn -# } + ecr_repo_lifecycle_policy = jsonencode({ + "rules" : [ + { + "rulePriority" : 1, + "description" : "Keep only the last 1 image", + "selection" : { + "tagStatus" : "any", + "countType" : "imageCountMoreThan", + "countNumber" : 1 + }, + "action" : { + "type" : "expire" + } + } + ] + }) +} resource "aws_iam_role" "app_role" { name = "climate_app_role" @@ -132,7 +167,7 @@ resource "aws_iam_role" "app_role" { }) } -resource "aws_iam_role_policy_attachment" "lambda_ecr_pull" { +resource "aws_iam_role_policy_attachment" "ecs_read" { role = aws_iam_role.app_role.name policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" } @@ -151,6 +186,39 @@ resource "aws_iam_role_policy_attachment" "app_role_policy_attach" { # Create api gateway +resource "aws_apigatewayv2_api" "api" { + name = "climate-analysis-api" + protocol_type = "HTTP" +} + +resource "aws_apigatewayv2_integration" "lambda_integration" { + api_id = aws_apigatewayv2_api.api.id + integration_type = "AWS_PROXY" + integration_uri = module.aws_lambda_function.lambda_function_arn + payload_format_version = "2.0" +} + +resource "aws_apigatewayv2_route" "root_route" { + api_id = aws_apigatewayv2_api.api.id + route_key = "ANY /{proxy+}" + target = "integrations/${aws_apigatewayv2_integration.lambda_integration.id}" +} + +resource "aws_apigatewayv2_stage" "default_stage" { + api_id = aws_apigatewayv2_api.api.id + name = "$default" + auto_deploy = true +} + +resource "aws_lambda_permission" "apigw_invoke" { + statement_id = "AllowAPIGatewayInvoke" + action = "lambda:InvokeFunction" + function_name = module.aws_lambda_function.lambda_function_name + principal = "apigateway.amazonaws.com" + source_arn = "${aws_apigatewayv2_api.api.execution_arn}/*/*" +} + +###--------------------------------------------- diff --git a/infra/terraform.tf b/infra/terraform.tf index fa17c8e..c8108ec 100644 --- a/infra/terraform.tf +++ b/infra/terraform.tf @@ -15,6 +15,10 @@ terraform { source = "kreuzwerker/docker" version = "3.6.1" } + archive = { + source = "hashicorp/archive" + version = "2.7.1" + } } required_version = ">= 1.2" diff --git a/requirements-lambda.txt b/requirements-lambda.txt new file mode 100644 index 0000000..a5cd7ec --- /dev/null +++ b/requirements-lambda.txt @@ -0,0 +1,82 @@ +affine==2.4.0 +aiobotocore==2.25.2 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.2 +aioitertools==0.12.0 +aiosignal==1.4.0 +anyio==4.11.0 +argon2-cffi==25.1.0 +attrs==25.4.0 +bcrypt==5.0.0 +Brotli==1.2.0 +cached-property==1.5.2 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +click-plugins==1.1.1.2 +decorator==5.2.1 +defusedxml==0.7.1 +executing==2.2.1 +fastapi==0.122.0 +fastapi-cli==0.0.16 +fastapi-cloud-cli==0.5.2 +fastar==0.8.0 +fasteners==0.20 +fastjsonschema==2.21.2 +fqdn==1.5.1 +frozenlist==1.7.0 +fsspec==2025.10.0 +h11==0.16.0 +httpcore==1.0.9 +httptools==0.7.1 +httpx==0.28.1 +idna==3.11 +importlib_metadata==8.7.0 +importlib_resources==6.5.2 +iso8601==2.1.0 +isoduration==20.11.0 +jmespath==1.0.1 +json5==0.12.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema==4.25.1 +jsonschema-specifications==2025.9.1 +keystoneauth1==5.12.0 +kombu==5.6.1 +lazy_loader==0.4 +locket==1.0.0 +mangum==0.19.0 +multidict==6.6.3 +multimethod==2.0.2 +overrides==7.7.0 +packaging==25.0 +prometheus_client==0.23.1 +pydantic==2.12.5 +pydantic_core==2.41.5 +pydantic-settings==2.12.0 +pyjwt==2.10.1 +python-dotenv==1.2.1 +rfc3339_validator==0.1.4 +rfc3986==2.0.0 +rfc3986-validator==0.1.1 +rfc3987-syntax==1.1.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +starlette==0.50.0 +tenacity==9.1.2 +typing_extensions==4.15.0 +typing-inspection==0.4.2 +typing_utils==0.1.0 +tzdata==2025.2 +urllib3==2.5.0 +uvicorn==0.38.0 +vine==5.1.0 +watchfiles==1.1.1 +webcolors==25.10.0 +webencodings==0.5.1 +Werkzeug==3.1.3 +wrapt==1.17.3 +yappi==1.7.3 +zict==3.0.0 +zipp==3.23.0 +zstandard==0.25.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8e8da38 Binary files /dev/null and b/requirements.txt differ diff --git a/src/api.py b/src/api.py index 791a956..d022a1a 100644 --- a/src/api.py +++ b/src/api.py @@ -3,7 +3,8 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from src.data import read_raw_co2 +from mangum import Mangum + from src.utils.location import get_location app = FastAPI( @@ -22,7 +23,7 @@ ) -@app.get("/refresh-data/") +@app.get("/refresh-data") async def refresh_data(year_range: int = 4, loc_range: int = 5): utc_now = datetime.now(timezone.utc) year_from = utc_now.year - year_range @@ -33,5 +34,10 @@ async def refresh_data(year_range: int = 4, loc_range: int = 5): lat_max = lat + loc_range lon_min = lon - loc_range lon_max = lon + loc_range - # For now starting with co2 only - co2_df = read_raw_co2(year_from, lat_min, lat_max, lon_min, lon_max) + # test + return {lat_min: lat_min, lat_max: lat_max, lon_min: lon_min, lon_max: lon_max} + + +handler = Mangum( + app +) diff --git a/src/data/load.py b/src/data/load.py index 57287ab..2ca55d4 100644 --- a/src/data/load.py +++ b/src/data/load.py @@ -8,7 +8,7 @@ import h5py import earthaccess -from src.config import DATA_DIR, TEMPANOMALIES, CO2 +from src.config import DATA_DIR, TEMPANOMALIES, CO2, settings def load_co2() -> pd.DataFrame: @@ -120,12 +120,13 @@ def read_remote_co2( Coords: time, latitude, longitude Parameters: - year_from (int): Year from which to read data - lat_min (float): Minimum latitude - lat_max (float): Maximum latitude - lon_min (float): Minimum longitude - lon_max (float): Maximum longitude - limit (int): Maximum number of files to download + year_from: Year from which to read data + lat_min: Minimum latitude + lat_max: Maximum latitude + lon_min: Minimum longitude + lon_max: Maximum longitude + limit: Maximum number of files to download + locally: To save processing raw files locally or work with s3 Returns: pd.DataFrame: DataFrame containing xco2 (ppm) by lat, lon, time """