diff --git a/arpa-exporter/poetry.lock b/arpa-exporter/poetry.lock index 8f55ba0dd..1d04c7df4 100644 --- a/arpa-exporter/poetry.lock +++ b/arpa-exporter/poetry.lock @@ -1965,30 +1965,30 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy [[package]] name = "ruff" -version = "0.9.10" +version = "0.10.0" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.9.10-py3-none-linux_armv6l.whl", hash = "sha256:eb4d25532cfd9fe461acc83498361ec2e2252795b4f40b17e80692814329e42d"}, - {file = "ruff-0.9.10-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:188a6638dab1aa9bb6228a7302387b2c9954e455fb25d6b4470cb0641d16759d"}, - {file = "ruff-0.9.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:5284dcac6b9dbc2fcb71fdfc26a217b2ca4ede6ccd57476f52a587451ebe450d"}, - {file = "ruff-0.9.10-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47678f39fa2a3da62724851107f438c8229a3470f533894b5568a39b40029c0c"}, - {file = "ruff-0.9.10-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:99713a6e2766b7a17147b309e8c915b32b07a25c9efd12ada79f217c9c778b3e"}, - {file = "ruff-0.9.10-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:524ee184d92f7c7304aa568e2db20f50c32d1d0caa235d8ddf10497566ea1a12"}, - {file = "ruff-0.9.10-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:df92aeac30af821f9acf819fc01b4afc3dfb829d2782884f8739fb52a8119a16"}, - {file = "ruff-0.9.10-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de42e4edc296f520bb84954eb992a07a0ec5a02fecb834498415908469854a52"}, - {file = "ruff-0.9.10-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d257f95b65806104b6b1ffca0ea53f4ef98454036df65b1eda3693534813ecd1"}, - {file = "ruff-0.9.10-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b60dec7201c0b10d6d11be00e8f2dbb6f40ef1828ee75ed739923799513db24c"}, - {file = "ruff-0.9.10-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:d838b60007da7a39c046fcdd317293d10b845001f38bcb55ba766c3875b01e43"}, - {file = "ruff-0.9.10-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ccaf903108b899beb8e09a63ffae5869057ab649c1e9231c05ae354ebc62066c"}, - {file = "ruff-0.9.10-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f9567d135265d46e59d62dc60c0bfad10e9a6822e231f5b24032dba5a55be6b5"}, - {file = "ruff-0.9.10-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5f202f0d93738c28a89f8ed9eaba01b7be339e5d8d642c994347eaa81c6d75b8"}, - {file = "ruff-0.9.10-py3-none-win32.whl", hash = "sha256:bfb834e87c916521ce46b1788fbb8484966e5113c02df216680102e9eb960029"}, - {file = "ruff-0.9.10-py3-none-win_amd64.whl", hash = "sha256:f2160eeef3031bf4b17df74e307d4c5fb689a6f3a26a2de3f7ef4044e3c484f1"}, - {file = "ruff-0.9.10-py3-none-win_arm64.whl", hash = "sha256:5fd804c0327a5e5ea26615550e706942f348b197d5475ff34c19733aee4b2e69"}, - {file = "ruff-0.9.10.tar.gz", hash = "sha256:9bacb735d7bada9cfb0f2c227d3658fc443d90a727b47f206fb33f52f3c0eac7"}, + {file = "ruff-0.10.0-py3-none-linux_armv6l.whl", hash = "sha256:46a2aa0eaae5048e5f804f0be9489d8a661633e23277b7293089e70d5c1a35c4"}, + {file = "ruff-0.10.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:775a6bc61af9dd0a2e1763406522a137e62aabb743d8b43ed95f019cdd1526c7"}, + {file = "ruff-0.10.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8b03e6fcd39d20f0004f9956f0ed5eadc404d3a299f9d9286323884e3b663730"}, + {file = "ruff-0.10.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:621101d1af80248827f2409a78c8177c8319986a57b4663613b9c72f8617bfcd"}, + {file = "ruff-0.10.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e2dfe85cb6bfbd4259801e7d4982f2a72bdbd5749dc73a09d68a6dbf77f2209a"}, + {file = "ruff-0.10.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43ac3879a20c22fdc57e559f0bb27f0c71828656841d0b42d3505b1e5b3a83c8"}, + {file = "ruff-0.10.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:ef5e3aac421bbc62f8a7aab21edd49a359ed42205f7a5091a74386bca1efa293"}, + {file = "ruff-0.10.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9f4f62d7fac8b748fce67ad308116b4d4cc1a9f964b4804fc5408fbd06e13ba9"}, + {file = "ruff-0.10.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:02f9f6205c5b0d626f98da01a0e75b724a64c21c554bba24b12522c9e9ba6a04"}, + {file = "ruff-0.10.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46a97f3d55f68464c48d1e929a8582c7e5bb80ac73336bbc7b0da894d8e6cd9e"}, + {file = "ruff-0.10.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a0b811197d0dc96c13d610f8cfdc56030b405bcff5c2f10eab187b329da0ca4a"}, + {file = "ruff-0.10.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a13a3fda0870c1c964b47ff5d73805ae80d2a9de93ee2d185d453b8fddf85a84"}, + {file = "ruff-0.10.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:6ceb8d9f062e90ddcbad929f6136edf764bbf6411420a07e8357602ea28cd99f"}, + {file = "ruff-0.10.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:c41d07d573617ed2f287ea892af2446fd8a8d877481e8e1ba6928e020665d240"}, + {file = "ruff-0.10.0-py3-none-win32.whl", hash = "sha256:76e2de0cbdd587e373cd3b4050d2c45babdd7014c1888a6f121c29525c748a15"}, + {file = "ruff-0.10.0-py3-none-win_amd64.whl", hash = "sha256:f943acdecdcc6786a8d1dad455dd9f94e6d57ccc115be4993f9b52ef8316027a"}, + {file = "ruff-0.10.0-py3-none-win_arm64.whl", hash = "sha256:935a943bdbd9ff0685acd80d484ea91088e27617537b5f7ef8907187d19d28d0"}, + {file = "ruff-0.10.0.tar.gz", hash = "sha256:fa1554e18deaf8aa097dbcfeafaf38b17a2a1e98fdc18f50e62e8a836abee392"}, ] [[package]] @@ -2043,14 +2043,14 @@ tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] [[package]] name = "structlog" -version = "25.1.0" +version = "25.2.0" description = "Structured Logging for Python" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "structlog-25.1.0-py3-none-any.whl", hash = "sha256:843fe4f254540329f380812cbe612e1af5ec5b8172205ae634679cd35a6d6321"}, - {file = "structlog-25.1.0.tar.gz", hash = "sha256:2ef2a572e0e27f09664965d31a576afe64e46ac6084ef5cec3c2b8cd6e4e3ad3"}, + {file = "structlog-25.2.0-py3-none-any.whl", hash = "sha256:0fecea2e345d5d491b72f3db2e5fcd6393abfc8cd06a4851f21fcd4d1a99f437"}, + {file = "structlog-25.2.0.tar.gz", hash = "sha256:d9f9776944207d1035b8b26072b9b140c63702fd7aa57c2f85d28ab701bd8e92"}, ] [package.extras] @@ -2295,4 +2295,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = "^3.13" -content-hash = "8253eff526c30a5064bd0c8d5966509eb7d68f8b50ca331ced4bb3b03a619c77" +content-hash = "192aab2c7ac7ebd18e169395d300fcfbbb720284fd549e85acf633765f0c7ed1" diff --git a/arpa-exporter/pyproject.toml b/arpa-exporter/pyproject.toml index 3e476f039..35ba0264d 100644 --- a/arpa-exporter/pyproject.toml +++ b/arpa-exporter/pyproject.toml @@ -9,13 +9,13 @@ package-mode = false python = "^3.13" boto3 = "^1.37.11" ddtrace = "^2.21.3" -structlog = "^25.1.0" +structlog = "^25.2.0" pydantic = "^2.10.3" chevron="^0.14.0" [tool.poetry.group.dev.dependencies] pytest = "^8.3.5" -ruff = "^0.9.10" +ruff = "^0.10.0" ipython = "^9.0.2" pytest-cov = "^6.0.0" mypy = "^1.15.0" diff --git a/packages/server/__tests__/arpa_reporter/server/services/full-file-export.spec.js b/packages/server/__tests__/arpa_reporter/server/services/full-file-export.spec.js index 00ea87fbf..4c2d7d619 100644 --- a/packages/server/__tests__/arpa_reporter/server/services/full-file-export.spec.js +++ b/packages/server/__tests__/arpa_reporter/server/services/full-file-export.spec.js @@ -115,7 +115,7 @@ describe('FullFileExport', () => { expect(command.Bucket).to.equal(process.env.AUDIT_REPORT_BUCKET); }); - it('shouldRecreateArchive raises an error if the S3 client throws an unknown error', async () => { + it('shouldRecreateArchive throw an error if the S3 client throws an unknown error', async () => { const uploads = [ fixtures.uploads.upload1, fixtures.uploads.upload2, diff --git a/packages/server/src/arpa_reporter/services/full-file-export.js b/packages/server/src/arpa_reporter/services/full-file-export.js index 279841338..47b03312c 100644 --- a/packages/server/src/arpa_reporter/services/full-file-export.js +++ b/packages/server/src/arpa_reporter/services/full-file-export.js @@ -1,5 +1,7 @@ const { SendMessageCommand } = require('@aws-sdk/client-sqs'); -const { HeadObjectCommand, PutObjectCommand, NotFound } = require('@aws-sdk/client-s3'); +const { + HeadObjectCommand, PutObjectCommand, NotFound, +} = require('@aws-sdk/client-s3'); const converter = require('json-2-csv'); const path = require('path'); const moment = require('moment'); @@ -79,6 +81,7 @@ async function getMetadataLastModified(organizationId, logger = log) { const s3 = aws.getS3Client(); const Key = metadataFileKey(organizationId); const baseParams = { Bucket: process.env.AUDIT_REPORT_BUCKET, Key }; + logger.child(baseParams); let headObject; @@ -89,7 +92,7 @@ async function getMetadataLastModified(organizationId, logger = log) { logger.info('metadata file does not exist'); return null; } - logger.error(error, 'failed to get existing metadata file'); + logger.error(error, 'failed to get metadata file due to an exception with s3'); throw error; } diff --git a/terraform/modules/gost_api/storage.tf b/terraform/modules/gost_api/storage.tf index d7c248561..2bf497c8c 100644 --- a/terraform/modules/gost_api/storage.tf +++ b/terraform/modules/gost_api/storage.tf @@ -67,9 +67,23 @@ module "access_arpa_reports_bucket_policy" { actions = [ "s3:PutObject", "s3:GetObject", - "s3:HeadObject", ] resources = ["${module.arpa_audit_reports_bucket.bucket_arn}/*"] + }, + { + sid = "ListBucketObjects" + effect = "Allow" + actions = [ + "s3:ListBucket", + ] + resources = [module.arpa_audit_reports_bucket.bucket_arn] + conditions = [ + { + test = "StringLike" + variable = "s3:prefix" + values = ["full-file-export/*"] + } + ] } ] } diff --git a/terraform/prod.tfvars b/terraform/prod.tfvars index 09871c2f3..1acfdf4ae 100644 --- a/terraform/prod.tfvars +++ b/terraform/prod.tfvars @@ -57,7 +57,7 @@ website_feature_flags = { newGrantsDetailPageEnabled = true, shareTerminologyEnabled = true, followNotesEnabled = false, - enableFullFileExport = false, + enableFullFileExport = true, } // Google Analytics Account ID: 233192355, Property ID: 321194851, Stream ID: 3802896350