diff --git a/ansible/deploy-fastpath.yml b/ansible/deploy-fastpath.yml new file mode 100644 index 00000000..74179cfd --- /dev/null +++ b/ansible/deploy-fastpath.yml @@ -0,0 +1,25 @@ +--- +- name: Deploy fastpath + hosts: + - fastpath.dev.ooni.io + become: true + roles: + - role: bootstrap + - role: dehydrated + vars: + ssl_domains: + - "{{ inventory_hostname }}" + tls_cert_dir: /var/lib/dehydrated/certs + - role: prometheus_node_exporter + vars: + node_exporter_port: 9100 + node_exporter_host: "0.0.0.0" + prometheus_nginx_proxy_config: + - location: /metrics/node_exporter + proxy_pass: http://127.0.0.1:9100/metrics + - role: geerlingguy.docker + docker_users: + - fastpath + - ubuntu + docker_package_state: latest + - role: fastpath diff --git a/ansible/host_vars/fastpath.dev.ooni.io/vars.yml b/ansible/host_vars/fastpath.dev.ooni.io/vars.yml new file mode 100644 index 00000000..30c753dd --- /dev/null +++ b/ansible/host_vars/fastpath.dev.ooni.io/vars.yml @@ -0,0 +1,4 @@ +s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_dev') }}" +clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" +bucket_name: "ooni-data-eu-fra-test" +collector_id: "3" \ No newline at end of file diff --git a/ansible/inventory b/ansible/inventory index 9009752d..3a23e524 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -42,3 +42,7 @@ monitoringproxy.prod.ooni.io [openvpn] openvpn1.htz-fsn.prod.ooni.nu openvpn2.htz-fsn.prod.ooni.nu + +[aws-backend] +fastpath.dev.ooni.io +# fastpath.prod.ooni.io \ No newline at end of file diff --git a/ansible/requirements/ansible-galaxy.yml b/ansible/requirements/ansible-galaxy.yml index 3b29d9ba..997d7a83 100644 --- a/ansible/requirements/ansible-galaxy.yml +++ b/ansible/requirements/ansible-galaxy.yml @@ -1,15 +1,21 @@ -- src: willshersystems.sshd - version: v0.25.0 -- src: nginxinc.nginx - version: 0.24.3 -- src: geerlingguy.certbot - version: 5.2.0 -- src: artis3n.tailscale - version: v4.5.0 -- src: https://github.com/idealista/clickhouse_role - scm: git - version: 3.5.1 - name: idealista.clickhouse_role -- src: https://github.com/ooni/airflow-role.git - scm: git - name: ooni.airflow_role \ No newline at end of file +roles: + - src: willshersystems.sshd + version: v0.25.0 + - src: nginxinc.nginx + version: 0.24.3 + - src: geerlingguy.certbot + version: 5.2.0 + - src: artis3n.tailscale + version: v4.5.0 + - src: https://github.com/idealista/clickhouse_role + scm: git + version: 3.5.1 + name: idealista.clickhouse_role + - src: https://github.com/ooni/airflow-role.git + scm: git + name: ooni.airflow_role + - src: geerlingguy.docker # installs docker + version: 7.4.7 +collections: + - name: community.docker # manages containers + version: 4.6.1 \ No newline at end of file diff --git a/ansible/roles/fastpath/defaults/main.yml b/ansible/roles/fastpath/defaults/main.yml new file mode 100644 index 00000000..1a15aea5 --- /dev/null +++ b/ansible/roles/fastpath/defaults/main.yml @@ -0,0 +1,8 @@ +tls_cert_dir: /var/lib/dehydrated/certs + +# Fastpath user +fastpath_user: fastpath +fastpath_home: "/opt/{{ fastpath_user }}" + +# Fastpath settings +clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" \ No newline at end of file diff --git a/ansible/roles/fastpath/handlers/main.yml b/ansible/roles/fastpath/handlers/main.yml new file mode 100644 index 00000000..594ebfe9 --- /dev/null +++ b/ansible/roles/fastpath/handlers/main.yml @@ -0,0 +1,27 @@ +- name: test nginx config + command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf + listen: + - restart nginx + - reload nginx + +- name: restart nginx + service: + name: nginx + state: restarted + +- name: reload nginx + service: + name: nginx + state: reloaded + +- name: reload nftables + tags: nftables + ansible.builtin.systemd_service: + name: nftables + state: reloaded + +- name: restart docker + tags: docker + ansible.builtin.systemd_service: + name: docker + state: restarted \ No newline at end of file diff --git a/ansible/roles/fastpath/tasks/main.yml b/ansible/roles/fastpath/tasks/main.yml new file mode 100644 index 00000000..85087916 --- /dev/null +++ b/ansible/roles/fastpath/tasks/main.yml @@ -0,0 +1,156 @@ +--- +# For prometheus scrape requests +- name: Flush all handlers + meta: flush_handlers + +- name: Allow traffic on port 9100 + become: true + tags: prometheus-proxy + blockinfile: + path: /etc/ooni/nftables/tcp/9100.nft + create: yes + block: | + add rule inet filter input tcp dport 9100 counter accept comment "node exporter" + notify: + - reload nftables + +# For incoming fastpath traffic +- name: Allow traffic on port 8472 + become: true + tags: fastpath + blockinfile: + path: /etc/ooni/nftables/tcp/8472.nft + create: yes + block: | + add rule inet filter input tcp dport 8472 counter accept comment "fastpath" + notify: + - reload nftables + +# Docker seems to have problems with nftables, so this command will translate all iptables +# commands to nftables commands +- name: Update alternatives for iptables + tags: docker + become: yes + ansible.builtin.command: "update-alternatives --set iptables /usr/sbin/iptables-nft" + notify: + - restart docker + +- name: Update alternatives for iptables + tags: docker + become: yes + ansible.builtin.command: "update-alternatives --set ip6tables /usr/sbin/ip6tables-nft" + notify: + - restart docker + +- name: Flush all handlers # Required to apply iptables settings before docker runs + meta: flush_handlers + +### Create fastpath user +- name: Ensure the fastpath group exists + ansible.builtin.group: + name: "{{ fastpath_user }}" + state: present + become: yes +- name: Create the fastpath user + ansible.builtin.user: + name: "{{ fastpath_user }}" + home: "{{ fastpath_home }}" + shell: "/bin/bash" + group: "{{ fastpath_user }}" + create_home: yes + system: yes + become: yes +- name: Set ownership of the fastpath directory + ansible.builtin.file: + path: "{{ fastpath_home }}" + owner: "{{ fastpath_user }}" + group: "{{ fastpath_user }}" + state: directory + mode: '0755' + become: yes + +### Run fastpath +- name: Make sure that the fastpath configuration directory exists + ansible.builtin.file: + path: /opt/{{fastpath_user}}/backend/fastpath/ + state: directory + mode: '0700' + owner: "{{fastpath_user}}" + group: "{{fastpath_user}}" + +- name: Create configuration file + tags: fastpath + template: + src: templates/fastpath.conf + dest: "/opt/{{fastpath_user}}/backend/fastpath/fastpath.conf" + mode: 0444 + owner: "{{fastpath_user}}" + become: yes + +- name: Ensure ooniapi directory existence + ansible.builtin.file: + path: /var/lib/ooniapi + state: directory + mode: '0700' + owner: "{{fastpath_user}}" + group: "{{fastpath_user}}" + +- name: Ensure fastpath is running + community.docker.docker_container: + name: fastpath + image: ooni/fastpath:v0.87 + state: started + published_ports: + - "8472:8472" + volumes: + - /opt/{{fastpath_user}}/backend/fastpath/fastpath.conf:/etc/ooni/fastpath.conf + - /var/lib/ooniapi:/var/lib/ooniapi + +### API Uploader set up +- name: configure api uploader using s3 bucket + tags: uploader + template: + src: templates/api-uploader.conf + dest: /etc/ooni/api-uploader.conf + +- name: Install measurement uploader script + tags: uploader + template: + src: templates/ooni_api_uploader.py + dest: /usr/bin/ooni_api_uploader.py + owner: "{{fastpath_user}}" + group: "{{fastpath_user}}" + mode: "0755" + +- name: Install dependencies for uploader + tags: uploader + apt: + pkg: + - python3-statsd + - python3-boto3 + - python3-clickhouse-driver + - python3-ujson + - python3-systemd + +- name: Install uploder service + tags: uploader + template: + src: templates/ooni-api-uploader.service + dest: /etc/systemd/system/ooni-api-uploader.service + mode: 0644 + owner: root + +- name: Install uploader timer + tags: uploader + template: + src: templates/ooni-api-uploader.timer + dest: /etc/systemd/system/ooni-api-uploader.timer + mode: 0644 + owner: root + +- name: Ensure uploader timer runs + tags: uploader + systemd: + name: ooni-api-uploader.timer + state: started + enabled: yes diff --git a/ansible/roles/fastpath/templates/api-uploader.conf b/ansible/roles/fastpath/templates/api-uploader.conf new file mode 100644 index 00000000..82b8c26e --- /dev/null +++ b/ansible/roles/fastpath/templates/api-uploader.conf @@ -0,0 +1,10 @@ +# OONI API measurement uploader - Python ini format +# Deployed by ansible, see roles/ooni-backend/templates/api-uploader.conf +[DEFAULT] +# arn:aws:iam::676739448697:user/ooni-pipeline, AWS: OONI Open Data +aws_access_key_id = AKIAJURD7T4DTN5JMJ5Q +aws_secret_access_key = {{ s3_ooni_open_data_access_key }} +bucket_name = {{ bucket_name }} +msmt_spool_dir = /var/lib/ooniapi/measurements +collector_id = {{ collector_id }} +db_uri = {{ clickhouse_url }} \ No newline at end of file diff --git a/ansible/roles/fastpath/templates/fastpath.conf b/ansible/roles/fastpath/templates/fastpath.conf new file mode 100644 index 00000000..ac997db3 --- /dev/null +++ b/ansible/roles/fastpath/templates/fastpath.conf @@ -0,0 +1,21 @@ +[DEFAULT] +# Collector hostnames, comma separated +collectors = localhost + + +{% if psql_uri is defined %} +# The password is already made public +db_uri = {{ psql_uri }} +{% else %} +db_uri = +{% endif %} + +# S3 access credentials +# Currently unused +s3_access_key = +s3_secret_key = + + +clickhouse_url = {{clickhouse_url}} + +msmt_spool_dir = /var/lib/ooniapi/measurements \ No newline at end of file diff --git a/ansible/roles/fastpath/templates/ooni-api-uploader.service b/ansible/roles/fastpath/templates/ooni-api-uploader.service new file mode 100644 index 00000000..3aff1e53 --- /dev/null +++ b/ansible/roles/fastpath/templates/ooni-api-uploader.service @@ -0,0 +1,11 @@ +[Unit] +Description=Upload raw OONI measurements to S3 +Wants=ooni-api-uploader.timer + +[Service] +Type=oneshot +ExecStart=/usr/bin/ooni_api_uploader.py + +[Install] +WantedBy=multi-user.target + diff --git a/ansible/roles/fastpath/templates/ooni-api-uploader.timer b/ansible/roles/fastpath/templates/ooni-api-uploader.timer new file mode 100644 index 00000000..b0946c58 --- /dev/null +++ b/ansible/roles/fastpath/templates/ooni-api-uploader.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Upload raw OONI measurements to S3 +Requires=ooni-api-uploader.service + +[Timer] +Unit=ooni-api-uploader.service +# run every hour at 10 minutes past the hour +OnCalendar=*-*-* *:10 +#OnCalendar=*-*-* *:00/5 + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/fastpath/templates/ooni_api_uploader.py b/ansible/roles/fastpath/templates/ooni_api_uploader.py new file mode 100644 index 00000000..f1a7d8a6 --- /dev/null +++ b/ansible/roles/fastpath/templates/ooni_api_uploader.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +""" +Uploads OONI API measurements to S3 +Reads /etc/ooni/api.conf +""" + +from configparser import ConfigParser +from pathlib import Path +from pathlib import PosixPath as PP +from datetime import datetime, timedelta +from typing import List, Dict +import gzip +import logging +import tarfile +import yaml + +import ujson +import boto3 +import statsd # debdeps: python3-statsd + +# debdeps: python3-clickhouse-driver +from clickhouse_driver import Client as Clickhouse + + +metrics = statsd.StatsClient("127.0.0.1", 8125, prefix="ooni_api_uploader") +log = logging.getLogger("ooni_api_uploader") + +try: + from systemd.journal import JournalHandler # debdeps: python3-systemd + + log.addHandler(JournalHandler(SYSLOG_IDENTIFIER="ooni_api_uploader")) +except ImportError: + pass + +log.setLevel(logging.DEBUG) + + +def create_s3_client(conf): + session = boto3.Session( + aws_access_key_id=conf.get("aws_access_key_id"), + aws_secret_access_key=conf.get("aws_secret_access_key"), + ) + return session.resource("s3") + + +def read_conf(): + cf = "/etc/ooni/api-uploader.conf" + log.info(f"Starting. Reading {cf}") + conf = ConfigParser() + conf.read(cf) + return conf["DEFAULT"] + + +def connect_to_db(conf): + default = "clickhouse://api:api@localhost/default" + uri = conf.get("db_uri", default) + log.info(f"Connecting to database on {uri}") + conn = Clickhouse.from_url(uri) + return conn + + +@metrics.timer("update_db_table") +def update_db_table(conn, lookup_list, jsonl_s3path): + for d in lookup_list: + d["s3path"] = jsonl_s3path + + q = "INSERT INTO jsonl (report_id, input, s3path, linenum, measurement_uid) VALUES" + log.info(f"Writing {len(lookup_list)} rows to DB") + conn.execute(q, lookup_list) + + +@metrics.timer("upload_measurement") +def upload_to_s3(s3, bucket_name: str, tarf: PP, s3path: str) -> None: + obj = s3.Object(bucket_name, s3path) + log.info(f"Uploading {s3path}") + obj.put(Body=tarf.read_bytes()) + + +@metrics.timer("fill_postcan") +def fill_postcan(hourdir: PP, postcanf: PP) -> List[PP]: + msmt_files = sorted(f for f in hourdir.iterdir() if f.suffix == ".post") + if not msmt_files: + log.info(f"Nothing to fill {postcanf.name}") + return [] + log.info(f"Filling {postcanf.name}") + measurements = [] + postcan_byte_thresh = 20 * 1000 * 1000 + # Open postcan + with tarfile.open(str(postcanf), "w:gz") as tar: + for msmt_f in msmt_files: + # Add a msmt and delete the msmt file + metrics.incr("msmt_count") + tar.add(str(msmt_f)) + measurements.append(msmt_f) + tarsize = postcanf.stat().st_size + if tarsize > postcan_byte_thresh: + log.info(f"Reached {tarsize} bytes") + return measurements + + return measurements + + +@metrics.timer("fill_jsonl") +def fill_jsonl(measurements: List[PP], jsonlf: PP) -> List[Dict]: + log.info(f"Filling {jsonlf.name}") + # report_id, input, 2020092119_IT_tor.n0.0.jsonl.gz + lookup_list = [] + with gzip.open(jsonlf, "w") as jf: + for linenum, msmt_f in enumerate(measurements): + try: + post = ujson.load(msmt_f.open()) + except Exception: + log.error("Unable to parse measurement") + jf.write(b"{}\n") + continue + + fmt = post.get("format", "").lower() + msm = None + if fmt == "json": + msm = post.get("content", {}) + elif fmt == "yaml": + try: + msm = yaml.load(msm, Loader=yaml.CLoader) + except Exception: + pass + + if msm is None: + log.error("Unable to parse measurement") + jf.write(b"{}\n") + continue + + jf.write(ujson.dumps(msm).encode()) + jf.write(b"\n") + + rid = msm.get("report_id") or "" + input = msm.get("input") or "" + msmt_uid = msmt_f.name[:-5] + d = dict( + report_id=rid, input=input, measurement_uid=msmt_uid, linenum=linenum + ) + lookup_list.append(d) + + return lookup_list + + +def delete_msmt_posts(measurements: List[PP]) -> None: + log.info(f"Deleting {len(measurements)} measurements") + for msmt_f in measurements: + msmt_f.unlink() + + +@metrics.timer("total_run_time") +def main(): + conf = read_conf() + bucket_name = conf.get("bucket_name") + spooldir = Path(conf.get("msmt_spool_dir")) + format_char = "n" + collector_id = conf.get("collector_id") + identity = f"{format_char}{collector_id}" + log.info(f"Uploader {collector_id} starting") + assert collector_id, "collector_id is not set" + log.info(f"Using bucket {bucket_name} and spool {spooldir}") + + s3 = create_s3_client(conf) + + db_conn = connect_to_db(conf) + + # Scan spool directories, by age + idir = spooldir / "incoming" + threshold = datetime.utcnow() - timedelta(hours=1) + for hourdir in sorted(idir.iterdir()): + if not hourdir.is_dir() or hourdir.suffix == ".tmp": + continue + try: + tstamp, cc, testname = hourdir.name.split("_") + except Exception: + continue + if len(tstamp) != 10: + continue + hourdir_time = datetime.strptime(tstamp, "%Y%m%d%H") + if hourdir_time > threshold: + log.info(f"Stopping before {hourdir_time}") + break + + log.info(f"Processing {hourdir}") + # Split msmts across multiple postcans and jsonl files + can_cnt = 0 + while True: + # Compress raw POSTs into a tar.gz postcan + postcanf = hourdir.with_suffix(f".{identity}.{can_cnt}.tar.gz") + jsonlf = hourdir.with_suffix(f".{identity}.{can_cnt}.jsonl.gz") + msmfiles = fill_postcan(hourdir, postcanf) + if len(msmfiles) == 0: + break + # Also create jsonl file and delete msmt POSTs + lookup_list = fill_jsonl(msmfiles, jsonlf) + delete_msmt_posts(msmfiles) + + # Upload current postcan to S3 + postcan_s3path = ( + f"raw/{tstamp[:8]}/{tstamp[8:10]}/{cc}/{testname}/{postcanf.name}" + ) + jsonl_s3path = ( + f"raw/{tstamp[:8]}/{tstamp[8:10]}/{cc}/{testname}/{jsonlf.name}" + ) + if conf.get("run_mode", "") == "DESTROY_DATA": + log.info("Testbed mode: Destroying postcans!") + else: + upload_to_s3(s3, bucket_name, postcanf, postcan_s3path) + upload_to_s3(s3, bucket_name, jsonlf, jsonl_s3path) + update_db_table(db_conn, lookup_list, jsonl_s3path) + + postcanf.unlink() + jsonlf.unlink() + + can_cnt += 1 + metrics.incr("postcan_count") + + # Delete whole hourly directory + for f in sorted(hourdir.iterdir()): + f.unlink() + hourdir.rmdir() + + log.info("Exiting") + + +if __name__ == "__main__": + main() diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 0cbf9538..d6ad5e07 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -341,10 +341,9 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" - service_name = "ooniprobe" - repo = "ooni/backend" - # TODO change to master when https://github.com/ooni/backend/pull/969 is merged - branch_name = "report-to-ecs" + service_name = "ooniprobe" + repo = "ooni/backend" + branch_name = "fastpath-writes-to-disk" trigger_path = "ooniapi/services/ooniprobe/**" buildspec_path = "ooniapi/services/ooniprobe/buildspec.yml" codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn @@ -382,6 +381,7 @@ module "ooniapi_ooniprobe" { task_environment = { FASTPATH_URL = format("http://fastpath.%s.ooni.io:8472", local.environment) FAILED_REPORTS_BUCKET = aws_s3_bucket.ooniprobe_failed_reports.bucket + COLLECTOR_ID = 3 # use a different one in prod } ooniapi_service_security_groups = [ @@ -479,7 +479,7 @@ module "ooni_clickhouse_proxy" { from_port = 9000, to_port = 9000, protocol = "tcp", - cidr_blocks = module.network.vpc_subnet_private[*].cidr_block, + cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, [format("%s/32", module.ooni_fastpath.aws_instance_public_ip)]), }, { // For the prometheus proxy: from_port = 9200, @@ -590,6 +590,89 @@ resource "aws_route53_record" "monitoring_proxy_alias" { ] } + +### Fastpath +module "ooni_fastpath" { + source = "../../modules/ec2" + + stage = local.environment + + vpc_id = module.network.vpc_id + subnet_id = module.network.vpc_subnet_public[0].id + private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block + dns_zone_ooni_io = local.dns_zone_ooni_io + + key_name = module.adm_iam_roles.oonidevops_key_name + instance_type = "t3a.small" + + name = "oonifastpath" + ingress_rules = [{ + from_port = 22, + to_port = 22, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 8472, + to_port = 8472, + protocol = "tcp", + cidr_blocks = module.network.vpc_subnet_private[*].cidr_block, + }, { + from_port = 9100, + to_port = 9100, + protocol = "tcp" + cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] + }] + + egress_rules = [{ + from_port = 0, + to_port = 0, + protocol = "-1", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 0, + to_port = 0, + protocol = "-1", + ipv6_cidr_blocks = ["::/0"], + }] + + sg_prefix = "oonifastpath" + tg_prefix = "fstp" + + disk_size = 150 + + tags = merge( + local.tags, + { Name = "ooni-tier0-fastpath" } + ) +} + +resource "aws_route53_record" "fastpath_alias" { + zone_id = local.dns_zone_ooni_io + name = "fastpath.${local.environment}.ooni.io" + type = "CNAME" + ttl = 300 + + records = [ + module.ooni_fastpath.aws_instance_public_dns + ] +} + +module "fastpath_builder" { + source = "../../modules/ooni_docker_build" + trigger_tag = "" + + service_name = "fastpath" + repo = "ooni/backend" + branch_name = "fastpath-writes-to-disk" + buildspec_path = "fastpath/buildspec.yml" + trigger_path = "fastpath/**" + codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn + + codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket + + ecs_cluster_name = module.ooniapi_cluster.cluster_name +} + #### OONI Run service module "ooniapi_oonirun_deployer" { diff --git a/tf/modules/ec2/main.tf b/tf/modules/ec2/main.tf index b3d7885f..b73bdec6 100644 --- a/tf/modules/ec2/main.tf +++ b/tf/modules/ec2/main.tf @@ -90,6 +90,11 @@ resource "aws_instance" "ooni_ec2" { ignore_changes = [ user_data, launch_template ] } + root_block_device { + volume_size = var.disk_size # Size in GB + volume_type = "gp2" + } + tags = merge(var.tags, {MonitoringActive = var.monitoring_active}) } diff --git a/tf/modules/ec2/outputs.tf b/tf/modules/ec2/outputs.tf index 458bf405..3acd1293 100644 --- a/tf/modules/ec2/outputs.tf +++ b/tf/modules/ec2/outputs.tf @@ -12,4 +12,8 @@ output "ec2_sg_id" { output "aws_instance_private_ip" { value = aws_instance.ooni_ec2.private_ip +} + +output "aws_instance_public_ip" { + value = aws_instance.ooni_ec2.public_ip } \ No newline at end of file diff --git a/tf/modules/ec2/variables.tf b/tf/modules/ec2/variables.tf index a4db445f..f8b30d78 100644 --- a/tf/modules/ec2/variables.tf +++ b/tf/modules/ec2/variables.tf @@ -68,4 +68,10 @@ variable "monitoring_active" { description = "If the monitoring system should consider this machine. Set it to 'true' to activate it, anything else to deactivate it" default = "true" type = string +} + +variable "disk_size" { + description = "Available disk space for this machine, in GB. Defaults to 8gb" + default = 8 + type = number } \ No newline at end of file diff --git a/tf/modules/ooni_docker_build/main.tf b/tf/modules/ooni_docker_build/main.tf new file mode 100644 index 00000000..8beb5ad7 --- /dev/null +++ b/tf/modules/ooni_docker_build/main.tf @@ -0,0 +1,268 @@ +## CodeBuild and CodePipeline for OONI API Services + +data "aws_caller_identity" "current" {} + +locals { + account_id = data.aws_caller_identity.current.account_id +} + +resource "aws_iam_policy" "codebuild" { + description = "Policy used in trust relationship with CodeBuild" + name = "codebuild-${var.service_name}-${var.aws_region}" + path = "/service-role/" + + policy = <