Update localhost refs

analysis/analysis/analysis.py

@@ -61,7 +61,9 @@
 
 log = logging.getLogger("analysis")
 metrics = setup_metrics(name="analysis")
-DB_URI = "clickhouse://api:api@localhost/default"
+DEFAULT_DB_URI = "clickhouse://api:api@localhost/default"
+CONF_FILE = Path("/etc/ooni/analysis.conf")
+DEV_CONF_FILE = Path(os.getcwd()) / "analysis.conf"
 
 
 def parse_args() -> Namespace:
@@ -81,32 +83,37 @@ def parse_args() -> Namespace:
     # ap.add_argument("--", action="store_true", help="")
     ap.add_argument("--devel", action="store_true", help="Devel mode")
     ap.add_argument("--stdout", action="store_true", help="Log to stdout")
-    ap.add_argument("--db-uri", help="Override DB URI", default=DB_URI)
+    ap.add_argument("--db-uri", help="Override DB URI", default=DEFAULT_DB_URI)
+
     return ap.parse_args()
 
 
-def main() -> None:
+def setup():
     global conf
-    log.info("Analysis starting")
-    # cp = ConfigParser()
-    # with open("/etc/ooni/analysis.conf") as f:
-    #     cp.read_file(f)
 
     conf = parse_args()
+
+    # Set up logs
     if conf.devel or conf.stdout or not has_systemd:
         format = "%(relativeCreated)d %(process)d %(levelname)s %(name)s %(message)s"
         logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format=format)
-
     else:
         log.addHandler(JournalHandler(SYSLOG_IDENTIFIER="analysis"))
         log.setLevel(logging.DEBUG)
 
     log.info("Logging started")
+
     conf.output_directory = (
         Path("./var/lib/analysis") if conf.devel else Path("/var/lib/analysis")
     )
     os.makedirs(conf.output_directory, exist_ok=True)
 
+
+def main() -> None:
+    global conf
+    log.info("Analysis starting")
+    setup()
+
     try:
         if conf.update_citizenlab:
             update_citizenlab_test_lists(conf)

analysis/analysis/citizenlab_test_lists_updater.py

@@ -101,7 +101,7 @@ def update_citizenlab_table(conf: Namespace, citizenlab: list) -> None:
     """Overwrite citizenlab_flip and swap tables atomically"""
     if conf.dry_run:
         return
-    click = Clickhouse("localhost", user="citizenlab")
+    click = Clickhouse.from_url(conf.db_uri)
     log.info("Emptying Clickhouse citizenlab_flip table")
     q = "TRUNCATE TABLE citizenlab_flip"
     click.execute(q)

analysis/analysis/clickhouse_feeder.py

@@ -23,6 +23,7 @@
 import time
 from configparser import ConfigParser
 from pathlib import Path
+from typing import Any, Dict
 
 from filelock import FileLock  # debdeps: python3-filelock
 import psycopg2  # debdeps: python3-psycopg2
@@ -47,9 +48,13 @@ def setup_pg_connection(c):
     return conn
 
 
-def setup_click_connection(c):
+def setup_click_connection(c : Dict[str, Any]):
     log.info("Connecting to Clickhouse")
-    return Clickhouse("localhost")
+    url  = c.get("clickhouse_url")
+    if url is not None:
+        return Clickhouse.from_url(url)
+    else: 
+        return Clickhouse("localhost")
 
 
 @metrics.timer("sync_clickhouse_fastpath")

analysis/debian/ooni-update-asn-metadata.service

@@ -4,7 +4,8 @@ Wants=ooni-update-asn-metadata.timer
 
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/analysis --update-asnmeta
+EnvironmentFile=/etc/ooni/analysis.env
+ExecStart=/usr/bin/analysis --update-asnmeta ${EXTRA_ARGS}
 
 [Install]
 WantedBy=multi-user.target

analysis/debian/ooni-update-citizenlab.service

@@ -4,7 +4,8 @@ Wants=ooni-update-citizenlab.timer
 
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/analysis --update-citizenlab
+EnvironmentFile=/etc/ooni/analysis.env
+ExecStart=/usr/bin/analysis --update-citizenlab ${EXTRA_ARGS}
 
 [Install]
 WantedBy=multi-user.target

analysis/debian/ooni-update-fingerprints.service

@@ -4,7 +4,8 @@ Wants=ooni-update-fingerprints.timer
 
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/analysis --update-fingerprints
+EnvironmentFile=/etc/ooni/analysis.env
+ExecStart=/usr/bin/analysis --update-fingerprints ${EXTRA_ARGS}
 
 [Install]
 WantedBy=multi-user.target