add intervals data to clickhouse

update intervals schema

Author: jdhayhurst

config/clickhouse/schema/intervals_log.sql

@@ -0,0 +1,20 @@
+create database if not exists ot;
+
+CREATE TABLE if not exists ot.intervals_log 
+(  
+    `chromosome` Enum8('1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', 'X', 'Y'),
+	`start` UInt32,
+    `end` UInt32,
+    `geneId` String,
+    `biosampleName` LowCardinality(String),
+    `biosampleId` LowCardinality(String),
+    `intervalType` LowCardinality(String),
+    `distanceToTSS` UInt32,
+    `score` Float64,
+    `resourceScore` Array(Tuple(name LowCardinality(String), value Float64)),
+    `datasourceId` LowCardinality(String),
+    `pmid` String,
+    `studyId` String,
+) 
+engine = Log;
+;

config/clickhouse/scripts/intervals.sql

@@ -0,0 +1,12 @@
+create database if not exists ot;
+
+create table if not exists ot.intervals engine = MergeTree ()
+order by
+    (chromosome, start, end) as (
+        select
+            *
+        from
+            ot.intervals_log
+    );
+
+drop table ot.intervals_log;

config/config.yaml

@@ -221,6 +221,7 @@ steps:
     - name: explode load all datasets
       foreach:
         - aotf
+        - intervals
         - literature
         - w2v
         - sentences

config/datasets.yaml

@@ -336,6 +336,10 @@ clickhouse:
     table: associations_otf_log
     input_dir: view/association_otf
     postload_script: config/clickhouse/scripts/aotf.sql
+  intervals:
+    table: intervals_log
+    input_dir: output/intervals
+    postload_script: config/clickhouse/scripts/intervals.sql
   literature:
     table: literature_log
     input_dir: output/literature

deployment/variables.tf

@@ -87,7 +87,7 @@ variable "open_search_jvm_options" {
 variable "clickhouse_image_tag" {
   description = "Clickhouse image tag"
   type        = string
-  default     = "23.3.1.2823"
+  default     = "25.6.3.116"
 }
 
 variable "data_location_source" {