FlipsideCrypto · austinFlipside · Aug 21, 2024 · Aug 21, 2024
diff --git a/.github/workflows/dbt_run_streamline_history_adhoc.yml b/.github/workflows/dbt_run_streamline_history_adhoc.yml
@@ -32,7 +32,7 @@ on:
           - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_core_history" "ethereum_models,tag:streamline_core_complete"
           - dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120,"row_limit":2000000}' -m "ethereum_models,tag:streamline_decoded_logs_history" "ethereum_models,tag:streamline_decoded_logs_complete"
           - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_beacon_history" "ethereum_models,tag:streamline_beacon_complete"
-          - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_balances_history" "ethereum_models,tag:streamline_balances_complete"
+          - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_balances_history" "ethereum_models,tag:streamline_balances_complete" "ethereum_models,tag:streamline_native_balances_history"
           - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_abis_history" "ethereum_models,tag:streamline_abis_complete"
           - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_reads_history" "ethereum_models,tag:streamline_reads_complete tag:streamline_reads_curated"
 env:

diff --git a/.github/workflows/dbt_temp_native_balances.yml b/.github/workflows/dbt_temp_native_balances.yml
@@ -0,0 +1,47 @@
+name: dbt_temp_native_balances
+run-name: dbt_temp_native_balances
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs “At minute 45 past every 12th hour.” (see https://crontab.guru)
+    - cron: '45 */12 * * *'
+
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run --threads 2 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":240}' -m "ethereum_models,tag:streamline_native_balances_history" models/streamline/silver/balances/complete/streamline__complete_eth_balances.sql
diff --git a/macros/streamline/models.sql b/macros/streamline/models.sql
@@ -11,7 +11,7 @@
 SELECT
     l.block_number,
     l._log_id,
-    a.abi AS abi,
+    A.abi AS abi,
     OBJECT_CONSTRUCT(
         'topics',
         l.topics,
@@ -23,11 +23,11 @@ SELECT
 FROM
     {{ ref("silver__logs") }}
     l
-    INNER JOIN {{ ref("silver__complete_event_abis") }}
-    a
-    ON a.parent_contract_address = l.contract_address
-    and a.event_signature = l.topics[0]::string
-    and l.block_number between a.start_block and a.end_block
+    INNER JOIN {{ ref("silver__complete_event_abis") }} A
+    ON A.parent_contract_address = l.contract_address
+    AND A.event_signature = l.topics [0] :: STRING
+    AND l.block_number BETWEEN A.start_block
+    AND A.end_block
 WHERE
     (
         l.block_number BETWEEN {{ start }}
@@ -62,14 +62,12 @@ WHERE
         start,
         stop
     ) %}
-
-WITH look_back AS (
-
+    WITH look_back AS (
         SELECT
             block_number
         FROM
             {{ ref("_24_hour_lookback") }}
-)
+    )
 SELECT
     t.block_number,
     t.tx_hash,
@@ -101,8 +99,11 @@ FROM
     AND t.block_number BETWEEN A.start_block
     AND A.end_block
 WHERE
-    (t.block_number BETWEEN {{ start }} AND {{ stop }})
-    and t.block_number < (
+    (
+        t.block_number BETWEEN {{ start }}
+        AND {{ stop }}
+    )
+    AND t.block_number < (
         SELECT
             block_number
         FROM
@@ -115,14 +116,17 @@ WHERE
         FROM
             {{ ref("streamline__complete_decode_traces") }}
         WHERE
-            (block_number BETWEEN {{ start }} AND {{ stop }})
-            and block_number < (
+            (
+                block_number BETWEEN {{ start }}
+                AND {{ stop }}
+            )
+            AND block_number < (
                 SELECT
                     block_number
                 FROM
                     look_back
-            ))
-
+            )
+    )
 {% endmacro %}
 
 {% macro streamline_external_table_query(
@@ -153,7 +157,7 @@ WHERE
                 )
             ) AS id,
             s.{{ partition_name }},
-            s.value AS value
+            s.value AS VALUE
         FROM
             {{ source(
                 "bronze_streamline",
@@ -212,7 +216,7 @@ SELECT
         )
     ) AS id,
     s.{{ partition_name }},
-    s.value AS value
+    s.value AS VALUE
 FROM
     {{ source(
         "bronze_streamline",
@@ -243,3 +247,77 @@ WHERE
     )
 {% endmacro %}
 
+{% macro native_balances_history(
+        start,
+        stop
+    ) %}
+    WITH look_back AS (
+        SELECT
+            block_number
+        FROM
+            {{ ref("_max_block_by_date") }}
+            qualify ROW_NUMBER() over (
+                ORDER BY
+                    block_number DESC
+            ) = 3
+    ),
+    traces AS (
+        SELECT
+            block_number,
+            from_address,
+            to_address
+        FROM
+            {{ ref('silver__traces') }}
+        WHERE
+            eth_value > 0
+            AND block_number < (
+                SELECT
+                    block_number
+                FROM
+                    look_back
+            )
+            AND block_number BETWEEN {{ start }}
+            AND {{ stop }}
+    ),
+    stacked AS (
+        SELECT
+            DISTINCT block_number,
+            from_address AS address
+        FROM
+            traces
+        WHERE
+            from_address IS NOT NULL
+            AND from_address <> '0x0000000000000000000000000000000000000000'
+        UNION
+        SELECT
+            DISTINCT block_number,
+            to_address AS address
+        FROM
+            traces
+        WHERE
+            to_address IS NOT NULL
+            AND to_address <> '0x0000000000000000000000000000000000000000'
+    )
+SELECT
+    block_number,
+    address
+FROM
+    stacked
+WHERE
+    block_number IS NOT NULL
+EXCEPT
+SELECT
+    block_number,
+    address
+FROM
+    {{ ref("streamline__complete_eth_balances") }}
+WHERE
+    block_number < (
+        SELECT
+            block_number
+        FROM
+            look_back
+    )
+    AND block_number BETWEEN {{ start }}
+    AND {{ stop }}
+{% endmacro %}
diff --git a/...ne/silver/balances/history/native/streamline__native_balances_history_0000000_1000000.sql b/...ne/silver/balances/history/native/streamline__native_balances_history_0000000_1000000.sql
@@ -0,0 +1,12 @@
+{{ config (
+    materialized = "view",
+    post_hook = [if_data_call_function( func = "{{this.schema}}.udf_get_eth_balances(object_construct('node_name','quicknode', 'sql_source', '{{this.identifier}}'))", target = "{{this.schema}}.{{this.identifier}}" ),if_data_call_wait()],
+    tags = ['streamline_native_balances_history']
+) }}
+
+{% set start = this.identifier.split("_") [-2] %}
+{% set stop = this.identifier.split("_") [-1] %}
+{{ native_balances_history(
+    start,
+    stop
+) }}
diff --git a/.../silver/balances/history/native/streamline__native_balances_history_10000001_11000000.sql b/.../silver/balances/history/native/streamline__native_balances_history_10000001_11000000.sql
@@ -0,0 +1,12 @@
+{{ config (
+    materialized = "view",
+    post_hook = [if_data_call_function( func = "{{this.schema}}.udf_get_eth_balances(object_construct('node_name','quicknode', 'sql_source', '{{this.identifier}}'))", target = "{{this.schema}}.{{this.identifier}}" ),if_data_call_wait()],
+    tags = ['streamline_native_balances_history']
+) }}
+
+{% set start = this.identifier.split("_") [-2] %}
+{% set stop = this.identifier.split("_") [-1] %}
+{{ native_balances_history(
+    start,
+    stop
+) }}
diff --git a/...ne/silver/balances/history/native/streamline__native_balances_history_1000001_2000000.sql b/...ne/silver/balances/history/native/streamline__native_balances_history_1000001_2000000.sql
@@ -0,0 +1,12 @@
+{{ config (
+    materialized = "view",
+    post_hook = [if_data_call_function( func = "{{this.schema}}.udf_get_eth_balances(object_construct('node_name','quicknode', 'sql_source', '{{this.identifier}}'))", target = "{{this.schema}}.{{this.identifier}}" ),if_data_call_wait()],
+    tags = ['streamline_native_balances_history']
+) }}
+
+{% set start = this.identifier.split("_") [-2] %}
+{% set stop = this.identifier.split("_") [-1] %}
+{{ native_balances_history(
+    start,
+    stop
+) }}
diff --git a/.../silver/balances/history/native/streamline__native_balances_history_11000001_12000000.sql b/.../silver/balances/history/native/streamline__native_balances_history_11000001_12000000.sql
@@ -0,0 +1,12 @@
+{{ config (
+    materialized = "view",
+    post_hook = [if_data_call_function( func = "{{this.schema}}.udf_get_eth_balances(object_construct('node_name','quicknode', 'sql_source', '{{this.identifier}}'))", target = "{{this.schema}}.{{this.identifier}}" ),if_data_call_wait()],
+    tags = ['streamline_native_balances_history']
+) }}
+
+{% set start = this.identifier.split("_") [-2] %}
+{% set stop = this.identifier.split("_") [-1] %}
+{{ native_balances_history(
+    start,
+    stop
+) }}
diff --git a/.../silver/balances/history/native/streamline__native_balances_history_12000001_13000000.sql b/.../silver/balances/history/native/streamline__native_balances_history_12000001_13000000.sql
@@ -0,0 +1,12 @@
+{{ config (
+    materialized = "view",
+    post_hook = [if_data_call_function( func = "{{this.schema}}.udf_get_eth_balances(object_construct('node_name','quicknode', 'sql_source', '{{this.identifier}}'))", target = "{{this.schema}}.{{this.identifier}}" ),if_data_call_wait()],
+    tags = ['streamline_native_balances_history']
+) }}
+
+{% set start = this.identifier.split("_") [-2] %}
+{% set stop = this.identifier.split("_") [-1] %}
+{{ native_balances_history(
+    start,
+    stop
+) }}
diff --git a/.../silver/balances/history/native/streamline__native_balances_history_13000001_14000000.sql b/.../silver/balances/history/native/streamline__native_balances_history_13000001_14000000.sql
@@ -0,0 +1,12 @@
+{{ config (
+    materialized = "view",
+    post_hook = [if_data_call_function( func = "{{this.schema}}.udf_get_eth_balances(object_construct('node_name','quicknode', 'sql_source', '{{this.identifier}}'))", target = "{{this.schema}}.{{this.identifier}}" ),if_data_call_wait()],
+    tags = ['streamline_native_balances_history']
+) }}
+
+{% set start = this.identifier.split("_") [-2] %}
+{% set stop = this.identifier.split("_") [-1] %}
+{{ native_balances_history(
+    start,
+    stop
+) }}
diff --git a/.../silver/balances/history/native/streamline__native_balances_history_14000001_15000000.sql b/.../silver/balances/history/native/streamline__native_balances_history_14000001_15000000.sql
@@ -0,0 +1,12 @@
+{{ config (
+    materialized = "view",
+    post_hook = [if_data_call_function( func = "{{this.schema}}.udf_get_eth_balances(object_construct('node_name','quicknode', 'sql_source', '{{this.identifier}}'))", target = "{{this.schema}}.{{this.identifier}}" ),if_data_call_wait()],
+    tags = ['streamline_native_balances_history']
+) }}
+
+{% set start = this.identifier.split("_") [-2] %}
+{% set stop = this.identifier.split("_") [-1] %}
+{{ native_balances_history(
+    start,
+    stop
+) }}
diff --git a/.../silver/balances/history/native/streamline__native_balances_history_15000001_16000000.sql b/.../silver/balances/history/native/streamline__native_balances_history_15000001_16000000.sql
@@ -0,0 +1,12 @@
+{{ config (
+    materialized = "view",
+    post_hook = [if_data_call_function( func = "{{this.schema}}.udf_get_eth_balances(object_construct('node_name','quicknode', 'sql_source', '{{this.identifier}}'))", target = "{{this.schema}}.{{this.identifier}}" ),if_data_call_wait()],
+    tags = ['streamline_native_balances_history']
+) }}
+
+{% set start = this.identifier.split("_") [-2] %}
+{% set stop = this.identifier.split("_") [-1] %}
+{{ native_balances_history(
+    start,
+    stop
+) }}
diff --git a/.../silver/balances/history/native/streamline__native_balances_history_16000001_17000000.sql b/.../silver/balances/history/native/streamline__native_balances_history_16000001_17000000.sql
@@ -0,0 +1,12 @@
+{{ config (
+    materialized = "view",
+    post_hook = [if_data_call_function( func = "{{this.schema}}.udf_get_eth_balances(object_construct('node_name','quicknode', 'sql_source', '{{this.identifier}}'))", target = "{{this.schema}}.{{this.identifier}}" ),if_data_call_wait()],
+    tags = ['streamline_native_balances_history']
+) }}
+
+{% set start = this.identifier.split("_") [-2] %}
+{% set stop = this.identifier.split("_") [-1] %}
+{{ native_balances_history(
+    start,
+    stop
+) }}
diff --git a/.../silver/balances/history/native/streamline__native_balances_history_17000001_18000000.sql b/.../silver/balances/history/native/streamline__native_balances_history_17000001_18000000.sql
@@ -0,0 +1,12 @@
+{{ config (
+    materialized = "view",
+    post_hook = [if_data_call_function( func = "{{this.schema}}.udf_get_eth_balances(object_construct('node_name','quicknode', 'sql_source', '{{this.identifier}}'))", target = "{{this.schema}}.{{this.identifier}}" ),if_data_call_wait()],
+    tags = ['streamline_native_balances_history']
+) }}
+
+{% set start = this.identifier.split("_") [-2] %}
+{% set stop = this.identifier.split("_") [-1] %}
+{{ native_balances_history(
+    start,
+    stop
+) }}
diff --git a/.../silver/balances/history/native/streamline__native_balances_history_18000001_19000000.sql b/.../silver/balances/history/native/streamline__native_balances_history_18000001_19000000.sql
@@ -0,0 +1,12 @@
+{{ config (
+    materialized = "view",
+    post_hook = [if_data_call_function( func = "{{this.schema}}.udf_get_eth_balances(object_construct('node_name','quicknode', 'sql_source', '{{this.identifier}}'))", target = "{{this.schema}}.{{this.identifier}}" ),if_data_call_wait()],
+    tags = ['streamline_native_balances_history']
+) }}
+
+{% set start = this.identifier.split("_") [-2] %}
+{% set stop = this.identifier.split("_") [-1] %}
+{{ native_balances_history(
+    start,
+    stop
+) }}
diff --git a/.../silver/balances/history/native/streamline__native_balances_history_19000001_20000000.sql b/.../silver/balances/history/native/streamline__native_balances_history_19000001_20000000.sql
@@ -0,0 +1,12 @@
+{{ config (
+    materialized = "view",
+    post_hook = [if_data_call_function( func = "{{this.schema}}.udf_get_eth_balances(object_construct('node_name','quicknode', 'sql_source', '{{this.identifier}}'))", target = "{{this.schema}}.{{this.identifier}}" ),if_data_call_wait()],
+    tags = ['streamline_native_balances_history']
+) }}
+
+{% set start = this.identifier.split("_") [-2] %}
+{% set stop = this.identifier.split("_") [-1] %}
+{{ native_balances_history(
+    start,
+    stop
+) }}