diff --git a/README.md b/README.md
index b15b4e1..6b07970 100644
--- a/README.md
+++ b/README.md
@@ -36,26 +36,28 @@ just dev
## Notebooks
-| Notebook | Description |
-| --------------------------------------------------------------------- | ------------------------------------------------------ |
-| [Blob Inclusion](notebooks/01-blob-inclusion.ipynb) | Blob inclusion patterns per block and epoch |
-| [Blob Flow](notebooks/02-blob-flow.ipynb) | Blob flow across validators, builders, and relays |
-| [Column Propagation](notebooks/03-column-propagation.ipynb) | Column propagation timing across 128 data columns |
-| [Mempool Visibility](notebooks/04-mempool-visibility.ipynb) | Transaction visibility in the public mempool |
-| [MEV Pipeline](notebooks/05-mev-pipeline.ipynb) | MEV bidding timing, relay/builder performance |
-| [Block/Column Timing](notebooks/06-block-column-timing.ipynb) | Block arrival to column propagation delay |
-| [Propagation Anomalies](notebooks/07-propagation-anomalies.ipynb) | Blocks that propagated slower than expected |
+| Notebook | Description |
+|-------------------------------------------------------------------|----------------------------------------------------|
+| [Blob Inclusion](notebooks/01-blob-inclusion.ipynb) | Blob inclusion patterns per block and epoch |
+| [Blob Flow](notebooks/02-blob-flow.ipynb) | Blob flow across validators, builders, and relays |
+| [Column Propagation](notebooks/03-column-propagation.ipynb) | Column propagation timing across 128 data columns |
+| [Mempool Visibility](notebooks/04-mempool-visibility.ipynb) | Transaction visibility in the public mempool |
+| [MEV Pipeline](notebooks/05-mev-pipeline.ipynb) | MEV bidding timing, relay/builder performance |
+| [Block/Column Timing](notebooks/06-block-column-timing.ipynb) | Block arrival to column propagation delay |
+| [Propagation Anomalies](notebooks/07-propagation-anomalies.ipynb) | Blocks that propagated slower than expected |
+| [Attestation Inclusion](notebooks/10-attestation-inclusion.ipynb) | Inclusion of attestations based on network metrics |
## Architecture
```
-pipeline.yaml # Central config: dates, queries, notebooks
-queries/ # ClickHouse query modules -> Parquet
-├── blob_inclusion.py # fetch_blobs_per_slot(), fetch_blocks_blob_epoch(), ...
-├── blob_flow.py # fetch_blob_flow()
-├── column_propagation.py # fetch_col_first_seen()
-├── mempool_visibility.py # fetch_tx_per_slot(), fetch_mempool_coverage(), ...
+pipeline.yaml # Central config: dates, queries, notebooks
+queries/ # ClickHouse query modules -> Parquet
+├── blob_inclusion.py # fetch_blobs_per_slot(), fetch_blocks_blob_epoch(), ...
+├── blob_flow.py # fetch_blob_flow()
+├── column_propagation.py # fetch_col_first_seen()
+├── mempool_visibility.py # fetch_tx_per_slot(), fetch_mempool_coverage(), ...
└── block_production_timeline.py # fetch_block_production_timeline()
+└── att_propagation.py # fetch_attestations_arrivals(), ...
scripts/
├── pipeline.py # Coordinator: config loading, hash computation, staleness
├── fetch_data.py # CLI: ClickHouse -> notebooks/data/*.parquet
@@ -233,7 +235,6 @@ just preview
## Adding New Analyses
1. **Create query function** in `queries/`:
-
```python
def fetch_my_data(client, target_date: str, output_path: Path, network: str) -> int:
query = f"SELECT ... WHERE slot_start_date_time >= '{target_date}' ..."
@@ -242,8 +243,24 @@ just preview
df.to_parquet(output_path, index=False)
return len(df)
```
+ __NOTE: don't forget to expose the new methods under `./queries/__init__.py`:__
+ ```python
+ from queries.my_module import fetch_my_data
+
+ __all__ = [
+ ...
+ "fetch_my_data",
+ ...
+ ]
+ ```
+
+2. **Create notebook** `notebooks/04-my-analysis.ipynb`:
+ - Add a cell tagged "parameters" with `target_date = None`
+ - Use `loaders.load_parquet("my_data")` to load data (The name of the file should match the one from the field `queries/my_data/output_file` in the `./pipeline.yaml`file)
+ - Create Plotly visualizations
+
-2. **Register in `pipeline.yaml`**:
+3. **Register in `pipeline.yaml`** to automate the deployment:
```yaml
queries:
@@ -260,11 +277,6 @@ just preview
queries: [my_data]
```
-3. **Create notebook** `notebooks/04-my-analysis.ipynb`:
- - Add a cell tagged "parameters" with `target_date = None`
- - Use `loaders.load_parquet("my_data")` to load data
- - Create Plotly visualizations
-
4. **Fetch and render**:
```bash
just fetch && just render && just build
diff --git a/notebooks/10-attestation-propagation.ipynb b/notebooks/10-attestation-propagation.ipynb
new file mode 100644
index 0000000..501a46e
--- /dev/null
+++ b/notebooks/10-attestation-propagation.ipynb
@@ -0,0 +1,1260 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "0",
+ "metadata": {},
+ "source": [
+ "Analysis of Ethereum consensus layer attestation propagation across the p2p network. Covers single attestations, aggregate attestations, and their relationship to block arrival timing, using telemetry from a distributed set of control nodes."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Imports\n",
+ "import polars as pl\n",
+ "import plotly.express as px\n",
+ "from IPython.display import display\n",
+ "\n",
+ "from loaders import load_parquet\n",
+ "from utils import render_table\n",
+ "\n",
+ "# Global Variables\n",
+ "target_date = None # Use this as a default for the automation and the rendering of the page\n",
+ "\n",
+ "# one from the list [\"save\", \"show\"]\n",
+ "# default to \"show\" for the website rendering\n",
+ "render_method = \"show\"\n",
+ "\n",
+ "def render_plot_fn(fig, method: str, path: str):\n",
+ " if method == \"image\" and path != \"\":\n",
+ " fig.write_image(path)\n",
+ " elif method == \"show\":\n",
+ " fig.show()\n",
+ " else:\n",
+ " raise f\"method {method} not supported\"\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load the raw datasets coming from the automated queries\n",
+ "# Single attestations\n",
+ "attestation_df = load_parquet(\"attestation_arrivals\", target_date)\n",
+ "attestation_df = pl.from_pandas(attestation_df)\n",
+ "\n",
+ "# Aggregations\n",
+ "pd_df = load_parquet(\"aggregation_broadcast_info\", target_date)\n",
+ "aggregations_df = pl.from_pandas(pd_df)\n",
+ "\n",
+ "# Blocks and data-columns\n",
+ "block_and_column_df = load_parquet(\"block_and_column_broadcast_info\", target_date)\n",
+ "block_and_column_df = (\n",
+ " pl.from_pandas(block_and_column_df)\n",
+ " .rename({\"bs.slot\": \"slot\"})\n",
+ ")\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3",
+ "metadata": {},
+ "source": [
+ "## Attestation Arrivals\n",
+ "\n",
+ "Distribution of single attestations by the time they were first observed on the network, measured from slot start. Attestations are expected to be broadcast within the first 4 seconds of a slot. The following graph shows the boundaries for each of the slot duties with vertical lines at seconds 8 (purple) and 12 (red). The split between included and non-included attestations reveals how much of the attestation traffic never makes it into a block."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Histogram over the first arrival of the attestations\n",
+ "\n",
+ "df = (\n",
+ " attestation_df\n",
+ " .with_columns(\n",
+ " total_attestations = pl.lit(len(attestation_df)),\n",
+ " tag = pl.when(\n",
+ " pl.col(\"block_slot\").gt(0)).then(pl.lit(\"included_att\"))\n",
+ " .otherwise(pl.lit(\"non_included_att\"))\n",
+ " )\n",
+ " .group_by([\"latency_bucket\", \"tag\"])\n",
+ " .agg(\n",
+ " attestations = pl.col(\"slot\").count(),\n",
+ " percentage = pl.col(\"slot\").count() * 100 / pl.col(\"total_attestations\").max(),\n",
+ " )\n",
+ " .sort([\"latency_bucket\"])\n",
+ ")\n",
+ "\n",
+ "# the histogram of when where the attestations seen\n",
+ "h = px.bar(\n",
+ " df,\n",
+ " x=\"latency_bucket\",\n",
+ " y=\"percentage\",\n",
+ " color=\"tag\",\n",
+ ")\n",
+ "h.update_xaxes(range=[0, 12])\n",
+ "h.update_layout(\n",
+ " title=\"Histogram of when attestations where seen for the first time\",\n",
+ " xaxis_title_text=\"Seconds since slot started\",\n",
+ " yaxis_title_text=\"% of Attestations\",\n",
+ " width=1200,\n",
+ " height=800,\n",
+ ")\n",
+ "h.update_xaxes(range=[0, 18])\n",
+ "h.add_vline(x=8, line_width=3, line_dash=\"dash\", line_color=\"purple\")\n",
+ "h.add_vline(x=12, line_width=3, line_dash=\"dash\", line_color=\"red\")\n",
+ "render_plot_fn(h, render_method, \"./images/agg/att_histogram_by_first_seen_tot.png\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5",
+ "metadata": {},
+ "source": [
+ "### Never included attestations \n",
+ "Attestations that were seen on the p2p network, but that weren't included into a block."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# render the never included attestations\n",
+ "\n",
+ "render_table(\n",
+ " attestation_df\n",
+ " .filter(pl.col(\"block_slot\").lt(1))\n",
+ " .group_by([\"com_idx\"])\n",
+ " .agg(attestations = pl.col(\"val_idx\").count())\n",
+ ")\n",
+ "\n",
+ "render_table(\n",
+ " attestation_df\n",
+ " .filter(pl.col(\"block_slot\").lt(1))\n",
+ " .select([\"slot\", \"val_idx\", \"att_first_seen_wb\", \"att_broadcast_p50\", \"inclusion_delay\"])\n",
+ ")\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7",
+ "metadata": {},
+ "source": [
+ "## Attestation first seen by inclusion delay\n",
+ "\n",
+ "Same histogram as above, colored by how many slots later each attestation was eventually included in a block. This reveals whether propagation timing predicts inclusion quality."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# same histogram, but using the inclusion time as legend\n",
+ "df_2 = (\n",
+ " attestation_df\n",
+ " .with_columns(\n",
+ " total_attestations = pl.lit(len(attestation_df)),\n",
+ " )\n",
+ " .group_by([\"latency_bucket\", \"inclusion_range\"])\n",
+ " .agg(\n",
+ " attestations = pl.col(\"slot\").count(),\n",
+ " percentage = pl.col(\"slot\").count() * 100 / pl.col(\"total_attestations\").max(),\n",
+ " inclusion= pl.col(\"inclusion_delay\").mean(),\n",
+ " )\n",
+ " .sort(\"inclusion\", \"latency_bucket\", \"percentage\", descending=True)\n",
+ ")\n",
+ "\n",
+ "h_2 = px.bar(\n",
+ " df_2,\n",
+ " x=\"latency_bucket\",\n",
+ " y=\"percentage\",\n",
+ " color=\"inclusion_range\",\n",
+ " opacity=0.9,\n",
+ ")\n",
+ "h_2.update_layout(\n",
+ " title=\"Histogram of when attestations where seen for the first time and their inclusion\",\n",
+ " xaxis_title_text=\"Seconds since slot started\",\n",
+ " yaxis_title_text=\"% of Attestations\",\n",
+ " barmode='stack',\n",
+ " width=1200,\n",
+ " height=800,\n",
+ ")\n",
+ "h_2.update_xaxes(range=[0, 14])\n",
+ "h_2.add_vline(x=8, line_width=3, line_dash=\"dash\", line_color=\"purple\")\n",
+ "h_2.add_vline(x=12, line_width=3, line_dash=\"dash\", line_color=\"red\")\n",
+ "render_plot_fn(h, render_method, f\"./images/agg/att_histogram_by_first_seen_by_inclusion_tot.png\")\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9",
+ "metadata": {},
+ "source": [
+ "## Attestation Arrivals by Network Coverage\n",
+ "\n",
+ "Attestation propagation from the perspective of the full control node set. Instead of first-seen, this uses the time at which p50 and p90 of control nodes had received each attestation."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "10",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Histogram over the first arrival of the attestations\n",
+ "def arrival_of_attestations_over_percentile_with_inclusion(att_df: pl.DataFrame, perc: str = 'p50'):\n",
+ " df_2 = (\n",
+ " att_df\n",
+ " .with_columns(\n",
+ " total_attestations = pl.lit(len(att_df)),\n",
+ " )\n",
+ " .group_by([f\"broadcast_{perc}_bucket\", \"inclusion_range\"])\n",
+ " .agg(\n",
+ " attestations = pl.col(\"slot\").count(),\n",
+ " percentage = pl.col(\"slot\").count() * 100 / pl.col(\"total_attestations\").max(),\n",
+ " inclusion= pl.col(\"inclusion_delay\").mean(),\n",
+ " broadcast= pl.col(f\"att_broadcast_{perc}\").mean(),\n",
+ " )\n",
+ " .sort(\"inclusion\", \"broadcast\", \"percentage\", descending=True)\n",
+ " )\n",
+ " h_2 = px.bar(\n",
+ " df_2,\n",
+ " x=f\"broadcast_{perc}_bucket\",\n",
+ " y=\"percentage\",\n",
+ " color=\"inclusion_range\",\n",
+ " opacity=0.9,\n",
+ " )\n",
+ " h_2.update_layout(\n",
+ " title=f\"Histogram of when attestations where seen by {perc} and their inclusion\",\n",
+ " xaxis_title_text=f\"Seconds\",\n",
+ " yaxis_title_text=\"% of Attestations\",\n",
+ " barmode='stack',\n",
+ " width=1200,\n",
+ " height=800,\n",
+ " )\n",
+ " h_2.update_xaxes(range=[0, 10])\n",
+ " h_2.add_vline(x=4, line_width=3, line_dash=\"dash\", line_color=\"purple\")\n",
+ " h_2.add_vline(x=8, line_width=3, line_dash=\"dash\", line_color=\"red\")\n",
+ " render_plot_fn(h_2, render_method, f\"./images/agg/att_histogram_by_{perc}_and_inclusion_tot.png\")\n",
+ "\n",
+ "arrival_of_attestations_over_percentile_with_inclusion(attestation_df, \"p50\")\n",
+ "arrival_of_attestations_over_percentile_with_inclusion(attestation_df, \"p90\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "11",
+ "metadata": {},
+ "source": [
+ "## Attestation propagation spread (p50 / p90)\n",
+ "\n",
+ "Distribution of attestation propagation times across the network, colored by inclusion delay. The p50 broadcast time captures when half the control nodes have seen the attestation, while p90 captures near-full network coverage. A wide tail to the right suggests a subset of attestations is slow to reach the broader network, potentially impacting aggregator quality."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "12",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Histogram over the first arrival of the attestations\n",
+ "def arrival_of_attestations_over_percentile(att_df: pl.DataFrame, perc: str = 'p50'):\n",
+ " df = (\n",
+ " att_df\n",
+ " .with_columns(\n",
+ " total_attestations = pl.lit(len(att_df)),\n",
+ " )\n",
+ " .group_by([f\"broadcast_{perc}_bucket_wfs\"])\n",
+ " .agg(\n",
+ " attestations = pl.col(\"slot\").count(),\n",
+ " percentage = pl.col(\"slot\").count() * 100 / pl.col(\"total_attestations\").max(),\n",
+ " broadcast= pl.col(f\"att_broadcast_{perc}\").mean(),\n",
+ " )\n",
+ " .sort([\"broadcast\"], descending=False)\n",
+ " )\n",
+ " \n",
+ " # the histogram of when where the attestations seen\n",
+ " f = px.bar(\n",
+ " df,\n",
+ " x=f\"broadcast_{perc}_bucket_wfs\",\n",
+ " y=\"percentage\",\n",
+ " )\n",
+ " f.update_layout(\n",
+ " title=f\"Histogram of when attestations where seen by {perc} of control nodes\",\n",
+ " xaxis_title_text=f\"{perc} propagation (s)\",\n",
+ " yaxis_title_text=\"% of Attestations\",\n",
+ " width=1200,\n",
+ " height=800,\n",
+ " )\n",
+ " f.update_xaxes(range=[0, 14])\n",
+ " f.add_vline(x=8, line_width=3, line_dash=\"dash\", line_color=\"purple\")\n",
+ " render_plot_fn(f, render_method, f\"./images/agg/att_histogram_first_seen_by_{perc}_tot.png\")\n",
+ " \n",
+ "arrival_of_attestations_over_percentile(attestation_df, \"p50\")\n",
+ "arrival_of_attestations_over_percentile(attestation_df, \"p90\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "13",
+ "metadata": {},
+ "source": [
+ "## Attestation network coverage without inclusion split\n",
+ "\n",
+ "Plain view of p50 and p90 propagation times (no inclusion coloring). Useful for reading off the raw propagation percentile distribution: where does the bulk of the network see most attestations, and how much headroom exists before the 8s and 12s protocol deadlines."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "14",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# same histogram, but using the inclusion time as legend\n",
+ "df_2 = (\n",
+ " attestation_df\n",
+ " .with_columns(\n",
+ " total_attestations = pl.lit(len(attestation_df)),\n",
+ " )\n",
+ " .group_by([\"broadcast_p50_bucket_wfs\", \"inclusion_range\"])\n",
+ " .agg(\n",
+ " attestations = pl.col(\"slot\").count(),\n",
+ " percentage = pl.col(\"slot\").count() * 100 / pl.col(\"total_attestations\").max(),\n",
+ " inclusion= pl.col(\"inclusion_delay\").mean(),\n",
+ " broadcast= pl.col(\"att_broadcast_p50\").mean(),\n",
+ " )\n",
+ " .sort(\"inclusion\", \"broadcast\", \"percentage\", descending=True)\n",
+ ")\n",
+ "\n",
+ "h_2 = px.bar(\n",
+ " df_2,\n",
+ " x=\"broadcast_p50_bucket_wfs\",\n",
+ " y=\"percentage\",\n",
+ " color=\"inclusion_range\",\n",
+ " opacity=0.9,\n",
+ ")\n",
+ "h_2.update_layout(\n",
+ " title=\"P50 histogram of the attestation propagation and their inclusion\",\n",
+ " xaxis_title_text=\"Seconds\",\n",
+ " yaxis_title_text=\"% of Attestations\",\n",
+ " barmode='stack',\n",
+ " width=1200,\n",
+ " height=800,\n",
+ ")\n",
+ "h_2.update_xaxes(range=[0, 12])\n",
+ "h_2.add_vline(x=8, line_width=3, line_dash=\"dash\", line_color=\"purple\")\n",
+ "h_2.add_vline(x=12, line_width=3, line_dash=\"dash\", line_color=\"red\")\n",
+ "render_plot_fn(h_2, render_method, \"./images/agg/att_histogram_by_p50_by_inclusion_tot.png\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "15",
+ "metadata": {},
+ "source": [
+ "## P50 propagation vs inclusion delay\n",
+ "\n",
+ "P50 broadcast time histogram using the inclusion delay as legend. The graph shows whether attestations that take longer to spread across the network are also included in later blocks."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "16",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# correlation between first seen and propagation by P50/P90/P95\n",
+ "def histogram_per_first_att_seen_and_broadcast_percentile(att_df: pl.DataFrame, percentile: str):\n",
+ " df = (\n",
+ " att_df\n",
+ " .with_columns(\n",
+ " total_attestations = pl.lit(len(att_df)),\n",
+ " )\n",
+ " .group_by([\"latency_bucket\", f\"broadcast_{percentile}_bucket_g\"])\n",
+ " .agg(\n",
+ " attestations = pl.col(\"slot\").count(),\n",
+ " percentage = pl.col(\"slot\").count() * 100 / pl.col(\"total_attestations\").max(),\n",
+ " broadcast= pl.col(f\"att_broadcast_{percentile}\").mean(),\n",
+ " )\n",
+ " .sort(f\"broadcast\", \"latency_bucket\", \"percentage\", descending=True)\n",
+ " )\n",
+ " \n",
+ " g = px.bar(\n",
+ " df,\n",
+ " x=\"latency_bucket\",\n",
+ " y=\"percentage\",\n",
+ " color=f\"broadcast_{percentile}_bucket_g\",\n",
+ " opacity=0.9,\n",
+ " )\n",
+ " g.update_layout(\n",
+ " title=f\"Histogram of attestations first time seen by propagation {percentile}\",\n",
+ " xaxis_title_text=\"Seconds\",\n",
+ " yaxis_title_text=\"% of Attestations\",\n",
+ " barmode='stack',\n",
+ " width=1200,\n",
+ " height=800,\n",
+ " )\n",
+ " g.update_xaxes(range=[0, 14])\n",
+ " g.add_vline(x=8, line_width=3, line_dash=\"dash\", line_color=\"purple\")\n",
+ " g.add_vline(x=12, line_width=3, line_dash=\"dash\", line_color=\"red\")\n",
+ " render_plot_fn(g, render_method, f\"./images/agg/att_histogram_by_first_seen_and_propagation{percentile}_tot.png\")\n",
+ " \n",
+ "histogram_per_first_att_seen_and_broadcast_percentile(attestation_df, \"p50\")\n",
+ "histogram_per_first_att_seen_and_broadcast_percentile(attestation_df, \"p90\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "17",
+ "metadata": {},
+ "source": [
+ "## Attestation first seen vs propagation spread\n",
+ "\n",
+ "Stacked histogram where the horizontal axis shows when each attestation was first observed by any control node and color axis shows the corresponding network-wide propagation bucket (p50 or p90). Identifies attestations that appeared early to one node but spread slowly."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "18",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# CDF of the whole broadcast duration (P50 and P90) based on the time that they were seen\n",
+ "import plotly.io as pio\n",
+ "from IPython.display import Image\n",
+ "\n",
+ "df_2 = (\n",
+ " attestation_df\n",
+ " .with_columns(\n",
+ " total_attestations = pl.lit(len(attestation_df)),\n",
+ " )\n",
+ " .group_by([\"latency_bucket\", \"inclusion_range\"])\n",
+ " .agg(\n",
+ " attestations = pl.col(\"slot\").count(),\n",
+ " percentage = pl.col(\"slot\").count() * 100 / pl.col(\"total_attestations\").max(),\n",
+ " inclusion= pl.col(\"inclusion_delay\").mean(),\n",
+ " )\n",
+ " .sort(\"inclusion\", \"latency_bucket\", \"percentage\", descending=True)\n",
+ ")\n",
+ "\n",
+ "h_3 = px.bar(\n",
+ " df_2,\n",
+ " x=\"latency_bucket\",\n",
+ " y=\"percentage\",\n",
+ " color=\"inclusion_range\",\n",
+ ")\n",
+ "h_3.update_layout(\n",
+ " title=\"Histogram of when attestations where seen for the first time\",\n",
+ " xaxis_title_text=\"Seconds since slot started\",\n",
+ " yaxis_title_text=\"% of Attestations\",\n",
+ " barmode='stack',\n",
+ " width=1200,\n",
+ " height=800,\n",
+ ")\n",
+ "h_3.update_xaxes(range=[0, 12])\n",
+ "h_3.add_vline(x=8, line_width=3, line_dash=\"dash\", line_color=\"purple\")\n",
+ "h_3.add_vline(x=12, line_width=3, line_dash=\"dash\", line_color=\"red\")\n",
+ "render_plot_fn(h_3, render_method, f\"./images/agg/att_propagation_cdf_tot.png\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "19",
+ "metadata": {},
+ "source": [
+ "# Aggregations\n",
+ "Aggregate attestations are signed by a randomly selected aggregator from each committee and carry a bitfield covering all attesting validators in that committee. They are expected on the network from around second 8 of each slot, when aggregators publish. The charts below examine their first-seen timing, network propagation speed, and relationship to block arrival.\n",
+ "\n",
+ "## Aggregation first seen\n",
+ "\n",
+ "Histogram of when aggregate attestations were first observed by any control node, measured in seconds from slot start. The bulk of aggregations should appear around 8–9s."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "20",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Aggregations display first time seen (assuming that we have the timings for each messsage_id)\n",
+ "df = (\n",
+ " aggregations_df\n",
+ " .with_columns(\n",
+ " total_aggregations = pl.lit(len(aggregations_df)),\n",
+ " )\n",
+ " .group_by([\"latency_bucket\"])\n",
+ " .agg(\n",
+ " percentage = pl.col(\"slot\").count() * 100 / pl.col(\"total_aggregations\").max(),\n",
+ " )\n",
+ " .sort(\"latency_bucket\", descending=False)\n",
+ ")\n",
+ "\n",
+ "g = px.bar(\n",
+ " df,\n",
+ " x=\"latency_bucket\",\n",
+ " y=\"percentage\",\n",
+ " opacity=0.9,\n",
+ ")\n",
+ "g.update_layout(\n",
+ " title = \"Histogram of first time seen the aggregations\",\n",
+ " xaxis_title_text=\"Seconds since slot start\",\n",
+ " yaxis_title_text=\"% of Aggregations\",\n",
+ " barmode='stack',\n",
+ " width=1200,\n",
+ " height=800,\n",
+ ")\n",
+ "g.update_xaxes(range=[7, 16])\n",
+ "render_plot_fn(g, render_method, \"./images/agg/aggregation_histogram_first_seen_tot.png\")\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "21",
+ "metadata": {},
+ "source": [
+ "## Aggregation first seen by aggregate quality\n",
+ "\n",
+ "Same histogram broken down by the number of bits set in the aggregation bitfield. Higher bit counts indicate larger, more useful aggregates. This reveals whether compact aggregates arrive earlier or later than their more complete counterparts."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "22",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Aggregations display first time seen (based on the number of aggregated bits that they have on the legend)\n",
+ "\n",
+ "df = (\n",
+ " aggregations_df\n",
+ " .with_columns(\n",
+ " total_aggregations = pl.lit(len(aggregations_df)),\n",
+ " )\n",
+ " .group_by([\"latency_bucket\", \"aggregated_bits_bucket\"])\n",
+ " .agg(\n",
+ " percentage = pl.col(\"slot\").count() * 100 / pl.col(\"total_aggregations\").max(),\n",
+ " )\n",
+ " .sort([\"latency_bucket\", \"aggregated_bits_bucket\"], descending=False)\n",
+ ")\n",
+ "\n",
+ "g = px.bar(\n",
+ " df,\n",
+ " x=\"latency_bucket\",\n",
+ " y=\"percentage\",\n",
+ " color=\"aggregated_bits_bucket\",\n",
+ " opacity=0.9,\n",
+ ")\n",
+ "g.update_layout(\n",
+ " title = \"Histogram of first time seen the aggregations\",\n",
+ " xaxis_title_text=\"Seconds since slot start\",\n",
+ " yaxis_title_text=\"% of Aggregations\",\n",
+ " barmode='stack',\n",
+ " width=1200,\n",
+ " height=800,\n",
+ ")\n",
+ "g.update_xaxes(range=[7, 16])\n",
+ "render_plot_fn(g, render_method, \"./images/agg/aggregation_histogram_first_seen_by_agg_bits_tot.png\")\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "23",
+ "metadata": {},
+ "source": [
+ "## Aggregation network propagation (p50 / p90)\n",
+ "\n",
+ "Distribution of how long it took for each aggregation to be seen by 50% and 90% of control nodes, measured from slot start. Analogous to the attestation propagation charts but shifted to the 8–16s window. The gap between p50 and p90 indicates how evenly the aggregation gossip reaches the network."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "24",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Aggregations display propagation percentile (based on the number of aggregated bits that they have)\n",
+ "def arrival_of_aggregations_over_percentile(agg_df: pl.DataFrame, perc: str = 'p50'):\n",
+ " df = (\n",
+ " agg_df\n",
+ " .with_columns(\n",
+ " total_aggregations = pl.lit(len(agg_df)),\n",
+ " )\n",
+ " .group_by([f\"broadcast_{perc}_bucket_wfs\"])\n",
+ " .agg(\n",
+ " percentage = pl.col(\"slot\").count() * 100 / pl.col(\"total_aggregations\").max(),\n",
+ " broadcast= pl.col(f\"agg_broadcast_{perc}\").mean(),\n",
+ " )\n",
+ " .sort([\"broadcast\"], descending=False)\n",
+ " )\n",
+ " \n",
+ " # the histogram of when where the attestations seen\n",
+ " g = px.bar(\n",
+ " df,\n",
+ " x=f\"broadcast_{perc}_bucket_wfs\",\n",
+ " y=\"percentage\",\n",
+ " )\n",
+ " g.update_layout(\n",
+ " title=f\"Histogram of when aggregations where seen by {perc} of control nodes\",\n",
+ " xaxis_title_text=f\"{perc} propagation (s)\",\n",
+ " yaxis_title_text=\"% of Aggregations\",\n",
+ " width=1200,\n",
+ " height=800,\n",
+ " )\n",
+ " g.update_xaxes(range=[7, 20])\n",
+ " g.add_vline(x=8, line_width=3, line_dash=\"dash\", line_color=\"purple\")\n",
+ " g.add_vline(x=12, line_width=3, line_dash=\"dash\", line_color=\"red\")\n",
+ " render_plot_fn(g, render_method, f\"./images/agg/aggregation_histogram_{perc}_tot.png\")\n",
+ " \n",
+ "arrival_of_aggregations_over_percentile(aggregations_df, \"p50\")\n",
+ "arrival_of_aggregations_over_percentile(aggregations_df, \"p90\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "25",
+ "metadata": {},
+ "source": [
+ "## Aggregation propagation by aggregate quality\n",
+ "\n",
+ "P50/p90 propagation histogram broken down by aggregated bit-count. Helps identify whether high-quality aggregates (more bits set) propagate faster or slower than sparse ones, which could indicate prioritization effects in the gossip layer or correlation with specific aggregator clients."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "26",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Aggregations display propagation percentile (based on the number of aggregated bits that they have)\n",
+ "def arrival_of_aggregations_over_percentile(agg_df: pl.DataFrame, perc: str = 'p50'):\n",
+ " df = (\n",
+ " agg_df\n",
+ " .with_columns(\n",
+ " total_aggregations = pl.lit(len(agg_df)),\n",
+ " )\n",
+ " .group_by([f\"broadcast_{perc}_bucket_wfs\", \"aggregated_bits_bucket\"])\n",
+ " .agg(\n",
+ " percentage = pl.col(\"slot\").count() * 100 / pl.col(\"total_aggregations\").max(),\n",
+ " broadcast= pl.col(f\"agg_broadcast_{perc}\").mean(),\n",
+ " )\n",
+ " .sort([\"broadcast\", \"aggregated_bits_bucket\"], descending=False)\n",
+ " )\n",
+ " \n",
+ " # the histogram of when where the attestations seen\n",
+ " g = px.bar(\n",
+ " df,\n",
+ " x=f\"broadcast_{perc}_bucket_wfs\",\n",
+ " y=\"percentage\",\n",
+ " color=\"aggregated_bits_bucket\",\n",
+ " opacity=0.9,\n",
+ " )\n",
+ " g.update_layout(\n",
+ " title=f\"Histogram of when aggregations where seen by {perc} of control nodes\",\n",
+ " xaxis_title_text=f\"{perc} propagation (s)\",\n",
+ " yaxis_title_text=\"% of Aggregations\",\n",
+ " barmode='stack', \n",
+ " width=1200,\n",
+ " height=800,\n",
+ " )\n",
+ " g.update_xaxes(range=[7, 20])\n",
+ " g.add_vline(x=8, line_width=3, line_dash=\"dash\", line_color=\"purple\")\n",
+ " g.add_vline(x=12, line_width=3, line_dash=\"dash\", line_color=\"red\")\n",
+ " render_plot_fn(g, render_method, f\"./images/agg/aggregation_histogram_{perc}_by_agg_bits_tot.png\")\n",
+ " \n",
+ "arrival_of_aggregations_over_percentile(aggregations_df, \"p50\")\n",
+ "arrival_of_aggregations_over_percentile(aggregations_df, \"p90\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "27",
+ "metadata": {},
+ "source": [
+ "## Aggregation propagation duration\n",
+ "\n",
+ "Raw propagation duration for each aggregation message, as the time from when the first node saw it until 50% or 90% of control nodes had received it. Unlike the slot-start-relative charts above, this measures within-network spread speed independent of when in the slot the aggregation was published."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "28",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Propagation CDF / Histogram\n",
+ "def plot_aggregation_propagation_time(agg_df: pl.DataFrame, perc: str = 'p50'):\n",
+ " df = (\n",
+ " agg_df\n",
+ " .with_columns(\n",
+ " total_aggregations = pl.lit(len(agg_df)),\n",
+ " )\n",
+ " .group_by([f\"broadcast_{perc}_bucket\"])\n",
+ " .agg(\n",
+ " percentage = pl.col(\"slot\").count() * 100 / pl.col(\"total_aggregations\").max(),\n",
+ " broadcast= pl.col(f\"agg_broadcast_{perc}\").mean(),\n",
+ " )\n",
+ " .sort([\"broadcast\"], descending=False)\n",
+ " )\n",
+ " \n",
+ " # the histogram of when where the attestations seen\n",
+ " g = px.bar(\n",
+ " df,\n",
+ " x=f\"broadcast_{perc}_bucket\",\n",
+ " y=\"percentage\",\n",
+ " opacity=0.9,\n",
+ " )\n",
+ " g.update_layout(\n",
+ " title=f\"Histogram the {perc} propagation of aggregations\",\n",
+ " xaxis_title_text=f\"{perc} propagation (s)\",\n",
+ " yaxis_title_text=\"% of Aggregations\",\n",
+ " barmode='stack', \n",
+ " width=1200,\n",
+ " height=800,\n",
+ " )\n",
+ " g.update_xaxes(range=[0, 8])\n",
+ " render_plot_fn(g, render_method, f\"./images/agg/aggregation_propagation_histogram_{perc}_tot.png\")\n",
+ " \n",
+ "plot_aggregation_propagation_time(aggregations_df, \"p50\")\n",
+ "plot_aggregation_propagation_time(aggregations_df, \"p90\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "29",
+ "metadata": {},
+ "source": [
+ "## Aggregation propagation by slot\n",
+ "\n",
+ "P50 propagation time for aggregations broken down by slot. Shows whether propagation quality is stable across slots or varies. E.g., slots with re-orgs, missed blocks, or heavy network load may exhibit different aggregation propagation patterns."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "30",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def arrival_of_aggregations_over_percentile_and_slot(\n",
+ " agg_df: pl.DataFrame,\n",
+ " perc: str = 'p50',\n",
+ "):\n",
+ " df = (\n",
+ " agg_df\n",
+ " .with_columns(\n",
+ " total_aggregations = pl.lit(len(agg_df)),\n",
+ " )\n",
+ " .group_by([f\"broadcast_{perc}_bucket_wfs\", \"slot\"])\n",
+ " .agg(\n",
+ " percentage = pl.col(\"slot\").count() * 100 / pl.col(\"total_aggregations\").max(),\n",
+ " broadcast= pl.col(f\"agg_broadcast_{perc}\").mean(),\n",
+ " )\n",
+ " .sort([\"broadcast\"], descending=False)\n",
+ " )\n",
+ " \n",
+ " # the histogram of when where the attestations seen\n",
+ " g = px.bar(\n",
+ " df,\n",
+ " x=f\"broadcast_{perc}_bucket_wfs\",\n",
+ " y=\"percentage\",\n",
+ " color=\"slot\",\n",
+ " opacity=0.9,\n",
+ " )\n",
+ " g.update_layout(\n",
+ " title=f\"Histogram of when aggregations where seen by {perc} of control nodes\",\n",
+ " xaxis_title_text=f\"{perc} propagation (s)\",\n",
+ " yaxis_title_text=\"% of Aggregations\",\n",
+ " barmode='stack', \n",
+ " width=1200,\n",
+ " height=800,\n",
+ " )\n",
+ " g.update_xaxes(range=[7, 20])\n",
+ " g.add_vline(x=8, line_width=3, line_dash=\"dash\", line_color=\"purple\")\n",
+ " g.add_vline(x=12, line_width=3, line_dash=\"dash\", line_color=\"red\")\n",
+ " g.write_image(f\"./images/agg/aggregation_histogram_{perc}__by_slot_tot.png\")\n",
+ " \n",
+ "arrival_of_aggregations_over_percentile_and_slot(aggregations_df, \"p50\")\n",
+ "arrival_of_aggregations_over_percentile_and_slot(aggregations_df, \"p90\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "31",
+ "metadata": {},
+ "source": [
+ "## Aggregation first seen vs propagation spread"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "32",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def histogram_per_first_att_seen_and_broadcast_percentile(att_df: pl.DataFrame, perc: str):\n",
+ " df = (\n",
+ " att_df\n",
+ " .with_columns(\n",
+ " total = pl.lit(len(att_df)),\n",
+ " )\n",
+ " .group_by([\"latency_bucket\", f\"broadcast_{perc}_bucket_g\"])\n",
+ " .agg(\n",
+ " attestations = pl.col(\"slot\").count(),\n",
+ " percentage = pl.col(\"slot\").count() * 100 / pl.col(\"total\").max(),\n",
+ " broadcast= pl.col(f\"agg_broadcast_{perc}\").mean(),\n",
+ " )\n",
+ " .sort(f\"broadcast\", \"latency_bucket\", \"percentage\", descending=True)\n",
+ " )\n",
+ " \n",
+ " g = px.bar(\n",
+ " df,\n",
+ " x=\"latency_bucket\",\n",
+ " y=\"percentage\",\n",
+ " color=f\"broadcast_{perc}_bucket_g\",\n",
+ " opacity=0.9,\n",
+ " )\n",
+ " g.update_layout(\n",
+ " title=f\"Histogram of aggregations first time seen by propagation {perc}\",\n",
+ " xaxis_title_text=\"Seconds since slot started\",\n",
+ " yaxis_title_text=\"% of Aggregations\",\n",
+ " barmode='stack',\n",
+ " width=1200,\n",
+ " height=800,\n",
+ " )\n",
+ " g.update_xaxes(range=[7, 13])\n",
+ " g.add_vline(x=8, line_width=3, line_dash=\"dash\", line_color=\"purple\")\n",
+ " g.add_vline(x=12, line_width=3, line_dash=\"dash\", line_color=\"red\")\n",
+ " render_plot_fn(g, render_method, f\"./images/agg/agg_histogram_first_seen_and_{perc}.png\")\n",
+ " \n",
+ "histogram_per_first_att_seen_and_broadcast_percentile(aggregations_df, \"p50\")\n",
+ "histogram_per_first_att_seen_and_broadcast_percentile(aggregations_df, \"p90\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "33",
+ "metadata": {},
+ "source": [
+ "## Aggregations per slot\n",
+ "\n",
+ "Count of unique aggregation message IDs and unique aggregator indices observed per slot."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "34",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# count of aggregations per slot\n",
+ "\n",
+ "df = (\n",
+ " aggregations_df\n",
+ " .group_by([\"slot\"])\n",
+ " .agg(\n",
+ " unique_msgs=pl.col(\"message_id\").count(),\n",
+ " unique_aggregators=pl.col(\"aggregator_index\").count(),\n",
+ " )\n",
+ " .sort([\"slot\"])\n",
+ ")\n",
+ "\n",
+ "g = px.line(\n",
+ " df.unpivot(index=[\"slot\"], on=[\"unique_msgs\", \"unique_aggregators\"], value_name=\"unique_items\", variable_name=\"aggregations\"),\n",
+ " x=\"slot\",\n",
+ " y=\"unique_items\",\n",
+ " color=\"aggregations\",\n",
+ ")\n",
+ "g.update_layout()\n",
+ "render_plot_fn(g, render_method, \"./images/agg/unique_aggregators_and_msg_ids_per_slot.png\")\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "35",
+ "metadata": {},
+ "source": [
+ "## Full Slot Timeline\n",
+ "\n",
+ "Blocks, data columns (PeerDAS), unaggregated attestations, and aggregate attestations are combined into a single timeline view spanning two consecutive slots. Block and column events from the next slot are shifted by +12s to show the full lifecycle: attestations propagate and get aggregated, then the next proposer publishes a block that includes them."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "36",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "block_flat_df = (\n",
+ " block_and_column_df\n",
+ " .select([ \n",
+ " \"slot\", \n",
+ " \"block_first_seen\", \"block_latency_bucket\",\n",
+ " \"block_broadcast_p50\", \"block_broadcast_p50_bucket\", \"block_broadcast_p50_bucket_wfs\", \"block_broadcast_p50_bucket_g\",\n",
+ " \"block_broadcast_p90\", \"block_broadcast_p90_bucket\", \"block_broadcast_p90_bucket_wfs\", \"block_broadcast_p90_bucket_g\",\n",
+ " ])\n",
+ " .with_columns(\n",
+ " type=pl.lit(\"blocks\"),\n",
+ " )\n",
+ " .rename({\n",
+ " \"block_first_seen\": \"first_seen\",\n",
+ " \"block_latency_bucket\": \"latency_bucket\",\n",
+ " \"block_broadcast_p50\": \"broadcast_p50\",\n",
+ " \"block_broadcast_p50_bucket\": \"broadcast_p50_bucket\",\n",
+ " \"block_broadcast_p50_bucket_wfs\": \"broadcast_p50_bucket_wfs\",\n",
+ " \"block_broadcast_p50_bucket_g\": \"broadcast_p50_bucket_g\",\n",
+ " \"block_broadcast_p90\": \"broadcast_p90\",\n",
+ " \"block_broadcast_p90_bucket\": \"broadcast_p90_bucket\",\n",
+ " \"block_broadcast_p90_bucket_wfs\": \"broadcast_p90_bucket_wfs\",\n",
+ " \"block_broadcast_p90_bucket_g\": \"broadcast_p90_bucket_g\",\n",
+ " })\n",
+ " .sort([\"slot\", \"first_seen\"])\n",
+ " .unique()\n",
+ ")\n",
+ "block_flat_df = block_flat_df.with_columns(total=pl.lit(len(block_flat_df)))\n",
+ "\n",
+ "columns_flat_df = (\n",
+ " block_and_column_df\n",
+ " .select([ \n",
+ " \"slot\", \n",
+ " \"column_first_seen\", \"column_latency_bucket\",\n",
+ " \"column_broadcast_p50\", \"column_broadcast_p50_bucket\", \"column_broadcast_p50_bucket_wfs\", \"column_broadcast_p50_bucket_g\",\n",
+ " \"column_broadcast_p90\", \"column_broadcast_p90_bucket\", \"column_broadcast_p90_bucket_wfs\", \"column_broadcast_p90_bucket_g\",\n",
+ " ])\n",
+ " .with_columns(\n",
+ " type=pl.lit(\"columns\"),\n",
+ " total=pl.lit(len(aggregations_df)),\n",
+ " )\n",
+ " .rename({\n",
+ " \"column_first_seen\": \"first_seen\",\n",
+ " \"column_latency_bucket\": \"latency_bucket\",\n",
+ " \"column_broadcast_p50\": \"broadcast_p50\",\n",
+ " \"column_broadcast_p50_bucket\": \"broadcast_p50_bucket\",\n",
+ " \"column_broadcast_p50_bucket_wfs\": \"broadcast_p50_bucket_wfs\",\n",
+ " \"column_broadcast_p50_bucket_g\": \"broadcast_p50_bucket_g\",\n",
+ " \"column_broadcast_p90\": \"broadcast_p90\",\n",
+ " \"column_broadcast_p90_bucket\": \"broadcast_p90_bucket\",\n",
+ " \"column_broadcast_p90_bucket_wfs\": \"broadcast_p90_bucket_wfs\",\n",
+ " \"column_broadcast_p90_bucket_g\": \"broadcast_p90_bucket_g\",\n",
+ " })\n",
+ " .sort([\"slot\", \"first_seen\"])\n",
+ " .unique()\n",
+ ")\n",
+ "columns_flat_df = columns_flat_df.with_columns(total=pl.lit(len(columns_flat_df)))\n",
+ "\n",
+ "attestations_flat_df = (\n",
+ " attestation_df\n",
+ " .select([ \n",
+ " \"slot\", \n",
+ " \"att_first_seen_wb\", \"latency_bucket\",\n",
+ " \"att_broadcast_p50\", \"broadcast_p50_bucket\", \"broadcast_p50_bucket_wfs\", \"broadcast_p50_bucket_g\",\n",
+ " \"att_broadcast_p90\", \"broadcast_p90_bucket\", \"broadcast_p90_bucket_wfs\", \"broadcast_p90_bucket_g\",\n",
+ " ])\n",
+ " .rename({\n",
+ " \"att_first_seen_wb\": \"first_seen\",\n",
+ " \"att_broadcast_p50\": \"broadcast_p50\",\n",
+ " \"att_broadcast_p90\": \"broadcast_p90\",\n",
+ " })\n",
+ " .with_columns(\n",
+ " type=pl.lit(\"attestations\"),\n",
+ " first_seen=pl.col(\"first_seen\").cast(pl.Float64),\n",
+ " broadcast_p50=pl.col(\"broadcast_p50\").cast(pl.Float64),\n",
+ " broadcast_p90=pl.col(\"broadcast_p90\").cast(pl.Float64),\n",
+ " total=pl.lit(len(attestation_df)),\n",
+ " )\n",
+ " .sort([\"slot\", \"first_seen\"])\n",
+ ")\n",
+ "\n",
+ "\n",
+ "aggregations_flat_df = (\n",
+ " aggregations_df\n",
+ " .select([ \n",
+ " \"slot\", \n",
+ " \"agg_first_seen_wb\", \"latency_bucket\",\n",
+ " \"agg_broadcast_p50\", \"broadcast_p50_bucket\", \"broadcast_p50_bucket_wfs\", \"broadcast_p50_bucket_g\",\n",
+ " \"agg_broadcast_p90\", \"broadcast_p90_bucket\", \"broadcast_p90_bucket_wfs\", \"broadcast_p90_bucket_g\",\n",
+ " ])\n",
+ " .rename({\n",
+ " \"agg_first_seen_wb\": \"first_seen\",\n",
+ " \"agg_broadcast_p50\": \"broadcast_p50\",\n",
+ " \"agg_broadcast_p90\": \"broadcast_p90\",\n",
+ " })\n",
+ " .with_columns(\n",
+ " slot=pl.col(\"slot\").cast(pl.UInt32),\n",
+ " type=pl.lit(\"aggregations\"),\n",
+ " first_seen=pl.col(\"first_seen\").cast(pl.Float64),\n",
+ " broadcast_p50=pl.col(\"broadcast_p50\").cast(pl.Float64),\n",
+ " broadcast_p90=pl.col(\"broadcast_p90\").cast(pl.Float64),\n",
+ " total=pl.lit(len(aggregations_df)),\n",
+ " )\n",
+ " .sort([\"slot\", \"first_seen\"])\n",
+ ")\n",
+ "\n",
+ "main_df = pl.concat([block_flat_df, columns_flat_df, attestations_flat_df, aggregations_flat_df]).sort(\"slot\")\n",
+ "main_df = main_df.unique()\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "37",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# only show the aggregations + next block arrivals + columns\n",
+ "next_block_flat_df = (\n",
+ " block_and_column_df\n",
+ " .select([ \n",
+ " \"previous_slot\",\n",
+ " \"block_first_seen\", \"block_latency_bucket\",\n",
+ " \"block_broadcast_p50\", \"block_broadcast_p50_bucket\", \"block_broadcast_p50_bucket_wfs\", \"block_broadcast_p50_bucket_g\",\n",
+ " \"block_broadcast_p90\", \"block_broadcast_p90_bucket\", \"block_broadcast_p90_bucket_wfs\", \"block_broadcast_p90_bucket_g\",\n",
+ " ])\n",
+ " .with_columns(\n",
+ " type=pl.lit(\"next_blocks\")\n",
+ " )\n",
+ " .rename({\n",
+ " \"previous_slot\": \"slot\",\n",
+ " \"block_first_seen\": \"first_seen\",\n",
+ " \"block_latency_bucket\": \"latency_bucket\",\n",
+ " \"block_broadcast_p50\": \"broadcast_p50\",\n",
+ " \"block_broadcast_p50_bucket\": \"broadcast_p50_bucket\",\n",
+ " \"block_broadcast_p50_bucket_wfs\": \"broadcast_p50_bucket_wfs\",\n",
+ " \"block_broadcast_p50_bucket_g\": \"broadcast_p50_bucket_g\",\n",
+ " \"block_broadcast_p90\": \"broadcast_p90\",\n",
+ " \"block_broadcast_p90_bucket\": \"broadcast_p90_bucket\",\n",
+ " \"block_broadcast_p90_bucket_wfs\": \"broadcast_p90_bucket_wfs\",\n",
+ " \"block_broadcast_p90_bucket_g\": \"broadcast_p90_bucket_g\",\n",
+ " })\n",
+ " .with_columns(\n",
+ " slot=pl.col(\"slot\").cast(pl.UInt32),\n",
+ " first_seen=pl.col(\"first_seen\")+12.0,\n",
+ " latency_bucket=pl.col(\"latency_bucket\")+12.0,\n",
+ " broadcast_p50_bucket_wfs=pl.col(\"broadcast_p50_bucket_wfs\")+12.0,\n",
+ " broadcast_p90_bucket_wfs=pl.col(\"broadcast_p90_bucket_wfs\")+12.0,\n",
+ " total=pl.lit(len(aggregations_flat_df)),\n",
+ " )\n",
+ " .sort([\"slot\", \"first_seen\"])\n",
+ ")\n",
+ "next_block_flat_df = next_block_flat_df.with_columns(total=pl.lit(len(block_flat_df)))\n",
+ "\n",
+ "\n",
+ "next_columns_flat_df = (\n",
+ " block_and_column_df\n",
+ " .select([ \n",
+ " \"slot\", \n",
+ " \"column_first_seen\", \"column_latency_bucket\",\n",
+ " \"column_broadcast_p50\", \"column_broadcast_p50_bucket\", \"column_broadcast_p50_bucket_wfs\", \"column_broadcast_p50_bucket_g\",\n",
+ " \"column_broadcast_p90\", \"column_broadcast_p90_bucket\", \"column_broadcast_p90_bucket_wfs\", \"column_broadcast_p90_bucket_g\",\n",
+ " ])\n",
+ " .with_columns(\n",
+ " type=pl.lit(\"next_columns\"),\n",
+ " total=pl.lit(len(aggregations_df)),\n",
+ " )\n",
+ " .rename({\n",
+ " \"column_first_seen\": \"first_seen\",\n",
+ " \"column_latency_bucket\": \"latency_bucket\",\n",
+ " \"column_broadcast_p50\": \"broadcast_p50\",\n",
+ " \"column_broadcast_p50_bucket\": \"broadcast_p50_bucket\",\n",
+ " \"column_broadcast_p50_bucket_wfs\": \"broadcast_p50_bucket_wfs\",\n",
+ " \"column_broadcast_p50_bucket_g\": \"broadcast_p50_bucket_g\",\n",
+ " \"column_broadcast_p90\": \"broadcast_p90\",\n",
+ " \"column_broadcast_p90_bucket\": \"broadcast_p90_bucket\",\n",
+ " \"column_broadcast_p90_bucket_wfs\": \"broadcast_p90_bucket_wfs\",\n",
+ " \"column_broadcast_p90_bucket_g\": \"broadcast_p90_bucket_g\",\n",
+ " })\n",
+ " .with_columns(\n",
+ " slot=pl.col(\"slot\").cast(pl.UInt32),\n",
+ " first_seen=pl.col(\"first_seen\")+12.0,\n",
+ " latency_bucket=pl.col(\"latency_bucket\")+12.0,\n",
+ " broadcast_p50_bucket_wfs=pl.col(\"broadcast_p50_bucket_wfs\")+12.0,\n",
+ " broadcast_p90_bucket_wfs=pl.col(\"broadcast_p90_bucket_wfs\")+12.0,\n",
+ " total=pl.lit(len(aggregations_flat_df)),\n",
+ " )\n",
+ " .sort([\"slot\", \"first_seen\"])\n",
+ " .unique()\n",
+ ")\n",
+ "next_columns_flat_df = next_columns_flat_df.with_columns(total=pl.lit(len(next_columns_flat_df)))\n",
+ "\n",
+ "main_df_2 = pl.concat([next_block_flat_df, next_columns_flat_df, aggregations_flat_df]).unique().sort(\"slot\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "38",
+ "metadata": {},
+ "source": [
+ "## Aggregations vs next block and column arrival\n",
+ "\n",
+ "Histogram comparing when aggregations were seen (first-seen for blocks/columns, p50 for aggregations) to when the next slot's block and data columns first arrived."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "39",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# display aggregations with next blocks\n",
+ "# we want to visualize the p50 aggregations with the following slot arrival (first seen)\n",
+ "block_aux_df = (\n",
+ " next_block_flat_df\n",
+ " .select([\"slot\", \"latency_bucket\", \"first_seen\", \"type\", \"total\"])\n",
+ " .rename({\n",
+ " \"latency_bucket\": \"broadcast_p50_bucket_wfs\",\n",
+ " \"first_seen\": \"broadcast_p50\",\n",
+ " })\n",
+ " .unique()\n",
+ ")\n",
+ "\n",
+ "columns_aux_df = (\n",
+ " next_columns_flat_df\n",
+ " .select([\"slot\", \"latency_bucket\", \"first_seen\", \"type\", \"total\"])\n",
+ " .rename({\n",
+ " \"latency_bucket\": \"broadcast_p50_bucket_wfs\",\n",
+ " \"first_seen\": \"broadcast_p50\",\n",
+ " })\n",
+ " .unique()\n",
+ ")\n",
+ "\n",
+ "aggregations_aux_df = (\n",
+ " aggregations_flat_df\n",
+ " .select([\"slot\", \"broadcast_p50_bucket_wfs\", \"broadcast_p50\", \"type\", \"total\"])\n",
+ ")\n",
+ "\n",
+ "df = (\n",
+ " pl.concat([block_aux_df, aggregations_aux_df, columns_aux_df])\n",
+ " .group_by([\"type\", \"broadcast_p50_bucket_wfs\"])\n",
+ " .agg(\n",
+ " percentage = pl.col(\"slot\").count() * 100 / pl.col(\"total\").max(),\n",
+ " broadcast= pl.col(f\"broadcast_p50\").mean(),\n",
+ " )\n",
+ " .sort([\"type\", \"broadcast\"], descending=False)\n",
+ ")\n",
+ "\n",
+ "# the histogram of when where the attestations seen\n",
+ "g = px.bar(\n",
+ " df,\n",
+ " x=f\"broadcast_p50_bucket_wfs\",\n",
+ " y=\"percentage\",\n",
+ " color=\"type\",\n",
+ ")\n",
+ "g.update_layout(\n",
+ " title=f\"Histogram of when aggregations were seen by p50 of control nodes and when the next block and columns were first seen\",\n",
+ " xaxis_title_text=f\"Seconds since the slot started\",\n",
+ " yaxis_title_text=\"% of messages\",\n",
+ " barmode='group',\n",
+ " width=1200,\n",
+ " height=800,\n",
+ ")\n",
+ "g.update_xaxes(range=[0, 20])\n",
+ "g.add_vline(x=8, line_width=3, line_dash=\"dash\", line_color=\"purple\")\n",
+ "g.add_vline(x=12, line_width=3, line_dash=\"dash\", line_color=\"red\")\n",
+ "render_plot_fn(g, render_method, \"./images/agg/aggregation_p50_and_next_block_correlation_histogram_tot.png\")\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "40",
+ "metadata": {},
+ "source": [
+ "## Aggregations vs next slot (p50 network perspective)\n",
+ "\n",
+ "Same cross-slot comparison using p50 propagation time for blocks and columns instead of first-seen. Shows whether the network-wide spread of aggregations aligns with the network-wide arrival of the following block."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "41",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# we want to visualize the p50 aggregations with the following slot arrival's p50\n",
+ "df = (\n",
+ " main_df_2\n",
+ " .group_by([f\"broadcast_p50_bucket_wfs\", \"type\"])\n",
+ " .agg(\n",
+ " percentage = pl.col(\"slot\").count() * 100 / pl.col(\"total\").max(),\n",
+ " broadcast= pl.col(f\"broadcast_p50\").mean(),\n",
+ " )\n",
+ " .sort([\"type\", \"broadcast\"], descending=False)\n",
+ ")\n",
+ "\n",
+ "# the histogram of when where the attestations seen\n",
+ "g = px.bar(\n",
+ " df,\n",
+ " x=f\"broadcast_p50_bucket_wfs\",\n",
+ " y=\"percentage\",\n",
+ " color=\"type\",\n",
+ ")\n",
+ "g.update_layout(\n",
+ " title=f\"Histogram of when aggregationsa and the next block + columns were seen by p50 of control nodes\",\n",
+ " xaxis_title_text=f\"Seconds since the slot started\",\n",
+ " yaxis_title_text=\"% of messages\",\n",
+ " barmode='group',\n",
+ " width=1200,\n",
+ " height=800,\n",
+ ")\n",
+ "g.update_xaxes(range=[0, 20])\n",
+ "g.add_vline(x=8, line_width=3, line_dash=\"dash\", line_color=\"purple\")\n",
+ "g.add_vline(x=12, line_width=3, line_dash=\"dash\", line_color=\"red\")\n",
+ "render_plot_fn(g, render_method, \"./images/agg/aggregation_p50_and_next_block_correlation_histogram_p50_tot.png\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.6"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/notebooks/utils.py b/notebooks/utils.py
new file mode 100644
index 0000000..665470d
--- /dev/null
+++ b/notebooks/utils.py
@@ -0,0 +1,46 @@
+import pandas as pd
+from typing import List
+from IPython.display import display, HTML
+
+
+def render_table(df: pd.DataFrame, columns: List[str] = None):
+ """
+ Renders the give dataframe as a table under a unified common style
+ Args:
+ df: dataframe to render
+ custom_colums: Custom naming for the columns. Get the names from the columns if None was given
+ """
+ cols = df.columns
+ if columns is not None:
+ if len(columns) != len(cols):
+ raise ValueError("Number of columns does not match")
+ else:
+ cols = columns
+
+ table = '''
+
+
+
+
+ '''
+ header = ""
+ for col in cols:
+ header += f"| {col.replace("_", " ").upper()} | """
+ table += header + '
'
+
+ for row in df.iter_rows(named=True):
+ row_str = ""
+ for col in df.columns:
+ row_str += f"| {row[col]} | "
+ table += row_str + '
'
+
+ table += '