Sat, Dec 27, 2025

Column propagation - 2025-12-27

Analysis of data column propagation timing across the 128 column subnets in PeerDAS.

View query 
SELECT
    slot,
    slot_start_date_time AS time,
    minIfOrNull(propagation_slot_start_diff, column_index = 0) AS c0,
    minIfOrNull(propagation_slot_start_diff, column_index = 1) AS c1,
    minIfOrNull(propagation_slot_start_diff, column_index = 2) AS c2,
    minIfOrNull(propagation_slot_start_diff, column_index = 3) AS c3,
    minIfOrNull(propagation_slot_start_diff, column_index = 4) AS c4,
    minIfOrNull(propagation_slot_start_diff, column_index = 5) AS c5,
    minIfOrNull(propagation_slot_start_diff, column_index = 6) AS c6,
    minIfOrNull(propagation_slot_start_diff, column_index = 7) AS c7,
    minIfOrNull(propagation_slot_start_diff, column_index = 8) AS c8,
    minIfOrNull(propagation_slot_start_diff, column_index = 9) AS c9,
    minIfOrNull(propagation_slot_start_diff, column_index = 10) AS c10,
    minIfOrNull(propagation_slot_start_diff, column_index = 11) AS c11,
    minIfOrNull(propagation_slot_start_diff, column_index = 12) AS c12,
    minIfOrNull(propagation_slot_start_diff, column_index = 13) AS c13,
    minIfOrNull(propagation_slot_start_diff, column_index = 14) AS c14,
    minIfOrNull(propagation_slot_start_diff, column_index = 15) AS c15,
    minIfOrNull(propagation_slot_start_diff, column_index = 16) AS c16,
    minIfOrNull(propagation_slot_start_diff, column_index = 17) AS c17,
    minIfOrNull(propagation_slot_start_diff, column_index = 18) AS c18,
    minIfOrNull(propagation_slot_start_diff, column_index = 19) AS c19,
    minIfOrNull(propagation_slot_start_diff, column_index = 20) AS c20,
    minIfOrNull(propagation_slot_start_diff, column_index = 21) AS c21,
    minIfOrNull(propagation_slot_start_diff, column_index = 22) AS c22,
    minIfOrNull(propagation_slot_start_diff, column_index = 23) AS c23,
    minIfOrNull(propagation_slot_start_diff, column_index = 24) AS c24,
    minIfOrNull(propagation_slot_start_diff, column_index = 25) AS c25,
    minIfOrNull(propagation_slot_start_diff, column_index = 26) AS c26,
    minIfOrNull(propagation_slot_start_diff, column_index = 27) AS c27,
    minIfOrNull(propagation_slot_start_diff, column_index = 28) AS c28,
    minIfOrNull(propagation_slot_start_diff, column_index = 29) AS c29,
    minIfOrNull(propagation_slot_start_diff, column_index = 30) AS c30,
    minIfOrNull(propagation_slot_start_diff, column_index = 31) AS c31,
    minIfOrNull(propagation_slot_start_diff, column_index = 32) AS c32,
    minIfOrNull(propagation_slot_start_diff, column_index = 33) AS c33,
    minIfOrNull(propagation_slot_start_diff, column_index = 34) AS c34,
    minIfOrNull(propagation_slot_start_diff, column_index = 35) AS c35,
    minIfOrNull(propagation_slot_start_diff, column_index = 36) AS c36,
    minIfOrNull(propagation_slot_start_diff, column_index = 37) AS c37,
    minIfOrNull(propagation_slot_start_diff, column_index = 38) AS c38,
    minIfOrNull(propagation_slot_start_diff, column_index = 39) AS c39,
    minIfOrNull(propagation_slot_start_diff, column_index = 40) AS c40,
    minIfOrNull(propagation_slot_start_diff, column_index = 41) AS c41,
    minIfOrNull(propagation_slot_start_diff, column_index = 42) AS c42,
    minIfOrNull(propagation_slot_start_diff, column_index = 43) AS c43,
    minIfOrNull(propagation_slot_start_diff, column_index = 44) AS c44,
    minIfOrNull(propagation_slot_start_diff, column_index = 45) AS c45,
    minIfOrNull(propagation_slot_start_diff, column_index = 46) AS c46,
    minIfOrNull(propagation_slot_start_diff, column_index = 47) AS c47,
    minIfOrNull(propagation_slot_start_diff, column_index = 48) AS c48,
    minIfOrNull(propagation_slot_start_diff, column_index = 49) AS c49,
    minIfOrNull(propagation_slot_start_diff, column_index = 50) AS c50,
    minIfOrNull(propagation_slot_start_diff, column_index = 51) AS c51,
    minIfOrNull(propagation_slot_start_diff, column_index = 52) AS c52,
    minIfOrNull(propagation_slot_start_diff, column_index = 53) AS c53,
    minIfOrNull(propagation_slot_start_diff, column_index = 54) AS c54,
    minIfOrNull(propagation_slot_start_diff, column_index = 55) AS c55,
    minIfOrNull(propagation_slot_start_diff, column_index = 56) AS c56,
    minIfOrNull(propagation_slot_start_diff, column_index = 57) AS c57,
    minIfOrNull(propagation_slot_start_diff, column_index = 58) AS c58,
    minIfOrNull(propagation_slot_start_diff, column_index = 59) AS c59,
    minIfOrNull(propagation_slot_start_diff, column_index = 60) AS c60,
    minIfOrNull(propagation_slot_start_diff, column_index = 61) AS c61,
    minIfOrNull(propagation_slot_start_diff, column_index = 62) AS c62,
    minIfOrNull(propagation_slot_start_diff, column_index = 63) AS c63,
    minIfOrNull(propagation_slot_start_diff, column_index = 64) AS c64,
    minIfOrNull(propagation_slot_start_diff, column_index = 65) AS c65,
    minIfOrNull(propagation_slot_start_diff, column_index = 66) AS c66,
    minIfOrNull(propagation_slot_start_diff, column_index = 67) AS c67,
    minIfOrNull(propagation_slot_start_diff, column_index = 68) AS c68,
    minIfOrNull(propagation_slot_start_diff, column_index = 69) AS c69,
    minIfOrNull(propagation_slot_start_diff, column_index = 70) AS c70,
    minIfOrNull(propagation_slot_start_diff, column_index = 71) AS c71,
    minIfOrNull(propagation_slot_start_diff, column_index = 72) AS c72,
    minIfOrNull(propagation_slot_start_diff, column_index = 73) AS c73,
    minIfOrNull(propagation_slot_start_diff, column_index = 74) AS c74,
    minIfOrNull(propagation_slot_start_diff, column_index = 75) AS c75,
    minIfOrNull(propagation_slot_start_diff, column_index = 76) AS c76,
    minIfOrNull(propagation_slot_start_diff, column_index = 77) AS c77,
    minIfOrNull(propagation_slot_start_diff, column_index = 78) AS c78,
    minIfOrNull(propagation_slot_start_diff, column_index = 79) AS c79,
    minIfOrNull(propagation_slot_start_diff, column_index = 80) AS c80,
    minIfOrNull(propagation_slot_start_diff, column_index = 81) AS c81,
    minIfOrNull(propagation_slot_start_diff, column_index = 82) AS c82,
    minIfOrNull(propagation_slot_start_diff, column_index = 83) AS c83,
    minIfOrNull(propagation_slot_start_diff, column_index = 84) AS c84,
    minIfOrNull(propagation_slot_start_diff, column_index = 85) AS c85,
    minIfOrNull(propagation_slot_start_diff, column_index = 86) AS c86,
    minIfOrNull(propagation_slot_start_diff, column_index = 87) AS c87,
    minIfOrNull(propagation_slot_start_diff, column_index = 88) AS c88,
    minIfOrNull(propagation_slot_start_diff, column_index = 89) AS c89,
    minIfOrNull(propagation_slot_start_diff, column_index = 90) AS c90,
    minIfOrNull(propagation_slot_start_diff, column_index = 91) AS c91,
    minIfOrNull(propagation_slot_start_diff, column_index = 92) AS c92,
    minIfOrNull(propagation_slot_start_diff, column_index = 93) AS c93,
    minIfOrNull(propagation_slot_start_diff, column_index = 94) AS c94,
    minIfOrNull(propagation_slot_start_diff, column_index = 95) AS c95,
    minIfOrNull(propagation_slot_start_diff, column_index = 96) AS c96,
    minIfOrNull(propagation_slot_start_diff, column_index = 97) AS c97,
    minIfOrNull(propagation_slot_start_diff, column_index = 98) AS c98,
    minIfOrNull(propagation_slot_start_diff, column_index = 99) AS c99,
    minIfOrNull(propagation_slot_start_diff, column_index = 100) AS c100,
    minIfOrNull(propagation_slot_start_diff, column_index = 101) AS c101,
    minIfOrNull(propagation_slot_start_diff, column_index = 102) AS c102,
    minIfOrNull(propagation_slot_start_diff, column_index = 103) AS c103,
    minIfOrNull(propagation_slot_start_diff, column_index = 104) AS c104,
    minIfOrNull(propagation_slot_start_diff, column_index = 105) AS c105,
    minIfOrNull(propagation_slot_start_diff, column_index = 106) AS c106,
    minIfOrNull(propagation_slot_start_diff, column_index = 107) AS c107,
    minIfOrNull(propagation_slot_start_diff, column_index = 108) AS c108,
    minIfOrNull(propagation_slot_start_diff, column_index = 109) AS c109,
    minIfOrNull(propagation_slot_start_diff, column_index = 110) AS c110,
    minIfOrNull(propagation_slot_start_diff, column_index = 111) AS c111,
    minIfOrNull(propagation_slot_start_diff, column_index = 112) AS c112,
    minIfOrNull(propagation_slot_start_diff, column_index = 113) AS c113,
    minIfOrNull(propagation_slot_start_diff, column_index = 114) AS c114,
    minIfOrNull(propagation_slot_start_diff, column_index = 115) AS c115,
    minIfOrNull(propagation_slot_start_diff, column_index = 116) AS c116,
    minIfOrNull(propagation_slot_start_diff, column_index = 117) AS c117,
    minIfOrNull(propagation_slot_start_diff, column_index = 118) AS c118,
    minIfOrNull(propagation_slot_start_diff, column_index = 119) AS c119,
    minIfOrNull(propagation_slot_start_diff, column_index = 120) AS c120,
    minIfOrNull(propagation_slot_start_diff, column_index = 121) AS c121,
    minIfOrNull(propagation_slot_start_diff, column_index = 122) AS c122,
    minIfOrNull(propagation_slot_start_diff, column_index = 123) AS c123,
    minIfOrNull(propagation_slot_start_diff, column_index = 124) AS c124,
    minIfOrNull(propagation_slot_start_diff, column_index = 125) AS c125,
    minIfOrNull(propagation_slot_start_diff, column_index = 126) AS c126,
    minIfOrNull(propagation_slot_start_diff, column_index = 127) AS c127
FROM libp2p_gossipsub_data_column_sidecar
WHERE event_date_time >= '2025-12-27' AND event_date_time < '2025-12-27'::date + INTERVAL 1 DAY
  AND slot_start_date_time >= '2025-12-27' AND slot_start_date_time < '2025-12-27'::date + INTERVAL 1 DAY
  AND meta_network_name = 'mainnet'
GROUP BY slot, slot_start_date_time
ORDER BY slot
Show code 
# Load column propagation data
df_col_first_seen = load_parquet("col_first_seen", target_date)

print(f"Slots with column data: {len(df_col_first_seen)}")

Slots with column data: 5361

Column first seen

Heatmap showing when each of the 128 data columns was first observed, measured in milliseconds from slot start. Consistent patterns across columns indicate healthy propagation; outliers may signal network issues.

Show code 
# Panel 1: Column first seen (ms into slot start) - 128 columns heatmap

# Reshape for heatmap: rows = columns (c0-c127), columns = time
col_names = [f"c{i}" for i in range(NUM_COLUMNS)]
df_cols = df_col_first_seen[col_names].T
df_cols.columns = df_col_first_seen["time"]

# Create slot lookup for hover data
slot_values = df_col_first_seen["slot"].values

# Build customdata: slot number for each column in the heatmap
customdata = np.array([[slot_values[j] for j in range(len(slot_values))] for _ in range(NUM_COLUMNS)])

fig = go.Figure(
    data=go.Heatmap(
        z=df_cols.values,
        x=df_cols.columns,
        y=[str(i) for i in range(NUM_COLUMNS)],
        zmin=1500,
        zmax=4000,
        colorbar=horizontal_colorbar("ms"),
        customdata=customdata,
        hovertemplate="<b>Slot:</b> %{customdata}<br><b>Time:</b> %{x}<br><b>Column Index:</b> %{y}<br><b>First Seen:</b> %{z} ms<extra></extra>",
    )
)
fig.update_layout(
    margin=dict(l=10, r=10, t=10, b=80),
    xaxis=dict(automargin=True),
    yaxis=dict(title="Column", automargin=True, autorange="reversed"),
    height=800,
)
fig.show()

Delta from fastest column (intraslot, ms)

Shows how much slower each column arrived compared to the fastest column in that slot. Highlights columns that consistently lag behind, which may indicate propagation bottlenecks.

Show code 
# Compute delta from min value per slot for each column
col_names = [f"c{i}" for i in range(NUM_COLUMNS)]
df_delta = df_col_first_seen.copy()

# Calculate row-wise minimum and subtract from each column
row_mins = df_delta[col_names].min(axis=1)
for col in col_names:
    df_delta[col] = df_delta[col] - row_mins

# Reshape for heatmap
df_delta_cols = df_delta[col_names].T
df_delta_cols.columns = df_delta["time"]

# Create slot lookup for hover data
slot_values = df_delta["slot"].values

# Build customdata: slot number for each column in the heatmap
customdata = np.array([[slot_values[j] for j in range(len(slot_values))] for _ in range(NUM_COLUMNS)])

fig = go.Figure(
    data=go.Heatmap(
        z=df_delta_cols.values,
        x=df_delta_cols.columns,
        y=[str(i) for i in range(NUM_COLUMNS)],
        colorscale="Inferno",
        reversescale=False,
        zmin=0,
        zmax=250,
        colorbar=horizontal_colorbar("ms"),
        customdata=customdata,
        hovertemplate="<b>Slot:</b> %{customdata}<br><b>Time:</b> %{x}<br><b>Column Index:</b> %{y}<br><b>Delta:</b> %{z} ms<extra></extra>",
    )
)
fig.update_layout(
    margin=dict(l=10, r=10, t=10, b=80),
    xaxis=dict(automargin=True),
    yaxis=dict(title="Column", automargin=True, autorange="reversed"),
    height=800,
)
fig.show()

Delta normalized (0-1)

Same delta data normalized to a 0–1 scale per slot, making it easier to compare relative propagation order regardless of absolute timing. Columns closer to 0 arrived first; those near 1 arrived last.

Show code 
# Normalize delta values to 0-1 range per slot
col_names = [f"c{i}" for i in range(NUM_COLUMNS)]
df_normalized = df_col_first_seen.copy()

# Calculate row-wise min and max, then normalize
row_mins = df_normalized[col_names].min(axis=1)
row_maxs = df_normalized[col_names].max(axis=1)
row_ranges = row_maxs - row_mins

for col in col_names:
    df_normalized[col] = (df_normalized[col] - row_mins) / row_ranges.replace(0, np.nan)

# Reshape for heatmap
df_norm_cols = df_normalized[col_names].T
df_norm_cols.columns = df_normalized["time"]

# Create slot lookup for hover data
slot_values = df_normalized["slot"].values

# Build customdata: slot number for each column in the heatmap
customdata = np.array([[slot_values[j] for j in range(len(slot_values))] for _ in range(NUM_COLUMNS)])

fig = go.Figure(
    data=go.Heatmap(
        z=df_norm_cols.values,
        x=df_norm_cols.columns,
        y=[str(i) for i in range(NUM_COLUMNS)],
        colorscale="YlGnBu",
        reversescale=True,
        zmin=0,
        zmax=1,
        colorbar=horizontal_colorbar("Normalized"),
        customdata=customdata,
        hovertemplate="<b>Slot:</b> %{customdata}<br><b>Time:</b> %{x}<br><b>Column Index:</b> %{y}<br><b>Normalized:</b> %{z:.2f}<extra></extra>",
    )
)
fig.update_layout(
    margin=dict(l=10, r=10, t=10, b=80),
    xaxis=dict(automargin=True),
    yaxis=dict(title="Column", automargin=True, autorange="reversed"),
    height=800,
)
fig.show()

Column arrival spread

Time between when the first column arrives and when the last column arrives for each slot. A wide spread indicates some columns arriving much later than others.

Show code 
# Compute column spread (max - min across all columns per slot)
col_names = [f"c{i}" for i in range(NUM_COLUMNS)]
df_spread = df_col_first_seen.copy()
df_spread["column_spread_ms"] = df_spread[col_names].max(axis=1) - df_spread[col_names].min(axis=1)

fig = px.histogram(
    df_spread,
    x="column_spread_ms",
    nbins=60,
    color_discrete_sequence=["#EF553B"],
)
fig.update_layout(
    margin=dict(l=60, r=30, t=30, b=60),
    xaxis=dict(title="Column spread (ms)"),
    yaxis=dict(title="Slots"),
    height=400,
)
fig.show(config={"responsive": True})
Show code 
# Summary statistics
stats = df_spread["column_spread_ms"].describe(percentiles=[0.5, 0.9, 0.95, 0.99])
print("Column spread (ms):")
print(f"  Median: {stats['50%']:.0f}")
print(f"  P90:    {stats['90%']:.0f}")
print(f"  P95:    {stats['95%']:.0f}")
print(f"  P99:    {stats['99%']:.0f}")
print(f"  Max:    {stats['max']:.0f}")

Column spread (ms):
  Median: 117
  P90:    183
  P95:    213
  P99:    397
  Max:    3190

Column spread over time

How column spread varies throughout the day. Useful for spotting periods of degraded propagation.

Show code 
fig = px.scatter(
    df_spread,
    x="time",
    y="column_spread_ms",
    opacity=0.5,
    color_discrete_sequence=["#EF553B"],
    hover_data={"slot": True, "column_spread_ms": ":.0f", "time": False},
)
fig.update_layout(
    margin=dict(l=60, r=30, t=30, b=60),
    xaxis=dict(title="Time (UTC)", tickformat="%H:%M"),
    yaxis=dict(title="Column spread (ms)"),
    height=400,
)
fig.show(config={"responsive": True})

Missing columns

Slots where specific columns were never observed. Missing columns indicate gaps in network coverage for that column subnet.

Show code 
# Missing columns heatmap - shows gaps in network coverage
col_names = [f"c{i}" for i in range(NUM_COLUMNS)]

# Create boolean mask: True (1) where column is missing (NaN)
df_missing = df_col_first_seen[col_names].isna().astype(int).T
df_missing.columns = df_col_first_seen["time"]

# Count missing data
total_missing = df_col_first_seen[col_names].isna().sum().sum()
slots_with_missing = (df_col_first_seen[col_names].isna().any(axis=1)).sum()
print(f"Total missing column observations: {total_missing:,}")
print(f"Slots with at least one missing column: {slots_with_missing:,} ({slots_with_missing/len(df_col_first_seen)*100:.1f}%)")

if total_missing > 0:
    # Create slot lookup for hover data
    slot_values = df_col_first_seen["slot"].values
    customdata = np.array([[slot_values[j] for j in range(len(slot_values))] for _ in range(NUM_COLUMNS)])

    fig = go.Figure(
        data=go.Heatmap(
            z=df_missing.values,
            x=df_missing.columns,
            y=[str(i) for i in range(NUM_COLUMNS)],
            colorscale=[[0, "#E8E8E8"], [1, "#1E1E1E"]],
            zmin=0,
            zmax=1,
            showscale=False,
            customdata=customdata,
            hovertemplate="<b>Slot:</b> %{customdata}<br><b>Time:</b> %{x}<br><b>Column:</b> %{y}<br><b>Status:</b> Missing<extra></extra>",
        )
    )
    fig.update_layout(
        margin=dict(l=10, r=10, t=10, b=80),
        xaxis=dict(automargin=True),
        yaxis=dict(title="Column", automargin=True, autorange="reversed"),
        height=800,
    )
    fig.show()
else:
    print("No missing columns detected.")

Total missing column observations: 794
Slots with at least one missing column: 403 (7.5%)