adapter: mz_dataflow_shutdown_durations_histogram

This commit adds a new `mz_dataflow_shutdown_durations_histogram_raw`
introspection source, as well as convenience views on top of it. The new
source and views follow the existing conventions of
`mz_peek_durations_histogram_raw`, which has the same format.

Author: teskje

src/adapter/src/catalog/builtin.rs

@@ -1297,6 +1297,12 @@ pub const MZ_PEEK_DURATIONS_HISTOGRAM_RAW: BuiltinLog = BuiltinLog {
     variant: LogVariant::Compute(ComputeLog::PeekDuration),
 };
 
+pub const MZ_DATAFLOW_SHUTDOWN_DURATIONS_HISTOGRAM_RAW: BuiltinLog = BuiltinLog {
+    name: "mz_dataflow_shutdown_durations_histogram_raw",
+    schema: MZ_INTERNAL_SCHEMA,
+    variant: LogVariant::Compute(ComputeLog::ShutdownDuration),
+};
+
 pub const MZ_ARRANGEMENT_HEAP_SIZE_RAW: BuiltinLog = BuiltinLog {
     name: "mz_arrangement_heap_size_raw",
     schema: MZ_INTERNAL_SCHEMA,
@@ -2639,6 +2645,28 @@ FROM mz_internal.mz_peek_durations_histogram_per_worker
 GROUP BY duration_ns",
 };
 
+pub const MZ_DATAFLOW_SHUTDOWN_DURATIONS_HISTOGRAM_PER_WORKER: BuiltinView = BuiltinView {
+    name: "mz_dataflow_shutdown_durations_histogram_per_worker",
+    schema: MZ_INTERNAL_SCHEMA,
+    sql: "CREATE VIEW mz_internal.mz_dataflow_shutdown_durations_histogram_per_worker AS SELECT
+    worker_id, duration_ns, pg_catalog.count(*) AS count
+FROM
+    mz_internal.mz_dataflow_shutdown_durations_histogram_raw
+GROUP BY
+    worker_id, duration_ns",
+};
+
+pub const MZ_DATAFLOW_SHUTDOWN_DURATIONS_HISTOGRAM: BuiltinView = BuiltinView {
+    name: "mz_dataflow_shutdown_durations_histogram",
+    schema: MZ_INTERNAL_SCHEMA,
+    sql: "CREATE VIEW mz_internal.mz_dataflow_shutdown_durations_histogram AS
+SELECT
+    duration_ns,
+    pg_catalog.sum(count) AS count
+FROM mz_internal.mz_dataflow_shutdown_durations_histogram_per_worker
+GROUP BY duration_ns",
+};
+
 pub const MZ_SCHEDULING_ELAPSED_PER_WORKER: BuiltinView = BuiltinView {
     name: "mz_scheduling_elapsed_per_worker",
     schema: MZ_INTERNAL_SCHEMA,
@@ -3760,6 +3788,7 @@ pub static BUILTINS_STATIC: Lazy<Vec<Builtin<NameReference>>> = Lazy::new(|| {
         Builtin::Log(&MZ_MESSAGE_COUNTS_SENT_RAW),
         Builtin::Log(&MZ_ACTIVE_PEEKS_PER_WORKER),
         Builtin::Log(&MZ_PEEK_DURATIONS_HISTOGRAM_RAW),
+        Builtin::Log(&MZ_DATAFLOW_SHUTDOWN_DURATIONS_HISTOGRAM_RAW),
         Builtin::Log(&MZ_ARRANGEMENT_HEAP_CAPACITY_RAW),
         Builtin::Log(&MZ_ARRANGEMENT_HEAP_ALLOCATIONS_RAW),
         Builtin::Log(&MZ_ARRANGEMENT_HEAP_SIZE_RAW),
@@ -3848,6 +3877,8 @@ pub static BUILTINS_STATIC: Lazy<Vec<Builtin<NameReference>>> = Lazy::new(|| {
         Builtin::View(&MZ_RECORDS_PER_DATAFLOW),
         Builtin::View(&MZ_PEEK_DURATIONS_HISTOGRAM_PER_WORKER),
         Builtin::View(&MZ_PEEK_DURATIONS_HISTOGRAM),
+        Builtin::View(&MZ_DATAFLOW_SHUTDOWN_DURATIONS_HISTOGRAM_PER_WORKER),
+        Builtin::View(&MZ_DATAFLOW_SHUTDOWN_DURATIONS_HISTOGRAM),
         Builtin::View(&MZ_SCHEDULING_ELAPSED_PER_WORKER),
         Builtin::View(&MZ_SCHEDULING_ELAPSED),
         Builtin::View(&MZ_SCHEDULING_PARKS_HISTOGRAM_PER_WORKER),

test/sqllogictest/cluster.slt

@@ -201,6 +201,8 @@ bar  mz_dataflow_operator_reachability_raw  mz_dataflow_operator_reachability_ra
 bar  mz_dataflow_operator_reachability_raw  mz_dataflow_operator_reachability_raw_u6_primary_idx  5  time  NULL  true
 bar  mz_dataflow_operators_per_worker  mz_dataflow_operators_per_worker_u6_primary_idx  1  id  NULL  false
 bar  mz_dataflow_operators_per_worker  mz_dataflow_operators_per_worker_u6_primary_idx  2  worker_id  NULL  false
+bar  mz_dataflow_shutdown_durations_histogram_raw  mz_dataflow_shutdown_durations_histogram_raw_u6_primary_idx  1  worker_id  NULL  false
+bar  mz_dataflow_shutdown_durations_histogram_raw  mz_dataflow_shutdown_durations_histogram_raw_u6_primary_idx  2  duration_ns  NULL  false
 bar  mz_message_counts_received_raw  mz_message_counts_received_raw_u6_primary_idx  1  channel_id  NULL  false
 bar  mz_message_counts_received_raw  mz_message_counts_received_raw_u6_primary_idx  2  from_worker_id  NULL  false
 bar  mz_message_counts_received_raw  mz_message_counts_received_raw_u6_primary_idx  3  to_worker_id  NULL  false
@@ -360,11 +362,10 @@ DROP CLUSTER foo, foo2, foo3, foo4 CASCADE
 
 # Test that introspection source indexes are created and dropped correctly
 
-# There are 19 introspection sources
 query I
 SELECT COUNT(name) FROM mz_indexes WHERE cluster_id = 'u1';
 ----
-22
+23
 
 query I
 SELECT COUNT(name) FROM mz_indexes WHERE cluster_id <> 'u1' AND cluster_id NOT LIKE 's%';
@@ -377,15 +378,15 @@ CREATE CLUSTER test REPLICAS (foo (SIZE '1'));
 query I
 SELECT COUNT(name) FROM mz_indexes;
 ----
-110
+114
 
 statement ok
 DROP CLUSTER test CASCADE
 
 query T
 SELECT COUNT(name) FROM mz_indexes;
 ----
-88
+91
 
 statement error nvalid SIZE: must provide a string value
 CREATE CLUSTER REPLICA default.size_1 SIZE;

test/sqllogictest/information_schema_tables.slt

@@ -377,6 +377,18 @@ mz_dataflow_operators_per_worker
 SOURCE
 materialize
 mz_internal
+mz_dataflow_shutdown_durations_histogram
+VIEW
+materialize
+mz_internal
+mz_dataflow_shutdown_durations_histogram_per_worker
+VIEW
+materialize
+mz_internal
+mz_dataflow_shutdown_durations_histogram_raw
+SOURCE
+materialize
+mz_internal
 mz_dataflows
 VIEW
 materialize

test/testdrive/catalog.td

@@ -530,6 +530,7 @@ mz_dataflow_addresses_per_worker             log   <null>
 mz_dataflow_channels_per_worker              log   <null>
 mz_dataflow_operator_reachability_raw        log   <null>
 mz_dataflow_operators_per_worker             log   <null>
+mz_dataflow_shutdown_durations_histogram_raw log   <null>
 mz_message_counts_received_raw               log   <null>
 mz_message_counts_sent_raw                   log   <null>
 mz_peek_durations_histogram_raw              log   <null>
@@ -577,17 +578,19 @@ mz_compute_import_frontiers
 mz_compute_operator_durations_histogram
 mz_compute_operator_durations_histogram_per_worker
 mz_dataflow_addresses
+mz_dataflow_arrangement_sizes
 mz_dataflow_channel_operators
 mz_dataflow_channel_operators_per_worker
 mz_dataflow_channels
 mz_dataflow_operator_dataflows
 mz_dataflow_operator_dataflows_per_worker
 mz_dataflow_operator_parents
 mz_dataflow_operator_parents_per_worker
-mz_dataflow_arrangement_sizes
 mz_dataflow_operator_reachability
 mz_dataflow_operator_reachability_per_worker
 mz_dataflow_operators
+mz_dataflow_shutdown_durations_histogram
+mz_dataflow_shutdown_durations_histogram_per_worker
 mz_dataflows
 mz_dataflows_per_worker
 mz_message_counts
@@ -625,7 +628,7 @@ test_table
 
 # There is one entry in mz_indexes for each field_number/expression of the index.
 > SELECT COUNT(id) FROM mz_indexes WHERE id LIKE 's%'
-88
+91
 
 # Create a second schema with the same table name as above
 > CREATE SCHEMA tester2

test/testdrive/divergent-dataflow-cancellation.td

@@ -93,7 +93,7 @@ contains: canceling statement due to statement timeout
 > SELECT count(*)
   FROM mz_internal.mz_compute_frontiers_per_worker
   WHERE worker_id = 0
-22
+23
 
 > SELECT count(*) FROM mz_internal.mz_compute_import_frontiers_per_worker
 0
@@ -113,6 +113,10 @@ contains: canceling statement due to statement timeout
 > SELECT count(*) FROM mz_internal.mz_dataflow_operators_per_worker
 0
 
+# This source never sees retractions.
+> SELECT count(*) > 0 FROM mz_internal.mz_dataflow_shutdown_durations_histogram_raw
+true
+
 > SELECT count(*) FROM mz_internal.mz_message_counts_received_raw
 0
 

test/testdrive/indexes.td

@@ -308,6 +308,7 @@ mz_dataflow_addresses_per_worker_s2_primary_idx             mz_dataflow_addresse
 mz_dataflow_channels_per_worker_s2_primary_idx              mz_dataflow_channels_per_worker              mz_introspection    {id,worker_id}
 mz_dataflow_operator_reachability_raw_s2_primary_idx        mz_dataflow_operator_reachability_raw        mz_introspection    {address,port,worker_id,update_type,time}
 mz_dataflow_operators_per_worker_s2_primary_idx             mz_dataflow_operators_per_worker             mz_introspection    {id,worker_id}
+mz_dataflow_shutdown_durations_histogram_raw_s2_primary_idx mz_dataflow_shutdown_durations_histogram_raw mz_introspection    {worker_id,duration_ns}
 mz_message_counts_received_raw_s2_primary_idx               mz_message_counts_received_raw               mz_introspection    {channel_id,from_worker_id,to_worker_id}
 mz_message_counts_sent_raw_s2_primary_idx                   mz_message_counts_sent_raw                   mz_introspection    {channel_id,from_worker_id,to_worker_id}
 mz_peek_durations_histogram_raw_s2_primary_idx              mz_peek_durations_histogram_raw              mz_introspection    {worker_id,duration_ns}

test/testdrive/logging.td

@@ -75,6 +75,9 @@ $ set-regex match=s\d+ replacement=SID
 > SELECT count(*) FROM (SELECT count (*) FROM mz_internal.mz_peek_durations_histogram);
 1
 
+> SELECT count(*) FROM (SELECT count (*) FROM mz_internal.mz_dataflow_shutdown_durations_histogram);
+1
+
 > SELECT count(*) FROM (SELECT count (*) FROM mz_internal.mz_records_per_dataflow);
 1