Use the generation time, not last seen time, for update timestamps

When we write an update, we have to include a timestamp that
represents both the the time that the client will use to fetch the
next diff and the timestamp used as a reference point when adding
updates to the network graph (the client actually uses the included
timestamp minus two weeks).

Because of the second use, updates were being generated with the
highest timestmap of all included updates, rounded down to the
nearest update-interval multiple (because of the first use).

In practice, because we expect to see many updates in any hour
window, this meant we were really calculating the generation
reference timestamp in a very indirect way. Here we simply change
to using the reference timestamp directly.

This updates regtest results when generating an initial sync
against an empty graph to include a real timestamp, rather than 0,
though this shouldn't impact clients' correctness.

Fixes #95

Author: TheBlueMatt

src/lib.rs

@@ -203,7 +203,9 @@ async fn calculate_delta<L: Deref + Clone>(network_graph: Arc<NetworkGraph<L>>,
 	serialization::serialize_delta_set(delta_set, node_delta_set, last_sync_timestamp)
 }
 
-fn serialize_delta<L: Deref + Clone>(serialization_details: &SerializationSet, serialization_version: u8, logger: L) -> SerializedResponse where L::Target: Logger {
+fn serialize_delta<L: Deref + Clone>(
+	serialization_details: &SerializationSet, serialization_time: u32, serialization_version: u8, logger: L,
+) -> SerializedResponse where L::Target: Logger {
 	let mut output: Vec<u8> = vec![];
 	let snapshot_interval = config::snapshot_generation_interval();
 
@@ -277,11 +279,9 @@ fn serialize_delta<L: Deref + Clone>(serialization_details: &SerializationSet, s
 
 	// always write the chain hash
 	serialization_details.chain_hash.write(&mut prefixed_output).unwrap();
-	// always write the latest seen timestamp
-	let latest_seen_timestamp = serialization_details.latest_seen;
-	let overflow_seconds = latest_seen_timestamp % snapshot_interval;
-	let serialized_seen_timestamp = latest_seen_timestamp.saturating_sub(overflow_seconds);
-	serialized_seen_timestamp.write(&mut prefixed_output).unwrap();
+	// always write the time we're generating this object
+	assert_eq!(serialization_time % snapshot_interval, 0);
+	serialization_time.write(&mut prefixed_output).unwrap();
 
 	if serialization_version >= 2 { // serialize the most common node features
 		for mutated_node_id in serialization_details.node_mutations.keys() {
@@ -389,7 +389,7 @@ fn serialize_delta<L: Deref + Clone>(serialization_details: &SerializationSet, s
 	prefixed_output.append(&mut output);
 
 	log_info!(logger, "duplicated node ids: {}", duplicate_node_ids);
-	log_info!(logger, "latest seen timestamp: {:?}", serialization_details.latest_seen);
+	log_info!(logger, "generated at: {}", serialization_time);
 
 	SerializedResponse {
 		data: prefixed_output,

src/serialization.rs

@@ -1,4 +1,3 @@
-use std::cmp::max;
 use std::collections::HashMap;
 use std::time::{SystemTime, UNIX_EPOCH};
 
@@ -16,7 +15,6 @@ pub(super) struct SerializationSet {
 	pub(super) full_update_defaults: DefaultUpdateValues,
 	pub(super) node_announcement_feature_defaults: Vec<NodeFeatures>,
 	pub(super) node_mutations: NodeDeltaSet,
-	pub(super) latest_seen: u32,
 	pub(super) chain_hash: ChainHash,
 }
 
@@ -130,7 +128,6 @@ pub(super) fn serialize_delta_set(channel_delta_set: DeltaSet, node_delta_set: N
 		node_announcement_feature_defaults: vec![],
 		node_mutations: Default::default(),
 		chain_hash: ChainHash::using_genesis_block(config::network()),
-		latest_seen: 0,
 	};
 
 	let mut full_update_histograms = FullUpdateValueHistograms {
@@ -165,7 +162,6 @@ pub(super) fn serialize_delta_set(channel_delta_set: DeltaSet, node_delta_set: N
 		};
 		let send_announcement = is_new_announcement || is_newly_included_announcement;
 		if send_announcement {
-			serialization_set.latest_seen = max(serialization_set.latest_seen, current_announcement_seen);
 			serialization_set.announcements.push(channel_delta.announcement.unwrap().announcement);
 		}
 
@@ -178,10 +174,6 @@ pub(super) fn serialize_delta_set(channel_delta_set: DeltaSet, node_delta_set: N
 					let latest_update = latest_update_delta.update;
 					assert_eq!(latest_update.short_channel_id, scid, "Update in DB had wrong SCID column");
 
-					// the returned seen timestamp should be the latest of all the returned
-					// announcements and latest updates
-					serialization_set.latest_seen = max(serialization_set.latest_seen, latest_update_delta.seen);
-
 					if let Some(update_delta) = updates.last_update_before_seen {
 						let mutated_properties = updates.mutated_properties;
 						if send_announcement || mutated_properties.len() == 5 || update_delta.seen <= non_incremental_previous_update_threshold_timestamp {

src/snapshot.rs

@@ -73,6 +73,7 @@ impl<L: Deref + Clone> Snapshotter<L> where L::Target: Logger {
 		let snapshot_generation_time = SystemTime::now();
 		let snapshot_generation_timestamp = snapshot_generation_time.duration_since(UNIX_EPOCH).unwrap().as_secs();
 		let reference_timestamp = Self::round_down_to_nearest_multiple(snapshot_generation_timestamp, snapshot_interval);
+		let serialized_timestamp: u32 = reference_timestamp.try_into().unwrap();
 		log_info!(self.logger, "Capturing snapshots at {} for: {}", snapshot_generation_timestamp, reference_timestamp);
 
 		// 2. sleep until the next round interval
@@ -121,8 +122,8 @@ impl<L: Deref + Clone> Snapshotter<L> where L::Target: Logger {
 				log_info!(self.logger, "Calculating {}-second snapshot", current_scope);
 				// calculate the snapshot
 				let delta = super::calculate_delta(network_graph_clone.clone(), current_last_sync_timestamp.clone() as u32, Some(reference_timestamp), self.logger.clone()).await;
-				let snapshot_v1 = super::serialize_delta(&delta, 1, self.logger.clone());
-				let snapshot_v2 = super::serialize_delta(&delta, 2, self.logger.clone());
+				let snapshot_v1 = super::serialize_delta(&delta, serialized_timestamp, 1, self.logger.clone());
+				let snapshot_v2 = super::serialize_delta(&delta, serialized_timestamp, 2, self.logger.clone());
 
 				// persist the snapshot and update the symlink
 				let snapshot_filename = format!("snapshot__calculated-at:{}__range:{}-scope__previous-sync:{}.lngossip", reference_timestamp, current_scope, current_last_sync_timestamp);