diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index 1b44339d..4d9cfcf5 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -38,3 +38,12 @@ runs: shell: bash run: | go mod download + + - name: Tidy Go modules + shell: bash + run: | + # Ensure go.mod/go.sum are up to date to prevent CI build failures + go mod tidy + if [ -f sn-manager/go.mod ]; then + (cd sn-manager && go mod tidy) + fi diff --git a/README.md b/README.md index 0aecceb2..8edccf03 100644 --- a/README.md +++ b/README.md @@ -205,125 +205,7 @@ enum SupernodeEventType { ## HTTP Gateway -The supernode provides an HTTP gateway that exposes the gRPC services via REST API on port `8002`. - -See `docs/gateway.md` for the full gateway guide and additional examples. - -### Example: GET /api/v1/status -Returns the current supernode status including system resources (CPU, memory, storage) and service information. - -- Query parameter `include_p2p_metrics=true` enables detailed P2P metrics and peer info. -- When omitted or false, peer count, peer addresses, and `p2p_metrics` are not included. - -```bash -curl "http://localhost:8002/api/v1/status" -``` - -Response (without P2P metrics): -```json -{ - "version": "1.0.0", - "uptime_seconds": "3600", - "resources": { - "cpu": { - "usage_percent": 15.2, - "cores": 8 - }, - "memory": { - "total_gb": 32.0, - "used_gb": 16.0, - "available_gb": 16.0, - "usage_percent": 50.0 - }, - "storage_volumes": [ - { - "path": "/", - "total_bytes": "500000000000", - "used_bytes": "250000000000", - "available_bytes": "250000000000", - "usage_percent": 50.0 - } - ], - "hardware_summary": "8 cores / 32GB RAM" - }, - "running_tasks": [ - { - "service_name": "cascade", - "task_ids": ["task1", "task2"], - "task_count": 2 - } - ], - "registered_services": ["cascade", "sense"], - "network": {}, - "rank": 6, - "ip_address": "192.168.1.100:4445" -} -``` - -To include P2P metrics and peer information: - -```bash -curl "http://localhost:8002/api/v1/status?include_p2p_metrics=true" -``` - -Response (with P2P metrics): - -```json -{ - "version": "1.0.0", - "uptime_seconds": "3600", - "resources": { /* ... */ }, - "running_tasks": [ /* ... */ ], - "registered_services": ["cascade", "sense"], - "network": { - "peers_count": 11, - "peer_addresses": [ - "lumera13z...@156.67.29.226:4445", - "lumera1s5...@18.216.80.56:4445" - ] - }, - "rank": 6, - "ip_address": "192.168.1.100:4445", - "p2p_metrics": { - "dht_metrics": { - "store_success_recent": [ /* ... */ ], - "batch_retrieve_recent": [ /* ... */ ], - "hot_path_banned_skips": 0, - "hot_path_ban_increments": 0 - }, - "network_handle_metrics": { "STORE": {"total": 42, "success": 40, "failure": 1, "timeout": 1} }, - "conn_pool_metrics": { "active": 12, "idle": 3 }, - "ban_list": [ /* ... */ ], - "database": { "p2p_db_size_mb": 123.4, "p2p_db_records_count": "1000" }, - "disk": { "all_mb": 102400, "used_mb": 51200, "free_mb": 51200 } - } -} -``` - -#### GET /api/v1/services -Returns the list of available services on the supernode. - -```bash -curl http://localhost:8002/api/v1/services -``` - -Response: -```json -{ - "services": ["cascade", "sense"] -} -``` - -The gateway automatically translates between HTTP/JSON and gRPC/protobuf formats, making it easy to integrate with web applications and monitoring tools. - -### API Documentation - -The gateway provides interactive API documentation via Swagger UI: - -- **Swagger UI**: http://localhost:8002/swagger-ui/ -- **OpenAPI Spec**: http://localhost:8002/swagger.json - -The Swagger UI provides an interactive interface to explore and test all available API endpoints. +See docs/gateway.md for the full gateway guide (endpoints, examples, Swagger links). ## CLI Commands diff --git a/docs/gateway.md b/docs/gateway.md index 9f7d5be3..638e2794 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -1,31 +1,40 @@ # Supernode HTTP Gateway -The HTTP gateway exposes the gRPC services via REST on port `8002` using grpc-gateway. +The Supernode exposes its gRPC services via an HTTP/JSON gateway on port `8002`. -## Endpoints +- Swagger UI: http://localhost:8002/swagger-ui/ +- OpenAPI Spec: http://localhost:8002/swagger.json -### GET `/api/v1/status` -Returns supernode status: system resources (CPU, memory, storage), service info, and optionally P2P metrics. +## Status API -- Query `include_p2p_metrics=true` enables detailed P2P metrics and peer info. -- When omitted or false, peer count, peer addresses, and `p2p_metrics` are not included. +GET `/api/v1/status` -Examples: +Returns the current supernode status including system resources (CPU, memory, storage), running tasks, registered services, network info, and codec configuration. +- Query `include_p2p_metrics=true` adds detailed P2P metrics and peer information. + +Example: ```bash -# Lightweight status curl "http://localhost:8002/api/v1/status" +``` -# Include P2P metrics and peer info +With P2P metrics: +```bash curl "http://localhost:8002/api/v1/status?include_p2p_metrics=true" ``` -Example responses are shown in the main README under the SupernodeService section. +## Services API -## API Documentation +GET `/api/v1/services` -- Swagger UI: `http://localhost:8002/swagger-ui/` -- OpenAPI Spec: `http://localhost:8002/swagger.json` +Returns the list of available services and methods exposed by this supernode. + +Example: +```bash +curl http://localhost:8002/api/v1/services +``` -The Swagger UI provides an interactive interface to explore and test all available API endpoints. +## Notes +- The gateway translates between HTTP/JSON and gRPC/protobuf, enabling easy integration with web tooling and monitoring. +- Interactive exploration is available via Swagger UI. diff --git a/gen/supernode/supernode.pb.go b/gen/supernode/supernode.pb.go index 5410f5c6..a14551a3 100644 --- a/gen/supernode/supernode.pb.go +++ b/gen/supernode/supernode.pb.go @@ -7,12 +7,11 @@ package supernode import ( - reflect "reflect" - sync "sync" - _ "google.golang.org/genproto/googleapis/api/annotations" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) const ( @@ -226,6 +225,7 @@ type StatusResponse struct { Rank int32 `protobuf:"varint,7,opt,name=rank,proto3" json:"rank,omitempty"` // Rank in the top supernodes list (0 if not in top list) IpAddress string `protobuf:"bytes,8,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"` // Supernode IP address with port (e.g., "192.168.1.1:4445") P2PMetrics *StatusResponse_P2PMetrics `protobuf:"bytes,9,opt,name=p2p_metrics,json=p2pMetrics,proto3" json:"p2p_metrics,omitempty"` + Codec *StatusResponse_CodecConfig `protobuf:"bytes,10,opt,name=codec,proto3" json:"codec,omitempty"` } func (x *StatusResponse) Reset() { @@ -321,6 +321,13 @@ func (x *StatusResponse) GetP2PMetrics() *StatusResponse_P2PMetrics { return nil } +func (x *StatusResponse) GetCodec() *StatusResponse_CodecConfig { + if x != nil { + return x.Codec + } + return nil +} + // System resource information type StatusResponse_Resources struct { state protoimpl.MessageState @@ -593,6 +600,124 @@ func (x *StatusResponse_P2PMetrics) GetDisk() *StatusResponse_P2PMetrics_DiskSta return nil } +// RaptorQ codec configuration (effective values) +type StatusResponse_CodecConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + SymbolSize uint32 `protobuf:"varint,1,opt,name=symbol_size,json=symbolSize,proto3" json:"symbol_size,omitempty"` // bytes (typically 65535) + Redundancy uint32 `protobuf:"varint,2,opt,name=redundancy,proto3" json:"redundancy,omitempty"` // repair factor (percent-like scalar; 5 = default) + MaxMemoryMb uint64 `protobuf:"varint,3,opt,name=max_memory_mb,json=maxMemoryMb,proto3" json:"max_memory_mb,omitempty"` // memory cap for native decoder + Concurrency uint32 `protobuf:"varint,4,opt,name=concurrency,proto3" json:"concurrency,omitempty"` // native decoder parallelism + Profile string `protobuf:"bytes,5,opt,name=profile,proto3" json:"profile,omitempty"` // selected profile: edge|standard|perf + HeadroomPct int32 `protobuf:"varint,6,opt,name=headroom_pct,json=headroomPct,proto3" json:"headroom_pct,omitempty"` // reserved memory percentage (0-90) + MemLimitMb uint64 `protobuf:"varint,7,opt,name=mem_limit_mb,json=memLimitMb,proto3" json:"mem_limit_mb,omitempty"` // detected memory limit (MB) + MemLimitSource string `protobuf:"bytes,8,opt,name=mem_limit_source,json=memLimitSource,proto3" json:"mem_limit_source,omitempty"` // detection source (cgroup/meminfo) + EffectiveCores int32 `protobuf:"varint,9,opt,name=effective_cores,json=effectiveCores,proto3" json:"effective_cores,omitempty"` // detected cores/quota + CpuLimitSource string `protobuf:"bytes,10,opt,name=cpu_limit_source,json=cpuLimitSource,proto3" json:"cpu_limit_source,omitempty"` // detection source (cgroups/NumCPU) +} + +func (x *StatusResponse_CodecConfig) Reset() { + *x = StatusResponse_CodecConfig{} + mi := &file_supernode_supernode_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StatusResponse_CodecConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StatusResponse_CodecConfig) ProtoMessage() {} + +func (x *StatusResponse_CodecConfig) ProtoReflect() protoreflect.Message { + mi := &file_supernode_supernode_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StatusResponse_CodecConfig.ProtoReflect.Descriptor instead. +func (*StatusResponse_CodecConfig) Descriptor() ([]byte, []int) { + return file_supernode_supernode_proto_rawDescGZIP(), []int{4, 4} +} + +func (x *StatusResponse_CodecConfig) GetSymbolSize() uint32 { + if x != nil { + return x.SymbolSize + } + return 0 +} + +func (x *StatusResponse_CodecConfig) GetRedundancy() uint32 { + if x != nil { + return x.Redundancy + } + return 0 +} + +func (x *StatusResponse_CodecConfig) GetMaxMemoryMb() uint64 { + if x != nil { + return x.MaxMemoryMb + } + return 0 +} + +func (x *StatusResponse_CodecConfig) GetConcurrency() uint32 { + if x != nil { + return x.Concurrency + } + return 0 +} + +func (x *StatusResponse_CodecConfig) GetProfile() string { + if x != nil { + return x.Profile + } + return "" +} + +func (x *StatusResponse_CodecConfig) GetHeadroomPct() int32 { + if x != nil { + return x.HeadroomPct + } + return 0 +} + +func (x *StatusResponse_CodecConfig) GetMemLimitMb() uint64 { + if x != nil { + return x.MemLimitMb + } + return 0 +} + +func (x *StatusResponse_CodecConfig) GetMemLimitSource() string { + if x != nil { + return x.MemLimitSource + } + return "" +} + +func (x *StatusResponse_CodecConfig) GetEffectiveCores() int32 { + if x != nil { + return x.EffectiveCores + } + return 0 +} + +func (x *StatusResponse_CodecConfig) GetCpuLimitSource() string { + if x != nil { + return x.CpuLimitSource + } + return "" +} + type StatusResponse_Resources_CPU struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -604,7 +729,7 @@ type StatusResponse_Resources_CPU struct { func (x *StatusResponse_Resources_CPU) Reset() { *x = StatusResponse_Resources_CPU{} - mi := &file_supernode_supernode_proto_msgTypes[9] + mi := &file_supernode_supernode_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -616,7 +741,7 @@ func (x *StatusResponse_Resources_CPU) String() string { func (*StatusResponse_Resources_CPU) ProtoMessage() {} func (x *StatusResponse_Resources_CPU) ProtoReflect() protoreflect.Message { - mi := &file_supernode_supernode_proto_msgTypes[9] + mi := &file_supernode_supernode_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -659,7 +784,7 @@ type StatusResponse_Resources_Memory struct { func (x *StatusResponse_Resources_Memory) Reset() { *x = StatusResponse_Resources_Memory{} - mi := &file_supernode_supernode_proto_msgTypes[10] + mi := &file_supernode_supernode_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -671,7 +796,7 @@ func (x *StatusResponse_Resources_Memory) String() string { func (*StatusResponse_Resources_Memory) ProtoMessage() {} func (x *StatusResponse_Resources_Memory) ProtoReflect() protoreflect.Message { - mi := &file_supernode_supernode_proto_msgTypes[10] + mi := &file_supernode_supernode_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -729,7 +854,7 @@ type StatusResponse_Resources_Storage struct { func (x *StatusResponse_Resources_Storage) Reset() { *x = StatusResponse_Resources_Storage{} - mi := &file_supernode_supernode_proto_msgTypes[11] + mi := &file_supernode_supernode_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -741,7 +866,7 @@ func (x *StatusResponse_Resources_Storage) String() string { func (*StatusResponse_Resources_Storage) ProtoMessage() {} func (x *StatusResponse_Resources_Storage) ProtoReflect() protoreflect.Message { - mi := &file_supernode_supernode_proto_msgTypes[11] + mi := &file_supernode_supernode_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -806,7 +931,7 @@ type StatusResponse_P2PMetrics_DhtMetrics struct { func (x *StatusResponse_P2PMetrics_DhtMetrics) Reset() { *x = StatusResponse_P2PMetrics_DhtMetrics{} - mi := &file_supernode_supernode_proto_msgTypes[12] + mi := &file_supernode_supernode_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -818,7 +943,7 @@ func (x *StatusResponse_P2PMetrics_DhtMetrics) String() string { func (*StatusResponse_P2PMetrics_DhtMetrics) ProtoMessage() {} func (x *StatusResponse_P2PMetrics_DhtMetrics) ProtoReflect() protoreflect.Message { - mi := &file_supernode_supernode_proto_msgTypes[12] + mi := &file_supernode_supernode_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -876,7 +1001,7 @@ type StatusResponse_P2PMetrics_HandleCounters struct { func (x *StatusResponse_P2PMetrics_HandleCounters) Reset() { *x = StatusResponse_P2PMetrics_HandleCounters{} - mi := &file_supernode_supernode_proto_msgTypes[13] + mi := &file_supernode_supernode_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -888,7 +1013,7 @@ func (x *StatusResponse_P2PMetrics_HandleCounters) String() string { func (*StatusResponse_P2PMetrics_HandleCounters) ProtoMessage() {} func (x *StatusResponse_P2PMetrics_HandleCounters) ProtoReflect() protoreflect.Message { - mi := &file_supernode_supernode_proto_msgTypes[13] + mi := &file_supernode_supernode_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -948,7 +1073,7 @@ type StatusResponse_P2PMetrics_BanEntry struct { func (x *StatusResponse_P2PMetrics_BanEntry) Reset() { *x = StatusResponse_P2PMetrics_BanEntry{} - mi := &file_supernode_supernode_proto_msgTypes[14] + mi := &file_supernode_supernode_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -960,7 +1085,7 @@ func (x *StatusResponse_P2PMetrics_BanEntry) String() string { func (*StatusResponse_P2PMetrics_BanEntry) ProtoMessage() {} func (x *StatusResponse_P2PMetrics_BanEntry) ProtoReflect() protoreflect.Message { - mi := &file_supernode_supernode_proto_msgTypes[14] + mi := &file_supernode_supernode_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1030,7 +1155,7 @@ type StatusResponse_P2PMetrics_DatabaseStats struct { func (x *StatusResponse_P2PMetrics_DatabaseStats) Reset() { *x = StatusResponse_P2PMetrics_DatabaseStats{} - mi := &file_supernode_supernode_proto_msgTypes[15] + mi := &file_supernode_supernode_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1042,7 +1167,7 @@ func (x *StatusResponse_P2PMetrics_DatabaseStats) String() string { func (*StatusResponse_P2PMetrics_DatabaseStats) ProtoMessage() {} func (x *StatusResponse_P2PMetrics_DatabaseStats) ProtoReflect() protoreflect.Message { - mi := &file_supernode_supernode_proto_msgTypes[15] + mi := &file_supernode_supernode_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1085,7 +1210,7 @@ type StatusResponse_P2PMetrics_DiskStatus struct { func (x *StatusResponse_P2PMetrics_DiskStatus) Reset() { *x = StatusResponse_P2PMetrics_DiskStatus{} - mi := &file_supernode_supernode_proto_msgTypes[16] + mi := &file_supernode_supernode_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1097,7 +1222,7 @@ func (x *StatusResponse_P2PMetrics_DiskStatus) String() string { func (*StatusResponse_P2PMetrics_DiskStatus) ProtoMessage() {} func (x *StatusResponse_P2PMetrics_DiskStatus) ProtoReflect() protoreflect.Message { - mi := &file_supernode_supernode_proto_msgTypes[16] + mi := &file_supernode_supernode_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1147,7 +1272,7 @@ type StatusResponse_P2PMetrics_DhtMetrics_StoreSuccessPoint struct { func (x *StatusResponse_P2PMetrics_DhtMetrics_StoreSuccessPoint) Reset() { *x = StatusResponse_P2PMetrics_DhtMetrics_StoreSuccessPoint{} - mi := &file_supernode_supernode_proto_msgTypes[19] + mi := &file_supernode_supernode_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1159,7 +1284,7 @@ func (x *StatusResponse_P2PMetrics_DhtMetrics_StoreSuccessPoint) String() string func (*StatusResponse_P2PMetrics_DhtMetrics_StoreSuccessPoint) ProtoMessage() {} func (x *StatusResponse_P2PMetrics_DhtMetrics_StoreSuccessPoint) ProtoReflect() protoreflect.Message { - mi := &file_supernode_supernode_proto_msgTypes[19] + mi := &file_supernode_supernode_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1218,7 +1343,7 @@ type StatusResponse_P2PMetrics_DhtMetrics_BatchRetrievePoint struct { func (x *StatusResponse_P2PMetrics_DhtMetrics_BatchRetrievePoint) Reset() { *x = StatusResponse_P2PMetrics_DhtMetrics_BatchRetrievePoint{} - mi := &file_supernode_supernode_proto_msgTypes[20] + mi := &file_supernode_supernode_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1230,7 +1355,7 @@ func (x *StatusResponse_P2PMetrics_DhtMetrics_BatchRetrievePoint) String() strin func (*StatusResponse_P2PMetrics_DhtMetrics_BatchRetrievePoint) ProtoMessage() {} func (x *StatusResponse_P2PMetrics_DhtMetrics_BatchRetrievePoint) ProtoReflect() protoreflect.Message { - mi := &file_supernode_supernode_proto_msgTypes[20] + mi := &file_supernode_supernode_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1310,7 +1435,7 @@ var file_supernode_supernode_proto_rawDesc = []byte{ 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x73, 0x22, 0x84, 0x19, 0x0a, 0x0e, + 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x73, 0x22, 0xb4, 0x1c, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x75, 0x70, 0x74, 0x69, @@ -1339,179 +1464,206 @@ var file_supernode_supernode_proto_rawDesc = []byte{ 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x32, 0x50, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x0a, 0x70, 0x32, 0x70, - 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x1a, 0x82, 0x05, 0x0a, 0x09, 0x52, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x39, 0x0a, 0x03, 0x63, 0x70, 0x75, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2e, 0x43, 0x50, 0x55, 0x52, 0x03, 0x63, 0x70, 0x75, - 0x12, 0x42, 0x0a, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x2a, 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x73, 0x2e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x52, 0x06, 0x6d, 0x65, - 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x54, 0x0a, 0x0f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, - 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, + 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x3b, 0x0a, 0x05, 0x63, 0x6f, 0x64, 0x65, 0x63, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, + 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2e, 0x43, 0x6f, 0x64, 0x65, 0x63, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x05, 0x63, + 0x6f, 0x64, 0x65, 0x63, 0x1a, 0x82, 0x05, 0x0a, 0x09, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x73, 0x12, 0x39, 0x0a, 0x03, 0x63, 0x70, 0x75, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x27, 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x73, 0x2e, 0x43, 0x50, 0x55, 0x52, 0x03, 0x63, 0x70, 0x75, 0x12, 0x42, 0x0a, + 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x73, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x0e, 0x73, 0x74, 0x6f, 0x72, - 0x61, 0x67, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x68, 0x61, - 0x72, 0x64, 0x77, 0x61, 0x72, 0x65, 0x5f, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x68, 0x61, 0x72, 0x64, 0x77, 0x61, 0x72, 0x65, 0x53, 0x75, - 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x1a, 0x40, 0x0a, 0x03, 0x43, 0x50, 0x55, 0x12, 0x23, 0x0a, 0x0d, - 0x75, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x01, 0x52, 0x0c, 0x75, 0x73, 0x61, 0x67, 0x65, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, - 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x1a, 0x84, 0x01, 0x0a, 0x06, 0x4d, 0x65, 0x6d, 0x6f, - 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x67, 0x62, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x47, 0x62, 0x12, 0x17, 0x0a, - 0x07, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x67, 0x62, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x06, - 0x75, 0x73, 0x65, 0x64, 0x47, 0x62, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, - 0x62, 0x6c, 0x65, 0x5f, 0x67, 0x62, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x61, 0x76, - 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, 0x62, 0x12, 0x23, 0x0a, 0x0d, 0x75, 0x73, 0x61, - 0x67, 0x65, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, - 0x52, 0x0c, 0x75, 0x73, 0x61, 0x67, 0x65, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x1a, 0xab, - 0x01, 0x0a, 0x07, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, - 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x1f, - 0x0a, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, - 0x1d, 0x0a, 0x0a, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x09, 0x75, 0x73, 0x65, 0x64, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x27, - 0x0a, 0x0f, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, - 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, - 0x6c, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x75, 0x73, 0x61, 0x67, 0x65, - 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0c, - 0x75, 0x73, 0x61, 0x67, 0x65, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x1a, 0x6b, 0x0a, 0x0c, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x21, 0x0a, 0x0c, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, - 0x19, 0x0a, 0x08, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x07, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x61, - 0x73, 0x6b, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, - 0x74, 0x61, 0x73, 0x6b, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x1a, 0x51, 0x0a, 0x07, 0x4e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x65, 0x65, 0x72, 0x73, 0x5f, 0x63, 0x6f, - 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x73, - 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x61, 0x64, - 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x70, - 0x65, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x1a, 0xf3, 0x0e, 0x0a, - 0x0a, 0x50, 0x32, 0x50, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x50, 0x0a, 0x0b, 0x64, - 0x68, 0x74, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x2f, 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x32, 0x50, 0x4d, - 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x44, 0x68, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x73, 0x52, 0x0a, 0x64, 0x68, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x74, 0x0a, - 0x16, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x5f, - 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3e, 0x2e, + 0x65, 0x73, 0x2e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x52, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, + 0x79, 0x12, 0x54, 0x0a, 0x0f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x76, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x73, 0x75, 0x70, + 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2e, + 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, + 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x68, 0x61, 0x72, 0x64, 0x77, + 0x61, 0x72, 0x65, 0x5f, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0f, 0x68, 0x61, 0x72, 0x64, 0x77, 0x61, 0x72, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, + 0x72, 0x79, 0x1a, 0x40, 0x0a, 0x03, 0x43, 0x50, 0x55, 0x12, 0x23, 0x0a, 0x0d, 0x75, 0x73, 0x61, + 0x67, 0x65, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, + 0x52, 0x0c, 0x75, 0x73, 0x61, 0x67, 0x65, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x12, 0x14, + 0x0a, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x63, + 0x6f, 0x72, 0x65, 0x73, 0x1a, 0x84, 0x01, 0x0a, 0x06, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, + 0x19, 0x0a, 0x08, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x67, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x01, 0x52, 0x07, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x47, 0x62, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, + 0x65, 0x64, 0x5f, 0x67, 0x62, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x06, 0x75, 0x73, 0x65, + 0x64, 0x47, 0x62, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, + 0x5f, 0x67, 0x62, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x61, 0x76, 0x61, 0x69, 0x6c, + 0x61, 0x62, 0x6c, 0x65, 0x47, 0x62, 0x12, 0x23, 0x0a, 0x0d, 0x75, 0x73, 0x61, 0x67, 0x65, 0x5f, + 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0c, 0x75, + 0x73, 0x61, 0x67, 0x65, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x1a, 0xab, 0x01, 0x0a, 0x07, + 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x1f, 0x0a, 0x0b, 0x74, + 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, + 0x75, 0x73, 0x65, 0x64, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x09, 0x75, 0x73, 0x65, 0x64, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x61, + 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x42, + 0x79, 0x74, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x75, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x70, 0x65, + 0x72, 0x63, 0x65, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0c, 0x75, 0x73, 0x61, + 0x67, 0x65, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x1a, 0x6b, 0x0a, 0x0c, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x19, 0x0a, 0x08, + 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, + 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x61, 0x73, 0x6b, 0x5f, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x74, 0x61, 0x73, + 0x6b, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x1a, 0x51, 0x0a, 0x07, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, + 0x6b, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x65, 0x65, 0x72, 0x73, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x73, 0x43, 0x6f, 0x75, + 0x6e, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, + 0x73, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x70, 0x65, 0x65, 0x72, + 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x1a, 0xf3, 0x0e, 0x0a, 0x0a, 0x50, 0x32, + 0x50, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x50, 0x0a, 0x0b, 0x64, 0x68, 0x74, 0x5f, + 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x32, 0x50, 0x4d, 0x65, 0x74, 0x72, - 0x69, 0x63, 0x73, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x48, 0x61, 0x6e, 0x64, 0x6c, - 0x65, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x14, 0x6e, - 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x72, - 0x69, 0x63, 0x73, 0x12, 0x65, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x6e, 0x5f, 0x70, 0x6f, 0x6f, 0x6c, - 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, - 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x32, 0x50, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x50, 0x6f, 0x6f, 0x6c, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x6e, 0x50, - 0x6f, 0x6f, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x48, 0x0a, 0x08, 0x62, 0x61, - 0x6e, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x73, - 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x32, 0x50, 0x4d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x2e, 0x42, 0x61, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x62, 0x61, 0x6e, - 0x4c, 0x69, 0x73, 0x74, 0x12, 0x4e, 0x0a, 0x08, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, - 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x50, 0x32, 0x50, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x44, 0x61, 0x74, - 0x61, 0x62, 0x61, 0x73, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x08, 0x64, 0x61, 0x74, 0x61, - 0x62, 0x61, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x04, 0x64, 0x69, 0x73, 0x6b, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x32, - 0x50, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x04, 0x64, 0x69, 0x73, 0x6b, 0x1a, 0xc0, 0x05, 0x0a, 0x0a, 0x44, 0x68, - 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x73, 0x0a, 0x14, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x72, 0x65, 0x63, 0x65, 0x6e, 0x74, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, - 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x50, 0x32, 0x50, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x44, 0x68, 0x74, - 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x75, 0x63, - 0x63, 0x65, 0x73, 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x12, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x12, 0x76, 0x0a, - 0x15, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x5f, - 0x72, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x73, - 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x32, 0x50, 0x4d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x2e, 0x44, 0x68, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x42, 0x61, - 0x74, 0x63, 0x68, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x50, 0x6f, 0x69, 0x6e, 0x74, - 0x52, 0x13, 0x62, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x52, - 0x65, 0x63, 0x65, 0x6e, 0x74, 0x12, 0x31, 0x0a, 0x15, 0x68, 0x6f, 0x74, 0x5f, 0x70, 0x61, 0x74, - 0x68, 0x5f, 0x62, 0x61, 0x6e, 0x6e, 0x65, 0x64, 0x5f, 0x73, 0x6b, 0x69, 0x70, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x12, 0x68, 0x6f, 0x74, 0x50, 0x61, 0x74, 0x68, 0x42, 0x61, 0x6e, - 0x6e, 0x65, 0x64, 0x53, 0x6b, 0x69, 0x70, 0x73, 0x12, 0x35, 0x0a, 0x17, 0x68, 0x6f, 0x74, 0x5f, - 0x70, 0x61, 0x74, 0x68, 0x5f, 0x62, 0x61, 0x6e, 0x5f, 0x69, 0x6e, 0x63, 0x72, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x14, 0x68, 0x6f, 0x74, 0x50, 0x61, - 0x74, 0x68, 0x42, 0x61, 0x6e, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x1a, - 0x8f, 0x01, 0x0a, 0x11, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, + 0x69, 0x63, 0x73, 0x2e, 0x44, 0x68, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x0a, + 0x64, 0x68, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x74, 0x0a, 0x16, 0x6e, 0x65, + 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x6d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x73, 0x75, 0x70, + 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x32, 0x50, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, + 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x14, 0x6e, 0x65, 0x74, 0x77, + 0x6f, 0x72, 0x6b, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, + 0x12, 0x65, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x6e, 0x5f, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x6d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x73, 0x75, + 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x32, 0x50, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x50, 0x6f, 0x6f, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x6e, 0x50, 0x6f, 0x6f, 0x6c, + 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x48, 0x0a, 0x08, 0x62, 0x61, 0x6e, 0x5f, 0x6c, + 0x69, 0x73, 0x74, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x73, 0x75, 0x70, 0x65, + 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x32, 0x50, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, + 0x42, 0x61, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x62, 0x61, 0x6e, 0x4c, 0x69, 0x73, + 0x74, 0x12, 0x4e, 0x0a, 0x08, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, + 0x32, 0x50, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, + 0x73, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x08, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, + 0x65, 0x12, 0x43, 0x0a, 0x04, 0x64, 0x69, 0x73, 0x6b, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x2f, 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x32, 0x50, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x04, 0x64, 0x69, 0x73, 0x6b, 0x1a, 0xc0, 0x05, 0x0a, 0x0a, 0x44, 0x68, 0x74, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x73, 0x0a, 0x14, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x73, + 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x72, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, + 0x32, 0x50, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x44, 0x68, 0x74, 0x4d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, + 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x12, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x75, 0x63, + 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x12, 0x76, 0x0a, 0x15, 0x62, 0x61, + 0x74, 0x63, 0x68, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x5f, 0x72, 0x65, 0x63, + 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x73, 0x75, 0x70, 0x65, + 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x32, 0x50, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, + 0x44, 0x68, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, + 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x13, 0x62, + 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x52, 0x65, 0x63, 0x65, + 0x6e, 0x74, 0x12, 0x31, 0x0a, 0x15, 0x68, 0x6f, 0x74, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x62, + 0x61, 0x6e, 0x6e, 0x65, 0x64, 0x5f, 0x73, 0x6b, 0x69, 0x70, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x12, 0x68, 0x6f, 0x74, 0x50, 0x61, 0x74, 0x68, 0x42, 0x61, 0x6e, 0x6e, 0x65, 0x64, + 0x53, 0x6b, 0x69, 0x70, 0x73, 0x12, 0x35, 0x0a, 0x17, 0x68, 0x6f, 0x74, 0x5f, 0x70, 0x61, 0x74, + 0x68, 0x5f, 0x62, 0x61, 0x6e, 0x5f, 0x69, 0x6e, 0x63, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x14, 0x68, 0x6f, 0x74, 0x50, 0x61, 0x74, 0x68, 0x42, + 0x61, 0x6e, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x1a, 0x8f, 0x01, 0x0a, + 0x11, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x50, 0x6f, 0x69, + 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x12, + 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x08, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73, + 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x66, 0x75, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x0a, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x66, 0x75, 0x6c, 0x12, 0x21, 0x0a, 0x0c, 0x73, + 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x01, 0x52, 0x0b, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x61, 0x74, 0x65, 0x1a, 0xc8, + 0x01, 0x0a, 0x12, 0x42, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x6e, - 0x69, 0x78, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, 0x1e, - 0x0a, 0x0a, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x66, 0x75, 0x6c, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x0a, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x66, 0x75, 0x6c, 0x12, 0x21, - 0x0a, 0x0c, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x61, 0x74, - 0x65, 0x1a, 0xc8, 0x01, 0x0a, 0x12, 0x42, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x74, 0x72, 0x69, - 0x65, 0x76, 0x65, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, - 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x74, 0x69, 0x6d, - 0x65, 0x55, 0x6e, 0x69, 0x78, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x71, - 0x75, 0x69, 0x72, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72, 0x65, 0x71, - 0x75, 0x69, 0x72, 0x65, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6c, - 0x6f, 0x63, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x66, 0x6f, 0x75, 0x6e, - 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x5f, - 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x66, - 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1f, 0x0a, 0x0b, 0x64, - 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x0a, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x73, 0x1a, 0x74, 0x0a, 0x0e, - 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x12, 0x14, - 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x74, - 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x18, - 0x0a, 0x07, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x07, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x69, 0x6d, 0x65, - 0x6f, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, - 0x75, 0x74, 0x1a, 0x9d, 0x01, 0x0a, 0x08, 0x42, 0x61, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, - 0x0e, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x70, 0x12, - 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, - 0x6f, 0x72, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x0d, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x55, 0x6e, 0x69, - 0x78, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x61, 0x67, 0x65, 0x53, 0x65, 0x63, 0x6f, 0x6e, - 0x64, 0x73, 0x1a, 0x65, 0x0a, 0x0d, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x53, 0x74, - 0x61, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x0e, 0x70, 0x32, 0x70, 0x5f, 0x64, 0x62, 0x5f, 0x73, 0x69, - 0x7a, 0x65, 0x5f, 0x6d, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x70, 0x32, 0x70, - 0x44, 0x62, 0x53, 0x69, 0x7a, 0x65, 0x4d, 0x62, 0x12, 0x2f, 0x0a, 0x14, 0x70, 0x32, 0x70, 0x5f, - 0x64, 0x62, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x11, 0x70, 0x32, 0x70, 0x44, 0x62, 0x52, 0x65, 0x63, - 0x6f, 0x72, 0x64, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x1a, 0x55, 0x0a, 0x0a, 0x44, 0x69, 0x73, - 0x6b, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x0a, 0x06, 0x61, 0x6c, 0x6c, 0x5f, 0x6d, - 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x61, 0x6c, 0x6c, 0x4d, 0x62, 0x12, 0x17, - 0x0a, 0x07, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x6d, 0x62, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, - 0x06, 0x75, 0x73, 0x65, 0x64, 0x4d, 0x62, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x72, 0x65, 0x65, 0x5f, - 0x6d, 0x62, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x06, 0x66, 0x72, 0x65, 0x65, 0x4d, 0x62, - 0x1a, 0x7c, 0x0a, 0x19, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x48, 0x61, 0x6e, 0x64, 0x6c, - 0x65, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x49, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, - 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x32, 0x50, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, - 0x65, 0x72, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x42, - 0x0a, 0x14, 0x43, 0x6f, 0x6e, 0x6e, 0x50, 0x6f, 0x6f, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x32, 0xd7, 0x01, 0x0a, 0x10, 0x53, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, + 0x69, 0x78, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, + 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, + 0x65, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6c, 0x6f, 0x63, 0x61, + 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6e, 0x65, 0x74, + 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x66, 0x6f, 0x75, 0x6e, + 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x73, 0x1a, 0x74, 0x0a, 0x0e, 0x48, 0x61, 0x6e, + 0x64, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x74, + 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, + 0x6c, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x66, + 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x66, 0x61, + 0x69, 0x6c, 0x75, 0x72, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x1a, + 0x9d, 0x01, 0x0a, 0x08, 0x42, 0x61, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x70, 0x12, 0x12, 0x0a, 0x04, + 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, + 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x64, 0x5f, 0x61, 0x74, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x0d, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x55, 0x6e, 0x69, 0x78, 0x12, 0x1f, + 0x0a, 0x0b, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x0a, 0x61, 0x67, 0x65, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x1a, + 0x65, 0x0a, 0x0d, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, + 0x12, 0x23, 0x0a, 0x0e, 0x70, 0x32, 0x70, 0x5f, 0x64, 0x62, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, + 0x6d, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x70, 0x32, 0x70, 0x44, 0x62, 0x53, + 0x69, 0x7a, 0x65, 0x4d, 0x62, 0x12, 0x2f, 0x0a, 0x14, 0x70, 0x32, 0x70, 0x5f, 0x64, 0x62, 0x5f, + 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x11, 0x70, 0x32, 0x70, 0x44, 0x62, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x1a, 0x55, 0x0a, 0x0a, 0x44, 0x69, 0x73, 0x6b, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x0a, 0x06, 0x61, 0x6c, 0x6c, 0x5f, 0x6d, 0x62, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x61, 0x6c, 0x6c, 0x4d, 0x62, 0x12, 0x17, 0x0a, 0x07, 0x75, + 0x73, 0x65, 0x64, 0x5f, 0x6d, 0x62, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x06, 0x75, 0x73, + 0x65, 0x64, 0x4d, 0x62, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x72, 0x65, 0x65, 0x5f, 0x6d, 0x62, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x06, 0x66, 0x72, 0x65, 0x65, 0x4d, 0x62, 0x1a, 0x7c, 0x0a, + 0x19, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x49, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x73, 0x75, + 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x32, 0x50, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x73, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x42, 0x0a, 0x14, 0x43, + 0x6f, 0x6e, 0x6e, 0x50, 0x6f, 0x6f, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, + 0xf0, 0x02, 0x0a, 0x0b, 0x43, 0x6f, 0x64, 0x65, 0x63, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, + 0x1f, 0x0a, 0x0b, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x53, 0x69, 0x7a, 0x65, + 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x64, 0x75, 0x6e, 0x64, 0x61, 0x6e, 0x63, 0x79, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x72, 0x65, 0x64, 0x75, 0x6e, 0x64, 0x61, 0x6e, 0x63, 0x79, + 0x12, 0x22, 0x0a, 0x0d, 0x6d, 0x61, 0x78, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x6d, + 0x62, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x6d, 0x61, 0x78, 0x4d, 0x65, 0x6d, 0x6f, + 0x72, 0x79, 0x4d, 0x62, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, + 0x6e, 0x63, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, + 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, + 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, + 0x12, 0x21, 0x0a, 0x0c, 0x68, 0x65, 0x61, 0x64, 0x72, 0x6f, 0x6f, 0x6d, 0x5f, 0x70, 0x63, 0x74, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x68, 0x65, 0x61, 0x64, 0x72, 0x6f, 0x6f, 0x6d, + 0x50, 0x63, 0x74, 0x12, 0x20, 0x0a, 0x0c, 0x6d, 0x65, 0x6d, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, + 0x5f, 0x6d, 0x62, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x6d, 0x65, 0x6d, 0x4c, 0x69, + 0x6d, 0x69, 0x74, 0x4d, 0x62, 0x12, 0x28, 0x0a, 0x10, 0x6d, 0x65, 0x6d, 0x5f, 0x6c, 0x69, 0x6d, + 0x69, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0e, 0x6d, 0x65, 0x6d, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, + 0x27, 0x0a, 0x0f, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x6f, 0x72, + 0x65, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, + 0x69, 0x76, 0x65, 0x43, 0x6f, 0x72, 0x65, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x63, 0x70, 0x75, 0x5f, + 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x0a, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0e, 0x63, 0x70, 0x75, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x53, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x32, 0xd7, 0x01, 0x0a, 0x10, 0x53, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x58, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x18, 0x2e, 0x73, 0x75, 0x70, 0x65, 0x72, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, @@ -1543,7 +1695,7 @@ func file_supernode_supernode_proto_rawDescGZIP() []byte { return file_supernode_supernode_proto_rawDescData } -var file_supernode_supernode_proto_msgTypes = make([]protoimpl.MessageInfo, 21) +var file_supernode_supernode_proto_msgTypes = make([]protoimpl.MessageInfo, 22) var file_supernode_supernode_proto_goTypes = []any{ (*StatusRequest)(nil), // 0: supernode.StatusRequest (*ListServicesRequest)(nil), // 1: supernode.ListServicesRequest @@ -1554,18 +1706,19 @@ var file_supernode_supernode_proto_goTypes = []any{ (*StatusResponse_ServiceTasks)(nil), // 6: supernode.StatusResponse.ServiceTasks (*StatusResponse_Network)(nil), // 7: supernode.StatusResponse.Network (*StatusResponse_P2PMetrics)(nil), // 8: supernode.StatusResponse.P2PMetrics - (*StatusResponse_Resources_CPU)(nil), // 9: supernode.StatusResponse.Resources.CPU - (*StatusResponse_Resources_Memory)(nil), // 10: supernode.StatusResponse.Resources.Memory - (*StatusResponse_Resources_Storage)(nil), // 11: supernode.StatusResponse.Resources.Storage - (*StatusResponse_P2PMetrics_DhtMetrics)(nil), // 12: supernode.StatusResponse.P2PMetrics.DhtMetrics - (*StatusResponse_P2PMetrics_HandleCounters)(nil), // 13: supernode.StatusResponse.P2PMetrics.HandleCounters - (*StatusResponse_P2PMetrics_BanEntry)(nil), // 14: supernode.StatusResponse.P2PMetrics.BanEntry - (*StatusResponse_P2PMetrics_DatabaseStats)(nil), // 15: supernode.StatusResponse.P2PMetrics.DatabaseStats - (*StatusResponse_P2PMetrics_DiskStatus)(nil), // 16: supernode.StatusResponse.P2PMetrics.DiskStatus - nil, // 17: supernode.StatusResponse.P2PMetrics.NetworkHandleMetricsEntry - nil, // 18: supernode.StatusResponse.P2PMetrics.ConnPoolMetricsEntry - (*StatusResponse_P2PMetrics_DhtMetrics_StoreSuccessPoint)(nil), // 19: supernode.StatusResponse.P2PMetrics.DhtMetrics.StoreSuccessPoint - (*StatusResponse_P2PMetrics_DhtMetrics_BatchRetrievePoint)(nil), // 20: supernode.StatusResponse.P2PMetrics.DhtMetrics.BatchRetrievePoint + (*StatusResponse_CodecConfig)(nil), // 9: supernode.StatusResponse.CodecConfig + (*StatusResponse_Resources_CPU)(nil), // 10: supernode.StatusResponse.Resources.CPU + (*StatusResponse_Resources_Memory)(nil), // 11: supernode.StatusResponse.Resources.Memory + (*StatusResponse_Resources_Storage)(nil), // 12: supernode.StatusResponse.Resources.Storage + (*StatusResponse_P2PMetrics_DhtMetrics)(nil), // 13: supernode.StatusResponse.P2PMetrics.DhtMetrics + (*StatusResponse_P2PMetrics_HandleCounters)(nil), // 14: supernode.StatusResponse.P2PMetrics.HandleCounters + (*StatusResponse_P2PMetrics_BanEntry)(nil), // 15: supernode.StatusResponse.P2PMetrics.BanEntry + (*StatusResponse_P2PMetrics_DatabaseStats)(nil), // 16: supernode.StatusResponse.P2PMetrics.DatabaseStats + (*StatusResponse_P2PMetrics_DiskStatus)(nil), // 17: supernode.StatusResponse.P2PMetrics.DiskStatus + nil, // 18: supernode.StatusResponse.P2PMetrics.NetworkHandleMetricsEntry + nil, // 19: supernode.StatusResponse.P2PMetrics.ConnPoolMetricsEntry + (*StatusResponse_P2PMetrics_DhtMetrics_StoreSuccessPoint)(nil), // 20: supernode.StatusResponse.P2PMetrics.DhtMetrics.StoreSuccessPoint + (*StatusResponse_P2PMetrics_DhtMetrics_BatchRetrievePoint)(nil), // 21: supernode.StatusResponse.P2PMetrics.DhtMetrics.BatchRetrievePoint } var file_supernode_supernode_proto_depIdxs = []int32{ 3, // 0: supernode.ListServicesResponse.services:type_name -> supernode.ServiceInfo @@ -1573,27 +1726,28 @@ var file_supernode_supernode_proto_depIdxs = []int32{ 6, // 2: supernode.StatusResponse.running_tasks:type_name -> supernode.StatusResponse.ServiceTasks 7, // 3: supernode.StatusResponse.network:type_name -> supernode.StatusResponse.Network 8, // 4: supernode.StatusResponse.p2p_metrics:type_name -> supernode.StatusResponse.P2PMetrics - 9, // 5: supernode.StatusResponse.Resources.cpu:type_name -> supernode.StatusResponse.Resources.CPU - 10, // 6: supernode.StatusResponse.Resources.memory:type_name -> supernode.StatusResponse.Resources.Memory - 11, // 7: supernode.StatusResponse.Resources.storage_volumes:type_name -> supernode.StatusResponse.Resources.Storage - 12, // 8: supernode.StatusResponse.P2PMetrics.dht_metrics:type_name -> supernode.StatusResponse.P2PMetrics.DhtMetrics - 17, // 9: supernode.StatusResponse.P2PMetrics.network_handle_metrics:type_name -> supernode.StatusResponse.P2PMetrics.NetworkHandleMetricsEntry - 18, // 10: supernode.StatusResponse.P2PMetrics.conn_pool_metrics:type_name -> supernode.StatusResponse.P2PMetrics.ConnPoolMetricsEntry - 14, // 11: supernode.StatusResponse.P2PMetrics.ban_list:type_name -> supernode.StatusResponse.P2PMetrics.BanEntry - 15, // 12: supernode.StatusResponse.P2PMetrics.database:type_name -> supernode.StatusResponse.P2PMetrics.DatabaseStats - 16, // 13: supernode.StatusResponse.P2PMetrics.disk:type_name -> supernode.StatusResponse.P2PMetrics.DiskStatus - 19, // 14: supernode.StatusResponse.P2PMetrics.DhtMetrics.store_success_recent:type_name -> supernode.StatusResponse.P2PMetrics.DhtMetrics.StoreSuccessPoint - 20, // 15: supernode.StatusResponse.P2PMetrics.DhtMetrics.batch_retrieve_recent:type_name -> supernode.StatusResponse.P2PMetrics.DhtMetrics.BatchRetrievePoint - 13, // 16: supernode.StatusResponse.P2PMetrics.NetworkHandleMetricsEntry.value:type_name -> supernode.StatusResponse.P2PMetrics.HandleCounters - 0, // 17: supernode.SupernodeService.GetStatus:input_type -> supernode.StatusRequest - 1, // 18: supernode.SupernodeService.ListServices:input_type -> supernode.ListServicesRequest - 4, // 19: supernode.SupernodeService.GetStatus:output_type -> supernode.StatusResponse - 2, // 20: supernode.SupernodeService.ListServices:output_type -> supernode.ListServicesResponse - 19, // [19:21] is the sub-list for method output_type - 17, // [17:19] is the sub-list for method input_type - 17, // [17:17] is the sub-list for extension type_name - 17, // [17:17] is the sub-list for extension extendee - 0, // [0:17] is the sub-list for field type_name + 9, // 5: supernode.StatusResponse.codec:type_name -> supernode.StatusResponse.CodecConfig + 10, // 6: supernode.StatusResponse.Resources.cpu:type_name -> supernode.StatusResponse.Resources.CPU + 11, // 7: supernode.StatusResponse.Resources.memory:type_name -> supernode.StatusResponse.Resources.Memory + 12, // 8: supernode.StatusResponse.Resources.storage_volumes:type_name -> supernode.StatusResponse.Resources.Storage + 13, // 9: supernode.StatusResponse.P2PMetrics.dht_metrics:type_name -> supernode.StatusResponse.P2PMetrics.DhtMetrics + 18, // 10: supernode.StatusResponse.P2PMetrics.network_handle_metrics:type_name -> supernode.StatusResponse.P2PMetrics.NetworkHandleMetricsEntry + 19, // 11: supernode.StatusResponse.P2PMetrics.conn_pool_metrics:type_name -> supernode.StatusResponse.P2PMetrics.ConnPoolMetricsEntry + 15, // 12: supernode.StatusResponse.P2PMetrics.ban_list:type_name -> supernode.StatusResponse.P2PMetrics.BanEntry + 16, // 13: supernode.StatusResponse.P2PMetrics.database:type_name -> supernode.StatusResponse.P2PMetrics.DatabaseStats + 17, // 14: supernode.StatusResponse.P2PMetrics.disk:type_name -> supernode.StatusResponse.P2PMetrics.DiskStatus + 20, // 15: supernode.StatusResponse.P2PMetrics.DhtMetrics.store_success_recent:type_name -> supernode.StatusResponse.P2PMetrics.DhtMetrics.StoreSuccessPoint + 21, // 16: supernode.StatusResponse.P2PMetrics.DhtMetrics.batch_retrieve_recent:type_name -> supernode.StatusResponse.P2PMetrics.DhtMetrics.BatchRetrievePoint + 14, // 17: supernode.StatusResponse.P2PMetrics.NetworkHandleMetricsEntry.value:type_name -> supernode.StatusResponse.P2PMetrics.HandleCounters + 0, // 18: supernode.SupernodeService.GetStatus:input_type -> supernode.StatusRequest + 1, // 19: supernode.SupernodeService.ListServices:input_type -> supernode.ListServicesRequest + 4, // 20: supernode.SupernodeService.GetStatus:output_type -> supernode.StatusResponse + 2, // 21: supernode.SupernodeService.ListServices:output_type -> supernode.ListServicesResponse + 20, // [20:22] is the sub-list for method output_type + 18, // [18:20] is the sub-list for method input_type + 18, // [18:18] is the sub-list for extension type_name + 18, // [18:18] is the sub-list for extension extendee + 0, // [0:18] is the sub-list for field type_name } func init() { file_supernode_supernode_proto_init() } @@ -1607,7 +1761,7 @@ func file_supernode_supernode_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_supernode_supernode_proto_rawDesc, NumEnums: 0, - NumMessages: 21, + NumMessages: 22, NumExtensions: 0, NumServices: 1, }, diff --git a/gen/supernode/supernode.swagger.json b/gen/supernode/supernode.swagger.json index e29dcbae..eadb5fbc 100644 --- a/gen/supernode/supernode.swagger.json +++ b/gen/supernode/supernode.swagger.json @@ -315,6 +315,59 @@ } } }, + "StatusResponseCodecConfig": { + "type": "object", + "properties": { + "symbolSize": { + "type": "integer", + "format": "int64", + "title": "bytes (typically 65535)" + }, + "redundancy": { + "type": "integer", + "format": "int64", + "title": "repair factor (percent-like scalar; 5 = default)" + }, + "maxMemoryMb": { + "type": "string", + "format": "uint64", + "title": "memory cap for native decoder" + }, + "concurrency": { + "type": "integer", + "format": "int64", + "title": "native decoder parallelism" + }, + "profile": { + "type": "string", + "title": "selected profile: edge|standard|perf" + }, + "headroomPct": { + "type": "integer", + "format": "int32", + "title": "reserved memory percentage (0-90)" + }, + "memLimitMb": { + "type": "string", + "format": "uint64", + "title": "detected memory limit (MB)" + }, + "memLimitSource": { + "type": "string", + "title": "detection source (cgroup/meminfo)" + }, + "effectiveCores": { + "type": "integer", + "format": "int32", + "title": "detected cores/quota" + }, + "cpuLimitSource": { + "type": "string", + "title": "detection source (cgroups/NumCPU)" + } + }, + "title": "RaptorQ codec configuration (effective values)" + }, "StatusResponseNetwork": { "type": "object", "properties": { @@ -513,6 +566,9 @@ }, "p2pMetrics": { "$ref": "#/definitions/StatusResponseP2PMetrics" + }, + "codec": { + "$ref": "#/definitions/StatusResponseCodecConfig" } }, "title": "The StatusResponse represents system status with clear organization" diff --git a/pkg/codec/README.md b/pkg/codec/README.md new file mode 100644 index 00000000..3ae6557c --- /dev/null +++ b/pkg/codec/README.md @@ -0,0 +1,116 @@ +# Codec (RaptorQ) Guide + +## Table of Contents +- [Overview](#overview) +- [What It Does](#what-it-does) +- [Behavioural Note](#behavioural-note) +- [What We Don’t Do](#what-we-dont-do) +- [Quick Defaults (no env)](#quick-defaults-no-env) +- [Change Behavior (simple rules)](#change-behavior-simple-rules) +- [Profiles (behavior overview)](#profiles-behavior-overview) +- [Where It Applies](#where-it-applies) +- [Integration](#integration) +- [Notes](#notes) +- [Log Observability](#log-observability) +- [P2P interaction (high‑level)](#p2p-interaction-highlevel) +- [Examples (no env)](#examples-no-env) +- [Minimal Usage (Decode)](#minimal-usage-decode) +- [Minimal Usage (Encode)](#minimal-usage-encode) +- [Troubleshooting](#troubleshooting) + +## Overview +- Thin, well‑scoped API around RaptorQ for encoding/decoding cascade artefacts. +- Safe defaults and memory‑conscious behavior tuned for files up to 1 GB. + +## What It Does +- Per‑request processor: create/free a processor per Encode/Decode to bound native memory. +- Decode hygiene: write symbols to disk and drop in‑memory buffers before decoding to lower peaks. +- Layout file: stored as `_raptorq_layout.json` in the request’s symbols directory. +- Directory layout: + - Encode → `//` + - Decode → `//` (decoded output sits alongside) + +## Behavioural Note +- Decode mutates input: it deletes entries from `DecodeRequest.Symbols` after flushing to disk to free memory. + +## What We Don’t Do +- Require env setup: runs fine without any env vars. If set, env vars override the defaults. +- Full pre‑fetch of symbols: progressive retrieval avoids over‑fetching and reduces memory pressure. +- Long‑lived processors: we create/free a processor per request to bound native memory. + +## Quick Defaults (no env) +- Profile: perf (fastest defaults) +- Headroom: `LUMERA_RQ_MEM_HEADROOM_PCT=40` → usable_mem = limit × (1−0.40) +- Max memory: `min(0.6 × usable_mem, 16 GiB)` +- Concurrency: `min(8, effective_cores)`, then reduced so `max_memory_mb / concurrency ≥ 512 MB` +- Symbol size: `65535` • Redundancy: `5` + +## Change Behavior (simple rules) +- Pick profile: set `CODEC_PROFILE=edge|standard|perf` (perf is default) +- Reserve headroom: set `LUMERA_RQ_MEM_HEADROOM_PCT` (0–90, default 40) +- Override knobs directly (take precedence over profile): + - `LUMERA_RQ_MAX_MEMORY_MB`, `LUMERA_RQ_CONCURRENCY`, `LUMERA_RQ_SYMBOL_SIZE`, `LUMERA_RQ_REDUNDANCY` +- Detection sources: memory from cgroups v2/v1 (fallback `/proc/meminfo`), CPU from cgroup quota or `runtime.NumCPU()` + +## Profiles (behavior overview) + +| Profile | Default selection | Max memory (default) | Concurrency (default) | CPU cap | Min per‑worker MB | Symbol size | Redundancy | Env overrides | +|-----------|-------------------|-----------------------------------------------|--------------------------------------|-------------------------------|-------------------|-------------|------------|----------------------------------------| +| `edge` | Only when forced | `min(usable_mem, 1 GiB)` | `2` | Capped by effective cores | `≥ 512` | 65535 | 5 | `LUMERA_RQ_*`, `CODEC_PROFILE=edge` | +| `standard`| Only when forced | `min(0.6 × usable_mem, 4 GiB)` | `4` | Capped by effective cores | `≥ 512` | 65535 | 5 | `LUMERA_RQ_*`, `CODEC_PROFILE=standard`| +| `perf` | Default | `min(0.6 × usable_mem, 16 GiB)` | `8` | Capped by effective cores | `≥ 512` | 65535 | 5 | `LUMERA_RQ_*`, `CODEC_PROFILE=perf` | + +- usable_mem = memory_limit × (1 − `LUMERA_RQ_MEM_HEADROOM_PCT`/100). Default headroom = 40%. +- Effective cores: derived from cgroup CPU quota (v2 `cpu.max`, v1 `cpu.cfs_*`) or `runtime.NumCPU()`. +- Per‑worker memory: if `max_memory_mb / concurrency < 512`, concurrency is reduced until the target is met (down to 1). +- Env overrides: any of `LUMERA_RQ_SYMBOL_SIZE`, `LUMERA_RQ_REDUNDANCY`, `LUMERA_RQ_MAX_MEMORY_MB`, `LUMERA_RQ_CONCURRENCY` supersede the profile defaults. + +## Where It Applies +- Encode: `pkg/codec/raptorq.go::Encode()` → compute block size, `EncodeFile()`, read layout +- Decode: `pkg/codec/decode.go::Decode()` → write symbols + layout to disk, `DecodeSymbols()` +- Progressive decode: `supernode/supernode/services/cascade/progressive_decode.go` escalates required symbols (9%, 25%, 50%, 75%, 100%) + +## Integration +- Adaptors: `supernode/supernode/services/cascade/adaptors/rq.go` bridge the codec to higher‑level services +- Download flow: `supernode/supernode/services/cascade/download.go` uses progressive decode and verifies final file hash + +## Notes +- Error code vs message: treat the error code as authoritative; message is context only +- Disk usage: ensure `` has space for symbols and the final file +- Cleanup: callers delete the decode temp dir (`DecodeTmpDir`) +- File size: defaults suit ≤1 GiB; use overrides or profiles for larger files + +## Log Observability +- On processor creation we log: symbol_size, redundancy_factor, max_memory_mb, concurrency, profile, headroom_pct, mem_limit and source, effective_cores and source. + +## P2P interaction (high‑level) +- Codec redundancy and DHT replication are independent; progressive decode needs “enough” distinct symbols, not all of them + +## Examples (no env) +- 16 GiB limit, 8 cores → usable=9.6 GiB → max=5.76 GiB → conc=8 → ~720 MB/worker +- 8 GiB limit, 4 cores → usable=4.8 GiB → max=2.88 GiB → conc=4 → ~720 MB/worker +- 2 GiB limit, 2 cores → usable=1.2 GiB → max=0.72 GiB → conc=1 (to keep ≥512 MB/worker) + +## Minimal Usage (Decode) +```go +resp, err := codecImpl.Decode(ctx, codec.DecodeRequest{ + ActionID: actionID, + Layout: layout, + Symbols: symbols, // map[id]payload +}) +// resp.Path is the reconstructed file; resp.DecodeTmpDir holds symbols + layout +``` + +## Minimal Usage (Encode) +```go +resp, err := codecImpl.Encode(ctx, codec.EncodeRequest{ + TaskID: taskID, + Path: inputPath, + DataSize: sizeBytes, +}) +// resp.SymbolsDir contains symbols; resp.Metadata holds the layout to publish +``` + +## Troubleshooting +- “memory limit exceeded” with integrity text: treat the error code as authoritative; the message may include integrity hints. +- Decode failures: progressive helper escalates symbol count automatically; non‑integrity errors bubble up immediately. diff --git a/pkg/codec/codec.go b/pkg/codec/codec.go index 39029569..23bfbc21 100644 --- a/pkg/codec/codec.go +++ b/pkg/codec/codec.go @@ -1,5 +1,18 @@ //go:generate mockgen -destination=codec_mock.go -package=codec -source=codec.go +// Package codec provides an abstraction over the RaptorQ encoding/decoding engine +// used by the supernode for cascade artefacts. It centralizes resource tuning and +// safe decode/encode workflows. +// +// - Memory/Concurrency: The concrete implementation (raptorQ) reads runtime overrides +// from environment variables to tame memory pressure on different deployments: +// * LUMERA_RQ_SYMBOL_SIZE (uint16, default 65535) +// * LUMERA_RQ_REDUNDANCY (uint8, default 5) +// * LUMERA_RQ_MAX_MEMORY_MB (uint64, default 16384) +// * LUMERA_RQ_CONCURRENCY (uint64, default 4) +// - Decode Memory Hygiene: Symbols passed in memory are written to disk immediately +// and dropped from RAM during decode to minimize overlapping heap and native +// allocations. package codec import ( diff --git a/pkg/codec/decode.go b/pkg/codec/decode.go index 7aae541e..d54a766c 100644 --- a/pkg/codec/decode.go +++ b/pkg/codec/decode.go @@ -7,7 +7,6 @@ import ( "os" "path/filepath" - raptorq "github.com/LumeraProtocol/rq-go" "github.com/LumeraProtocol/supernode/v2/pkg/logtrace" ) @@ -18,8 +17,8 @@ type DecodeRequest struct { } type DecodeResponse struct { - Path string - DecodeTmpDir string + Path string + DecodeTmpDir string } func (rq *raptorQ) Decode(ctx context.Context, req DecodeRequest) (DecodeResponse, error) { @@ -30,7 +29,8 @@ func (rq *raptorQ) Decode(ctx context.Context, req DecodeRequest) (DecodeRespons } logtrace.Info(ctx, "RaptorQ decode request received", fields) - processor, err := raptorq.NewDefaultRaptorQProcessor() + // Use env-configurable processor to allow memory/concurrency tuning per deployment + processor, err := newProcessor(ctx) if err != nil { fields[logtrace.FieldError] = err.Error() return DecodeResponse{}, fmt.Errorf("create RaptorQ processor: %w", err) @@ -43,30 +43,39 @@ func (rq *raptorQ) Decode(ctx context.Context, req DecodeRequest) (DecodeRespons return DecodeResponse{}, fmt.Errorf("mkdir %s: %w", symbolsDir, err) } - // Write symbols to disk - for id, data := range req.Symbols { - symbolPath := filepath.Join(symbolsDir, id) - if err := os.WriteFile(symbolPath, data, 0o644); err != nil { - fields[logtrace.FieldError] = err.Error() - return DecodeResponse{}, fmt.Errorf("write symbol %s: %w", id, err) - } - } - logtrace.Info(ctx, "symbols written to disk", fields) + // Write symbols to disk as soon as possible to reduce heap residency. + // Prior versions kept symbols in memory until after decode, which could + // exacerbate memory pressure. We now persist to disk immediately and drop each entry + // from the map to allow GC to reclaim memory sooner. This helps avoid spikes that + // could coincide with RaptorQ allocations (cf. memory limit exceeded reports). + for id, data := range req.Symbols { + symbolPath := filepath.Join(symbolsDir, id) + if err := os.WriteFile(symbolPath, data, 0o644); err != nil { + fields[logtrace.FieldError] = err.Error() + return DecodeResponse{}, fmt.Errorf("write symbol %s: %w", id, err) + } + // Drop the in-memory copy promptly; safe to delete during range + delete(req.Symbols, id) + } + logtrace.Info(ctx, "symbols written to disk", fields) - // ---------- write layout.json ---------- ←★ - layoutPath := filepath.Join(symbolsDir, "layout.json") - layoutBytes, err := json.Marshal(req.Layout) - if err != nil { - fields[logtrace.FieldError] = err.Error() - return DecodeResponse{}, fmt.Errorf("marshal layout: %w", err) - } + // ---------- write layout file ---------- + // Use a conventional filename to match rq-go documentation examples. + // The library consumes the explicit path, so name is not strictly required, but + // aligning with `_raptorq_layout.json` aids operators/debuggers. + layoutPath := filepath.Join(symbolsDir, "_raptorq_layout.json") + layoutBytes, err := json.Marshal(req.Layout) + if err != nil { + fields[logtrace.FieldError] = err.Error() + return DecodeResponse{}, fmt.Errorf("marshal layout: %w", err) + } if err := os.WriteFile(layoutPath, layoutBytes, 0o644); err != nil { fields[logtrace.FieldError] = err.Error() return DecodeResponse{}, fmt.Errorf("write layout file: %w", err) } logtrace.Info(ctx, "layout.json written", fields) - // Decode + // Decode the symbols into an output file using the provided layout. outputPath := filepath.Join(symbolsDir, "output") if err := processor.DecodeSymbols(symbolsDir, outputPath, layoutPath); err != nil { fields[logtrace.FieldError] = err.Error() diff --git a/pkg/codec/raptorq.go b/pkg/codec/raptorq.go index 52de1eab..3437b466 100644 --- a/pkg/codec/raptorq.go +++ b/pkg/codec/raptorq.go @@ -4,8 +4,12 @@ import ( "context" "encoding/json" "fmt" + "math" "os" "path/filepath" + "runtime" + "strconv" + "strings" raptorq "github.com/LumeraProtocol/rq-go" "github.com/LumeraProtocol/supernode/v2/pkg/logtrace" @@ -22,6 +26,300 @@ func NewRaptorQCodec(dir string) Codec { } +// newProcessor constructs a RaptorQ processor using environment overrides when provided. +// +// Environment variables: +// - LUMERA_RQ_SYMBOL_SIZE (uint16, default 65535) +// - LUMERA_RQ_REDUNDANCY (uint8, default 5) +// - LUMERA_RQ_MAX_MEMORY_MB (uint64, default 16384) +// - LUMERA_RQ_CONCURRENCY (uint64, default 4) +// +// The goal is to allow deployments to tune memory and concurrency to avoid pressure spikes +// without changing code paths, while keeping the prior defaults when unset. +func newProcessor(ctx context.Context) (*raptorq.RaptorQProcessor, error) { + // 1) Detect resources (memory/CPU) + memLimitMB, memSource := detectMemoryLimitMB() + effCores, cpuSource := detectEffectiveCores() + + // 2) Apply headroom knob and compute usable memory + headroomPct := readInt("LUMERA_RQ_MEM_HEADROOM_PCT", 40, 0, 90) + usableMemMB := computeUsableMem(memLimitMB, headroomPct) + + // 3) Select profile (forced via env or inferred) + profile := selectProfile(os.Getenv("CODEC_PROFILE")) + + // 4) Compute default limits for the chosen profile + defMaxMemMB, defConcurrency := defaultLimitsForProfile(profile, usableMemMB) + + // 5) Adjust concurrency by effective cores/quotas + defConcurrency = adjustConcurrency(defConcurrency, effCores) + + // 6) Ensure per-worker memory target (>=512MB) by reducing concurrency if needed + defConcurrency = rebalancePerWorkerMem(defMaxMemMB, defConcurrency, 512) + + // 7) Read env overrides (env wins) + symbolSize := uint16(readUint("LUMERA_RQ_SYMBOL_SIZE", uint64(raptorq.DefaultSymbolSize), 1024, 65535)) + redundancy := uint8(readUint("LUMERA_RQ_REDUNDANCY", 5, 1, 32)) + maxMemMB := readUint("LUMERA_RQ_MAX_MEMORY_MB", defMaxMemMB, 256, 1<<20) + concurrency := readUint("LUMERA_RQ_CONCURRENCY", defConcurrency, 1, 1024) + + // 8) Log final configuration + logtrace.Info(ctx, "RaptorQ processor config", logtrace.Fields{ + "symbol_size": symbolSize, + "redundancy_factor": redundancy, + "max_memory_mb": maxMemMB, + "concurrency": concurrency, + "profile": profile, + "headroom_pct": headroomPct, + "mem_limit_mb": memLimitMB, + "mem_limit_source": memSource, + "effective_cores": effCores, + "cpu_limit_source": cpuSource, + }) + + // 9) Construct processor + return raptorq.NewRaptorQProcessor(symbolSize, redundancy, maxMemMB, concurrency) +} + +// RaptorQConfig describes the effective codec configuration derived from env, resources and defaults. +type RaptorQConfig struct { + SymbolSize uint16 + Redundancy uint8 + MaxMemoryMB uint64 + Concurrency uint64 + Profile string + HeadroomPct int + MemLimitMB uint64 + MemLimitSource string + EffectiveCores int + CpuLimitSource string +} + +// CurrentConfig computes the current effective RaptorQ configuration without allocating a processor. +func CurrentConfig(ctx context.Context) RaptorQConfig { + // 1) Detect resources + memLimitMB, memSource := detectMemoryLimitMB() + effCores, cpuSource := detectEffectiveCores() + + // 2) Apply headroom and select profile + headroomPct := readInt("LUMERA_RQ_MEM_HEADROOM_PCT", 40, 0, 90) + usableMemMB := computeUsableMem(memLimitMB, headroomPct) + profile := selectProfile(os.Getenv("CODEC_PROFILE")) + + // 3) Compute defaults and adjust by cores and per‑worker budget + defMaxMemMB, defConcurrency := defaultLimitsForProfile(profile, usableMemMB) + defConcurrency = adjustConcurrency(defConcurrency, effCores) + defConcurrency = rebalancePerWorkerMem(defMaxMemMB, defConcurrency, 512) + + // 4) Apply env overrides + symbolSize := uint16(readUint("LUMERA_RQ_SYMBOL_SIZE", uint64(raptorq.DefaultSymbolSize), 1024, 65535)) + redundancy := uint8(readUint("LUMERA_RQ_REDUNDANCY", 5, 1, 32)) + maxMemMB := readUint("LUMERA_RQ_MAX_MEMORY_MB", defMaxMemMB, 256, 1<<20) + concurrency := readUint("LUMERA_RQ_CONCURRENCY", defConcurrency, 1, 1024) + + return RaptorQConfig{ + SymbolSize: symbolSize, + Redundancy: redundancy, + MaxMemoryMB: maxMemMB, + Concurrency: concurrency, + Profile: profile, + HeadroomPct: headroomPct, + MemLimitMB: memLimitMB, + MemLimitSource: memSource, + EffectiveCores: effCores, + CpuLimitSource: cpuSource, + } +} + +// detectMemoryLimitMB attempts to determine the memory limit (MB) from cgroups, falling back to MemTotal. +func detectMemoryLimitMB() (uint64, string) { + // cgroup v2: /sys/fs/cgroup/memory.max + if b, err := os.ReadFile("/sys/fs/cgroup/memory.max"); err == nil { + s := strings.TrimSpace(string(b)) + if s != "max" { + if v, err := strconv.ParseUint(s, 10, 64); err == nil && v > 0 { + return v / (1024 * 1024), "cgroupv2:memory.max" + } + } + } + // cgroup v1: /sys/fs/cgroup/memory/memory.limit_in_bytes + if b, err := os.ReadFile("/sys/fs/cgroup/memory/memory.limit_in_bytes"); err == nil { + s := strings.TrimSpace(string(b)) + if v, err := strconv.ParseUint(s, 10, 64); err == nil && v > 0 { + // Some systems report a huge number when unlimited; treat > 1PB as unlimited + if v > 1<<50 { + // unlimited; fallthrough to MemTotal + } else { + return v / (1024 * 1024), "cgroupv1:memory.limit_in_bytes" + } + } + } + // Fallback: /proc/meminfo MemTotal + if b, err := os.ReadFile("/proc/meminfo"); err == nil { + lines := strings.Split(string(b), "\n") + for _, ln := range lines { + if strings.HasPrefix(ln, "MemTotal:") { + f := strings.Fields(ln) + if len(f) >= 2 { + if kb, err := strconv.ParseUint(f[1], 10, 64); err == nil { + return kb / 1024, "meminfo:MemTotal" + } + } + } + } + } + return 0, "unknown" +} + +// detectEffectiveCores attempts to determine CPU quota; returns cores and source. +func detectEffectiveCores() (int, string) { + // cgroup v2: /sys/fs/cgroup/cpu.max: "max" or " " + if b, err := os.ReadFile("/sys/fs/cgroup/cpu.max"); err == nil { + parts := strings.Fields(strings.TrimSpace(string(b))) + if len(parts) == 2 && parts[0] != "max" { + if quota, err1 := strconv.ParseUint(parts[0], 10, 64); err1 == nil { + if period, err2 := strconv.ParseUint(parts[1], 10, 64); err2 == nil && period > 0 { + cores := int(float64(quota) / float64(period)) + if cores < 1 { + cores = 1 + } + return cores, "cgroupv2:cpu.max" + } + } + } + } + // cgroup v1: /sys/fs/cgroup/cpu/cpu.cfs_quota_us and cpu.cfs_period_us + if qb, err1 := os.ReadFile("/sys/fs/cgroup/cpu/cpu.cfs_quota_us"); err1 == nil { + if pb, err2 := os.ReadFile("/sys/fs/cgroup/cpu/cpu.cfs_period_us"); err2 == nil { + qStr := strings.TrimSpace(string(qb)) + pStr := strings.TrimSpace(string(pb)) + if qStr != "-1" { + if quota, errA := strconv.ParseUint(qStr, 10, 64); errA == nil { + if period, errB := strconv.ParseUint(pStr, 10, 64); errB == nil && period > 0 { + cores := int(float64(quota) / float64(period)) + if cores < 1 { + cores = 1 + } + return cores, "cgroupv1:cpu.cfs_quota_us/period" + } + } + } + } + } + return 0, "runtime.NumCPU" +} + +func minNonZero(a, b uint64) uint64 { + if a == 0 { + return b + } + if b == 0 { + return a + } + if a < b { + return a + } + return b +} + +// readUint reads an unsigned integer from env with bounds and a default. +func readUint(env string, def uint64, min uint64, max uint64) uint64 { + if v, ok := os.LookupEnv(env); ok { + if n, err := strconv.ParseUint(v, 10, 64); err == nil { + if n < min { + return min + } + if max > 0 && n > max { + return max + } + return n + } + } + return def +} + +// readInt reads an integer from env with bounds and a default. +func readInt(env string, def int, min int, max int) int { + if v, ok := os.LookupEnv(env); ok { + if n, err := strconv.Atoi(v); err == nil { + if n < min { + return min + } + if max > 0 && n > max { + return max + } + return n + } + } + return def +} + +// computeUsableMem applies a headroom percentage to a memory limit. +func computeUsableMem(memLimitMB uint64, headroomPct int) uint64 { + if headroomPct <= 0 || memLimitMB == 0 { + return memLimitMB + } + return uint64(math.Max(0, float64(memLimitMB)*(1.0-float64(headroomPct)/100.0))) +} + +// selectProfile decides which profile to use based on a forced value or memory limit. +func selectProfile(forced string) string { + p := strings.ToLower(strings.TrimSpace(forced)) + switch p { + case "edge", "standard", "perf": + return p + } + // Default to perf when not forced + return "perf" +} + +// defaultLimitsForProfile returns default max memory and concurrency for a profile. +func defaultLimitsForProfile(profile string, usableMemMB uint64) (uint64, uint64) { + switch profile { + case "edge": + mm := minNonZero(usableMemMB, 1024) + if mm == 0 { + mm = 1024 + } + return mm, 2 + case "standard": + mm := uint64(math.Min(float64(usableMemMB)*0.6, 4*1024)) + if mm < 1024 { + mm = 1024 + } + return mm, 4 + default: // perf + mm := uint64(math.Min(float64(usableMemMB)*0.6, 16*1024)) + return mm, 8 + } +} + +// adjustConcurrency caps concurrency by effective cores. +func adjustConcurrency(defConcurrency uint64, effCores int) uint64 { + cpu := runtime.NumCPU() + if effCores > 0 { + cpu = effCores + } + if cpu < 1 { + cpu = 1 + } + if defConcurrency > uint64(cpu) { + return uint64(cpu) + } + return defConcurrency +} + +// rebalancePerWorkerMem reduces concurrency until each worker has at least minPerWorkerMB. +func rebalancePerWorkerMem(defMaxMemMB uint64, defConcurrency uint64, minPerWorkerMB uint64) uint64 { + if defConcurrency == 0 { + return 1 + } + for defMaxMemMB/defConcurrency < minPerWorkerMB && defConcurrency > 1 { + defConcurrency-- + } + return defConcurrency +} + func (rq *raptorQ) Encode(ctx context.Context, req EncodeRequest) (EncodeResponse, error) { /* ---------- 1. initialise RaptorQ processor ---------- */ fields := logtrace.Fields{ @@ -32,7 +330,8 @@ func (rq *raptorQ) Encode(ctx context.Context, req EncodeRequest) (EncodeRespons "data-size": req.DataSize, } - processor, err := raptorq.NewDefaultRaptorQProcessor() + // Use env-configurable processor to allow memory/concurrency tuning per deployment + processor, err := newProcessor(ctx) if err != nil { return EncodeResponse{}, fmt.Errorf("create RaptorQ processor: %w", err) } diff --git a/pkg/codecconfig/config.go b/pkg/codecconfig/config.go new file mode 100644 index 00000000..f4ac8f05 --- /dev/null +++ b/pkg/codecconfig/config.go @@ -0,0 +1,250 @@ +package codecconfig + +import ( + "context" + "math" + "os" + "runtime" + "strconv" + "strings" +) + +// Config describes the effective codec configuration derived from env, resources and defaults. +type Config struct { + SymbolSize uint16 + Redundancy uint8 + MaxMemoryMB uint64 + Concurrency uint64 + Profile string + HeadroomPct int + MemLimitMB uint64 + MemLimitSource string + EffectiveCores int + CpuLimitSource string +} + +// Defaults mirrored from rq-go and current project conventions. +const ( + defaultSymbolSize = 65535 + defaultRedundancy = 5 + minPerWorkerMB = 512 +) + +// Current computes the current effective codec configuration without requiring cgo. +func Current(ctx context.Context) Config { + memLimitMB, memSource := detectMemoryLimitMB() + effCores, cpuSource := detectEffectiveCores() + + headroomPct := readInt("LUMERA_RQ_MEM_HEADROOM_PCT", 40, 0, 90) + usableMemMB := computeUsableMem(memLimitMB, headroomPct) + + profile := selectProfile(os.Getenv("CODEC_PROFILE")) + // Support alternative name for profile if provided + if p := strings.TrimSpace(os.Getenv("LUMERA_RQ_PROFILE")); p != "" { + profile = selectProfile(p) + } + + defMaxMemMB, defConcurrency := defaultLimitsForProfile(profile, usableMemMB) + defConcurrency = adjustConcurrency(defConcurrency, effCores) + defConcurrency = rebalancePerWorkerMem(defMaxMemMB, defConcurrency, minPerWorkerMB) + + symbolSize := uint16(readUint("LUMERA_RQ_SYMBOL_SIZE", uint64(defaultSymbolSize), 1024, 65535)) + redundancy := uint8(readUint("LUMERA_RQ_REDUNDANCY", uint64(defaultRedundancy), 1, 32)) + maxMemMB := readUint("LUMERA_RQ_MAX_MEMORY_MB", defMaxMemMB, 256, 1<<20) + concurrency := readUint("LUMERA_RQ_CONCURRENCY", defConcurrency, 1, 1024) + + return Config{ + SymbolSize: symbolSize, + Redundancy: redundancy, + MaxMemoryMB: maxMemMB, + Concurrency: concurrency, + Profile: profile, + HeadroomPct: headroomPct, + MemLimitMB: memLimitMB, + MemLimitSource: memSource, + EffectiveCores: effCores, + CpuLimitSource: cpuSource, + } +} + +func minNonZero(a, b uint64) uint64 { + if a == 0 { + return b + } + if b == 0 { + return a + } + if a < b { + return a + } + return b +} + +func readUint(env string, def uint64, min uint64, max uint64) uint64 { + if v, ok := os.LookupEnv(env); ok { + if n, err := strconv.ParseUint(v, 10, 64); err == nil { + if n < min { + return min + } + if max > 0 && n > max { + return max + } + return n + } + } + return def +} + +func readInt(env string, def int, min int, max int) int { + if v, ok := os.LookupEnv(env); ok { + if n, err := strconv.Atoi(v); err == nil { + if n < min { + return min + } + if max > 0 && n > max { + return max + } + return n + } + } + return def +} + +func computeUsableMem(memLimitMB uint64, headroomPct int) uint64 { + if headroomPct <= 0 || memLimitMB == 0 { + return memLimitMB + } + return uint64(math.Max(0, float64(memLimitMB)*(1.0-float64(headroomPct)/100.0))) +} + +func selectProfile(forced string) string { + p := strings.ToLower(strings.TrimSpace(forced)) + switch p { + case "edge", "standard", "perf": + return p + } + // Default to perf when not forced + return "perf" +} + +func defaultLimitsForProfile(profile string, usableMemMB uint64) (uint64, uint64) { + switch profile { + case "edge": + mm := minNonZero(usableMemMB, 1024) + if mm == 0 { + mm = 1024 + } + return mm, 2 + case "standard": + mm := uint64(math.Min(float64(usableMemMB)*0.6, 4*1024)) + if mm < 1024 { + mm = 1024 + } + return mm, 4 + default: // perf + mm := uint64(math.Min(float64(usableMemMB)*0.6, 16*1024)) + return mm, 8 + } +} + +func adjustConcurrency(defConcurrency uint64, effCores int) uint64 { + cpu := runtime.NumCPU() + if effCores > 0 { + cpu = effCores + } + if cpu < 1 { + cpu = 1 + } + if defConcurrency > uint64(cpu) { + return uint64(cpu) + } + return defConcurrency +} + +func rebalancePerWorkerMem(defMaxMemMB uint64, defConcurrency uint64, minPerWorkerMB uint64) uint64 { + if defConcurrency == 0 { + return 1 + } + for defMaxMemMB/defConcurrency < minPerWorkerMB && defConcurrency > 1 { + defConcurrency-- + } + return defConcurrency +} + +// detectMemoryLimitMB attempts to determine the memory limit (MB) from cgroups, falling back to MemTotal. +func detectMemoryLimitMB() (uint64, string) { + // cgroup v2: /sys/fs/cgroup/memory.max + if b, err := os.ReadFile("/sys/fs/cgroup/memory.max"); err == nil { + s := strings.TrimSpace(string(b)) + if s != "max" { + if v, err := strconv.ParseUint(s, 10, 64); err == nil && v > 0 { + return v / (1024 * 1024), "cgroupv2:memory.max" + } + } + } + // cgroup v1: /sys/fs/cgroup/memory/memory.limit_in_bytes + if b, err := os.ReadFile("/sys/fs/cgroup/memory/memory.limit_in_bytes"); err == nil { + s := strings.TrimSpace(string(b)) + if v, err := strconv.ParseUint(s, 10, 64); err == nil && v > 0 { + // Some systems report a huge number when unlimited; treat > 1PB as unlimited + if v > 1<<50 { + // unlimited; fallthrough to MemTotal + } else { + return v / (1024 * 1024), "cgroupv1:memory.limit_in_bytes" + } + } + } + // Fallback: /proc/meminfo MemTotal + if b, err := os.ReadFile("/proc/meminfo"); err == nil { + lines := strings.Split(string(b), "\n") + for _, ln := range lines { + if strings.HasPrefix(ln, "MemTotal:") { + f := strings.Fields(ln) + if len(f) >= 2 { + if kb, err := strconv.ParseUint(f[1], 10, 64); err == nil { + return kb / 1024, "meminfo:MemTotal" + } + } + } + } + } + return 0, "unknown" +} + +// detectEffectiveCores attempts to determine CPU quota; returns cores and source. +func detectEffectiveCores() (int, string) { + // cgroup v2: /sys/fs/cgroup/cpu.max: "max" or " " + if b, err := os.ReadFile("/sys/fs/cgroup/cpu.max"); err == nil { + parts := strings.Fields(strings.TrimSpace(string(b))) + if len(parts) == 2 && parts[0] != "max" { + if quota, err1 := strconv.ParseUint(parts[0], 10, 64); err1 == nil { + if period, err2 := strconv.ParseUint(parts[1], 10, 64); err2 == nil && period > 0 { + cores := int(float64(quota) / float64(period)) + if cores < 1 { + cores = 1 + } + return cores, "cgroupv2:cpu.max" + } + } + } + } + // cgroup v1: /sys/fs/cgroup/cpu/cpu.cfs_quota_us and cpu.cfs_period_us + if qb, err1 := os.ReadFile("/sys/fs/cgroup/cpu/cpu.cfs_quota_us"); err1 == nil { + if pb, err2 := os.ReadFile("/sys/fs/cgroup/cpu/cpu.cfs_period_us"); err2 == nil { + qStr := strings.TrimSpace(string(qb)) + pStr := strings.TrimSpace(string(pb)) + if qStr != "-1" { + if quota, errA := strconv.ParseUint(qStr, 10, 64); errA == nil { + if period, errB := strconv.ParseUint(pStr, 10, 64); errB == nil && period > 0 { + cores := int(float64(quota) / float64(period)) + if cores < 1 { + cores = 1 + } + return cores, "cgroupv1:cpu.cfs_quota_us/period" + } + } + } + } + } + return 0, "runtime.NumCPU" +} diff --git a/proto/supernode/supernode.proto b/proto/supernode/supernode.proto index c68e8fce..eb7cba4a 100644 --- a/proto/supernode/supernode.proto +++ b/proto/supernode/supernode.proto @@ -157,4 +157,20 @@ message StatusResponse { } P2PMetrics p2p_metrics = 9; + + // RaptorQ codec configuration (effective values) + message CodecConfig { + uint32 symbol_size = 1; // bytes (typically 65535) + uint32 redundancy = 2; // repair factor (percent-like scalar; 5 = default) + uint64 max_memory_mb = 3; // memory cap for native decoder + uint32 concurrency = 4; // native decoder parallelism + string profile = 5; // selected profile: edge|standard|perf + int32 headroom_pct = 6; // reserved memory percentage (0-90) + uint64 mem_limit_mb = 7; // detected memory limit (MB) + string mem_limit_source = 8;// detection source (cgroup/meminfo) + int32 effective_cores = 9; // detected cores/quota + string cpu_limit_source = 10; // detection source (cgroups/NumCPU) + } + + CodecConfig codec = 10; } diff --git a/sdk/README.md b/sdk/README.md index b0aecb20..fb56a610 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -221,6 +221,12 @@ if err != nil { // taskID can be used to track the download progress ``` +Behavior notes: +- The SDK determines the final filename from on-chain cascade metadata and creates the path: `outputDir//`. +- Events will include both the supernode endpoint (`event.KeySupernode`) and its Cosmos address (`event.KeySupernodeAddress`). +- On success, both `SDKOutputPathReceived` and `SDKDownloadSuccessful` include `event.KeyOutputPath`. +- Failures include reason-coded suffixes in the message (e.g., `| reason=timeout`) and set `event.KeyMessage` accordingly. + **Parameters:** - `ctx context.Context`: Context for the operation - `actionID string`: ID of the action to download @@ -381,7 +387,7 @@ The SDK provides an event system to monitor task progress through event subscrip - `SDKRegistrationAttempt`: Attempting to register with a supernode - `SDKRegistrationFailure`: Registration with supernode failed - `SDKRegistrationSuccessful`: Successfully registered with supernode -- `SDKTaskTxHashReceived`: Transaction hash received from supernode +- `SDKTaskTxHashReceived`: Transaction hash received from supernode (includes endpoint in `KeySupernode` and Cosmos address in `KeySupernodeAddress`) - `SDKTaskCompleted`: Task completed successfully - `SDKTaskFailed`: Task failed with error - `SDKConnectionEstablished`: Connection to supernode established @@ -392,9 +398,9 @@ The SDK provides an event system to monitor task progress through event subscrip - `SDKProcessingFailed`: Processing failed (reason=stream_recv|missing_final_response) - `SDKProcessingTimeout`: Processing exceeded time budget and was cancelled - `SDKDownloadAttempt`: Attempting to download from supernode -- `SDKDownloadFailure`: Download attempt failed -- `SDKOutputPathReceived`: File download path received -- `SDKDownloadSuccessful`: Download completed successfully +- `SDKDownloadFailure`: Download attempt failed (message may include `| reason=timeout|canceled`; `KeyMessage` mirrors the reason) +- `SDKOutputPathReceived`: File download path received (includes `KeyOutputPath` and supernode identity keys) +- `SDKDownloadSuccessful`: Download completed successfully (includes `KeyOutputPath` and supernode identity keys) **Supernode Events (forwarded from supernodes):** - `SupernodeActionRetrieved`: Action retrieved from blockchain @@ -421,8 +427,8 @@ Events may include additional data accessible through these keys: - `event.KeyError`: Error message (for failure events) - `event.KeyCount`: Count of items (e.g., supernodes found) -- `event.KeySupernode`: Supernode endpoint -- `event.KeySupernodeAddress`: Supernode cosmos address +- `event.KeySupernode`: Supernode gRPC endpoint +- `event.KeySupernodeAddress`: Supernode Cosmos address - `event.KeyIteration`: Attempt iteration number - `event.KeyTxHash`: Transaction hash - `event.KeyMessage`: Event message diff --git a/sdk/adapters/supernodeservice/adapter.go b/sdk/adapters/supernodeservice/adapter.go index b0c43c86..e55cc845 100644 --- a/sdk/adapters/supernodeservice/adapter.go +++ b/sdk/adapters/supernodeservice/adapter.go @@ -350,9 +350,9 @@ func (a *cascadeAdapter) CascadeSupernodeRegister(ctx context.Context, in *Casca } func (a *cascadeAdapter) GetSupernodeStatus(ctx context.Context) (SupernodeStatusresponse, error) { - // Gate P2P metrics via context option to keep API backward compatible - req := &supernode.StatusRequest{IncludeP2PMetrics: includeP2PMetrics(ctx)} - resp, err := a.statusClient.GetStatus(ctx, req) + // Gate P2P metrics via context option to keep API backward compatible + req := &supernode.StatusRequest{IncludeP2PMetrics: includeP2PMetrics(ctx)} + resp, err := a.statusClient.GetStatus(ctx, req) if err != nil { a.logger.Error(ctx, "Failed to get supernode status", "error", err) return SupernodeStatusresponse{}, fmt.Errorf("failed to get supernode status: %w", err) @@ -419,6 +419,7 @@ func (a *cascadeAdapter) CascadeSupernodeDownload( if in.EventLogger != nil { in.EventLogger(ctx, toSdkEvent(x.Event.EventType), x.Event.Message, event.EventData{ + event.KeyTaskID: in.TaskID, event.KeyActionID: in.ActionID, event.KeyEventType: x.Event.EventType, event.KeyMessage: x.Event.Message, @@ -453,7 +454,7 @@ func (a *cascadeAdapter) CascadeSupernodeDownload( // toSdkEvent converts a supernode-side enum value into an internal SDK EventType. func toSdkEvent(e cascade.SupernodeEventType) event.EventType { - switch e { + switch e { case cascade.SupernodeEventType_ACTION_RETRIEVED: return event.SupernodeActionRetrieved case cascade.SupernodeEventType_ACTION_FEE_VERIFIED: @@ -478,10 +479,10 @@ func toSdkEvent(e cascade.SupernodeEventType) event.EventType { return event.SupernodeActionFinalized case cascade.SupernodeEventType_ARTEFACTS_DOWNLOADED: return event.SupernodeArtefactsDownloaded - case cascade.SupernodeEventType_FINALIZE_SIMULATED: - return event.SupernodeFinalizeSimulated - case cascade.SupernodeEventType_FINALIZE_SIMULATION_FAILED: - return event.SupernodeFinalizeSimulationFailed + case cascade.SupernodeEventType_FINALIZE_SIMULATED: + return event.SupernodeFinalizeSimulated + case cascade.SupernodeEventType_FINALIZE_SIMULATION_FAILED: + return event.SupernodeFinalizeSimulationFailed default: return event.SupernodeUnknown } @@ -511,9 +512,9 @@ func parseSuccessRate(msg string) (float64, bool) { } func toSdkSupernodeStatus(resp *supernode.StatusResponse) *SupernodeStatusresponse { - result := &SupernodeStatusresponse{} - result.Version = resp.Version - result.UptimeSeconds = resp.UptimeSeconds + result := &SupernodeStatusresponse{} + result.Version = resp.Version + result.UptimeSeconds = resp.UptimeSeconds // Convert Resources data if resp.Resources != nil { @@ -568,117 +569,117 @@ func toSdkSupernodeStatus(resp *supernode.StatusResponse) *SupernodeStatusrespon copy(result.Network.PeerAddresses, resp.Network.PeerAddresses) } - // Copy rank and IP address - result.Rank = resp.Rank - result.IPAddress = resp.IpAddress - - // Map optional P2P metrics - if resp.P2PMetrics != nil { - // DHT metrics - if resp.P2PMetrics.DhtMetrics != nil { - // Store success recent - for _, p := range resp.P2PMetrics.DhtMetrics.StoreSuccessRecent { - result.P2PMetrics.DhtMetrics.StoreSuccessRecent = append(result.P2PMetrics.DhtMetrics.StoreSuccessRecent, struct { - TimeUnix int64 - Requests int32 - Successful int32 - SuccessRate float64 - }{ - TimeUnix: p.TimeUnix, - Requests: p.Requests, - Successful: p.Successful, - SuccessRate: p.SuccessRate, - }) - } - // Batch retrieve recent - for _, p := range resp.P2PMetrics.DhtMetrics.BatchRetrieveRecent { - result.P2PMetrics.DhtMetrics.BatchRetrieveRecent = append(result.P2PMetrics.DhtMetrics.BatchRetrieveRecent, struct { - TimeUnix int64 - Keys int32 - Required int32 - FoundLocal int32 - FoundNetwork int32 - DurationMS int64 - }{ - TimeUnix: p.TimeUnix, - Keys: p.Keys, - Required: p.Required, - FoundLocal: p.FoundLocal, - FoundNetwork: p.FoundNetwork, - DurationMS: p.DurationMs, - }) - } - result.P2PMetrics.DhtMetrics.HotPathBannedSkips = resp.P2PMetrics.DhtMetrics.HotPathBannedSkips - result.P2PMetrics.DhtMetrics.HotPathBanIncrements = resp.P2PMetrics.DhtMetrics.HotPathBanIncrements - } - - // Network handle metrics - if resp.P2PMetrics.NetworkHandleMetrics != nil { - if result.P2PMetrics.NetworkHandleMetrics == nil { - result.P2PMetrics.NetworkHandleMetrics = map[string]struct{ - Total int64 - Success int64 - Failure int64 - Timeout int64 - }{} - } - for k, v := range resp.P2PMetrics.NetworkHandleMetrics { - result.P2PMetrics.NetworkHandleMetrics[k] = struct{ - Total int64 - Success int64 - Failure int64 - Timeout int64 - }{ - Total: v.Total, - Success: v.Success, - Failure: v.Failure, - Timeout: v.Timeout, - } - } - } - - // Conn pool metrics - if resp.P2PMetrics.ConnPoolMetrics != nil { - if result.P2PMetrics.ConnPoolMetrics == nil { - result.P2PMetrics.ConnPoolMetrics = map[string]int64{} - } - for k, v := range resp.P2PMetrics.ConnPoolMetrics { - result.P2PMetrics.ConnPoolMetrics[k] = v - } - } - - // Ban list - for _, b := range resp.P2PMetrics.BanList { - result.P2PMetrics.BanList = append(result.P2PMetrics.BanList, struct { - ID string - IP string - Port uint32 - Count int32 - CreatedAtUnix int64 - AgeSeconds int64 - }{ - ID: b.Id, - IP: b.Ip, - Port: b.Port, - Count: b.Count, - CreatedAtUnix: b.CreatedAtUnix, - AgeSeconds: b.AgeSeconds, - }) - } - - // Database - if resp.P2PMetrics.Database != nil { - result.P2PMetrics.Database.P2PDBSizeMB = resp.P2PMetrics.Database.P2PDbSizeMb - result.P2PMetrics.Database.P2PDBRecordsCount = resp.P2PMetrics.Database.P2PDbRecordsCount - } - - // Disk - if resp.P2PMetrics.Disk != nil { - result.P2PMetrics.Disk.AllMB = resp.P2PMetrics.Disk.AllMb - result.P2PMetrics.Disk.UsedMB = resp.P2PMetrics.Disk.UsedMb - result.P2PMetrics.Disk.FreeMB = resp.P2PMetrics.Disk.FreeMb - } - } - - return result + // Copy rank and IP address + result.Rank = resp.Rank + result.IPAddress = resp.IpAddress + + // Map optional P2P metrics + if resp.P2PMetrics != nil { + // DHT metrics + if resp.P2PMetrics.DhtMetrics != nil { + // Store success recent + for _, p := range resp.P2PMetrics.DhtMetrics.StoreSuccessRecent { + result.P2PMetrics.DhtMetrics.StoreSuccessRecent = append(result.P2PMetrics.DhtMetrics.StoreSuccessRecent, struct { + TimeUnix int64 + Requests int32 + Successful int32 + SuccessRate float64 + }{ + TimeUnix: p.TimeUnix, + Requests: p.Requests, + Successful: p.Successful, + SuccessRate: p.SuccessRate, + }) + } + // Batch retrieve recent + for _, p := range resp.P2PMetrics.DhtMetrics.BatchRetrieveRecent { + result.P2PMetrics.DhtMetrics.BatchRetrieveRecent = append(result.P2PMetrics.DhtMetrics.BatchRetrieveRecent, struct { + TimeUnix int64 + Keys int32 + Required int32 + FoundLocal int32 + FoundNetwork int32 + DurationMS int64 + }{ + TimeUnix: p.TimeUnix, + Keys: p.Keys, + Required: p.Required, + FoundLocal: p.FoundLocal, + FoundNetwork: p.FoundNetwork, + DurationMS: p.DurationMs, + }) + } + result.P2PMetrics.DhtMetrics.HotPathBannedSkips = resp.P2PMetrics.DhtMetrics.HotPathBannedSkips + result.P2PMetrics.DhtMetrics.HotPathBanIncrements = resp.P2PMetrics.DhtMetrics.HotPathBanIncrements + } + + // Network handle metrics + if resp.P2PMetrics.NetworkHandleMetrics != nil { + if result.P2PMetrics.NetworkHandleMetrics == nil { + result.P2PMetrics.NetworkHandleMetrics = map[string]struct { + Total int64 + Success int64 + Failure int64 + Timeout int64 + }{} + } + for k, v := range resp.P2PMetrics.NetworkHandleMetrics { + result.P2PMetrics.NetworkHandleMetrics[k] = struct { + Total int64 + Success int64 + Failure int64 + Timeout int64 + }{ + Total: v.Total, + Success: v.Success, + Failure: v.Failure, + Timeout: v.Timeout, + } + } + } + + // Conn pool metrics + if resp.P2PMetrics.ConnPoolMetrics != nil { + if result.P2PMetrics.ConnPoolMetrics == nil { + result.P2PMetrics.ConnPoolMetrics = map[string]int64{} + } + for k, v := range resp.P2PMetrics.ConnPoolMetrics { + result.P2PMetrics.ConnPoolMetrics[k] = v + } + } + + // Ban list + for _, b := range resp.P2PMetrics.BanList { + result.P2PMetrics.BanList = append(result.P2PMetrics.BanList, struct { + ID string + IP string + Port uint32 + Count int32 + CreatedAtUnix int64 + AgeSeconds int64 + }{ + ID: b.Id, + IP: b.Ip, + Port: b.Port, + Count: b.Count, + CreatedAtUnix: b.CreatedAtUnix, + AgeSeconds: b.AgeSeconds, + }) + } + + // Database + if resp.P2PMetrics.Database != nil { + result.P2PMetrics.Database.P2PDBSizeMB = resp.P2PMetrics.Database.P2PDbSizeMb + result.P2PMetrics.Database.P2PDBRecordsCount = resp.P2PMetrics.Database.P2PDbRecordsCount + } + + // Disk + if resp.P2PMetrics.Disk != nil { + result.P2PMetrics.Disk.AllMB = resp.P2PMetrics.Disk.AllMb + result.P2PMetrics.Disk.UsedMB = resp.P2PMetrics.Disk.UsedMb + result.P2PMetrics.Disk.FreeMB = resp.P2PMetrics.Disk.FreeMb + } + } + + return result } diff --git a/sdk/docs/cascade-timeouts.md b/sdk/docs/cascade-timeouts.md index 716804bc..b84348c1 100644 --- a/sdk/docs/cascade-timeouts.md +++ b/sdk/docs/cascade-timeouts.md @@ -121,7 +121,7 @@ This approach requires no request‑struct changes and preserves existing call s - Health checks use `connectionTimeout = 10s` during supernode discovery. - gRPC client connect behavior: adds a `30s` deadline if none is present, waits up to `ConnWaitTime = 10s` per attempt with retries. -- Downloads use a separate `downloadTimeout = 5m` envelope. +- Downloads use a separate `downloadTimeout = 5m` envelope (per-attempt). On timeout during download, the SDK emits `SDKDownloadFailure` with a reason-coded message `| reason=timeout` and sets `event.KeyMessage = "timeout"`. ## Operational Guidance @@ -149,7 +149,7 @@ This approach requires no request‑struct changes and preserves existing call s ## Events -- Upload phase timeout: `SDKUploadTimeout`. +- Upload phase timeout: classified as `SDKUploadFailed` with message suffix `| reason=timeout`. - Processing phase timeout: `SDKProcessingTimeout`. # Cascade Registration Timeouts and Networking @@ -178,7 +178,7 @@ This document describes how the SDK applies timeouts and deadlines during cascad ## Events -- `SDKUploadTimeout` — emitted when the upload phase exceeds its time budget. +- Upload phase timeout — emitted as `SDKUploadFailed` with `reason=timeout` in the message and `KeyMessage = "timeout"`. - `SDKProcessingTimeout` — emitted when the post-upload processing exceeds its time budget. ## Files and Constants diff --git a/sdk/net/impl.go b/sdk/net/impl.go index ab0f7b28..f09f8649 100644 --- a/sdk/net/impl.go +++ b/sdk/net/impl.go @@ -140,12 +140,12 @@ func (c *supernodeClient) GetSupernodeStatus(ctx context.Context) (*supernodeser // Download downloads the cascade action file func (c *supernodeClient) Download(ctx context.Context, in *supernodeservice.CascadeSupernodeDownloadRequest, opts ...grpc.CallOption) (*supernodeservice.CascadeSupernodeDownloadResponse, error) { - resp, err := c.cascadeClient.CascadeSupernodeDownload(ctx, in, opts...) - if err != nil { - return nil, fmt.Errorf("get artefacts failed: %w", err) - } + resp, err := c.cascadeClient.CascadeSupernodeDownload(ctx, in, opts...) + if err != nil { + return nil, fmt.Errorf("download failed: %w", err) + } - return resp, nil + return resp, nil } // Close closes the connection to the supernode diff --git a/sdk/task/cascade.go b/sdk/task/cascade.go index a33a3acc..3eb0b057 100644 --- a/sdk/task/cascade.go +++ b/sdk/task/cascade.go @@ -127,10 +127,11 @@ func (t *CascadeTask) attemptRegistration(ctx context.Context, _ int, sn lumera. return fmt.Errorf("upload rejected by %s: %s", sn.CosmosAddress, resp.Message) } - t.LogEvent(ctx, event.SDKTaskTxHashReceived, "txhash received", event.EventData{ - event.KeyTxHash: resp.TxHash, - event.KeySupernode: sn.CosmosAddress, - }) + t.LogEvent(ctx, event.SDKTaskTxHashReceived, "txhash received", event.EventData{ + event.KeyTxHash: resp.TxHash, + event.KeySupernode: sn.GrpcEndpoint, + event.KeySupernodeAddress: sn.CosmosAddress, + }) return nil } diff --git a/sdk/task/download.go b/sdk/task/download.go index 9d405ddb..95a1fa84 100644 --- a/sdk/task/download.go +++ b/sdk/task/download.go @@ -2,8 +2,10 @@ package task import ( "context" + stderrors "errors" "fmt" "os" + "path/filepath" "time" "github.com/LumeraProtocol/supernode/v2/sdk/adapters/lumera" @@ -84,11 +86,12 @@ func (t *CascadeDownloadTask) downloadFromSupernodes(ctx context.Context, supern result, batchErrors := t.attemptConcurrentDownload(ctx, supernodes[i:i+batchSize], clientFactory, req, i) if result != nil { - // Success! Log and return + // Success! Log and return (include final output path) t.LogEvent(ctx, event.SDKDownloadSuccessful, "download successful", event.EventData{ event.KeySupernode: result.SupernodeEndpoint, event.KeySupernodeAddress: result.SupernodeAddress, event.KeyIteration: result.Iteration, + event.KeyOutputPath: t.outputPath, }) return nil } @@ -123,6 +126,12 @@ func (t *CascadeDownloadTask) attemptDownload( } defer client.Close(ctx) + // Emit connection established for observability (parity with StartCascade) + t.LogEvent(ctx, event.SDKConnectionEstablished, "connection established", event.EventData{ + event.KeySupernode: sn.GrpcEndpoint, + event.KeySupernodeAddress: sn.CosmosAddress, + }) + req.EventLogger = func(ctx context.Context, evt event.EventType, msg string, data event.EventData) { t.LogEvent(ctx, evt, msg, data) } @@ -136,8 +145,9 @@ func (t *CascadeDownloadTask) attemptDownload( } t.LogEvent(ctx, event.SDKOutputPathReceived, "file downloaded", event.EventData{ - event.KeyOutputPath: resp.OutputPath, - event.KeySupernode: sn.CosmosAddress, + event.KeyOutputPath: resp.OutputPath, + event.KeySupernode: sn.GrpcEndpoint, + event.KeySupernodeAddress: sn.CosmosAddress, }) return nil @@ -148,6 +158,7 @@ type downloadResult struct { SupernodeAddress string SupernodeEndpoint string Iteration int + TempPath string } // attemptConcurrentDownload tries to download from multiple supernodes concurrently @@ -159,10 +170,11 @@ func (t *CascadeDownloadTask) attemptConcurrentDownload( req *supernodeservice.CascadeSupernodeDownloadRequest, baseIteration int, ) (*downloadResult, []error) { - // Remove existing file if it exists to allow overwrite (do this once before concurrent attempts) - if _, err := os.Stat(req.OutputPath); err == nil { - if removeErr := os.Remove(req.OutputPath); removeErr != nil { - return nil, []error{fmt.Errorf("failed to remove existing file %s: %w", req.OutputPath, removeErr)} + // Remove existing final file if it exists to allow overwrite (once per batch) + finalOutputPath := req.OutputPath + if _, err := os.Stat(finalOutputPath); err == nil { + if removeErr := os.Remove(finalOutputPath); removeErr != nil { + return nil, []error{fmt.Errorf("failed to remove existing file %s: %w", finalOutputPath, removeErr)} } } @@ -178,6 +190,9 @@ func (t *CascadeDownloadTask) attemptConcurrentDownload( } resultCh := make(chan attemptResult, len(batch)) + // Track per-attempt temporary output paths for safe concurrent writes + tmpPaths := make([]string, len(batch)) + // Start concurrent download attempts for idx, sn := range batch { iteration := baseIteration + idx + 1 @@ -192,14 +207,18 @@ func (t *CascadeDownloadTask) attemptConcurrentDownload( go func(sn lumera.Supernode, idx int, iter int) { // Create a copy of the request for this goroutine reqCopy := &supernodeservice.CascadeSupernodeDownloadRequest{ - ActionID: req.ActionID, - TaskID: req.TaskID, - OutputPath: req.OutputPath, + ActionID: req.ActionID, + TaskID: req.TaskID, + // Use a unique temporary path per attempt to avoid concurrent writes + OutputPath: filepath.Join(filepath.Dir(finalOutputPath), fmt.Sprintf(".%s.part.%d", filepath.Base(finalOutputPath), idx)), Signature: req.Signature, } + tmpPaths[idx] = reqCopy.OutputPath err := t.attemptDownload(batchCtx, sn, factory, reqCopy) if err != nil { + // Best-effort cleanup of the partial file for this attempt + _ = os.Remove(reqCopy.OutputPath) resultCh <- attemptResult{ err: err, idx: idx, @@ -212,6 +231,7 @@ func (t *CascadeDownloadTask) attemptConcurrentDownload( SupernodeAddress: sn.CosmosAddress, SupernodeEndpoint: sn.GrpcEndpoint, Iteration: iter, + TempPath: reqCopy.OutputPath, }, idx: idx, } @@ -219,11 +239,24 @@ func (t *CascadeDownloadTask) attemptConcurrentDownload( } // Collect results - var errors []error - for i := range len(batch) { + var errs []error + for i := 0; i < len(batch); i++ { select { case result := <-resultCh: if result.success != nil { + // Attempt to move the temp file to the final destination atomically + if err := os.Rename(result.success.TempPath, finalOutputPath); err != nil { + // Treat rename failure as a batch error + cancelBatch() + // Drain remaining results to avoid goroutine leaks + go func() { + for j := i + 1; j < len(batch); j++ { + <-resultCh + } + }() + return nil, []error{fmt.Errorf("finalize download (rename) failed: %w", err)} + } + // Success! Cancel other attempts and return cancelBatch() // Drain remaining results to avoid goroutine leaks @@ -237,13 +270,23 @@ func (t *CascadeDownloadTask) attemptConcurrentDownload( // Log failure sn := batch[result.idx] - t.LogEvent(ctx, event.SDKDownloadFailure, "download from super-node failed", event.EventData{ - event.KeySupernode: sn.GrpcEndpoint, - event.KeySupernodeAddress: sn.CosmosAddress, - event.KeyIteration: baseIteration + result.idx + 1, - event.KeyError: result.err.Error(), - }) - errors = append(errors, result.err) + // Classify failure reason when possible + data := event.EventData{ + event.KeySupernode: sn.GrpcEndpoint, + event.KeySupernodeAddress: sn.CosmosAddress, + event.KeyIteration: baseIteration + result.idx + 1, + event.KeyError: result.err.Error(), + } + msg := "download from super-node failed" + if stderrors.Is(result.err, context.DeadlineExceeded) { + data[event.KeyMessage] = "timeout" + msg += " | reason=timeout" + } else if stderrors.Is(result.err, context.Canceled) { + data[event.KeyMessage] = "canceled" + msg += " | reason=canceled" + } + t.LogEvent(ctx, event.SDKDownloadFailure, msg, data) + errs = append(errs, result.err) case <-ctx.Done(): return nil, []error{ctx.Err()} @@ -251,5 +294,5 @@ func (t *CascadeDownloadTask) attemptConcurrentDownload( } // All attempts in this batch failed - return nil, errors + return nil, errs } diff --git a/supernode/node/supernode/gateway/server.go b/supernode/node/supernode/gateway/server.go index 5440a7f4..10cf0cc1 100644 --- a/supernode/node/supernode/gateway/server.go +++ b/supernode/node/supernode/gateway/server.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "encoding/json" "fmt" "net" "net/http" @@ -11,6 +12,7 @@ import ( "github.com/grpc-ecosystem/grpc-gateway/runtime" pb "github.com/LumeraProtocol/supernode/v2/gen/supernode" + "github.com/LumeraProtocol/supernode/v2/pkg/codecconfig" "github.com/LumeraProtocol/supernode/v2/pkg/logtrace" ) @@ -69,6 +71,16 @@ func (s *Server) Run(ctx context.Context) error { // Register Swagger endpoints httpMux.HandleFunc("/swagger.json", s.serveSwaggerJSON) httpMux.HandleFunc("/swagger-ui/", s.serveSwaggerUI) + // Expose current RaptorQ codec config as part of status surface + httpMux.HandleFunc("/api/v1/codec", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + cfg := codecconfig.Current(r.Context()) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(cfg) + }) httpMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/" { http.Redirect(w, r, "/swagger-ui/", http.StatusFound) diff --git a/supernode/node/supernode/server/status_server.go b/supernode/node/supernode/server/status_server.go index 62eba841..f4d2a6d6 100644 --- a/supernode/node/supernode/server/status_server.go +++ b/supernode/node/supernode/server/status_server.go @@ -6,6 +6,7 @@ import ( "google.golang.org/grpc" pb "github.com/LumeraProtocol/supernode/v2/gen/supernode" + "github.com/LumeraProtocol/supernode/v2/pkg/codecconfig" "github.com/LumeraProtocol/supernode/v2/supernode/services/common/supernode" ) @@ -52,39 +53,39 @@ func (s *SupernodeServer) RegisterService(serviceName string, desc *grpc.Service // GetStatus implements SupernodeService.GetStatus func (s *SupernodeServer) GetStatus(ctx context.Context, req *pb.StatusRequest) (*pb.StatusResponse, error) { - // Get status from the common service; gate P2P metrics by request flag - status, err := s.statusService.GetStatus(ctx, req.GetIncludeP2PMetrics()) + // Get status from the common service; gate P2P metrics by request flag + status, err := s.statusService.GetStatus(ctx, req.GetIncludeP2PMetrics()) if err != nil { return nil, err } // Convert to protobuf response - response := &pb.StatusResponse{ - Version: status.Version, - UptimeSeconds: status.UptimeSeconds, - Resources: &pb.StatusResponse_Resources{ - Cpu: &pb.StatusResponse_Resources_CPU{ - UsagePercent: status.Resources.CPU.UsagePercent, - Cores: status.Resources.CPU.Cores, - }, - Memory: &pb.StatusResponse_Resources_Memory{ - TotalGb: status.Resources.Memory.TotalGB, - UsedGb: status.Resources.Memory.UsedGB, - AvailableGb: status.Resources.Memory.AvailableGB, - UsagePercent: status.Resources.Memory.UsagePercent, - }, - StorageVolumes: make([]*pb.StatusResponse_Resources_Storage, 0, len(status.Resources.Storage)), - HardwareSummary: status.Resources.HardwareSummary, - }, - RunningTasks: make([]*pb.StatusResponse_ServiceTasks, 0, len(status.RunningTasks)), - RegisteredServices: status.RegisteredServices, - Network: &pb.StatusResponse_Network{ - PeersCount: status.Network.PeersCount, - PeerAddresses: status.Network.PeerAddresses, - }, - Rank: status.Rank, - IpAddress: status.IPAddress, - } + response := &pb.StatusResponse{ + Version: status.Version, + UptimeSeconds: status.UptimeSeconds, + Resources: &pb.StatusResponse_Resources{ + Cpu: &pb.StatusResponse_Resources_CPU{ + UsagePercent: status.Resources.CPU.UsagePercent, + Cores: status.Resources.CPU.Cores, + }, + Memory: &pb.StatusResponse_Resources_Memory{ + TotalGb: status.Resources.Memory.TotalGB, + UsedGb: status.Resources.Memory.UsedGB, + AvailableGb: status.Resources.Memory.AvailableGB, + UsagePercent: status.Resources.Memory.UsagePercent, + }, + StorageVolumes: make([]*pb.StatusResponse_Resources_Storage, 0, len(status.Resources.Storage)), + HardwareSummary: status.Resources.HardwareSummary, + }, + RunningTasks: make([]*pb.StatusResponse_ServiceTasks, 0, len(status.RunningTasks)), + RegisteredServices: status.RegisteredServices, + Network: &pb.StatusResponse_Network{ + PeersCount: status.Network.PeersCount, + PeerAddresses: status.Network.PeerAddresses, + }, + Rank: status.Rank, + IpAddress: status.IPAddress, + } // Convert storage information for _, storage := range status.Resources.Storage { @@ -99,85 +100,100 @@ func (s *SupernodeServer) GetStatus(ctx context.Context, req *pb.StatusRequest) } // Convert service tasks - for _, service := range status.RunningTasks { - serviceTask := &pb.StatusResponse_ServiceTasks{ - ServiceName: service.ServiceName, - TaskIds: service.TaskIDs, - TaskCount: service.TaskCount, - } - response.RunningTasks = append(response.RunningTasks, serviceTask) - } - - // Map optional P2P metrics - if req.GetIncludeP2PMetrics() { - pm := status.P2PMetrics - pbdht := &pb.StatusResponse_P2PMetrics_DhtMetrics{} - for _, p := range pm.DhtMetrics.StoreSuccessRecent { - pbdht.StoreSuccessRecent = append(pbdht.StoreSuccessRecent, &pb.StatusResponse_P2PMetrics_DhtMetrics_StoreSuccessPoint{ - TimeUnix: p.TimeUnix, - Requests: p.Requests, - Successful: p.Successful, - SuccessRate: p.SuccessRate, - }) - } - for _, p := range pm.DhtMetrics.BatchRetrieveRecent { - pbdht.BatchRetrieveRecent = append(pbdht.BatchRetrieveRecent, &pb.StatusResponse_P2PMetrics_DhtMetrics_BatchRetrievePoint{ - TimeUnix: p.TimeUnix, - Keys: p.Keys, - Required: p.Required, - FoundLocal: p.FoundLocal, - FoundNetwork: p.FoundNetwork, - DurationMs: p.DurationMS, - }) - } - pbdht.HotPathBannedSkips = pm.DhtMetrics.HotPathBannedSkips - pbdht.HotPathBanIncrements = pm.DhtMetrics.HotPathBanIncrements - - pbpm := &pb.StatusResponse_P2PMetrics{ - DhtMetrics: pbdht, - NetworkHandleMetrics: map[string]*pb.StatusResponse_P2PMetrics_HandleCounters{}, - ConnPoolMetrics: map[string]int64{}, - BanList: []*pb.StatusResponse_P2PMetrics_BanEntry{}, - Database: &pb.StatusResponse_P2PMetrics_DatabaseStats{}, - Disk: &pb.StatusResponse_P2PMetrics_DiskStatus{}, - } - - // Network handle metrics - for k, v := range pm.NetworkHandleMetrics { - pbpm.NetworkHandleMetrics[k] = &pb.StatusResponse_P2PMetrics_HandleCounters{ - Total: v.Total, - Success: v.Success, - Failure: v.Failure, - Timeout: v.Timeout, - } - } - // Conn pool metrics - for k, v := range pm.ConnPoolMetrics { - pbpm.ConnPoolMetrics[k] = v - } - // Ban list - for _, b := range pm.BanList { - pbpm.BanList = append(pbpm.BanList, &pb.StatusResponse_P2PMetrics_BanEntry{ - Id: b.ID, - Ip: b.IP, - Port: b.Port, - Count: b.Count, - CreatedAtUnix: b.CreatedAtUnix, - AgeSeconds: b.AgeSeconds, - }) - } - // Database - pbpm.Database.P2PDbSizeMb = pm.Database.P2PDBSizeMB - pbpm.Database.P2PDbRecordsCount = pm.Database.P2PDBRecordsCount - // Disk - pbpm.Disk.AllMb = pm.Disk.AllMB - pbpm.Disk.UsedMb = pm.Disk.UsedMB - pbpm.Disk.FreeMb = pm.Disk.FreeMB - - response.P2PMetrics = pbpm - } - - return response, nil + for _, service := range status.RunningTasks { + serviceTask := &pb.StatusResponse_ServiceTasks{ + ServiceName: service.ServiceName, + TaskIds: service.TaskIDs, + TaskCount: service.TaskCount, + } + response.RunningTasks = append(response.RunningTasks, serviceTask) + } + + // Map optional P2P metrics + if req.GetIncludeP2PMetrics() { + pm := status.P2PMetrics + pbdht := &pb.StatusResponse_P2PMetrics_DhtMetrics{} + for _, p := range pm.DhtMetrics.StoreSuccessRecent { + pbdht.StoreSuccessRecent = append(pbdht.StoreSuccessRecent, &pb.StatusResponse_P2PMetrics_DhtMetrics_StoreSuccessPoint{ + TimeUnix: p.TimeUnix, + Requests: p.Requests, + Successful: p.Successful, + SuccessRate: p.SuccessRate, + }) + } + for _, p := range pm.DhtMetrics.BatchRetrieveRecent { + pbdht.BatchRetrieveRecent = append(pbdht.BatchRetrieveRecent, &pb.StatusResponse_P2PMetrics_DhtMetrics_BatchRetrievePoint{ + TimeUnix: p.TimeUnix, + Keys: p.Keys, + Required: p.Required, + FoundLocal: p.FoundLocal, + FoundNetwork: p.FoundNetwork, + DurationMs: p.DurationMS, + }) + } + pbdht.HotPathBannedSkips = pm.DhtMetrics.HotPathBannedSkips + pbdht.HotPathBanIncrements = pm.DhtMetrics.HotPathBanIncrements + + pbpm := &pb.StatusResponse_P2PMetrics{ + DhtMetrics: pbdht, + NetworkHandleMetrics: map[string]*pb.StatusResponse_P2PMetrics_HandleCounters{}, + ConnPoolMetrics: map[string]int64{}, + BanList: []*pb.StatusResponse_P2PMetrics_BanEntry{}, + Database: &pb.StatusResponse_P2PMetrics_DatabaseStats{}, + Disk: &pb.StatusResponse_P2PMetrics_DiskStatus{}, + } + + // Network handle metrics + for k, v := range pm.NetworkHandleMetrics { + pbpm.NetworkHandleMetrics[k] = &pb.StatusResponse_P2PMetrics_HandleCounters{ + Total: v.Total, + Success: v.Success, + Failure: v.Failure, + Timeout: v.Timeout, + } + } + // Conn pool metrics + for k, v := range pm.ConnPoolMetrics { + pbpm.ConnPoolMetrics[k] = v + } + // Ban list + for _, b := range pm.BanList { + pbpm.BanList = append(pbpm.BanList, &pb.StatusResponse_P2PMetrics_BanEntry{ + Id: b.ID, + Ip: b.IP, + Port: b.Port, + Count: b.Count, + CreatedAtUnix: b.CreatedAtUnix, + AgeSeconds: b.AgeSeconds, + }) + } + // Database + pbpm.Database.P2PDbSizeMb = pm.Database.P2PDBSizeMB + pbpm.Database.P2PDbRecordsCount = pm.Database.P2PDBRecordsCount + // Disk + pbpm.Disk.AllMb = pm.Disk.AllMB + pbpm.Disk.UsedMb = pm.Disk.UsedMB + pbpm.Disk.FreeMb = pm.Disk.FreeMB + + response.P2PMetrics = pbpm + } + + // Populate codec configuration + cfg := codecconfig.Current(ctx) + response.Codec = &pb.StatusResponse_CodecConfig{ + SymbolSize: uint32(cfg.SymbolSize), + Redundancy: uint32(cfg.Redundancy), + MaxMemoryMb: cfg.MaxMemoryMB, + Concurrency: uint32(cfg.Concurrency), + Profile: cfg.Profile, + HeadroomPct: int32(cfg.HeadroomPct), + MemLimitMb: cfg.MemLimitMB, + MemLimitSource: cfg.MemLimitSource, + EffectiveCores: int32(cfg.EffectiveCores), + CpuLimitSource: cfg.CpuLimitSource, + } + + return response, nil } // ListServices implements SupernodeService.ListServices diff --git a/supernode/services/cascade/download.go b/supernode/services/cascade/download.go index 146c8016..c403b729 100644 --- a/supernode/services/cascade/download.go +++ b/supernode/services/cascade/download.go @@ -1,11 +1,11 @@ package cascade import ( - "bytes" - "context" - "fmt" - "os" - "sort" + "bytes" + "context" + "fmt" + "os" + "sort" actiontypes "github.com/LumeraProtocol/lumera/x/action/v1/types" "github.com/LumeraProtocol/supernode/v2/pkg/codec" @@ -13,7 +13,6 @@ import ( "github.com/LumeraProtocol/supernode/v2/pkg/errors" "github.com/LumeraProtocol/supernode/v2/pkg/logtrace" "github.com/LumeraProtocol/supernode/v2/pkg/utils" - "github.com/LumeraProtocol/supernode/v2/supernode/services/cascade/adaptors" "github.com/LumeraProtocol/supernode/v2/supernode/services/common" ) @@ -125,10 +124,10 @@ func (task *CascadeRegistrationTask) downloadArtifacts(ctx context.Context, acti } func (task *CascadeRegistrationTask) restoreFileFromLayout( - ctx context.Context, - layout codec.Layout, - dataHash string, - actionID string, + ctx context.Context, + layout codec.Layout, + dataHash string, + actionID string, ) (string, string, error) { fields := logtrace.Fields{ @@ -140,34 +139,18 @@ func (task *CascadeRegistrationTask) restoreFileFromLayout( } sort.Strings(allSymbols) - totalSymbols := len(allSymbols) - requiredSymbols := (totalSymbols*requiredSymbolPercent + 99) / 100 - - fields["totalSymbols"] = totalSymbols - fields["requiredSymbols"] = requiredSymbols - logtrace.Info(ctx, "symbols to be retrieved", fields) - - symbols, err := task.P2PClient.BatchRetrieve(ctx, allSymbols, requiredSymbols, actionID) - if err != nil { - fields[logtrace.FieldError] = err.Error() - logtrace.Error(ctx, "failed to retrieve symbols", fields) - return "", "", fmt.Errorf("failed to retrieve symbols: %w", err) - } + totalSymbols := len(allSymbols) + requiredSymbols := (totalSymbols*requiredSymbolPercent + 99) / 100 - fields["retrievedSymbols"] = len(symbols) - logtrace.Info(ctx, "symbols retrieved", fields) + fields["totalSymbols"] = totalSymbols + fields["requiredSymbols"] = requiredSymbols + logtrace.Info(ctx, "symbols to be retrieved", fields) - // 2. Decode symbols using RaptorQ - decodeInfo, err := task.RQ.Decode(ctx, adaptors.DecodeRequest{ - ActionID: actionID, - Symbols: symbols, - Layout: layout, - }) - if err != nil { - fields[logtrace.FieldError] = err.Error() - logtrace.Error(ctx, "failed to decode symbols", fields) - return "", "", fmt.Errorf("decode symbols using RaptorQ: %w", err) - } + // Progressive retrieval moved to helper for readability/testing + decodeInfo, err := task.retrieveAndDecodeProgressively(ctx, allSymbols, layout, actionID, fields) + if err != nil { + return "", "", err + } fileHash, err := crypto.HashFileIncrementally(decodeInfo.FilePath, 0) if err != nil { @@ -181,7 +164,8 @@ func (task *CascadeRegistrationTask) restoreFileFromLayout( return "", "", errors.New("file hash is nil") } - err = task.verifyDataHash(ctx, fileHash, dataHash, fields) + // Validate final payload hash against on-chain data hash + err = task.verifyDataHash(ctx, fileHash, dataHash, fields) if err != nil { logtrace.Error(ctx, "failed to verify hash", fields) fields[logtrace.FieldError] = err.Error() diff --git a/supernode/services/cascade/progressive_decode.go b/supernode/services/cascade/progressive_decode.go new file mode 100644 index 00000000..87c958be --- /dev/null +++ b/supernode/services/cascade/progressive_decode.go @@ -0,0 +1,102 @@ +package cascade + +import ( + "context" + "fmt" + "strings" + + "github.com/LumeraProtocol/supernode/v2/pkg/codec" + "github.com/LumeraProtocol/supernode/v2/pkg/logtrace" + "github.com/LumeraProtocol/supernode/v2/supernode/services/cascade/adaptors" +) + +// retrieveAndDecodeProgressively progressively retrieves symbols and attempts decode at +// increasing thresholds to avoid over-fetching and reduce memory pressure. +// +// The progressive retrieval + decode loop originally lived in +// download.go::restoreFileFromLayout. It is moved here to isolate the control-flow from +// the main task logic, making Download/restoreFileFromLayout easier to follow and test. +// +// Prior implementation fetched a fixed minimum percentage and failed +// immediately on decode errors. In cases with symbol set skew or peer inconsistency, this +// could lead to repeated failures or fetching too many symbols upfront, increasing memory +// pressure. This helper escalates in steps (9%, 25%, 50%, 75%, 100%). +func (task *CascadeRegistrationTask) retrieveAndDecodeProgressively( + ctx context.Context, + allSymbols []string, + layout codec.Layout, + actionID string, + fields logtrace.Fields, +) (adaptors.DecodeResponse, error) { + // Ensure base context fields are present for logs + if fields == nil { + fields = logtrace.Fields{} + } + fields[logtrace.FieldActionID] = actionID + + totalSymbols := len(allSymbols) + // escalate retrieval targets + percents := []int{requiredSymbolPercent, 25, 50, 75, 100} + seen := map[int]struct{}{} + ordered := make([]int, 0, len(percents)) + for _, p := range percents { + if p < requiredSymbolPercent { + continue + } + if _, ok := seen[p]; !ok { + seen[p] = struct{}{} + ordered = append(ordered, p) + } + } + + var lastErr error + for _, p := range ordered { + reqCount := (totalSymbols*p + 99) / 100 + fields["targetPercent"] = p + fields["targetCount"] = reqCount + logtrace.Info(ctx, "retrieving symbols for target percent", fields) + + symbols, err := task.P2PClient.BatchRetrieve(ctx, allSymbols, reqCount, actionID) + if err != nil { + fields[logtrace.FieldError] = err.Error() + logtrace.Error(ctx, "failed to retrieve symbols", fields) + return adaptors.DecodeResponse{}, fmt.Errorf("failed to retrieve symbols: %w", err) + } + fields["retrievedSymbols"] = len(symbols) + logtrace.Info(ctx, "symbols retrieved", fields) + + // Attempt decode + decodeInfo, err := task.RQ.Decode(ctx, adaptors.DecodeRequest{ + ActionID: actionID, + Symbols: symbols, + Layout: layout, + }) + if err == nil { + return decodeInfo, nil + } + + // Only escalate for probable insufficiency/integrity errors; otherwise, fail fast + errStr := err.Error() + if p >= 100 || !( + strings.Contains(errStr, "decoding failed") || + strings.Contains(strings.ToLower(errStr), "hash mismatch") || + strings.Contains(strings.ToLower(errStr), "insufficient") || + strings.Contains(strings.ToLower(errStr), "symbol")) { + fields[logtrace.FieldError] = errStr + logtrace.Error(ctx, "failed to decode symbols", fields) + return adaptors.DecodeResponse{}, fmt.Errorf("decode symbols using RaptorQ: %w", err) + } + + logtrace.Info(ctx, "decode failed; escalating symbol target", logtrace.Fields{ + "last_error": errStr, + }) + lastErr = err + } + + if lastErr != nil { + fields[logtrace.FieldError] = lastErr.Error() + logtrace.Error(ctx, "failed to decode symbols after escalation", fields) + return adaptors.DecodeResponse{}, fmt.Errorf("decode symbols using RaptorQ: %w", lastErr) + } + return adaptors.DecodeResponse{}, fmt.Errorf("decode symbols using RaptorQ: unknown failure") +}