diff --git a/monitor/cosmwasm.go b/monitor/cosmwasm.go index 1b975a3..f9a96a5 100644 --- a/monitor/cosmwasm.go +++ b/monitor/cosmwasm.go @@ -11,6 +11,7 @@ import ( "time" "github.com/rs/zerolog" + "github.com/slack-go/slack" "golang.org/x/sync/errgroup" "github.com/ojo-network/contractMonitor/config" @@ -53,6 +54,9 @@ type cosmwasmChecker struct { requestID int64 deviationID int64 medianID int64 + rateError bool + medianError bool + deviationError bool timeout time.Time logger zerolog.Logger mut sync.Mutex @@ -97,27 +101,24 @@ func StartCosmwasmServices(ctx context.Context, logger zerolog.Logger, config co return &cms } -func (cws *CosmwasmService) GetBalance(network string) (balance, denom, relayerAddress string, err error) { +func (cws *CosmwasmService) GetBalanceAttachment(network string) (*slack.Attachment, error) { service, found := cws.services[network] if !found { - err = fmt.Errorf("network not found") - return + err := fmt.Errorf("network not found") + return nil, err } - balance, denom, relayerAddress = service.GetBalance() - return + return service.currentBalanceAttachment(), nil } -func (cws *CosmwasmService) GetIDS(network string) (rate int64, median int64, deviation int64, contractAddress string, err error) { +func (cws *CosmwasmService) GetIdAttachment(network string) (*slack.Attachment, error) { service, found := cws.services[network] if !found { - err = fmt.Errorf("network not found") - return + err := fmt.Errorf("network not found") + return nil, err } - rate, median, deviation = service.GetIds() - contractAddress = service.contractAddress - return + return service.currentRequestIDAttachment(), nil } func (cws *CosmwasmService) SetTimeout(network string, timeout time.Duration) (err error) { @@ -192,6 +193,12 @@ func (c *cosmwasmChecker) startCron(ctx context.Context, duration time.Duration) Msg("Error in querying request ids") } + // check for rate, median or deviation error + msg := c.createStaleRequestIDAttachment() + if msg != nil { + slackChan <- *msg + } + time.Sleep(duration) } } @@ -246,12 +253,9 @@ func (c *cosmwasmChecker) checkBalance(ctx context.Context) error { if amount <= c.warning { if time.Now().After(c.timeout) { - slackChan <- createLowBalanceAttachment( - (amount <= c.warning) && (amount > c.threshold), + slackChan <- c.createLowBalanceAttachment( + amount, balance.Amount, - c.denom, - c.relayerAddress, - c.network, ) } } @@ -263,6 +267,10 @@ func (c *cosmwasmChecker) checkBalance(ctx context.Context) error { func (c *cosmwasmChecker) checkQuery(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) post := time.Now().After(c.timeout) + if !post { + return nil + } + g.Go( func() error { num, err := c.returnLatestID(ctx, rate) @@ -271,18 +279,11 @@ func (c *cosmwasmChecker) checkQuery(ctx context.Context) error { } if num <= c.requestID { - if post { - slackChan <- createStaleRequestIDAttachment( - StaleRateRequestID, - c.contractAddress, - c.network, - c.requestID, - num, - ) - return nil - } + c.rateError = true + return nil } + // update the latest request id c.requestID = num return nil }, @@ -297,19 +298,11 @@ func (c *cosmwasmChecker) checkQuery(ctx context.Context) error { } if num <= c.deviationID { - if post { - slackChan <- createStaleRequestIDAttachment( - StaleDeviationRequestID, - c.contractAddress, - c.network, - c.deviationID, - num, - ) - return nil - } + c.deviationError = true + return nil } - // update to latest id + // update to latest deviation id c.deviationID = num return nil }, @@ -323,20 +316,12 @@ func (c *cosmwasmChecker) checkQuery(ctx context.Context) error { if err != nil { return err } - if post { - if num <= c.medianID { - slackChan <- createStaleRequestIDAttachment( - StaleMedianRequestID, - c.contractAddress, - c.network, - c.medianID, - num, - ) - return nil - } + if num <= c.medianID { + c.medianError = true + return nil } - // update to latest id + // update to latest median id c.medianID = num return nil }, diff --git a/monitor/events.go b/monitor/events.go index 07e7451..cf99f7b 100644 --- a/monitor/events.go +++ b/monitor/events.go @@ -60,20 +60,20 @@ func handleSlashCommand(cms *CosmwasmService, command *slack.SlashCommand) error network := commands[0] switch command.Command { case "/balance": - balance, denom, relayerAddress, err := cms.GetBalance(network) + msg, err := cms.GetBalanceAttachment(network) if err != nil { return err } - slackChan <- balanceAttachment(balance, denom, relayerAddress, network) + slackChan <- *msg case "/relayerstatus": - rid, mid, did, address, err := cms.GetIDS(network) + msg, err := cms.GetIdAttachment(network) if err != nil { return err } - slackChan <- requestIDAttachment(address, network, rid, mid, did) + slackChan <- *msg case "/timeout": if len(commands) < 2 { diff --git a/monitor/slack.go b/monitor/slack.go index a07d4f3..0da5190 100644 --- a/monitor/slack.go +++ b/monitor/slack.go @@ -9,25 +9,26 @@ import ( "github.com/slack-go/slack" ) -func createLowBalanceAttachment(warning bool, balance, denom, relayerAddress, network string) slack.Attachment { +func (c *cosmwasmChecker) createLowBalanceAttachment(amount int64, balance string) slack.Attachment { + warning := (amount <= c.warning) && (amount > c.threshold) attachment := slack.Attachment{ - Pretext: fmt.Sprintf("*Network*: %s\n*Relayer*: %s", network, Relayer), + Pretext: fmt.Sprintf("*Network*: %s\n*Relayer*: %s", c.network, Relayer), Title: fmt.Sprintf(":exclamation: %s", LowBalance), Color: "danger", Fields: []slack.AttachmentField{ { Title: "Relayer Address", - Value: fmt.Sprintf("```%s```", relayerAddress), + Value: fmt.Sprintf("```%s```", c.relayerAddress), Short: false, }, { Title: "Current balance", - Value: fmt.Sprintf("```%s%s```", balance, denom), + Value: fmt.Sprintf("```%s%s```", balance, c.denom), Short: true, }, { Title: "Network", - Value: fmt.Sprintf("```%s```", network), + Value: fmt.Sprintf("```%s```", c.network), Short: true, }, }, @@ -42,30 +43,24 @@ func createLowBalanceAttachment(warning bool, balance, denom, relayerAddress, ne return attachment } -func createStaleRequestIDAttachment(requestTitle, contractAddress, network string, oldRequestID, currentRequestID int64) slack.Attachment { +func (c *cosmwasmChecker) createStaleRequestIDAttachment() *slack.Attachment { + if !(c.rateError || c.medianError || c.deviationError) { + // no errors + return nil + } + attachment := slack.Attachment{ - Pretext: fmt.Sprintf("*Network*: %s\n*Relayer*: %s", network, Relayer), - Title: fmt.Sprintf(":exclamation: %s", requestTitle), + Pretext: fmt.Sprintf("*Network*: %s\n*Relayer*: %s", c.network, Relayer), Color: "danger", Fields: []slack.AttachmentField{ { Title: "Contract Address", - Value: fmt.Sprintf("```%s```", contractAddress), + Value: fmt.Sprintf("```%s```", c.contractAddress), Short: false, }, - { - Title: "Current Request ID", - Value: fmt.Sprintf("```%d```", currentRequestID), - Short: true, - }, - { - Title: "Old Request ID", - Value: fmt.Sprintf("```%d```", oldRequestID), - Short: true, - }, { Title: "Network", - Value: fmt.Sprintf("```%s```", network), + Value: fmt.Sprintf("```%s```", c.network), Short: false, }, }, @@ -73,28 +68,67 @@ func createStaleRequestIDAttachment(requestTitle, contractAddress, network strin Ts: json.Number(strconv.FormatInt(time.Now().Unix(), 10)), } - return attachment + title := ":exclamation: Stale" + if c.rateError { + attachment.Fields = append(attachment.Fields, slack.AttachmentField{ + Title: "Stale Request ID", + Value: fmt.Sprintf("```%d```", c.requestID), + Short: true, + }) + + title = fmt.Sprintf("%s |%s|", title, "Request") + c.rateError = false + } + + if c.medianError { + attachment.Fields = append(attachment.Fields, slack.AttachmentField{ + Title: "Stale Median ID", + Value: fmt.Sprintf("```%d```", c.medianID), + Short: true, + }) + + title = fmt.Sprintf("%s |%s|", title, "Median") + + c.medianError = false + } + + if c.deviationError { + attachment.Fields = append(attachment.Fields, slack.AttachmentField{ + Title: "Stale Deviation ID", + Value: fmt.Sprintf("```%d```", c.deviationID), + Short: true, + }) + + title = fmt.Sprintf("%s |%s|", title, "Deviation") + + c.deviationError = false + } + attachment.Title = fmt.Sprintf("%s %s", title, "ID") + + return &attachment } -func balanceAttachment(balance, denom, relayerAddress, network string) slack.Attachment { +func (c *cosmwasmChecker) currentBalanceAttachment() *slack.Attachment { + c.mut.Lock() + defer c.mut.Unlock() attachment := slack.Attachment{ - Pretext: fmt.Sprintf("*Network*: %s\n*Relayer*: %s", network, Relayer), + Pretext: fmt.Sprintf("*Network*: %s\n*Relayer*: %s", c.network, Relayer), Title: fmt.Sprint(CurrentBalance), Color: "good", Fields: []slack.AttachmentField{ { Title: "Relayer Address", - Value: fmt.Sprintf("```%s```", relayerAddress), + Value: fmt.Sprintf("```%s```", c.relayerAddress), Short: false, }, { Title: "Current balance", - Value: fmt.Sprintf("```%s%s```", balance, denom), + Value: fmt.Sprintf("```%s%s```", c.balance, c.denom), Short: true, }, { Title: "Network", - Value: fmt.Sprintf("```%s```", network), + Value: fmt.Sprintf("```%s```", c.network), Short: true, }, }, @@ -102,39 +136,40 @@ func balanceAttachment(balance, denom, relayerAddress, network string) slack.Att Ts: json.Number(strconv.FormatInt(time.Now().Unix(), 10)), } - return attachment + return &attachment } -func requestIDAttachment(contractAddress, network string, currentRequestID, medianID, deviationID int64) slack.Attachment { +func (c *cosmwasmChecker) currentRequestIDAttachment() *slack.Attachment { + c.mut.Lock() + defer c.mut.Unlock() attachment := slack.Attachment{ - Pretext: fmt.Sprintf("*Network*: %s\n*Relayer*: %s", network, Relayer), + Pretext: fmt.Sprintf("*Network*: %s\n*Relayer*: %s", c.network, Relayer), Title: fmt.Sprint(RequestIDS), Color: "good", Fields: []slack.AttachmentField{ { Title: "Contract Address", - Value: fmt.Sprintf("```%s```", contractAddress), + Value: fmt.Sprintf("```%s```", c.contractAddress), Short: false, }, { Title: "Current Request ID", - Value: fmt.Sprintf("```%d```", currentRequestID), + Value: fmt.Sprintf("```%d```", c.requestID), Short: true, }, { Title: "Current Median ID", - Value: fmt.Sprintf("```%d```", medianID), + Value: fmt.Sprintf("```%d```", c.medianID), Short: true, }, { Title: "Current Deviation ID", - Value: fmt.Sprintf("```%d```", deviationID), + Value: fmt.Sprintf("```%d```", c.deviationID), Short: true, }, - { Title: "Network", - Value: fmt.Sprintf("```%s```", network), + Value: fmt.Sprintf("```%s```", c.network), Short: false, }, }, @@ -142,7 +177,7 @@ func requestIDAttachment(contractAddress, network string, currentRequestID, medi Ts: json.Number(strconv.FormatInt(time.Now().Unix(), 10)), } - return attachment + return &attachment } func postErr(err error) slack.Attachment {