From b15d743da361cc873cf4993d1699245ee0e1882a Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 16 Oct 2024 10:14:10 -0700 Subject: [PATCH 01/18] Add txn logging --- src/webhooks/index.ts | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/webhooks/index.ts b/src/webhooks/index.ts index 77693b13..4bf9866f 100644 --- a/src/webhooks/index.ts +++ b/src/webhooks/index.ts @@ -9,38 +9,47 @@ import { kafkactlWebhook } from './kafka-control-plane/kafka-control-plane'; import { sentryOptionsWebhook } from './sentry-options/sentry-options'; import { webpackWebhook } from './webpack/webpack'; +type WebhookHandler = ( + request: FastifyRequest, + reply: FastifyReply +) => Promise; + // Error handling wrapper function export async function handleRoute( - handler, + handler: WebhookHandler, request: FastifyRequest, - reply: FastifyReply + reply: FastifyReply, + name: string ): Promise { + const tx = Sentry.startTransaction({ + op: 'webhooks', + name: 'webhooks.' + name, + }); try { await handler(request, reply); } catch (err) { - console.error(err); Sentry.captureException(err); reply.code(400).send('Bad Request'); - return; } + tx.finish(); } // Function that maps routes to their respective handlers export async function routeHandlers(server: Fastify, _options): Promise { server.post('/metrics/bootstrap-dev-env/webhook', (request, reply) => - handleRoute(bootstrapWebhook, request, reply) + handleRoute(bootstrapWebhook, request, reply, 'bootstrap-dev-env') ); server.post('/metrics/gocd/webhook', (request, reply) => - handleRoute(gocdWebhook, request, reply) + handleRoute(gocdWebhook, request, reply, 'gocd') ); server.post('/metrics/kafka-control-plane/webhook', (request, reply) => - handleRoute(kafkactlWebhook, request, reply) + handleRoute(kafkactlWebhook, request, reply, 'kafka-control-plane') ); server.post('/metrics/sentry-options/webhook', (request, reply) => - handleRoute(sentryOptionsWebhook, request, reply) + handleRoute(sentryOptionsWebhook, request, reply, 'sentry-options') ); server.post('/metrics/webpack/webhook', (request, reply) => - handleRoute(webpackWebhook, request, reply) + handleRoute(webpackWebhook, request, reply, 'webpack') ); // Default handler for invalid routes From 20b59c3247be30e9e8252e8622b84bb2c9313efd Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 16 Oct 2024 10:17:03 -0700 Subject: [PATCH 02/18] Delete unused index.ts files --- src/webhooks/bootstrap-dev-env/index.ts | 1 - src/webhooks/gocd/index.ts | 1 - src/webhooks/kafka-control-plane/index.ts | 1 - src/webhooks/sentry-options/index.ts | 1 - src/webhooks/webpack/index.ts | 1 - 5 files changed, 5 deletions(-) delete mode 100644 src/webhooks/bootstrap-dev-env/index.ts delete mode 100644 src/webhooks/gocd/index.ts delete mode 100644 src/webhooks/kafka-control-plane/index.ts delete mode 100644 src/webhooks/sentry-options/index.ts delete mode 100644 src/webhooks/webpack/index.ts diff --git a/src/webhooks/bootstrap-dev-env/index.ts b/src/webhooks/bootstrap-dev-env/index.ts deleted file mode 100644 index abe4c848..00000000 --- a/src/webhooks/bootstrap-dev-env/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { bootstrapWebhook } from './bootstrap-dev-env'; diff --git a/src/webhooks/gocd/index.ts b/src/webhooks/gocd/index.ts deleted file mode 100644 index fe90ed99..00000000 --- a/src/webhooks/gocd/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { gocdWebhook } from './gocd'; diff --git a/src/webhooks/kafka-control-plane/index.ts b/src/webhooks/kafka-control-plane/index.ts deleted file mode 100644 index bf5ecdd6..00000000 --- a/src/webhooks/kafka-control-plane/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { kafkactlWebhook } from './kafka-control-plane'; diff --git a/src/webhooks/sentry-options/index.ts b/src/webhooks/sentry-options/index.ts deleted file mode 100644 index 77b320b0..00000000 --- a/src/webhooks/sentry-options/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { sentryOptionsWebhook } from './sentry-options'; diff --git a/src/webhooks/webpack/index.ts b/src/webhooks/webpack/index.ts deleted file mode 100644 index 5b3a1952..00000000 --- a/src/webhooks/webpack/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { webpackWebhook } from './webpack'; From 575eb6f94ea379eda589b8c2859cf1cb7e7f2e27 Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 16 Oct 2024 10:19:15 -0700 Subject: [PATCH 03/18] Fix tests --- src/webhooks/index.test.ts | 2 +- src/webhooks/index.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/webhooks/index.test.ts b/src/webhooks/index.test.ts index 2ee6a6e6..a3e4f0d9 100644 --- a/src/webhooks/index.test.ts +++ b/src/webhooks/index.test.ts @@ -88,7 +88,7 @@ describe('cron jobs testing', function () { } ); const reply = new MockReply() as FastifyReply; - await handleRoute(mockError, {} as FastifyRequest, reply); + await handleRoute(mockError, {} as FastifyRequest, reply, ''); expect(mockError).toHaveBeenCalled(); expect(reply.statusCode).toBe(400); }); diff --git a/src/webhooks/index.ts b/src/webhooks/index.ts index 4bf9866f..48f541c9 100644 --- a/src/webhooks/index.ts +++ b/src/webhooks/index.ts @@ -1,3 +1,5 @@ +import '@sentry/tracing'; + import * as Sentry from '@sentry/node'; import { FastifyReply, FastifyRequest } from 'fastify'; From e38691d9fa063e9716112973f3e153fb0ef673c6 Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 16 Oct 2024 10:28:17 -0700 Subject: [PATCH 04/18] Add http status to txn --- src/webhooks/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webhooks/index.ts b/src/webhooks/index.ts index 48f541c9..941f3918 100644 --- a/src/webhooks/index.ts +++ b/src/webhooks/index.ts @@ -32,6 +32,7 @@ export async function handleRoute( } catch (err) { Sentry.captureException(err); reply.code(400).send('Bad Request'); + tx.setHttpStatus(400); } tx.finish(); } From 36272242c7afdeba8dc692b160a7e10c8bdd7b5a Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 16 Oct 2024 10:23:42 -0700 Subject: [PATCH 05/18] Deprecate old Slack handler --- src/buildServer.ts | 7 ----- src/slack/README.md | 7 ----- src/slack/index.ts | 38 -------------------------- src/webhooks/notifier/notifier.test.ts | 0 src/webhooks/notifier/notifier.ts | 0 5 files changed, 52 deletions(-) delete mode 100644 src/slack/README.md delete mode 100644 src/slack/index.ts create mode 100644 src/webhooks/notifier/notifier.test.ts create mode 100644 src/webhooks/notifier/notifier.ts diff --git a/src/buildServer.ts b/src/buildServer.ts index 21dbd43b..09909212 100644 --- a/src/buildServer.ts +++ b/src/buildServer.ts @@ -15,7 +15,6 @@ import { loadBrain } from '@utils/loadBrain'; import { SENTRY_DSN } from './config'; import { routeJobs } from './jobs'; -import { SlackRouter } from './slack'; export async function buildServer( logger: boolean | { prettyPrint: boolean } = { @@ -96,11 +95,5 @@ export async function buildServer( // Endpoints for Cloud Scheduler webhooks (Cron Jobs) server.register(routeJobs, { prefix: '/jobs' }); - server.post<{ Params: { service: string } }>( - '/slack/:service/webhook', - {}, - SlackRouter(server) - ); - return server; } diff --git a/src/slack/README.md b/src/slack/README.md deleted file mode 100644 index 50096cff..00000000 --- a/src/slack/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Middleman Webhooks for Slack - -Handlers in this folder can be sent messages to be sent to Sentry Slack channels. - -Webhooks can be added to this directory with the name `custom-name` and will be served at `/slack/custom-name/webhook`. - -To do so, create a directory at this level with the name `custom-name` containing an index.ts file that exports the webhook handler. diff --git a/src/slack/index.ts b/src/slack/index.ts deleted file mode 100644 index 0c630876..00000000 --- a/src/slack/index.ts +++ /dev/null @@ -1,38 +0,0 @@ -import path from 'path'; - -import * as Sentry from '@sentry/node'; - -// Code taken from src/webhooks/index.ts -export function SlackRouter(_server) { - return async function (request, reply) { - const rootDir = __dirname; - let handler; - - try { - const handlerPath = path.resolve(__dirname, request.params.service); - - // Prevent directory traversals - if (!handlerPath.startsWith(rootDir)) { - throw new Error('Invalid service'); - } - - ({ handler } = require(handlerPath)); - if (!handler) { - throw new Error('Invalid service'); - } - } catch (err) { - console.error(err); - Sentry.captureException(err); - reply.callNotFound(); - return; - } - - try { - return await handler(request, reply); - } catch (err) { - console.error(err); - Sentry.captureException(err); - return reply.code(400).send('Bad Request'); - } - }; -} diff --git a/src/webhooks/notifier/notifier.test.ts b/src/webhooks/notifier/notifier.test.ts new file mode 100644 index 00000000..e69de29b diff --git a/src/webhooks/notifier/notifier.ts b/src/webhooks/notifier/notifier.ts new file mode 100644 index 00000000..e69de29b From 53802aca0cc796412db45a6db526f07b786ba386 Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 16 Oct 2024 14:10:40 -0700 Subject: [PATCH 06/18] Add generic-notifier --- src/config/secrets.ts | 10 ++ src/types/index.ts | 21 ++++ src/webhooks/README.md | 4 + .../generic-notifier/generic-notifier.test.ts | 118 ++++++++++++++++++ .../generic-notifier/generic-notifier.ts | 82 ++++++++++++ src/webhooks/index.ts | 4 + src/webhooks/notifier/notifier.test.ts | 0 src/webhooks/notifier/notifier.ts | 0 .../generic-notifier/testAdminPayload.json | 5 + .../generic-notifier/testBadPayload.json | 5 + .../generic-notifier/testMegaPayload.json | 5 + .../generic-notifier/testPayload.json | 18 +++ test/utils/createGenericMessageRequest.ts | 29 +++++ 13 files changed, 301 insertions(+) create mode 100644 src/config/secrets.ts create mode 100644 src/webhooks/generic-notifier/generic-notifier.test.ts create mode 100644 src/webhooks/generic-notifier/generic-notifier.ts delete mode 100644 src/webhooks/notifier/notifier.test.ts delete mode 100644 src/webhooks/notifier/notifier.ts create mode 100644 test/payloads/generic-notifier/testAdminPayload.json create mode 100644 test/payloads/generic-notifier/testBadPayload.json create mode 100644 test/payloads/generic-notifier/testMegaPayload.json create mode 100644 test/payloads/generic-notifier/testPayload.json create mode 100644 test/utils/createGenericMessageRequest.ts diff --git a/src/config/secrets.ts b/src/config/secrets.ts new file mode 100644 index 00000000..76cb4ffd --- /dev/null +++ b/src/config/secrets.ts @@ -0,0 +1,10 @@ +/* + +This file contains secrets used for verifying incoming events from different HTTP sources. + +*/ + +export const EVENT_NOTIFIER_SECRETS = { + // Follow the pattern below to add a new secret + // exampleService: process.env.EXAMPLE_SERVICE_SECRET, +}; diff --git a/src/types/index.ts b/src/types/index.ts index 584d9f8b..0513bb83 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,5 +1,6 @@ import { IncomingMessage, Server, ServerResponse } from 'http'; +import { EventAlertType } from '@datadog/datadog-api-client/dist/packages/datadog-api-client-v1'; import { FastifyInstance } from 'fastify'; // e.g. the return type of `buildServer` @@ -26,3 +27,23 @@ export interface KafkaControlPlaneResponse { title: string; body: string; } + +export type GenericEvent = { + source: string; + timestamp: number; + service_name?: string; // Official service registry name if applicable + data: { + title: string; + message: string; + channels: { + slack?: string[]; // list of Slack Channels + datadog?: string[]; // list of DD Monitors + jira?: string[]; // list of Jira Projects + bigquery?: string; + }; + tags?: string[]; // Not used for Slack + misc: { + alertType?: EventAlertType; // Datadog alert type + }; + }; +}; diff --git a/src/webhooks/README.md b/src/webhooks/README.md index a0c1fb47..4ae748d4 100644 --- a/src/webhooks/README.md +++ b/src/webhooks/README.md @@ -30,3 +30,7 @@ Make sure to write the appropriate tests for the new webhook as well, by creatin ## Authentication Auth is handled via HMAC SHA256 signing. Each webhook expects a HMAC SHA hash sent in the `x-` header. Requests are validated by locally computing the expected HMAC SHA hash using a local secret (from an env variable) and comparing the values. `@/utils/auth/extractAndVerifySignature.ts` provides a utility function for authenticating requests. + +## Generic Event Notifier + +Handlers in the folder `notifier` can be used to send messages to be sent to Sentry Slack channels. diff --git a/src/webhooks/generic-notifier/generic-notifier.test.ts b/src/webhooks/generic-notifier/generic-notifier.test.ts new file mode 100644 index 00000000..eede31eb --- /dev/null +++ b/src/webhooks/generic-notifier/generic-notifier.test.ts @@ -0,0 +1,118 @@ +import * as Sentry from '@sentry/node'; + +import testAdminPayload from '@test/payloads/generic-notifier/testAdminPayload.json'; +import testBadPayload from '@test/payloads/generic-notifier/testBadPayload.json'; +import testMegaPayload from '@test/payloads/generic-notifier/testMegaPayload.json'; +import testPayload from '@test/payloads/generic-notifier/testPayload.json'; +import { createNotifierRequest } from '@test/utils/createGenericMessageRequest'; + +import { buildServer } from '@/buildServer'; +import { bolt } from '@api/slack'; + +import { messageSlack } from './generic-notifier'; + +describe('generic messages webhook', function () { + let fastify; + beforeEach(async function () { + fastify = await buildServer(false); + }); + + afterEach(function () { + fastify.close(); + jest.clearAllMocks(); + }); + + it('correctly inserts generic notifier when stage starts', async function () { + const response = await createNotifierRequest(fastify, testPayload); + + expect(response.statusCode).toBe(200); + }); + + it('returns 400 for invalid signature', async function () { + const response = await fastify.inject({ + method: 'POST', + url: '/event-notifier/v1', + headers: { + 'x-infra-hub-signature': 'invalid', + }, + payload: testPayload, + }); + expect(response.statusCode).toBe(400); + }); + + it('returns 400 for no signature', async function () { + const response = await fastify.inject({ + method: 'POST', + url: '/event-notifier/v1', + payload: testPayload, + }); + expect(response.statusCode).toBe(400); + }); + + describe('messageSlack tests', function () { + afterEach(function () { + jest.clearAllMocks(); + }); + + it('handles bad fields and reports to Sentry', async function () { + const sentryCaptureExceptionSpy = jest.spyOn(Sentry, 'captureException'); + const sentrySetContextSpy = jest.spyOn(Sentry, 'setContext'); + // expected error + // @ts-expect-error + await messageSlack(testBadPayload); + await messageSlack(testMegaPayload); + expect(sentryCaptureExceptionSpy).toHaveBeenCalledTimes(2); + expect(sentrySetContextSpy).toHaveBeenCalledTimes(2); + expect(sentrySetContextSpy.mock.calls[0][0]).toEqual(`message_data`); + expect(sentrySetContextSpy.mock.calls[0][1]).toEqual({ + message: { + bad_key_name: 'not good', + source: 'infra-event-notifier', + title: 'this is a title', + }, + }); + expect(sentrySetContextSpy.mock.calls[1][0]).toEqual(`message_data`); + expect(sentrySetContextSpy.mock.calls[1][1]).toEqual({ + message: { + source: 'infra-event-notifier', + title: 'really, really big payload', + body: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit.\nSed fringilla venenatis ipsum eu vestibulum.\nNam ultricies, elit sed commodo eleifend, ipsum est tempus lorem, porttitor molestie urna sem in eros.\nSuspendisse non interdum sapien, vel commodo dui.\nNunc eu scelerisque augue.\nAliquam eget rhoncus leo.\nDonec nulla elit, aliquet ut porttitor at, sodales quis ex.\nMaecenas sit amet pretium neque.\nIn eu nisi vel purus mattis vehicula in sed mauris.\nPellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas.\nProin varius eget nisl sed vulputate.\nFusce quis nibh eu enim blandit bibendum.\nCras volutpat est erat, sit amet molestie ante commodo hendrerit.\nPellentesque in luctus augue.Lorem ipsum dolor sit amet, consectetur adipiscing elit.\nIn ac bibendum odio, condimentum finibus nisi.\nIn at enim vel elit aliquet dictum.\nNullam efficitur gravida gravida.\nNullam scelerisque euismod mi, non dictum elit posuere sed.\nSed dictum quam in ornare lacinia.\nDonec vulputate dictum tortor quis volutpat.\nCurabitur at nulla hendrerit, placerat nibh sed, gravida metus.\nMaecenas nec eros sollicitudin, consectetur nibh sit amet, fringilla nibh.\nUt efficitur convallis luctus.\nIn ultricies lectus non urna faucibus, a venenatis ipsum mollis.\nVivamus tincidunt interdum lorem vitae fermentum.\nVivamus ante nunc, facilisis sed velit egestas, maximus euismod enim.\nSed pretium et ligula ac suscipit.\nSed et metus ut orci faucibus lacinia.\nInteger vestibulum commodo blandit.\nIn dapibus libero nec mauris sagittis, vel accumsan erat feugiat.\nCurabitur tempus, arcu non accumsan faucibus, sapien nisl pharetra justo, id fringilla mauris sem eu ante.\nProin ac feugiat dolor.\nNulla auctor vestibulum tortor at placerat.\nCras tempus non tortor ut dictum.\nSed eleifend velit nisi, sit amet lacinia velit convallis non.\nCurabitur imperdiet tortor sit amet massa condimentum, nec cursus lectus placerat.\nUt egestas suscipit est, at ultricies tellus viverra ut.\nSed elementum dignissim nulla, a feugiat massa mollis quis.\nPellentesque a lobortis dolor.\nCras eleifend condimentum orci, a venenatis arcu feugiat at.\nSuspendisse condimentum neque metus, non faucibus libero ultricies nec.\nInteger quis orci at enim aliquet dapibus id quis nisl.\nMaecenas et convallis massa.\nSed ornare sagittis erat, et hendrerit neque auctor quis.\nSed dignissim erat nisl, sed pharetra mauris sollicitudin sed.\nNullam eu purus quis augue scelerisque aliquam id sed enim.\nMaecenas in posuere ex.\nVivamus quis sem faucibus, suscipit eros nec, bibendum purus.\nUt in urna orci.\nAenean id bibendum urna.\nDuis gravida ac massa vel egestas.\nFusce in erat hendrerit nunc tempus dictum.\nCras sapien diam, eleifend vitae commodo in, bibendum ut eros.\nNam sit amet massa tincidunt neque rutrum sodales.\nNunc vitae felis ut diam lobortis pellentesque.\nNunc nulla ante, sodales non tempor pretium, vulputate vel nisi.\nNulla facilisis dolor sit amet aliquam imperdiet.\nPellentesque lacinia augue eget nulla finibus tincidunt.\nNam urna tellus, aliquam sit amet velit vestibulum, dictum eleifend dolor.\nDonec eu commodo est, non tincidunt orci.\nCras vel nisl libero.\nPraesent cursus neque massa.\nNunc convallis vitae sem nec tincidunt.\nEtiam eget dui in ligula dictum sodales nec quis ante.\nCras sagittis, dui at tempus porttitor, lectus ipsum malesuada augue, vitae suscipit felis lacus facilisis nisl.\nDonec tristique tellus at aliquet accumsan.\nInteger elementum venenatis mollis.\nSuspendisse rutrum eros eget justo scelerisque facilisis.\nDonec vehicula neque at lectus interdum egestas.\nPellentesque dolor dui, feugiat ac placerat sit amet, lacinia sed massa.\nMaecenas aliquam, massa sodales ultrices pellentesque, est tortor pharetra metus, at congue libero ligula ut purus.\nVestibulum mattis scelerisque tellus, vitae volutpat velit ultrices nec.\nUt mollis hendrerit magna vitae mollis.\nNulla pharetra magna eros, quis facilisis arcu accumsan sodales.\nIn pharetra quam maximus turpis volutpat blandit.\nIn at dui nec velit sagittis tincidunt.\nIn felis orci, vulputate non luctus ac, molestie vitae urna.\nIn iaculis velit id convallis condimentum.', + }, + }); + }); + + it('writes to slack', async function () { + const postMessageSpy = jest.spyOn(bolt.client.chat, 'postMessage'); + await messageSlack(testPayload); + expect(postMessageSpy).toHaveBeenCalledTimes(1); + const message = postMessageSpy.mock.calls[0][0]; + expect(message).toEqual({ + blocks: [ + { + type: 'header', + text: { + type: 'plain_text', + text: 'this is a title', + }, + }, + { + type: 'section', + text: { + type: 'mrkdwn', + text: 'this is a text body', + }, + }, + ], + text: '', + channel: 'C07E9S96YPM', + unfurl_links: false, + }); + }); + + it('only writes kafka-control-plane changes', async function () { + const postMessageSpy = jest.spyOn(bolt.client.chat, 'postMessage'); + await messageSlack(testAdminPayload); + expect(postMessageSpy).toHaveBeenCalledTimes(0); + }); + }); +}); diff --git a/src/webhooks/generic-notifier/generic-notifier.ts b/src/webhooks/generic-notifier/generic-notifier.ts new file mode 100644 index 00000000..acdf9dda --- /dev/null +++ b/src/webhooks/generic-notifier/generic-notifier.ts @@ -0,0 +1,82 @@ +import { v1 } from '@datadog/datadog-api-client'; +import * as Sentry from '@sentry/node'; +import { FastifyReply, FastifyRequest } from 'fastify'; +import moment from 'moment-timezone'; + +import { GenericEvent } from '@types'; + +import { bolt } from '@/api/slack'; +import { DATADOG_API_INSTANCE } from '@/config'; +import { EVENT_NOTIFIER_SECRETS } from '@/config/secrets'; +import { extractAndVerifySignature } from '@/utils/auth/extractAndVerifySignature'; + +export async function genericEventNotifier( + request: FastifyRequest<{ Body: GenericEvent }>, + reply: FastifyReply +): Promise { + try { + // If the webhook secret is not defined, throw an error + const { body }: { body: GenericEvent } = request; + if ( + body.source === undefined || + EVENT_NOTIFIER_SECRETS[body.source] === undefined + ) { + throw new Error('Invalid source or missing secret'); + } + + const isVerified = await extractAndVerifySignature( + request, + reply, + 'x-infra-hub-signature', + EVENT_NOTIFIER_SECRETS[body.source] + ); + + if (!isVerified) { + // If the signature is not verified, return (since extractAndVerifySignature sends the response) + return; + } + + await messageSlack(body); + await sendEventToDatadog(body, moment().unix()); + reply.code(200).send('OK'); + return; + } catch (err) { + Sentry.captureException(err); + reply.code(500).send(); + return; + } +} + +export async function sendEventToDatadog( + message: GenericEvent, + timestamp: number +) { + if (message.data.channels.datadog) { + const params: v1.EventCreateRequest = { + title: message.data.title, + text: message.data.message, + alertType: message.data.misc.alertType, + dateHappened: timestamp, + tags: message.data.tags, + }; + await DATADOG_API_INSTANCE.createEvent({ body: params }); + } +} + +export async function messageSlack(message: GenericEvent) { + if (message.data.channels.slack) { + for (const channel of message.data.channels.slack) { + const text = message.data.message; + try { + await bolt.client.chat.postMessage({ + channel: channel, + text: text, + unfurl_links: false, + }); + } catch (err) { + Sentry.setContext('msg:', { text }); + Sentry.captureException(err); + } + } + } +} diff --git a/src/webhooks/index.ts b/src/webhooks/index.ts index 941f3918..5f02b7ff 100644 --- a/src/webhooks/index.ts +++ b/src/webhooks/index.ts @@ -6,6 +6,7 @@ import { FastifyReply, FastifyRequest } from 'fastify'; import { Fastify } from '@/types'; import { bootstrapWebhook } from './bootstrap-dev-env/bootstrap-dev-env'; +import { genericEventNotifier } from './generic-notifier/generic-notifier'; import { gocdWebhook } from './gocd/gocd'; import { kafkactlWebhook } from './kafka-control-plane/kafka-control-plane'; import { sentryOptionsWebhook } from './sentry-options/sentry-options'; @@ -54,6 +55,9 @@ export async function routeHandlers(server: Fastify, _options): Promise { server.post('/metrics/webpack/webhook', (request, reply) => handleRoute(webpackWebhook, request, reply, 'webpack') ); + server.post('/event-notifier/v1', (request, reply) => + handleRoute(genericEventNotifier, request, reply, 'webpack') + ); // Default handler for invalid routes server.all('/metrics/*/webhook', async (request, reply) => { diff --git a/src/webhooks/notifier/notifier.test.ts b/src/webhooks/notifier/notifier.test.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/src/webhooks/notifier/notifier.ts b/src/webhooks/notifier/notifier.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/test/payloads/generic-notifier/testAdminPayload.json b/test/payloads/generic-notifier/testAdminPayload.json new file mode 100644 index 00000000..296049a6 --- /dev/null +++ b/test/payloads/generic-notifier/testAdminPayload.json @@ -0,0 +1,5 @@ +{ + "source": "admin", + "title": "this is a title", + "body": "this is a text body" +} diff --git a/test/payloads/generic-notifier/testBadPayload.json b/test/payloads/generic-notifier/testBadPayload.json new file mode 100644 index 00000000..48f86c03 --- /dev/null +++ b/test/payloads/generic-notifier/testBadPayload.json @@ -0,0 +1,5 @@ +{ + "source": "infra-event-notifier", + "title": "this is a title", + "bad_key_name": "not good" +} diff --git a/test/payloads/generic-notifier/testMegaPayload.json b/test/payloads/generic-notifier/testMegaPayload.json new file mode 100644 index 00000000..0a44ffb9 --- /dev/null +++ b/test/payloads/generic-notifier/testMegaPayload.json @@ -0,0 +1,5 @@ +{ + "source": "infra-event-notifier", + "title": "really, really big payload", + "body": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.\nSed fringilla venenatis ipsum eu vestibulum.\nNam ultricies, elit sed commodo eleifend, ipsum est tempus lorem, porttitor molestie urna sem in eros.\nSuspendisse non interdum sapien, vel commodo dui.\nNunc eu scelerisque augue.\nAliquam eget rhoncus leo.\nDonec nulla elit, aliquet ut porttitor at, sodales quis ex.\nMaecenas sit amet pretium neque.\nIn eu nisi vel purus mattis vehicula in sed mauris.\nPellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas.\nProin varius eget nisl sed vulputate.\nFusce quis nibh eu enim blandit bibendum.\nCras volutpat est erat, sit amet molestie ante commodo hendrerit.\nPellentesque in luctus augue.Lorem ipsum dolor sit amet, consectetur adipiscing elit.\nIn ac bibendum odio, condimentum finibus nisi.\nIn at enim vel elit aliquet dictum.\nNullam efficitur gravida gravida.\nNullam scelerisque euismod mi, non dictum elit posuere sed.\nSed dictum quam in ornare lacinia.\nDonec vulputate dictum tortor quis volutpat.\nCurabitur at nulla hendrerit, placerat nibh sed, gravida metus.\nMaecenas nec eros sollicitudin, consectetur nibh sit amet, fringilla nibh.\nUt efficitur convallis luctus.\nIn ultricies lectus non urna faucibus, a venenatis ipsum mollis.\nVivamus tincidunt interdum lorem vitae fermentum.\nVivamus ante nunc, facilisis sed velit egestas, maximus euismod enim.\nSed pretium et ligula ac suscipit.\nSed et metus ut orci faucibus lacinia.\nInteger vestibulum commodo blandit.\nIn dapibus libero nec mauris sagittis, vel accumsan erat feugiat.\nCurabitur tempus, arcu non accumsan faucibus, sapien nisl pharetra justo, id fringilla mauris sem eu ante.\nProin ac feugiat dolor.\nNulla auctor vestibulum tortor at placerat.\nCras tempus non tortor ut dictum.\nSed eleifend velit nisi, sit amet lacinia velit convallis non.\nCurabitur imperdiet tortor sit amet massa condimentum, nec cursus lectus placerat.\nUt egestas suscipit est, at ultricies tellus viverra ut.\nSed elementum dignissim nulla, a feugiat massa mollis quis.\nPellentesque a lobortis dolor.\nCras eleifend condimentum orci, a venenatis arcu feugiat at.\nSuspendisse condimentum neque metus, non faucibus libero ultricies nec.\nInteger quis orci at enim aliquet dapibus id quis nisl.\nMaecenas et convallis massa.\nSed ornare sagittis erat, et hendrerit neque auctor quis.\nSed dignissim erat nisl, sed pharetra mauris sollicitudin sed.\nNullam eu purus quis augue scelerisque aliquam id sed enim.\nMaecenas in posuere ex.\nVivamus quis sem faucibus, suscipit eros nec, bibendum purus.\nUt in urna orci.\nAenean id bibendum urna.\nDuis gravida ac massa vel egestas.\nFusce in erat hendrerit nunc tempus dictum.\nCras sapien diam, eleifend vitae commodo in, bibendum ut eros.\nNam sit amet massa tincidunt neque rutrum sodales.\nNunc vitae felis ut diam lobortis pellentesque.\nNunc nulla ante, sodales non tempor pretium, vulputate vel nisi.\nNulla facilisis dolor sit amet aliquam imperdiet.\nPellentesque lacinia augue eget nulla finibus tincidunt.\nNam urna tellus, aliquam sit amet velit vestibulum, dictum eleifend dolor.\nDonec eu commodo est, non tincidunt orci.\nCras vel nisl libero.\nPraesent cursus neque massa.\nNunc convallis vitae sem nec tincidunt.\nEtiam eget dui in ligula dictum sodales nec quis ante.\nCras sagittis, dui at tempus porttitor, lectus ipsum malesuada augue, vitae suscipit felis lacus facilisis nisl.\nDonec tristique tellus at aliquet accumsan.\nInteger elementum venenatis mollis.\nSuspendisse rutrum eros eget justo scelerisque facilisis.\nDonec vehicula neque at lectus interdum egestas.\nPellentesque dolor dui, feugiat ac placerat sit amet, lacinia sed massa.\nMaecenas aliquam, massa sodales ultrices pellentesque, est tortor pharetra metus, at congue libero ligula ut purus.\nVestibulum mattis scelerisque tellus, vitae volutpat velit ultrices nec.\nUt mollis hendrerit magna vitae mollis.\nNulla pharetra magna eros, quis facilisis arcu accumsan sodales.\nIn pharetra quam maximus turpis volutpat blandit.\nIn at dui nec velit sagittis tincidunt.\nIn felis orci, vulputate non luctus ac, molestie vitae urna.\nIn iaculis velit id convallis condimentum." +} diff --git a/test/payloads/generic-notifier/testPayload.json b/test/payloads/generic-notifier/testPayload.json new file mode 100644 index 00000000..442b91cf --- /dev/null +++ b/test/payloads/generic-notifier/testPayload.json @@ -0,0 +1,18 @@ +{ + "source": "example-service", + "timestamp": 0, + "service_name": "official_service_name", + "data": { + "title": "This is an Example Notification", + "message": "Random text here", + "tags": [ + "source:example-service", "sentry-region:all", "sentry-user:bob" + ], + "misc": {}, + "channels": { + "slack": ["#C07GZR8LA82"], + "datadog": ["example-proj-id"], + "jira": ["INC"] + } + } +} \ No newline at end of file diff --git a/test/utils/createGenericMessageRequest.ts b/test/utils/createGenericMessageRequest.ts new file mode 100644 index 00000000..ec1dd76a --- /dev/null +++ b/test/utils/createGenericMessageRequest.ts @@ -0,0 +1,29 @@ +import { Fastify } from '@/types'; +import { createSignature } from '@/utils/auth/createSignature'; + +import { GenericEvent } from '../../src/types/index'; + +function createNotifierSignature(payload) { + return createSignature( + JSON.stringify(payload), + process.env.TESTING_SECRET || '', + (i) => i, + 'sha256' + ); +} + +export async function createNotifierRequest( + fastify: Fastify, + payload: GenericEvent +) { + const signature = createNotifierSignature(payload); + + return await fastify.inject({ + method: 'POST', + url: '/metrics/kafka-control-plane/webhook', + headers: { + 'x-infra-hub-signature': signature.toString(), + }, + payload: payload, + }); +} From 6f143856fc48ddff56e8fb981892bdf1055fa1c0 Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 16 Oct 2024 15:14:16 -0700 Subject: [PATCH 07/18] Fix tests --- .env.example | 1 + .env.test | 1 + src/config/secrets.ts | 5 +- .../generic-notifier/generic-notifier.test.ts | 78 ++++++------------- .../generic-notifier/generic-notifier.ts | 3 +- src/webhooks/index.ts | 2 +- .../generic-notifier/testBadPayload.json | 18 ++++- .../generic-notifier/testMegaPayload.json | 21 ++++- .../generic-notifier/testPayload.json | 4 +- test/utils/createGenericMessageRequest.ts | 4 +- 10 files changed, 67 insertions(+), 70 deletions(-) diff --git a/.env.example b/.env.example index 3d8fe1f3..a8e47468 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,7 @@ SLACK_BOT_USER_ACCESS_TOKEN='' GOCD_WEBHOOK_SECRET='' KAFKA_CONTROL_PLANE_WEBHOOK_SECRET='' SENTRY_OPTIONS_WEBHOOK_SECRET='' +EXAMPLE_SERVICE_SECRET='' # Silence some GCP noise DRY_RUN=true diff --git a/.env.test b/.env.test index f0a9afd1..69e7c647 100644 --- a/.env.test +++ b/.env.test @@ -19,6 +19,7 @@ SLACK_BOT_APP_ID="5678" GOCD_WEBHOOK_SECRET="webhooksecret" KAFKA_CONTROL_PLANE_WEBHOOK_SECRET="kcpwebhooksecret" SENTRY_OPTIONS_WEBHOOK_SECRET="sentryoptionswebhooksecret" +EXAMPLE_SERVICE_SECRET="examplewebhooksecret" # Other GOCD_SENTRYIO_FE_PIPELINE_NAME="getsentry-frontend" diff --git a/src/config/secrets.ts b/src/config/secrets.ts index 76cb4ffd..02af665a 100644 --- a/src/config/secrets.ts +++ b/src/config/secrets.ts @@ -6,5 +6,8 @@ This file contains secrets used for verifying incoming events from different HTT export const EVENT_NOTIFIER_SECRETS = { // Follow the pattern below to add a new secret - // exampleService: process.env.EXAMPLE_SERVICE_SECRET, + // 'example-service': process.env.EXAMPLE_SERVICE_SECRET, }; +if (process.env.ENV !== 'production') + EVENT_NOTIFIER_SECRETS['example-service'] = + process.env.EXAMPLE_SERVICE_SECRET; diff --git a/src/webhooks/generic-notifier/generic-notifier.test.ts b/src/webhooks/generic-notifier/generic-notifier.test.ts index eede31eb..43c58193 100644 --- a/src/webhooks/generic-notifier/generic-notifier.test.ts +++ b/src/webhooks/generic-notifier/generic-notifier.test.ts @@ -1,12 +1,8 @@ -import * as Sentry from '@sentry/node'; - -import testAdminPayload from '@test/payloads/generic-notifier/testAdminPayload.json'; -import testBadPayload from '@test/payloads/generic-notifier/testBadPayload.json'; -import testMegaPayload from '@test/payloads/generic-notifier/testMegaPayload.json'; import testPayload from '@test/payloads/generic-notifier/testPayload.json'; import { createNotifierRequest } from '@test/utils/createGenericMessageRequest'; import { buildServer } from '@/buildServer'; +import { DATADOG_API_INSTANCE } from '@/config'; import { bolt } from '@api/slack'; import { messageSlack } from './generic-notifier'; @@ -23,6 +19,10 @@ describe('generic messages webhook', function () { }); it('correctly inserts generic notifier when stage starts', async function () { + jest.spyOn(bolt.client.chat, 'postMessage').mockImplementation(jest.fn()); + jest + .spyOn(DATADOG_API_INSTANCE, 'createEvent') + .mockImplementation(jest.fn()); const response = await createNotifierRequest(fastify, testPayload); expect(response.statusCode).toBe(200); @@ -54,65 +54,33 @@ describe('generic messages webhook', function () { jest.clearAllMocks(); }); - it('handles bad fields and reports to Sentry', async function () { - const sentryCaptureExceptionSpy = jest.spyOn(Sentry, 'captureException'); - const sentrySetContextSpy = jest.spyOn(Sentry, 'setContext'); - // expected error - // @ts-expect-error - await messageSlack(testBadPayload); - await messageSlack(testMegaPayload); - expect(sentryCaptureExceptionSpy).toHaveBeenCalledTimes(2); - expect(sentrySetContextSpy).toHaveBeenCalledTimes(2); - expect(sentrySetContextSpy.mock.calls[0][0]).toEqual(`message_data`); - expect(sentrySetContextSpy.mock.calls[0][1]).toEqual({ - message: { - bad_key_name: 'not good', - source: 'infra-event-notifier', - title: 'this is a title', - }, - }); - expect(sentrySetContextSpy.mock.calls[1][0]).toEqual(`message_data`); - expect(sentrySetContextSpy.mock.calls[1][1]).toEqual({ - message: { - source: 'infra-event-notifier', - title: 'really, really big payload', - body: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit.\nSed fringilla venenatis ipsum eu vestibulum.\nNam ultricies, elit sed commodo eleifend, ipsum est tempus lorem, porttitor molestie urna sem in eros.\nSuspendisse non interdum sapien, vel commodo dui.\nNunc eu scelerisque augue.\nAliquam eget rhoncus leo.\nDonec nulla elit, aliquet ut porttitor at, sodales quis ex.\nMaecenas sit amet pretium neque.\nIn eu nisi vel purus mattis vehicula in sed mauris.\nPellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas.\nProin varius eget nisl sed vulputate.\nFusce quis nibh eu enim blandit bibendum.\nCras volutpat est erat, sit amet molestie ante commodo hendrerit.\nPellentesque in luctus augue.Lorem ipsum dolor sit amet, consectetur adipiscing elit.\nIn ac bibendum odio, condimentum finibus nisi.\nIn at enim vel elit aliquet dictum.\nNullam efficitur gravida gravida.\nNullam scelerisque euismod mi, non dictum elit posuere sed.\nSed dictum quam in ornare lacinia.\nDonec vulputate dictum tortor quis volutpat.\nCurabitur at nulla hendrerit, placerat nibh sed, gravida metus.\nMaecenas nec eros sollicitudin, consectetur nibh sit amet, fringilla nibh.\nUt efficitur convallis luctus.\nIn ultricies lectus non urna faucibus, a venenatis ipsum mollis.\nVivamus tincidunt interdum lorem vitae fermentum.\nVivamus ante nunc, facilisis sed velit egestas, maximus euismod enim.\nSed pretium et ligula ac suscipit.\nSed et metus ut orci faucibus lacinia.\nInteger vestibulum commodo blandit.\nIn dapibus libero nec mauris sagittis, vel accumsan erat feugiat.\nCurabitur tempus, arcu non accumsan faucibus, sapien nisl pharetra justo, id fringilla mauris sem eu ante.\nProin ac feugiat dolor.\nNulla auctor vestibulum tortor at placerat.\nCras tempus non tortor ut dictum.\nSed eleifend velit nisi, sit amet lacinia velit convallis non.\nCurabitur imperdiet tortor sit amet massa condimentum, nec cursus lectus placerat.\nUt egestas suscipit est, at ultricies tellus viverra ut.\nSed elementum dignissim nulla, a feugiat massa mollis quis.\nPellentesque a lobortis dolor.\nCras eleifend condimentum orci, a venenatis arcu feugiat at.\nSuspendisse condimentum neque metus, non faucibus libero ultricies nec.\nInteger quis orci at enim aliquet dapibus id quis nisl.\nMaecenas et convallis massa.\nSed ornare sagittis erat, et hendrerit neque auctor quis.\nSed dignissim erat nisl, sed pharetra mauris sollicitudin sed.\nNullam eu purus quis augue scelerisque aliquam id sed enim.\nMaecenas in posuere ex.\nVivamus quis sem faucibus, suscipit eros nec, bibendum purus.\nUt in urna orci.\nAenean id bibendum urna.\nDuis gravida ac massa vel egestas.\nFusce in erat hendrerit nunc tempus dictum.\nCras sapien diam, eleifend vitae commodo in, bibendum ut eros.\nNam sit amet massa tincidunt neque rutrum sodales.\nNunc vitae felis ut diam lobortis pellentesque.\nNunc nulla ante, sodales non tempor pretium, vulputate vel nisi.\nNulla facilisis dolor sit amet aliquam imperdiet.\nPellentesque lacinia augue eget nulla finibus tincidunt.\nNam urna tellus, aliquam sit amet velit vestibulum, dictum eleifend dolor.\nDonec eu commodo est, non tincidunt orci.\nCras vel nisl libero.\nPraesent cursus neque massa.\nNunc convallis vitae sem nec tincidunt.\nEtiam eget dui in ligula dictum sodales nec quis ante.\nCras sagittis, dui at tempus porttitor, lectus ipsum malesuada augue, vitae suscipit felis lacus facilisis nisl.\nDonec tristique tellus at aliquet accumsan.\nInteger elementum venenatis mollis.\nSuspendisse rutrum eros eget justo scelerisque facilisis.\nDonec vehicula neque at lectus interdum egestas.\nPellentesque dolor dui, feugiat ac placerat sit amet, lacinia sed massa.\nMaecenas aliquam, massa sodales ultrices pellentesque, est tortor pharetra metus, at congue libero ligula ut purus.\nVestibulum mattis scelerisque tellus, vitae volutpat velit ultrices nec.\nUt mollis hendrerit magna vitae mollis.\nNulla pharetra magna eros, quis facilisis arcu accumsan sodales.\nIn pharetra quam maximus turpis volutpat blandit.\nIn at dui nec velit sagittis tincidunt.\nIn felis orci, vulputate non luctus ac, molestie vitae urna.\nIn iaculis velit id convallis condimentum.', - }, - }); - }); - it('writes to slack', async function () { const postMessageSpy = jest.spyOn(bolt.client.chat, 'postMessage'); await messageSlack(testPayload); expect(postMessageSpy).toHaveBeenCalledTimes(1); const message = postMessageSpy.mock.calls[0][0]; expect(message).toEqual({ - blocks: [ - { - type: 'header', - text: { - type: 'plain_text', - text: 'this is a title', - }, - }, - { - type: 'section', - text: { - type: 'mrkdwn', - text: 'this is a text body', - }, - }, - ], - text: '', - channel: 'C07E9S96YPM', + channel: '#aaaaaa', + text: 'Random text here', unfurl_links: false, }); }); + }); - it('only writes kafka-control-plane changes', async function () { - const postMessageSpy = jest.spyOn(bolt.client.chat, 'postMessage'); - await messageSlack(testAdminPayload); - expect(postMessageSpy).toHaveBeenCalledTimes(0); - }); + it('checks that slack msg is sent', async function () { + const postMessageSpy = jest.spyOn(bolt.client.chat, 'postMessage'); + const response = await createNotifierRequest(fastify, testPayload); + + expect(postMessageSpy).toHaveBeenCalledTimes(1); + + expect(response.statusCode).toBe(200); + }); + it('checks that dd msg is sent', async function () { + const ddMessageSpy = jest.spyOn(DATADOG_API_INSTANCE, 'createEvent'); + const response = await createNotifierRequest(fastify, testPayload); + + expect(ddMessageSpy).toHaveBeenCalledTimes(1); + + expect(response.statusCode).toBe(200); }); }); diff --git a/src/webhooks/generic-notifier/generic-notifier.ts b/src/webhooks/generic-notifier/generic-notifier.ts index acdf9dda..e2a484ce 100644 --- a/src/webhooks/generic-notifier/generic-notifier.ts +++ b/src/webhooks/generic-notifier/generic-notifier.ts @@ -30,9 +30,9 @@ export async function genericEventNotifier( 'x-infra-hub-signature', EVENT_NOTIFIER_SECRETS[body.source] ); - if (!isVerified) { // If the signature is not verified, return (since extractAndVerifySignature sends the response) + reply.code(400).send('Invalid signature'); return; } @@ -41,6 +41,7 @@ export async function genericEventNotifier( reply.code(200).send('OK'); return; } catch (err) { + console.error(err); Sentry.captureException(err); reply.code(500).send(); return; diff --git a/src/webhooks/index.ts b/src/webhooks/index.ts index 5f02b7ff..586b5fc1 100644 --- a/src/webhooks/index.ts +++ b/src/webhooks/index.ts @@ -56,7 +56,7 @@ export async function routeHandlers(server: Fastify, _options): Promise { handleRoute(webpackWebhook, request, reply, 'webpack') ); server.post('/event-notifier/v1', (request, reply) => - handleRoute(genericEventNotifier, request, reply, 'webpack') + handleRoute(genericEventNotifier, request, reply, 'generic-notifier') ); // Default handler for invalid routes diff --git a/test/payloads/generic-notifier/testBadPayload.json b/test/payloads/generic-notifier/testBadPayload.json index 48f86c03..dfda7d2a 100644 --- a/test/payloads/generic-notifier/testBadPayload.json +++ b/test/payloads/generic-notifier/testBadPayload.json @@ -1,5 +1,15 @@ { - "source": "infra-event-notifier", - "title": "this is a title", - "bad_key_name": "not good" -} + "service_name": "official_service_name", + "data": { + "message": "Random text here", + "tags": [ + "source:example-service", "sentry-region:all", "sentry-user:bob" + ], + "misc": {}, + "channels": { + "slack": ["#C07GZR8LA82"], + "datadog": ["example-proj-id"], + "jira": ["INC"] + } + } +} \ No newline at end of file diff --git a/test/payloads/generic-notifier/testMegaPayload.json b/test/payloads/generic-notifier/testMegaPayload.json index 0a44ffb9..1f51f5ef 100644 --- a/test/payloads/generic-notifier/testMegaPayload.json +++ b/test/payloads/generic-notifier/testMegaPayload.json @@ -1,5 +1,18 @@ { - "source": "infra-event-notifier", - "title": "really, really big payload", - "body": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.\nSed fringilla venenatis ipsum eu vestibulum.\nNam ultricies, elit sed commodo eleifend, ipsum est tempus lorem, porttitor molestie urna sem in eros.\nSuspendisse non interdum sapien, vel commodo dui.\nNunc eu scelerisque augue.\nAliquam eget rhoncus leo.\nDonec nulla elit, aliquet ut porttitor at, sodales quis ex.\nMaecenas sit amet pretium neque.\nIn eu nisi vel purus mattis vehicula in sed mauris.\nPellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas.\nProin varius eget nisl sed vulputate.\nFusce quis nibh eu enim blandit bibendum.\nCras volutpat est erat, sit amet molestie ante commodo hendrerit.\nPellentesque in luctus augue.Lorem ipsum dolor sit amet, consectetur adipiscing elit.\nIn ac bibendum odio, condimentum finibus nisi.\nIn at enim vel elit aliquet dictum.\nNullam efficitur gravida gravida.\nNullam scelerisque euismod mi, non dictum elit posuere sed.\nSed dictum quam in ornare lacinia.\nDonec vulputate dictum tortor quis volutpat.\nCurabitur at nulla hendrerit, placerat nibh sed, gravida metus.\nMaecenas nec eros sollicitudin, consectetur nibh sit amet, fringilla nibh.\nUt efficitur convallis luctus.\nIn ultricies lectus non urna faucibus, a venenatis ipsum mollis.\nVivamus tincidunt interdum lorem vitae fermentum.\nVivamus ante nunc, facilisis sed velit egestas, maximus euismod enim.\nSed pretium et ligula ac suscipit.\nSed et metus ut orci faucibus lacinia.\nInteger vestibulum commodo blandit.\nIn dapibus libero nec mauris sagittis, vel accumsan erat feugiat.\nCurabitur tempus, arcu non accumsan faucibus, sapien nisl pharetra justo, id fringilla mauris sem eu ante.\nProin ac feugiat dolor.\nNulla auctor vestibulum tortor at placerat.\nCras tempus non tortor ut dictum.\nSed eleifend velit nisi, sit amet lacinia velit convallis non.\nCurabitur imperdiet tortor sit amet massa condimentum, nec cursus lectus placerat.\nUt egestas suscipit est, at ultricies tellus viverra ut.\nSed elementum dignissim nulla, a feugiat massa mollis quis.\nPellentesque a lobortis dolor.\nCras eleifend condimentum orci, a venenatis arcu feugiat at.\nSuspendisse condimentum neque metus, non faucibus libero ultricies nec.\nInteger quis orci at enim aliquet dapibus id quis nisl.\nMaecenas et convallis massa.\nSed ornare sagittis erat, et hendrerit neque auctor quis.\nSed dignissim erat nisl, sed pharetra mauris sollicitudin sed.\nNullam eu purus quis augue scelerisque aliquam id sed enim.\nMaecenas in posuere ex.\nVivamus quis sem faucibus, suscipit eros nec, bibendum purus.\nUt in urna orci.\nAenean id bibendum urna.\nDuis gravida ac massa vel egestas.\nFusce in erat hendrerit nunc tempus dictum.\nCras sapien diam, eleifend vitae commodo in, bibendum ut eros.\nNam sit amet massa tincidunt neque rutrum sodales.\nNunc vitae felis ut diam lobortis pellentesque.\nNunc nulla ante, sodales non tempor pretium, vulputate vel nisi.\nNulla facilisis dolor sit amet aliquam imperdiet.\nPellentesque lacinia augue eget nulla finibus tincidunt.\nNam urna tellus, aliquam sit amet velit vestibulum, dictum eleifend dolor.\nDonec eu commodo est, non tincidunt orci.\nCras vel nisl libero.\nPraesent cursus neque massa.\nNunc convallis vitae sem nec tincidunt.\nEtiam eget dui in ligula dictum sodales nec quis ante.\nCras sagittis, dui at tempus porttitor, lectus ipsum malesuada augue, vitae suscipit felis lacus facilisis nisl.\nDonec tristique tellus at aliquet accumsan.\nInteger elementum venenatis mollis.\nSuspendisse rutrum eros eget justo scelerisque facilisis.\nDonec vehicula neque at lectus interdum egestas.\nPellentesque dolor dui, feugiat ac placerat sit amet, lacinia sed massa.\nMaecenas aliquam, massa sodales ultrices pellentesque, est tortor pharetra metus, at congue libero ligula ut purus.\nVestibulum mattis scelerisque tellus, vitae volutpat velit ultrices nec.\nUt mollis hendrerit magna vitae mollis.\nNulla pharetra magna eros, quis facilisis arcu accumsan sodales.\nIn pharetra quam maximus turpis volutpat blandit.\nIn at dui nec velit sagittis tincidunt.\nIn felis orci, vulputate non luctus ac, molestie vitae urna.\nIn iaculis velit id convallis condimentum." -} + "source": "example-service", + "timestamp": 0, + "service_name": "official_service_name", + "data": { + "title": "This is an Example Notification", + "message": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.\nSed fringilla venenatis ipsum eu vestibulum.\nNam ultricies, elit sed commodo eleifend, ipsum est tempus lorem, porttitor molestie urna sem in eros.\nSuspendisse non interdum sapien, vel commodo dui.\nNunc eu scelerisque augue.\nAliquam eget rhoncus leo.\nDonec nulla elit, aliquet ut porttitor at, sodales quis ex.\nMaecenas sit amet pretium neque.\nIn eu nisi vel purus mattis vehicula in sed mauris.\nPellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas.\nProin varius eget nisl sed vulputate.\nFusce quis nibh eu enim blandit bibendum.\nCras volutpat est erat, sit amet molestie ante commodo hendrerit.\nPellentesque in luctus augue.Lorem ipsum dolor sit amet, consectetur adipiscing elit.\nIn ac bibendum odio, condimentum finibus nisi.\nIn at enim vel elit aliquet dictum.\nNullam efficitur gravida gravida.\nNullam scelerisque euismod mi, non dictum elit posuere sed.\nSed dictum quam in ornare lacinia.\nDonec vulputate dictum tortor quis volutpat.\nCurabitur at nulla hendrerit, placerat nibh sed, gravida metus.\nMaecenas nec eros sollicitudin, consectetur nibh sit amet, fringilla nibh.\nUt efficitur convallis luctus.\nIn ultricies lectus non urna faucibus, a venenatis ipsum mollis.\nVivamus tincidunt interdum lorem vitae fermentum.\nVivamus ante nunc, facilisis sed velit egestas, maximus euismod enim.\nSed pretium et ligula ac suscipit.\nSed et metus ut orci faucibus lacinia.\nInteger vestibulum commodo blandit.\nIn dapibus libero nec mauris sagittis, vel accumsan erat feugiat.\nCurabitur tempus, arcu non accumsan faucibus, sapien nisl pharetra justo, id fringilla mauris sem eu ante.\nProin ac feugiat dolor.\nNulla auctor vestibulum tortor at placerat.\nCras tempus non tortor ut dictum.\nSed eleifend velit nisi, sit amet lacinia velit convallis non.\nCurabitur imperdiet tortor sit amet massa condimentum, nec cursus lectus placerat.\nUt egestas suscipit est, at ultricies tellus viverra ut.\nSed elementum dignissim nulla, a feugiat massa mollis quis.\nPellentesque a lobortis dolor.\nCras eleifend condimentum orci, a venenatis arcu feugiat at.\nSuspendisse condimentum neque metus, non faucibus libero ultricies nec.\nInteger quis orci at enim aliquet dapibus id quis nisl.\nMaecenas et convallis massa.\nSed ornare sagittis erat, et hendrerit neque auctor quis.\nSed dignissim erat nisl, sed pharetra mauris sollicitudin sed.\nNullam eu purus quis augue scelerisque aliquam id sed enim.\nMaecenas in posuere ex.\nVivamus quis sem faucibus, suscipit eros nec, bibendum purus.\nUt in urna orci.\nAenean id bibendum urna.\nDuis gravida ac massa vel egestas.\nFusce in erat hendrerit nunc tempus dictum.\nCras sapien diam, eleifend vitae commodo in, bibendum ut eros.\nNam sit amet massa tincidunt neque rutrum sodales.\nNunc vitae felis ut diam lobortis pellentesque.\nNunc nulla ante, sodales non tempor pretium, vulputate vel nisi.\nNulla facilisis dolor sit amet aliquam imperdiet.\nPellentesque lacinia augue eget nulla finibus tincidunt.\nNam urna tellus, aliquam sit amet velit vestibulum, dictum eleifend dolor.\nDonec eu commodo est, non tincidunt orci.\nCras vel nisl libero.\nPraesent cursus neque massa.\nNunc convallis vitae sem nec tincidunt.\nEtiam eget dui in ligula dictum sodales nec quis ante.\nCras sagittis, dui at tempus porttitor, lectus ipsum malesuada augue, vitae suscipit felis lacus facilisis nisl.\nDonec tristique tellus at aliquet accumsan.\nInteger elementum venenatis mollis.\nSuspendisse rutrum eros eget justo scelerisque facilisis.\nDonec vehicula neque at lectus interdum egestas.\nPellentesque dolor dui, feugiat ac placerat sit amet, lacinia sed massa.\nMaecenas aliquam, massa sodales ultrices pellentesque, est tortor pharetra metus, at congue libero ligula ut purus.\nVestibulum mattis scelerisque tellus, vitae volutpat velit ultrices nec.\nUt mollis hendrerit magna vitae mollis.\nNulla pharetra magna eros, quis facilisis arcu accumsan sodales.\nIn pharetra quam maximus turpis volutpat blandit.\nIn at dui nec velit sagittis tincidunt.\nIn felis orci, vulputate non luctus ac, molestie vitae urna.\nIn iaculis velit id convallis condimentum.", + "tags": [ + "source:example-service", "sentry-region:all", "sentry-user:bob" + ], + "misc": {}, + "channels": { + "slack": ["#C07GZR8LA82"], + "datadog": ["example-proj-id"], + "jira": ["INC"] + } + } +} \ No newline at end of file diff --git a/test/payloads/generic-notifier/testPayload.json b/test/payloads/generic-notifier/testPayload.json index 442b91cf..77330298 100644 --- a/test/payloads/generic-notifier/testPayload.json +++ b/test/payloads/generic-notifier/testPayload.json @@ -10,9 +10,9 @@ ], "misc": {}, "channels": { - "slack": ["#C07GZR8LA82"], + "slack": ["#aaaaaa"], "datadog": ["example-proj-id"], - "jira": ["INC"] + "jira": ["TEST"] } } } \ No newline at end of file diff --git a/test/utils/createGenericMessageRequest.ts b/test/utils/createGenericMessageRequest.ts index ec1dd76a..b3709eb1 100644 --- a/test/utils/createGenericMessageRequest.ts +++ b/test/utils/createGenericMessageRequest.ts @@ -6,7 +6,7 @@ import { GenericEvent } from '../../src/types/index'; function createNotifierSignature(payload) { return createSignature( JSON.stringify(payload), - process.env.TESTING_SECRET || '', + process.env.EXAMPLE_SERVICE_SECRET || '', (i) => i, 'sha256' ); @@ -20,7 +20,7 @@ export async function createNotifierRequest( return await fastify.inject({ method: 'POST', - url: '/metrics/kafka-control-plane/webhook', + url: '/event-notifier/v1', headers: { 'x-infra-hub-signature': signature.toString(), }, From 2d50f636b725e8ec1d6c998cc98744b79af7e6a1 Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 16 Oct 2024 15:28:49 -0700 Subject: [PATCH 08/18] Add blocks to generic notifier --- src/types/index.ts | 2 ++ src/webhooks/generic-notifier/generic-notifier.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/types/index.ts b/src/types/index.ts index 0513bb83..ea3cf403 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,6 +1,7 @@ import { IncomingMessage, Server, ServerResponse } from 'http'; import { EventAlertType } from '@datadog/datadog-api-client/dist/packages/datadog-api-client-v1'; +import { Block, KnownBlock } from '@slack/types'; import { FastifyInstance } from 'fastify'; // e.g. the return type of `buildServer` @@ -44,6 +45,7 @@ export type GenericEvent = { tags?: string[]; // Not used for Slack misc: { alertType?: EventAlertType; // Datadog alert type + blocks?: (KnownBlock | Block)[]; // Optional Slack blocks }; }; }; diff --git a/src/webhooks/generic-notifier/generic-notifier.ts b/src/webhooks/generic-notifier/generic-notifier.ts index e2a484ce..f9d3f165 100644 --- a/src/webhooks/generic-notifier/generic-notifier.ts +++ b/src/webhooks/generic-notifier/generic-notifier.ts @@ -32,7 +32,6 @@ export async function genericEventNotifier( ); if (!isVerified) { // If the signature is not verified, return (since extractAndVerifySignature sends the response) - reply.code(400).send('Invalid signature'); return; } @@ -71,6 +70,7 @@ export async function messageSlack(message: GenericEvent) { try { await bolt.client.chat.postMessage({ channel: channel, + blocks: message.data.misc.blocks, text: text, unfurl_links: false, }); From 67ff8c2773423da977d75eab7e104f23863b07f8 Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 16 Oct 2024 15:43:30 -0700 Subject: [PATCH 09/18] Update readme --- src/webhooks/README.md | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/webhooks/README.md b/src/webhooks/README.md index 4ae748d4..21a95384 100644 --- a/src/webhooks/README.md +++ b/src/webhooks/README.md @@ -3,6 +3,36 @@ * Webhooks in "production" are deployed to a Google Cloud Run instance, in the project `super-big-data`. Why? (TODO insert why) * The webhook points to `https://product-eng-webhooks-vmrqv3f7nq-uw.a.run.app` +## Generic Event Notifier + +The folder `generic-notifier` provides a generic webhook which can be used to send messages to Sentry Slack channels and Sentry Datadog. Using this webhook is VERY simple. + +Simply, go to `@/config/secrets.ts` and add an entry to the `EVENT_NOTIFIER_SECRETS` object. This entry should contain a mapping from the name of your service (for example, `example-service`) to an environment variable. [TODO: Fill in how to set the prod env var here]. Make a PR with this change and get it approved & merged. + +Once this has been deployed, all you have to do is send a POST request to `https://product-eng-webhooks-vmrqv3f7nq-uw.a.run.app/event-notifier/v1` with a JSON payload in the format of the type `GenericEvent` defined in `@/types/index.ts`. Example: + +```json +{ + "source": "example-service", // This must match the mapping string you define in the EVENT_NOTIFIER_SECRETS obj + "timestamp": 0, + "service_name": "official_service_name", + "data": { + "title": "This is an Example Notification", + "message": "Random text here", + "tags": [ + "source:example-service", "sentry-region:all", "sentry-user:bob" + ], + "misc": {}, + "channels": { + "slack": ["C07EH2QGGQ5"], + "jira": ["TEST"] + } + } +} +``` + +Additionally, you must compute the HMAC SHA256 hash of the raw payload string computed with the secret key, and attach it to the `Authorization` header. EX: `Authorization: ` + ## Adding a webhook to GoCD event emitter * goto [gocd](deploy.getsentry.net) @@ -30,7 +60,3 @@ Make sure to write the appropriate tests for the new webhook as well, by creatin ## Authentication Auth is handled via HMAC SHA256 signing. Each webhook expects a HMAC SHA hash sent in the `x-` header. Requests are validated by locally computing the expected HMAC SHA hash using a local secret (from an env variable) and comparing the values. `@/utils/auth/extractAndVerifySignature.ts` provides a utility function for authenticating requests. - -## Generic Event Notifier - -Handlers in the folder `notifier` can be used to send messages to be sent to Sentry Slack channels. From 2628a1a9bb4da8d3a0ec212c2e3ac01c663560ec Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 16 Oct 2024 15:57:28 -0700 Subject: [PATCH 10/18] Add 400 error for invalid source field --- .../generic-notifier/generic-notifier.test.ts | 9 +++++++++ .../generic-notifier/generic-notifier.ts | 1 + .../generic-notifier/testInvalidPayload.json | 18 ++++++++++++++++++ 3 files changed, 28 insertions(+) create mode 100644 test/payloads/generic-notifier/testInvalidPayload.json diff --git a/src/webhooks/generic-notifier/generic-notifier.test.ts b/src/webhooks/generic-notifier/generic-notifier.test.ts index 43c58193..2995eeb0 100644 --- a/src/webhooks/generic-notifier/generic-notifier.test.ts +++ b/src/webhooks/generic-notifier/generic-notifier.test.ts @@ -1,3 +1,4 @@ +import testInvalidPayload from '@test/payloads/generic-notifier/testInvalidPayload.json'; import testPayload from '@test/payloads/generic-notifier/testPayload.json'; import { createNotifierRequest } from '@test/utils/createGenericMessageRequest'; @@ -28,6 +29,14 @@ describe('generic messages webhook', function () { expect(response.statusCode).toBe(200); }); + it('returns 400 for an invalid source', async function () { + const response = await fastify.inject({ + method: 'POST', + url: '/event-notifier/v1', + payload: testInvalidPayload, + }); + expect(response.statusCode).toBe(400); + }); it('returns 400 for invalid signature', async function () { const response = await fastify.inject({ method: 'POST', diff --git a/src/webhooks/generic-notifier/generic-notifier.ts b/src/webhooks/generic-notifier/generic-notifier.ts index f9d3f165..4fbc8aa4 100644 --- a/src/webhooks/generic-notifier/generic-notifier.ts +++ b/src/webhooks/generic-notifier/generic-notifier.ts @@ -21,6 +21,7 @@ export async function genericEventNotifier( body.source === undefined || EVENT_NOTIFIER_SECRETS[body.source] === undefined ) { + reply.code(400).send('Invalid source or missing secret'); throw new Error('Invalid source or missing secret'); } diff --git a/test/payloads/generic-notifier/testInvalidPayload.json b/test/payloads/generic-notifier/testInvalidPayload.json new file mode 100644 index 00000000..28f9defa --- /dev/null +++ b/test/payloads/generic-notifier/testInvalidPayload.json @@ -0,0 +1,18 @@ +{ + "source": "bad-source", + "timestamp": 0, + "service_name": "official_service_name", + "data": { + "title": "This is an Example Notification", + "message": "Random text here", + "tags": [ + "source:example-service", "sentry-region:all", "sentry-user:bob" + ], + "misc": {}, + "channels": { + "slack": ["#aaaaaa"], + "datadog": ["example-proj-id"], + "jira": ["TEST"] + } + } +} \ No newline at end of file From ad0745b1c62c6d017723f60f532a789cbb114a6b Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 16 Oct 2024 16:06:54 -0700 Subject: [PATCH 11/18] Update readme --- src/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/README.md b/src/README.md index 835ffd3d..aae002e7 100644 --- a/src/README.md +++ b/src/README.md @@ -20,6 +20,10 @@ Below are descriptions for how this application is organized. Each directory con ## Common Use Cases +## Generic Event Notifier + +You can use this service to send a message to Sentry Slack or Datadog. All you have to do is create a small PR to create a HMAC secret for your use case, and your service can send messages to Sentry Slack and Datadog via infra-hub. See [this README](webhooks/README.md) for more details. + ### Adding a New Webhook To add a new webhook, nagivate to `webhooks` and follow the directions there. Most of the logic should be self-contained within the `webhooks` directory, with handlers in `brain` being appropriate if the webhook is for receiving event streams. To send a message to external sources, use the APIs in `api`. From c61f87c94ee5625f34db1bb2bd84b614bad0a773 Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 13 Nov 2024 11:46:17 -0800 Subject: [PATCH 12/18] Change generic webhook API --- .github/workflows/ci.yml | 28 +++++- .github/workflows/migration.yml | 10 +- bin/deploy.sh | 1 + src/config/secrets.ts | 2 + src/types/index.ts | 47 ++++++---- src/utils/misc/serviceRegistry.ts | 8 ++ src/webhooks/README.md | 36 +++++--- .../generic-notifier/generic-notifier.test.ts | 42 +++++++-- .../generic-notifier/generic-notifier.ts | 91 +++++++++++++------ .../generic-notifier/testMegaPayload.json | 29 +++--- .../generic-notifier/testPayload.json | 34 ++++--- .../generic-notifier/testServicePayload.json | 12 +++ 12 files changed, 248 insertions(+), 92 deletions(-) create mode 100644 src/utils/misc/serviceRegistry.ts create mode 100644 test/payloads/generic-notifier/testServicePayload.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bcc129e..34eb8e3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,13 +58,22 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Install Git + run: | + sudo apt-get update + sudo apt-get install git -y + - name: Set up SSH agent (for service-registry) + uses: webfactory/ssh-agent@v0.8.0 + with: + ssh-private-key: ${{ secrets.SENTRY_INTERNAL_GH_SSH_PRIVATE_KEY }} + - name: Setup node uses: actions/setup-node@v4 with: node-version: '18' - name: yarn install - run: yarn install --immutable + run: yarn install --immutable && yarn up "service-registry@git+ssh://git@github.com:getsentry/service-registry#main" - name: tsc run: yarn build @@ -85,8 +94,13 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Set up SSH agent (for service-registry) + uses: webfactory/ssh-agent@v0.8.0 + with: + ssh-private-key: ${{ secrets.SENTRY_INTERNAL_GH_SSH_PRIVATE_KEY }} + - name: Builds docker image - run: docker build -t ci-tooling . + run: DOCKER_BUILDKIT=1 docker build --ssh default -t ci-tooling . build-deploy: name: build and deploy @@ -110,6 +124,14 @@ jobs: with: # for Sentry releases fetch-depth: 0 + - name: Install Git + run: | + sudo apt-get update + sudo apt-get install git -y + - name: Set up SSH agent (for service-registry) + uses: webfactory/ssh-agent@v0.8.0 + with: + ssh-private-key: ${{ secrets.SENTRY_INTERNAL_GH_SSH_PRIVATE_KEY }} - name: Setup node uses: actions/setup-node@v4 @@ -117,7 +139,7 @@ jobs: node-version: '18' - name: yarn install - run: yarn install --immutable + run: yarn install --immutable && yarn up "service-registry@git+ssh://git@github.com:getsentry/service-registry#main" - name: tsc run: yarn build diff --git a/.github/workflows/migration.yml b/.github/workflows/migration.yml index a66c1bda..7fca1419 100644 --- a/.github/workflows/migration.yml +++ b/.github/workflows/migration.yml @@ -33,6 +33,14 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Install Git + run: | + sudo apt-get update + sudo apt-get install git -y + - name: Set up SSH agent (for service-registry) + uses: webfactory/ssh-agent@v0.8.0 + with: + ssh-private-key: ${{ secrets.SENTRY_INTERNAL_GH_SSH_PRIVATE_KEY }} - name: Setup node uses: actions/setup-node@v4 @@ -59,7 +67,7 @@ jobs: - name: yarn install run: | - yarn install --immutable + yarn install --immutable && yarn up "service-registry@git+ssh://git@github.com:getsentry/service-registry#main" - name: Run migration env: diff --git a/bin/deploy.sh b/bin/deploy.sh index c0b62162..92edb3c7 100755 --- a/bin/deploy.sh +++ b/bin/deploy.sh @@ -27,6 +27,7 @@ GOCD_WEBHOOK_SECRET KAFKA_CONTROL_PLANE_WEBHOOK_SECRET SENTRY_OPTIONS_WEBHOOK_SECRET " +# TODO: Revamp this and make it easier to add secrets & deploy to GCP secrets="" for secret_name in $secret_names; do diff --git a/src/config/secrets.ts b/src/config/secrets.ts index 02af665a..c2ea6d5b 100644 --- a/src/config/secrets.ts +++ b/src/config/secrets.ts @@ -6,6 +6,8 @@ This file contains secrets used for verifying incoming events from different HTT export const EVENT_NOTIFIER_SECRETS = { // Follow the pattern below to add a new secret + // The secret will also need to be added in the deploy.sh script and in + // Google Secret manager // 'example-service': process.env.EXAMPLE_SERVICE_SECRET, }; if (process.env.ENV !== 'production') diff --git a/src/types/index.ts b/src/types/index.ts index ea3cf403..0416b9b8 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -29,23 +29,38 @@ export interface KafkaControlPlaneResponse { body: string; } +export interface SlackMessage { + type: 'slack'; + channels: string[]; + text: string; + blocks?: KnownBlock[] | Block[]; +} + +export interface DatadogEvent { + type: 'datadog'; + title: string; + text: string; + tags: string[]; + alertType: EventAlertType; +} + +export interface JiraEvent { + type: 'jira'; + projectId: string; + title: string; +} + export type GenericEvent = { source: string; timestamp: number; - service_name?: string; // Official service registry name if applicable - data: { - title: string; - message: string; - channels: { - slack?: string[]; // list of Slack Channels - datadog?: string[]; // list of DD Monitors - jira?: string[]; // list of Jira Projects - bigquery?: string; - }; - tags?: string[]; // Not used for Slack - misc: { - alertType?: EventAlertType; // Datadog alert type - blocks?: (KnownBlock | Block)[]; // Optional Slack blocks - }; - }; + data: (DatadogEvent | JiraEvent | SlackMessage | ServiceSlackMessage)[]; }; + +// Currently only used for Slack notifications since +// service registry only contains Slack channels (and not DD or Jira or others) +export interface ServiceSlackMessage { + type: 'service_notification'; + service_name: string; // Official service registry service id + text: string; + blocks?: KnownBlock[] | Block[]; +} diff --git a/src/utils/misc/serviceRegistry.ts b/src/utils/misc/serviceRegistry.ts new file mode 100644 index 00000000..564d7880 --- /dev/null +++ b/src/utils/misc/serviceRegistry.ts @@ -0,0 +1,8 @@ +import servicesData from 'service-registry/sentry_service_registry/config/combined/service_registry.json'; +import type { Service, ServiceRegistry } from 'service-registry/types/index'; + +const services: ServiceRegistry = servicesData; + +export function getService(serviceId: string): Service { + return services[serviceId]; +} diff --git a/src/webhooks/README.md b/src/webhooks/README.md index 21a95384..d2b85980 100644 --- a/src/webhooks/README.md +++ b/src/webhooks/README.md @@ -7,27 +7,35 @@ The folder `generic-notifier` provides a generic webhook which can be used to send messages to Sentry Slack channels and Sentry Datadog. Using this webhook is VERY simple. -Simply, go to `@/config/secrets.ts` and add an entry to the `EVENT_NOTIFIER_SECRETS` object. This entry should contain a mapping from the name of your service (for example, `example-service`) to an environment variable. [TODO: Fill in how to set the prod env var here]. Make a PR with this change and get it approved & merged. +Simply, go to `@/config/secrets.ts` and add an entry to the `EVENT_NOTIFIER_SECRETS` object. This entry should contain a mapping from the source of the message (for example, `example-service`) to an environment variable. As of now, you will also need to edit `bin/deploy.sh` to add the new secret to the deployment and also add the secret to Google Secret Manager. Make a PR with this change and get it approved & merged. -Once this has been deployed, all you have to do is send a POST request to `https://product-eng-webhooks-vmrqv3f7nq-uw.a.run.app/event-notifier/v1` with a JSON payload in the format of the type `GenericEvent` defined in `@/types/index.ts`. Example: +Once this has been deployed, all you have to do is send a POST request to `https://product-eng-webhooks-vmrqv3f7nq-uw.a.run.app/event-notifier/v1` with a JSON payload in the format of the type `GenericEvent` defined in `@/types/index.ts`. Currently, only Datadog and Slack messages are supported. Example: ```json { "source": "example-service", // This must match the mapping string you define in the EVENT_NOTIFIER_SECRETS obj "timestamp": 0, - "service_name": "official_service_name", - "data": { - "title": "This is an Example Notification", - "message": "Random text here", - "tags": [ - "source:example-service", "sentry-region:all", "sentry-user:bob" - ], - "misc": {}, - "channels": { - "slack": ["C07EH2QGGQ5"], - "jira": ["TEST"] + "data": [ + { + "type": "slack", // Basic Slack message + "text": "Random text here", + "channels": ["#aaaaaa"], + // Optionally, include Slack Blocks + "blocks": [] + }, { + "type": "service_notification", // Slack message using service registry information + "service_name": "eng_pipes_gh_notifications", + "text": "Random text here", + // Optionally, include Slack Blocks + "blocks": [] + }, { + "type": "datadog", // Datadog message + "title": "This is an Example Notification", + "text": "Random text here", + "tags": ["source:example-service", "sentry-region:all", "sentry-user:bob"], + "alertType": "info" } - } + ] } ``` diff --git a/src/webhooks/generic-notifier/generic-notifier.test.ts b/src/webhooks/generic-notifier/generic-notifier.test.ts index 2995eeb0..9db372ac 100644 --- a/src/webhooks/generic-notifier/generic-notifier.test.ts +++ b/src/webhooks/generic-notifier/generic-notifier.test.ts @@ -1,12 +1,14 @@ import testInvalidPayload from '@test/payloads/generic-notifier/testInvalidPayload.json'; import testPayload from '@test/payloads/generic-notifier/testPayload.json'; +import testServicePayload from '@test/payloads/generic-notifier/testServicePayload.json'; import { createNotifierRequest } from '@test/utils/createGenericMessageRequest'; import { buildServer } from '@/buildServer'; import { DATADOG_API_INSTANCE } from '@/config'; +import { GenericEvent, ServiceSlackMessage, SlackMessage } from '@/types'; import { bolt } from '@api/slack'; -import { messageSlack } from './generic-notifier'; +import { handleServiceSlackMessage, messageSlack } from './generic-notifier'; describe('generic messages webhook', function () { let fastify; @@ -24,7 +26,10 @@ describe('generic messages webhook', function () { jest .spyOn(DATADOG_API_INSTANCE, 'createEvent') .mockImplementation(jest.fn()); - const response = await createNotifierRequest(fastify, testPayload); + const response = await createNotifierRequest( + fastify, + testPayload as GenericEvent + ); expect(response.statusCode).toBe(200); }); @@ -65,7 +70,7 @@ describe('generic messages webhook', function () { it('writes to slack', async function () { const postMessageSpy = jest.spyOn(bolt.client.chat, 'postMessage'); - await messageSlack(testPayload); + await messageSlack(testPayload.data[0] as SlackMessage); expect(postMessageSpy).toHaveBeenCalledTimes(1); const message = postMessageSpy.mock.calls[0][0]; expect(message).toEqual({ @@ -75,18 +80,43 @@ describe('generic messages webhook', function () { }); }); }); + describe('handleServiceSlackMessage tests', function () { + afterEach(function () { + jest.clearAllMocks(); + }); + + it('writes to slack', async function () { + const postMessageSpy = jest.spyOn(bolt.client.chat, 'postMessage'); + await handleServiceSlackMessage( + testServicePayload.data[0] as ServiceSlackMessage + ); + expect(postMessageSpy).toHaveBeenCalledTimes(1); + const message = postMessageSpy.mock.calls[0][0]; + expect(message).toEqual({ + channel: 'feed-datdog', + text: 'Random text here', + unfurl_links: false, + }); + }); + }); it('checks that slack msg is sent', async function () { const postMessageSpy = jest.spyOn(bolt.client.chat, 'postMessage'); - const response = await createNotifierRequest(fastify, testPayload); + const response = await createNotifierRequest( + fastify, + testPayload as GenericEvent + ); - expect(postMessageSpy).toHaveBeenCalledTimes(1); + expect(postMessageSpy).toHaveBeenCalledTimes(2); expect(response.statusCode).toBe(200); }); it('checks that dd msg is sent', async function () { const ddMessageSpy = jest.spyOn(DATADOG_API_INSTANCE, 'createEvent'); - const response = await createNotifierRequest(fastify, testPayload); + const response = await createNotifierRequest( + fastify, + testPayload as GenericEvent + ); expect(ddMessageSpy).toHaveBeenCalledTimes(1); diff --git a/src/webhooks/generic-notifier/generic-notifier.ts b/src/webhooks/generic-notifier/generic-notifier.ts index 4fbc8aa4..17b2c2d9 100644 --- a/src/webhooks/generic-notifier/generic-notifier.ts +++ b/src/webhooks/generic-notifier/generic-notifier.ts @@ -1,15 +1,21 @@ import { v1 } from '@datadog/datadog-api-client'; import * as Sentry from '@sentry/node'; import { FastifyReply, FastifyRequest } from 'fastify'; -import moment from 'moment-timezone'; -import { GenericEvent } from '@types'; +import { + DatadogEvent, + GenericEvent, + ServiceSlackMessage, + SlackMessage, +} from '@types'; import { bolt } from '@/api/slack'; import { DATADOG_API_INSTANCE } from '@/config'; import { EVENT_NOTIFIER_SECRETS } from '@/config/secrets'; import { extractAndVerifySignature } from '@/utils/auth/extractAndVerifySignature'; +import { getService } from '../../utils/misc/serviceRegistry'; + export async function genericEventNotifier( request: FastifyRequest<{ Body: GenericEvent }>, reply: FastifyReply @@ -24,7 +30,6 @@ export async function genericEventNotifier( reply.code(400).send('Invalid source or missing secret'); throw new Error('Invalid source or missing secret'); } - const isVerified = await extractAndVerifySignature( request, reply, @@ -35,9 +40,15 @@ export async function genericEventNotifier( // If the signature is not verified, return (since extractAndVerifySignature sends the response) return; } - - await messageSlack(body); - await sendEventToDatadog(body, moment().unix()); + for (const message of body.data) { + if (message.type === 'slack') { + await messageSlack(message); + } else if (message.type === 'service_notification') { + await handleServiceSlackMessage(message); + } else if (message.type === 'datadog') { + await sendEventToDatadog(message, body.timestamp); + } + } reply.code(200).send('OK'); return; } catch (err) { @@ -49,36 +60,64 @@ export async function genericEventNotifier( } export async function sendEventToDatadog( - message: GenericEvent, + message: DatadogEvent, timestamp: number ) { - if (message.data.channels.datadog) { + try { const params: v1.EventCreateRequest = { - title: message.data.title, - text: message.data.message, - alertType: message.data.misc.alertType, + title: message.title, + text: message.text, + alertType: message.alertType, dateHappened: timestamp, - tags: message.data.tags, + tags: message.tags, }; await DATADOG_API_INSTANCE.createEvent({ body: params }); + } catch (err) { + Sentry.setContext('dd msg:', { text: message.text }); + Sentry.captureException(err); + } +} + +export async function messageSlack(message: SlackMessage) { + const channels = message.channels ?? []; + for (const channel of channels) { + try { + const args = { + channel: channel, + blocks: message.blocks, + text: message.text, + unfurl_links: false, + }; + if (message.blocks) { + args.blocks = message.blocks; + } + await bolt.client.chat.postMessage(args); + } catch (err) { + Sentry.setContext('slack msg:', { text: message.text }); + Sentry.captureException(err); + } } } -export async function messageSlack(message: GenericEvent) { - if (message.data.channels.slack) { - for (const channel of message.data.channels.slack) { - const text = message.data.message; - try { - await bolt.client.chat.postMessage({ - channel: channel, - blocks: message.data.misc.blocks, - text: text, - unfurl_links: false, - }); - } catch (err) { - Sentry.setContext('msg:', { text }); - Sentry.captureException(err); +export async function handleServiceSlackMessage(message: ServiceSlackMessage) { + const service = getService(message.service_name); + const channels = service.alert_slack_channels ?? []; + for (const channel of channels) { + try { + const args = { + channel: channel, + blocks: message.blocks, + text: message.text, + unfurl_links: false, + }; + if (message.blocks) { + args.blocks = message.blocks; } + await bolt.client.chat.postMessage(args); + } catch (err) { + Sentry.setContext('slack msg:', { text: message.text }); + Sentry.captureException(err); } } + // TODO: Add other types of notifications (Jira, DD, etc.) } diff --git a/test/payloads/generic-notifier/testMegaPayload.json b/test/payloads/generic-notifier/testMegaPayload.json index 1f51f5ef..4122309a 100644 --- a/test/payloads/generic-notifier/testMegaPayload.json +++ b/test/payloads/generic-notifier/testMegaPayload.json @@ -1,18 +1,21 @@ { "source": "example-service", "timestamp": 0, - "service_name": "official_service_name", - "data": { - "title": "This is an Example Notification", - "message": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.\nSed fringilla venenatis ipsum eu vestibulum.\nNam ultricies, elit sed commodo eleifend, ipsum est tempus lorem, porttitor molestie urna sem in eros.\nSuspendisse non interdum sapien, vel commodo dui.\nNunc eu scelerisque augue.\nAliquam eget rhoncus leo.\nDonec nulla elit, aliquet ut porttitor at, sodales quis ex.\nMaecenas sit amet pretium neque.\nIn eu nisi vel purus mattis vehicula in sed mauris.\nPellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas.\nProin varius eget nisl sed vulputate.\nFusce quis nibh eu enim blandit bibendum.\nCras volutpat est erat, sit amet molestie ante commodo hendrerit.\nPellentesque in luctus augue.Lorem ipsum dolor sit amet, consectetur adipiscing elit.\nIn ac bibendum odio, condimentum finibus nisi.\nIn at enim vel elit aliquet dictum.\nNullam efficitur gravida gravida.\nNullam scelerisque euismod mi, non dictum elit posuere sed.\nSed dictum quam in ornare lacinia.\nDonec vulputate dictum tortor quis volutpat.\nCurabitur at nulla hendrerit, placerat nibh sed, gravida metus.\nMaecenas nec eros sollicitudin, consectetur nibh sit amet, fringilla nibh.\nUt efficitur convallis luctus.\nIn ultricies lectus non urna faucibus, a venenatis ipsum mollis.\nVivamus tincidunt interdum lorem vitae fermentum.\nVivamus ante nunc, facilisis sed velit egestas, maximus euismod enim.\nSed pretium et ligula ac suscipit.\nSed et metus ut orci faucibus lacinia.\nInteger vestibulum commodo blandit.\nIn dapibus libero nec mauris sagittis, vel accumsan erat feugiat.\nCurabitur tempus, arcu non accumsan faucibus, sapien nisl pharetra justo, id fringilla mauris sem eu ante.\nProin ac feugiat dolor.\nNulla auctor vestibulum tortor at placerat.\nCras tempus non tortor ut dictum.\nSed eleifend velit nisi, sit amet lacinia velit convallis non.\nCurabitur imperdiet tortor sit amet massa condimentum, nec cursus lectus placerat.\nUt egestas suscipit est, at ultricies tellus viverra ut.\nSed elementum dignissim nulla, a feugiat massa mollis quis.\nPellentesque a lobortis dolor.\nCras eleifend condimentum orci, a venenatis arcu feugiat at.\nSuspendisse condimentum neque metus, non faucibus libero ultricies nec.\nInteger quis orci at enim aliquet dapibus id quis nisl.\nMaecenas et convallis massa.\nSed ornare sagittis erat, et hendrerit neque auctor quis.\nSed dignissim erat nisl, sed pharetra mauris sollicitudin sed.\nNullam eu purus quis augue scelerisque aliquam id sed enim.\nMaecenas in posuere ex.\nVivamus quis sem faucibus, suscipit eros nec, bibendum purus.\nUt in urna orci.\nAenean id bibendum urna.\nDuis gravida ac massa vel egestas.\nFusce in erat hendrerit nunc tempus dictum.\nCras sapien diam, eleifend vitae commodo in, bibendum ut eros.\nNam sit amet massa tincidunt neque rutrum sodales.\nNunc vitae felis ut diam lobortis pellentesque.\nNunc nulla ante, sodales non tempor pretium, vulputate vel nisi.\nNulla facilisis dolor sit amet aliquam imperdiet.\nPellentesque lacinia augue eget nulla finibus tincidunt.\nNam urna tellus, aliquam sit amet velit vestibulum, dictum eleifend dolor.\nDonec eu commodo est, non tincidunt orci.\nCras vel nisl libero.\nPraesent cursus neque massa.\nNunc convallis vitae sem nec tincidunt.\nEtiam eget dui in ligula dictum sodales nec quis ante.\nCras sagittis, dui at tempus porttitor, lectus ipsum malesuada augue, vitae suscipit felis lacus facilisis nisl.\nDonec tristique tellus at aliquet accumsan.\nInteger elementum venenatis mollis.\nSuspendisse rutrum eros eget justo scelerisque facilisis.\nDonec vehicula neque at lectus interdum egestas.\nPellentesque dolor dui, feugiat ac placerat sit amet, lacinia sed massa.\nMaecenas aliquam, massa sodales ultrices pellentesque, est tortor pharetra metus, at congue libero ligula ut purus.\nVestibulum mattis scelerisque tellus, vitae volutpat velit ultrices nec.\nUt mollis hendrerit magna vitae mollis.\nNulla pharetra magna eros, quis facilisis arcu accumsan sodales.\nIn pharetra quam maximus turpis volutpat blandit.\nIn at dui nec velit sagittis tincidunt.\nIn felis orci, vulputate non luctus ac, molestie vitae urna.\nIn iaculis velit id convallis condimentum.", - "tags": [ - "source:example-service", "sentry-region:all", "sentry-user:bob" - ], - "misc": {}, - "channels": { - "slack": ["#C07GZR8LA82"], - "datadog": ["example-proj-id"], - "jira": ["INC"] + "data": [ + { + "type": "slack", + "text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.\nSed fringilla venenatis ipsum eu vestibulum.\nNam ultricies, elit sed commodo eleifend, ipsum est tempus lorem, porttitor molestie urna sem in eros.\nSuspendisse non interdum sapien, vel commodo dui.\nNunc eu scelerisque augue.\nAliquam eget rhoncus leo.\nDonec nulla elit, aliquet ut porttitor at, sodales quis ex.\nMaecenas sit amet pretium neque.\nIn eu nisi vel purus mattis vehicula in sed mauris.\nPellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas.\nProin varius eget nisl sed vulputate.\nFusce quis nibh eu enim blandit bibendum.\nCras volutpat est erat, sit amet molestie ante commodo hendrerit.\nPellentesque in luctus augue.Lorem ipsum dolor sit amet, consectetur adipiscing elit.\nIn ac bibendum odio, condimentum finibus nisi.\nIn at enim vel elit aliquet dictum.\nNullam efficitur gravida gravida.\nNullam scelerisque euismod mi, non dictum elit posuere sed.\nSed dictum quam in ornare lacinia.\nDonec vulputate dictum tortor quis volutpat.\nCurabitur at nulla hendrerit, placerat nibh sed, gravida metus.\nMaecenas nec eros sollicitudin, consectetur nibh sit amet, fringilla nibh.\nUt efficitur convallis luctus.\nIn ultricies lectus non urna faucibus, a venenatis ipsum mollis.\nVivamus tincidunt interdum lorem vitae fermentum.\nVivamus ante nunc, facilisis sed velit egestas, maximus euismod enim.\nSed pretium et ligula ac suscipit.\nSed et metus ut orci faucibus lacinia.\nInteger vestibulum commodo blandit.\nIn dapibus libero nec mauris sagittis, vel accumsan erat feugiat.\nCurabitur tempus, arcu non accumsan faucibus, sapien nisl pharetra justo, id fringilla mauris sem eu ante.\nProin ac feugiat dolor.\nNulla auctor vestibulum tortor at placerat.\nCras tempus non tortor ut dictum.\nSed eleifend velit nisi, sit amet lacinia velit convallis non.\nCurabitur imperdiet tortor sit amet massa condimentum, nec cursus lectus placerat.\nUt egestas suscipit est, at ultricies tellus viverra ut.\nSed elementum dignissim nulla, a feugiat massa mollis quis.\nPellentesque a lobortis dolor.\nCras eleifend condimentum orci, a venenatis arcu feugiat at.\nSuspendisse condimentum neque metus, non faucibus libero ultricies nec.\nInteger quis orci at enim aliquet dapibus id quis nisl.\nMaecenas et convallis massa.\nSed ornare sagittis erat, et hendrerit neque auctor quis.\nSed dignissim erat nisl, sed pharetra mauris sollicitudin sed.\nNullam eu purus quis augue scelerisque aliquam id sed enim.\nMaecenas in posuere ex.\nVivamus quis sem faucibus, suscipit eros nec, bibendum purus.\nUt in urna orci.\nAenean id bibendum urna.\nDuis gravida ac massa vel egestas.\nFusce in erat hendrerit nunc tempus dictum.\nCras sapien diam, eleifend vitae commodo in, bibendum ut eros.\nNam sit amet massa tincidunt neque rutrum sodales.\nNunc vitae felis ut diam lobortis pellentesque.\nNunc nulla ante, sodales non tempor pretium, vulputate vel nisi.\nNulla facilisis dolor sit amet aliquam imperdiet.\nPellentesque lacinia augue eget nulla finibus tincidunt.\nNam urna tellus, aliquam sit amet velit vestibulum, dictum eleifend dolor.\nDonec eu commodo est, non tincidunt orci.\nCras vel nisl libero.\nPraesent cursus neque massa.\nNunc convallis vitae sem nec tincidunt.\nEtiam eget dui in ligula dictum sodales nec quis ante.\nCras sagittis, dui at tempus porttitor, lectus ipsum malesuada augue, vitae suscipit felis lacus facilisis nisl.\nDonec tristique tellus at aliquet accumsan.\nInteger elementum venenatis mollis.\nSuspendisse rutrum eros eget justo scelerisque facilisis.\nDonec vehicula neque at lectus interdum egestas.\nPellentesque dolor dui, feugiat ac placerat sit amet, lacinia sed massa.\nMaecenas aliquam, massa sodales ultrices pellentesque, est tortor pharetra metus, at congue libero ligula ut purus.\nVestibulum mattis scelerisque tellus, vitae volutpat velit ultrices nec.\nUt mollis hendrerit magna vitae mollis.\nNulla pharetra magna eros, quis facilisis arcu accumsan sodales.\nIn pharetra quam maximus turpis volutpat blandit.\nIn at dui nec velit sagittis tincidunt.\nIn felis orci, vulputate non luctus ac, molestie vitae urna.\nIn iaculis velit id convallis condimentum.", + "channels": ["#aaaaaa"] + }, { + "type": "datadog", + "title": "This is an Example Notification", + "text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.\nSed fringilla venenatis ipsum eu vestibulum.\nNam ultricies, elit sed commodo eleifend, ipsum est tempus lorem, porttitor molestie urna sem in eros.\nSuspendisse non interdum sapien, vel commodo dui.\nNunc eu scelerisque augue.\nAliquam eget rhoncus leo.\nDonec nulla elit, aliquet ut porttitor at, sodales quis ex.\nMaecenas sit amet pretium neque.\nIn eu nisi vel purus mattis vehicula in sed mauris.\nPellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas.\nProin varius eget nisl sed vulputate.\nFusce quis nibh eu enim blandit bibendum.\nCras volutpat est erat, sit amet molestie ante commodo hendrerit.\nPellentesque in luctus augue.Lorem ipsum dolor sit amet, consectetur adipiscing elit.\nIn ac bibendum odio, condimentum finibus nisi.\nIn at enim vel elit aliquet dictum.\nNullam efficitur gravida gravida.\nNullam scelerisque euismod mi, non dictum elit posuere sed.\nSed dictum quam in ornare lacinia.\nDonec vulputate dictum tortor quis volutpat.\nCurabitur at nulla hendrerit, placerat nibh sed, gravida metus.\nMaecenas nec eros sollicitudin, consectetur nibh sit amet, fringilla nibh.\nUt efficitur convallis luctus.\nIn ultricies lectus non urna faucibus, a venenatis ipsum mollis.\nVivamus tincidunt interdum lorem vitae fermentum.\nVivamus ante nunc, facilisis sed velit egestas, maximus euismod enim.\nSed pretium et ligula ac suscipit.\nSed et metus ut orci faucibus lacinia.\nInteger vestibulum commodo blandit.\nIn dapibus libero nec mauris sagittis, vel accumsan erat feugiat.\nCurabitur tempus, arcu non accumsan faucibus, sapien nisl pharetra justo, id fringilla mauris sem eu ante.\nProin ac feugiat dolor.\nNulla auctor vestibulum tortor at placerat.\nCras tempus non tortor ut dictum.\nSed eleifend velit nisi, sit amet lacinia velit convallis non.\nCurabitur imperdiet tortor sit amet massa condimentum, nec cursus lectus placerat.\nUt egestas suscipit est, at ultricies tellus viverra ut.\nSed elementum dignissim nulla, a feugiat massa mollis quis.\nPellentesque a lobortis dolor.\nCras eleifend condimentum orci, a venenatis arcu feugiat at.\nSuspendisse condimentum neque metus, non faucibus libero ultricies nec.\nInteger quis orci at enim aliquet dapibus id quis nisl.\nMaecenas et convallis massa.\nSed ornare sagittis erat, et hendrerit neque auctor quis.\nSed dignissim erat nisl, sed pharetra mauris sollicitudin sed.\nNullam eu purus quis augue scelerisque aliquam id sed enim.\nMaecenas in posuere ex.\nVivamus quis sem faucibus, suscipit eros nec, bibendum purus.\nUt in urna orci.\nAenean id bibendum urna.\nDuis gravida ac massa vel egestas.\nFusce in erat hendrerit nunc tempus dictum.\nCras sapien diam, eleifend vitae commodo in, bibendum ut eros.\nNam sit amet massa tincidunt neque rutrum sodales.\nNunc vitae felis ut diam lobortis pellentesque.\nNunc nulla ante, sodales non tempor pretium, vulputate vel nisi.\nNulla facilisis dolor sit amet aliquam imperdiet.\nPellentesque lacinia augue eget nulla finibus tincidunt.\nNam urna tellus, aliquam sit amet velit vestibulum, dictum eleifend dolor.\nDonec eu commodo est, non tincidunt orci.\nCras vel nisl libero.\nPraesent cursus neque massa.\nNunc convallis vitae sem nec tincidunt.\nEtiam eget dui in ligula dictum sodales nec quis ante.\nCras sagittis, dui at tempus porttitor, lectus ipsum malesuada augue, vitae suscipit felis lacus facilisis nisl.\nDonec tristique tellus at aliquet accumsan.\nInteger elementum venenatis mollis.\nSuspendisse rutrum eros eget justo scelerisque facilisis.\nDonec vehicula neque at lectus interdum egestas.\nPellentesque dolor dui, feugiat ac placerat sit amet, lacinia sed massa.\nMaecenas aliquam, massa sodales ultrices pellentesque, est tortor pharetra metus, at congue libero ligula ut purus.\nVestibulum mattis scelerisque tellus, vitae volutpat velit ultrices nec.\nUt mollis hendrerit magna vitae mollis.\nNulla pharetra magna eros, quis facilisis arcu accumsan sodales.\nIn pharetra quam maximus turpis volutpat blandit.\nIn at dui nec velit sagittis tincidunt.\nIn felis orci, vulputate non luctus ac, molestie vitae urna.\nIn iaculis velit id convallis condimentum.", + "tags": ["source:example-service", "sentry-region:all", "sentry-user:bob"], + "alertType": "info" + }, { + "type": "jira", + "title": "This is an Example Notification", + "projectId": "TEST" } - } + ] } \ No newline at end of file diff --git a/test/payloads/generic-notifier/testPayload.json b/test/payloads/generic-notifier/testPayload.json index 77330298..4b4da639 100644 --- a/test/payloads/generic-notifier/testPayload.json +++ b/test/payloads/generic-notifier/testPayload.json @@ -1,18 +1,26 @@ { "source": "example-service", "timestamp": 0, - "service_name": "official_service_name", - "data": { - "title": "This is an Example Notification", - "message": "Random text here", - "tags": [ - "source:example-service", "sentry-region:all", "sentry-user:bob" - ], - "misc": {}, - "channels": { - "slack": ["#aaaaaa"], - "datadog": ["example-proj-id"], - "jira": ["TEST"] + "data": [ + { + "type": "slack", + "text": "Random text here", + "channels": ["#aaaaaa"] + }, { + "type": "service_notification", + "service_name": "eng_pipes_gh_notifications", + "text": "Random text here", + "channels": ["#aaaaaa"] + }, { + "type": "datadog", + "title": "This is an Example Notification", + "text": "Random text here", + "tags": ["source:example-service", "sentry-region:all", "sentry-user:bob"], + "alertType": "info" + }, { + "type": "jira", + "title": "This is an Example Notification", + "projectId": "TEST" } - } + ] } \ No newline at end of file diff --git a/test/payloads/generic-notifier/testServicePayload.json b/test/payloads/generic-notifier/testServicePayload.json new file mode 100644 index 00000000..26e0521d --- /dev/null +++ b/test/payloads/generic-notifier/testServicePayload.json @@ -0,0 +1,12 @@ +{ + "source": "example-service", + "timestamp": 0, + "data": [ + { + "type": "service_notification", + "service_name": "eng_pipes_gh_notifications", + "text": "Random text here", + "channels": ["#aaaaaa"] + } + ] +} \ No newline at end of file From 3915a017ad14774abc44ee372ed6886ff5ca2bdb Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 13 Nov 2024 13:06:36 -0800 Subject: [PATCH 13/18] Add example service-registry config & change types --- service-registry/service_registry.json | 36 ++++++++++++++++++ service-registry/types/index.ts | 32 ++++++++++++++++ src/types/index.ts | 25 ++++++------ src/utils/misc/serviceRegistry.ts | 2 +- src/webhooks/README.md | 2 +- .../generic-notifier/generic-notifier.test.ts | 24 +----------- .../generic-notifier/generic-notifier.ts | 38 ++++--------------- 7 files changed, 92 insertions(+), 67 deletions(-) create mode 100644 service-registry/service_registry.json create mode 100644 service-registry/types/index.ts diff --git a/service-registry/service_registry.json b/service-registry/service_registry.json new file mode 100644 index 00000000..279a6560 --- /dev/null +++ b/service-registry/service_registry.json @@ -0,0 +1,36 @@ +{ + "example_service": { + "alert_slack_channels": [ + "slack-channel-id-or-name" + ], + "aspiring_domain_experts": [], + "component": "tool", + "dashboard": null, + "docs": { + "notion page": "https://www.notion.so/" + }, + "domain_experts": [ + { + "email": "example@sentry.io", + "name": "Example Person" + } + ], + "escalation": "https://sentry.pagerduty.com/", + "id": "example_service", + "name": "Example service", + "notes": null, + "production_readiness_docs": [], + "slack_channels": [ + "discuss-stuff" + ], + "slos": [], + "teams": [ + { + "display_name": "Team 1", + "id": "team1", + "tags": [] + } + ], + "tier": 1 + } +} \ No newline at end of file diff --git a/service-registry/types/index.ts b/service-registry/types/index.ts new file mode 100644 index 00000000..fff2bc55 --- /dev/null +++ b/service-registry/types/index.ts @@ -0,0 +1,32 @@ +export interface Team { + id: string; + display_name: string; + tags: string[]; +} + +export interface Expert { + email: string; + name: string; +} + +export interface Service { + id: string; + name: string; + tier: number | null; + component: string | null; + teams: Team[]; + slack_channels: string[]; + alert_slack_channels: string[]; + domain_experts: Expert[]; + escalation: string; + slos: string[]; + dashboard: string | null; + production_readiness_docs: string[]; + notes: string | null; + docs: Record; + aspiring_domain_experts: Expert[]; +} + +export type ServiceRegistry = { + [serviceName: string]: Service; +}; diff --git a/src/types/index.ts b/src/types/index.ts index 0416b9b8..9f81d896 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -29,13 +29,23 @@ export interface KafkaControlPlaneResponse { body: string; } -export interface SlackMessage { +interface BaseSlackMessage { type: 'slack'; - channels: string[]; text: string; blocks?: KnownBlock[] | Block[]; } +export interface SlackChannel extends BaseSlackMessage { + channels: string[]; +} + +// Currently service registry is only used for Slack notifications since +// it only contains Slack alert channels (and not DD or Jira or others) +export interface ServiceSlackChannel extends BaseSlackMessage { + service_name: string; +} +export type SlackMessage = SlackChannel | ServiceSlackChannel; + export interface DatadogEvent { type: 'datadog'; title: string; @@ -53,14 +63,5 @@ export interface JiraEvent { export type GenericEvent = { source: string; timestamp: number; - data: (DatadogEvent | JiraEvent | SlackMessage | ServiceSlackMessage)[]; + data: (DatadogEvent | JiraEvent | SlackMessage)[]; }; - -// Currently only used for Slack notifications since -// service registry only contains Slack channels (and not DD or Jira or others) -export interface ServiceSlackMessage { - type: 'service_notification'; - service_name: string; // Official service registry service id - text: string; - blocks?: KnownBlock[] | Block[]; -} diff --git a/src/utils/misc/serviceRegistry.ts b/src/utils/misc/serviceRegistry.ts index 564d7880..24f04592 100644 --- a/src/utils/misc/serviceRegistry.ts +++ b/src/utils/misc/serviceRegistry.ts @@ -1,4 +1,4 @@ -import servicesData from 'service-registry/sentry_service_registry/config/combined/service_registry.json'; +import servicesData from 'service-registry/service_registry.json'; import type { Service, ServiceRegistry } from 'service-registry/types/index'; const services: ServiceRegistry = servicesData; diff --git a/src/webhooks/README.md b/src/webhooks/README.md index d2b85980..a80f6ae3 100644 --- a/src/webhooks/README.md +++ b/src/webhooks/README.md @@ -5,7 +5,7 @@ ## Generic Event Notifier -The folder `generic-notifier` provides a generic webhook which can be used to send messages to Sentry Slack channels and Sentry Datadog. Using this webhook is VERY simple. +The folder `generic-notifier` provides a generic webhook which can be used to send messages to Sentry Slack channels and Sentry Datadog. Simply, go to `@/config/secrets.ts` and add an entry to the `EVENT_NOTIFIER_SECRETS` object. This entry should contain a mapping from the source of the message (for example, `example-service`) to an environment variable. As of now, you will also need to edit `bin/deploy.sh` to add the new secret to the deployment and also add the secret to Google Secret Manager. Make a PR with this change and get it approved & merged. diff --git a/src/webhooks/generic-notifier/generic-notifier.test.ts b/src/webhooks/generic-notifier/generic-notifier.test.ts index 9db372ac..e2912996 100644 --- a/src/webhooks/generic-notifier/generic-notifier.test.ts +++ b/src/webhooks/generic-notifier/generic-notifier.test.ts @@ -1,14 +1,13 @@ import testInvalidPayload from '@test/payloads/generic-notifier/testInvalidPayload.json'; import testPayload from '@test/payloads/generic-notifier/testPayload.json'; -import testServicePayload from '@test/payloads/generic-notifier/testServicePayload.json'; import { createNotifierRequest } from '@test/utils/createGenericMessageRequest'; import { buildServer } from '@/buildServer'; import { DATADOG_API_INSTANCE } from '@/config'; -import { GenericEvent, ServiceSlackMessage, SlackMessage } from '@/types'; +import { GenericEvent, SlackMessage } from '@/types'; import { bolt } from '@api/slack'; -import { handleServiceSlackMessage, messageSlack } from './generic-notifier'; +import { messageSlack } from './generic-notifier'; describe('generic messages webhook', function () { let fastify; @@ -80,25 +79,6 @@ describe('generic messages webhook', function () { }); }); }); - describe('handleServiceSlackMessage tests', function () { - afterEach(function () { - jest.clearAllMocks(); - }); - - it('writes to slack', async function () { - const postMessageSpy = jest.spyOn(bolt.client.chat, 'postMessage'); - await handleServiceSlackMessage( - testServicePayload.data[0] as ServiceSlackMessage - ); - expect(postMessageSpy).toHaveBeenCalledTimes(1); - const message = postMessageSpy.mock.calls[0][0]; - expect(message).toEqual({ - channel: 'feed-datdog', - text: 'Random text here', - unfurl_links: false, - }); - }); - }); it('checks that slack msg is sent', async function () { const postMessageSpy = jest.spyOn(bolt.client.chat, 'postMessage'); diff --git a/src/webhooks/generic-notifier/generic-notifier.ts b/src/webhooks/generic-notifier/generic-notifier.ts index 17b2c2d9..7642c708 100644 --- a/src/webhooks/generic-notifier/generic-notifier.ts +++ b/src/webhooks/generic-notifier/generic-notifier.ts @@ -2,12 +2,7 @@ import { v1 } from '@datadog/datadog-api-client'; import * as Sentry from '@sentry/node'; import { FastifyReply, FastifyRequest } from 'fastify'; -import { - DatadogEvent, - GenericEvent, - ServiceSlackMessage, - SlackMessage, -} from '@types'; +import { DatadogEvent, GenericEvent, SlackMessage } from '@types'; import { bolt } from '@/api/slack'; import { DATADOG_API_INSTANCE } from '@/config'; @@ -43,8 +38,6 @@ export async function genericEventNotifier( for (const message of body.data) { if (message.type === 'slack') { await messageSlack(message); - } else if (message.type === 'service_notification') { - await handleServiceSlackMessage(message); } else if (message.type === 'datadog') { await sendEventToDatadog(message, body.timestamp); } @@ -79,29 +72,13 @@ export async function sendEventToDatadog( } export async function messageSlack(message: SlackMessage) { - const channels = message.channels ?? []; - for (const channel of channels) { - try { - const args = { - channel: channel, - blocks: message.blocks, - text: message.text, - unfurl_links: false, - }; - if (message.blocks) { - args.blocks = message.blocks; - } - await bolt.client.chat.postMessage(args); - } catch (err) { - Sentry.setContext('slack msg:', { text: message.text }); - Sentry.captureException(err); - } + let channels: string[] = []; + if ('channels' in message) { + channels = message.channels ?? []; + } else if ('service_name' in message) { + const service = getService(message.service_name); + channels = service.alert_slack_channels ?? []; } -} - -export async function handleServiceSlackMessage(message: ServiceSlackMessage) { - const service = getService(message.service_name); - const channels = service.alert_slack_channels ?? []; for (const channel of channels) { try { const args = { @@ -119,5 +96,4 @@ export async function handleServiceSlackMessage(message: ServiceSlackMessage) { Sentry.captureException(err); } } - // TODO: Add other types of notifications (Jira, DD, etc.) } From 30cb490f51f2344b5dfac94b3c86a92616308a02 Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 13 Nov 2024 13:17:40 -0800 Subject: [PATCH 14/18] Fix tests --- .../service-registry}/service_registry.json | 0 {service-registry => src/service-registry}/types/index.ts | 0 src/utils/misc/serviceRegistry.ts | 4 ++-- test/payloads/generic-notifier/testPayload.json | 7 +++---- test/payloads/generic-notifier/testServicePayload.json | 7 +++---- 5 files changed, 8 insertions(+), 10 deletions(-) rename {service-registry => src/service-registry}/service_registry.json (100%) rename {service-registry => src/service-registry}/types/index.ts (100%) diff --git a/service-registry/service_registry.json b/src/service-registry/service_registry.json similarity index 100% rename from service-registry/service_registry.json rename to src/service-registry/service_registry.json diff --git a/service-registry/types/index.ts b/src/service-registry/types/index.ts similarity index 100% rename from service-registry/types/index.ts rename to src/service-registry/types/index.ts diff --git a/src/utils/misc/serviceRegistry.ts b/src/utils/misc/serviceRegistry.ts index 24f04592..ca9f4c2f 100644 --- a/src/utils/misc/serviceRegistry.ts +++ b/src/utils/misc/serviceRegistry.ts @@ -1,5 +1,5 @@ -import servicesData from 'service-registry/service_registry.json'; -import type { Service, ServiceRegistry } from 'service-registry/types/index'; +import servicesData from '@/service-registry/service_registry.json'; +import type { Service, ServiceRegistry } from '@/service-registry/types/index'; const services: ServiceRegistry = servicesData; diff --git a/test/payloads/generic-notifier/testPayload.json b/test/payloads/generic-notifier/testPayload.json index 4b4da639..72cbf248 100644 --- a/test/payloads/generic-notifier/testPayload.json +++ b/test/payloads/generic-notifier/testPayload.json @@ -7,10 +7,9 @@ "text": "Random text here", "channels": ["#aaaaaa"] }, { - "type": "service_notification", - "service_name": "eng_pipes_gh_notifications", - "text": "Random text here", - "channels": ["#aaaaaa"] + "type": "slack", + "service_name": "example_service", + "text": "Random text here" }, { "type": "datadog", "title": "This is an Example Notification", diff --git a/test/payloads/generic-notifier/testServicePayload.json b/test/payloads/generic-notifier/testServicePayload.json index 26e0521d..1f903553 100644 --- a/test/payloads/generic-notifier/testServicePayload.json +++ b/test/payloads/generic-notifier/testServicePayload.json @@ -3,10 +3,9 @@ "timestamp": 0, "data": [ { - "type": "service_notification", - "service_name": "eng_pipes_gh_notifications", - "text": "Random text here", - "channels": ["#aaaaaa"] + "type": "slack", + "service_name": "example_service", + "text": "Random text here" } ] } \ No newline at end of file From 7a308c7567b82d6a0ad2078ecf7cf58dbe4156e6 Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 13 Nov 2024 14:59:47 -0800 Subject: [PATCH 15/18] Del unused ssh-agent stuff --- .github/workflows/ci.yml | 23 ----------------------- .github/workflows/migration.yml | 4 ---- 2 files changed, 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34eb8e3d..dd25946b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,19 +62,12 @@ jobs: run: | sudo apt-get update sudo apt-get install git -y - - name: Set up SSH agent (for service-registry) - uses: webfactory/ssh-agent@v0.8.0 - with: - ssh-private-key: ${{ secrets.SENTRY_INTERNAL_GH_SSH_PRIVATE_KEY }} - name: Setup node uses: actions/setup-node@v4 with: node-version: '18' - - name: yarn install - run: yarn install --immutable && yarn up "service-registry@git+ssh://git@github.com:getsentry/service-registry#main" - - name: tsc run: yarn build - name: test @@ -94,11 +87,6 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up SSH agent (for service-registry) - uses: webfactory/ssh-agent@v0.8.0 - with: - ssh-private-key: ${{ secrets.SENTRY_INTERNAL_GH_SSH_PRIVATE_KEY }} - - name: Builds docker image run: DOCKER_BUILDKIT=1 docker build --ssh default -t ci-tooling . @@ -124,23 +112,12 @@ jobs: with: # for Sentry releases fetch-depth: 0 - - name: Install Git - run: | - sudo apt-get update - sudo apt-get install git -y - - name: Set up SSH agent (for service-registry) - uses: webfactory/ssh-agent@v0.8.0 - with: - ssh-private-key: ${{ secrets.SENTRY_INTERNAL_GH_SSH_PRIVATE_KEY }} - name: Setup node uses: actions/setup-node@v4 with: node-version: '18' - - name: yarn install - run: yarn install --immutable && yarn up "service-registry@git+ssh://git@github.com:getsentry/service-registry#main" - - name: tsc run: yarn build diff --git a/.github/workflows/migration.yml b/.github/workflows/migration.yml index 7fca1419..c772dc11 100644 --- a/.github/workflows/migration.yml +++ b/.github/workflows/migration.yml @@ -65,10 +65,6 @@ jobs: chmod +x cloud_sql_proxy ./cloud_sql_proxy -instances=$DB=tcp:5432 -dir . & - - name: yarn install - run: | - yarn install --immutable && yarn up "service-registry@git+ssh://git@github.com:getsentry/service-registry#main" - - name: Run migration env: DB_HOST: ${{ secrets.DB_HOST }} From 85ac0f90d5f8e6d169417f8d9e27d01f51183b9e Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 13 Nov 2024 15:02:48 -0800 Subject: [PATCH 16/18] Revert ci.yml changes --- .github/workflows/ci.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd25946b..2bcc129e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,16 +58,14 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Install Git - run: | - sudo apt-get update - sudo apt-get install git -y - - name: Setup node uses: actions/setup-node@v4 with: node-version: '18' + - name: yarn install + run: yarn install --immutable + - name: tsc run: yarn build - name: test @@ -88,7 +86,7 @@ jobs: - uses: actions/checkout@v3 - name: Builds docker image - run: DOCKER_BUILDKIT=1 docker build --ssh default -t ci-tooling . + run: docker build -t ci-tooling . build-deploy: name: build and deploy @@ -118,6 +116,9 @@ jobs: with: node-version: '18' + - name: yarn install + run: yarn install --immutable + - name: tsc run: yarn build From 7ff4b8331610ef3435c0c35cabfabfc82befe178 Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 13 Nov 2024 15:03:20 -0800 Subject: [PATCH 17/18] Revert migration.yml changes --- .github/workflows/migration.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/migration.yml b/.github/workflows/migration.yml index c772dc11..a66c1bda 100644 --- a/.github/workflows/migration.yml +++ b/.github/workflows/migration.yml @@ -33,14 +33,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Install Git - run: | - sudo apt-get update - sudo apt-get install git -y - - name: Set up SSH agent (for service-registry) - uses: webfactory/ssh-agent@v0.8.0 - with: - ssh-private-key: ${{ secrets.SENTRY_INTERNAL_GH_SSH_PRIVATE_KEY }} - name: Setup node uses: actions/setup-node@v4 @@ -65,6 +57,10 @@ jobs: chmod +x cloud_sql_proxy ./cloud_sql_proxy -instances=$DB=tcp:5432 -dir . & + - name: yarn install + run: | + yarn install --immutable + - name: Run migration env: DB_HOST: ${{ secrets.DB_HOST }} From bc4ed328caefd8e7248e743b0028832511604ad7 Mon Sep 17 00:00:00 2001 From: Brian Lou Date: Wed, 13 Nov 2024 15:06:32 -0800 Subject: [PATCH 18/18] Add TODO --- src/webhooks/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/webhooks/README.md b/src/webhooks/README.md index a80f6ae3..4aa7bec0 100644 --- a/src/webhooks/README.md +++ b/src/webhooks/README.md @@ -24,7 +24,7 @@ Once this has been deployed, all you have to do is send a POST request to `https "blocks": [] }, { "type": "service_notification", // Slack message using service registry information - "service_name": "eng_pipes_gh_notifications", + "service_name": "eng_pipes_gh_notifications", "text": "Random text here", // Optionally, include Slack Blocks "blocks": [] @@ -41,6 +41,8 @@ Once this has been deployed, all you have to do is send a POST request to `https Additionally, you must compute the HMAC SHA256 hash of the raw payload string computed with the secret key, and attach it to the `Authorization` header. EX: `Authorization: ` +TODO: Add the service registry configs to the deployed instance & replace the current dummy json at `@/service-registry/service_registry.json` with the actual service registry json. + ## Adding a webhook to GoCD event emitter * goto [gocd](deploy.getsentry.net)