Change generic webhook API

.github/workflows/ci.yml

@@ -58,13 +58,22 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Install Git
+        run: |
+          sudo apt-get update
+          sudo apt-get install git -y
+      - name: Set up SSH agent (for service-registry)
+        uses: webfactory/[email protected]
+        with:
+          ssh-private-key: ${{ secrets.SENTRY_INTERNAL_GH_SSH_PRIVATE_KEY }}
+
       - name: Setup node
         uses: actions/setup-node@v4
         with:
           node-version: '18'
 
       - name: yarn install
-        run: yarn install --immutable
+        run: yarn install --immutable && yarn up "service-registry@git+ssh://[email protected]:getsentry/service-registry#main"
 
       - name: tsc
         run: yarn build
@@ -85,8 +94,13 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Set up SSH agent (for service-registry)
+        uses: webfactory/[email protected]
+        with:
+          ssh-private-key: ${{ secrets.SENTRY_INTERNAL_GH_SSH_PRIVATE_KEY }}
+
       - name: Builds docker image
-        run: docker build -t ci-tooling .
+        run: DOCKER_BUILDKIT=1 docker build --ssh default -t ci-tooling .
 
   build-deploy:
     name: build and deploy
@@ -110,14 +124,22 @@ jobs:
         with:
           # for Sentry releases
           fetch-depth: 0
+      - name: Install Git
+        run: |
+          sudo apt-get update
+          sudo apt-get install git -y
+      - name: Set up SSH agent (for service-registry)
+        uses: webfactory/[email protected]
+        with:
+          ssh-private-key: ${{ secrets.SENTRY_INTERNAL_GH_SSH_PRIVATE_KEY }}
 
       - name: Setup node
         uses: actions/setup-node@v4
         with:
           node-version: '18'
 
       - name: yarn install
-        run: yarn install --immutable
+        run: yarn install --immutable && yarn up "service-registry@git+ssh://[email protected]:getsentry/service-registry#main"
 
       - name: tsc
         run: yarn build

.github/workflows/migration.yml

@@ -33,6 +33,14 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Install Git
+        run: |
+          sudo apt-get update
+          sudo apt-get install git -y
+      - name: Set up SSH agent (for service-registry)
+        uses: webfactory/[email protected]
+        with:
+          ssh-private-key: ${{ secrets.SENTRY_INTERNAL_GH_SSH_PRIVATE_KEY }}
 
       - name: Setup node
         uses: actions/setup-node@v4
@@ -59,7 +67,7 @@ jobs:
 
       - name: yarn install
         run: |
-          yarn install --immutable
+          yarn install --immutable && yarn up "service-registry@git+ssh://[email protected]:getsentry/service-registry#main"
 
       - name: Run migration
         env:

bin/deploy.sh

@@ -27,6 +27,7 @@ GOCD_WEBHOOK_SECRET
 KAFKA_CONTROL_PLANE_WEBHOOK_SECRET
 SENTRY_OPTIONS_WEBHOOK_SECRET
 "
+# TODO: Revamp this and make it easier to add secrets & deploy to GCP
 
 secrets=""
 for secret_name in $secret_names; do

src/config/secrets.ts

@@ -6,6 +6,8 @@ This file contains secrets used for verifying incoming events from different HTT
 
 export const EVENT_NOTIFIER_SECRETS = {
   // Follow the pattern below to add a new secret
+  // The secret will also need to be added in the deploy.sh script and in
+  // Google Secret manager
   // 'example-service': process.env.EXAMPLE_SERVICE_SECRET,
 };
 if (process.env.ENV !== 'production')

src/types/index.ts

@@ -29,23 +29,38 @@ export interface KafkaControlPlaneResponse {
   body: string;
 }
 
+export interface SlackMessage {
+  type: 'slack';
+  channels: string[];
+  text: string;
+  blocks?: KnownBlock[] | Block[];
+}
+
+export interface DatadogEvent {
+  type: 'datadog';
+  title: string;
+  text: string;
+  tags: string[];
+  alertType: EventAlertType;
+}
+
+export interface JiraEvent {
+  type: 'jira';
+  projectId: string;
+  title: string;
+}
+
 export type GenericEvent = {
   source: string;
   timestamp: number;
-  service_name?: string; // Official service registry name if applicable
-  data: {
-    title: string;
-    message: string;
-    channels: {
-      slack?: string[]; // list of Slack Channels
-      datadog?: string[]; // list of DD Monitors
-      jira?: string[]; // list of Jira Projects
-      bigquery?: string;
-    };
-    tags?: string[]; // Not used for Slack
-    misc: {
-      alertType?: EventAlertType; // Datadog alert type
-      blocks?: (KnownBlock | Block)[]; // Optional Slack blocks
-    };
-  };
+  data: (DatadogEvent | JiraEvent | SlackMessage | ServiceSlackMessage)[];
 };
+
+// Currently only used for Slack notifications since
+// service registry only contains Slack channels (and not DD or Jira or others)
+export interface ServiceSlackMessage {
+  type: 'service_notification';
+  service_name: string; // Official service registry service id
+  text: string;
+  blocks?: KnownBlock[] | Block[];
+}

src/utils/misc/serviceRegistry.ts

@@ -0,0 +1,8 @@
+import servicesData from 'service-registry/sentry_service_registry/config/combined/service_registry.json';
+import type { Service, ServiceRegistry } from 'service-registry/types/index';
+
+const services: ServiceRegistry = servicesData;
+
+export function getService(serviceId: string): Service {
+  return services[serviceId];
+}

src/webhooks/README.md

@@ -7,27 +7,35 @@
 
 The folder `generic-notifier` provides a generic webhook which can be used to send messages to Sentry Slack channels and Sentry Datadog. Using this webhook is VERY simple.
 
-Simply, go to `@/config/secrets.ts` and add an entry to the `EVENT_NOTIFIER_SECRETS` object. This entry should contain a mapping from the name of your service (for example, `example-service`) to an environment variable. [TODO: Fill in how to set the prod env var here]. Make a PR with this change and get it approved & merged.
+Simply, go to `@/config/secrets.ts` and add an entry to the `EVENT_NOTIFIER_SECRETS` object. This entry should contain a mapping from the source of the message (for example, `example-service`) to an environment variable. As of now, you will also need to edit `bin/deploy.sh` to add the new secret to the deployment and also add the secret to Google Secret Manager. Make a PR with this change and get it approved & merged.
 
-Once this has been deployed, all you have to do is send a POST request to `https://product-eng-webhooks-vmrqv3f7nq-uw.a.run.app/event-notifier/v1` with a JSON payload in the format of the type `GenericEvent` defined in `@/types/index.ts`. Example:
+Once this has been deployed, all you have to do is send a POST request to `https://product-eng-webhooks-vmrqv3f7nq-uw.a.run.app/event-notifier/v1` with a JSON payload in the format of the type `GenericEvent` defined in `@/types/index.ts`. Currently, only Datadog and Slack messages are supported. Example:
 
 ```json
 {
  "source": "example-service", // This must match the mapping string you define in the EVENT_NOTIFIER_SECRETS obj
  "timestamp": 0,
- "service_name": "official_service_name",
- "data": {
-  "title": "This is an Example Notification",
-  "message": "Random text here",
-  "tags": [
-   "source:example-service", "sentry-region:all", "sentry-user:bob"
-  ],
-  "misc": {},
-  "channels": {
-   "slack": ["C07EH2QGGQ5"],
-   "jira": ["TEST"]
+ "data": [
+  {
+   "type": "slack", // Basic Slack message
+   "text": "Random text here", 
+   "channels": ["#aaaaaa"],
+   // Optionally, include Slack Blocks
+   "blocks": []
+  }, {
+   "type": "service_notification", // Slack message using service registry information
+   "service_name": "eng_pipes_gh_notifications",
+   "text": "Random text here",
+   // Optionally, include Slack Blocks
+   "blocks": []
+  }, {
+   "type": "datadog", // Datadog message
+   "title": "This is an Example Notification",
+   "text": "Random text here",
+   "tags": ["source:example-service", "sentry-region:all", "sentry-user:bob"],
+   "alertType": "info"
   }
- }
+ ]
 }
 ```
 

src/webhooks/generic-notifier/generic-notifier.test.ts

@@ -1,12 +1,14 @@
 import testInvalidPayload from '@test/payloads/generic-notifier/testInvalidPayload.json';
 import testPayload from '@test/payloads/generic-notifier/testPayload.json';
+import testServicePayload from '@test/payloads/generic-notifier/testServicePayload.json';
 import { createNotifierRequest } from '@test/utils/createGenericMessageRequest';
 
 import { buildServer } from '@/buildServer';
 import { DATADOG_API_INSTANCE } from '@/config';
+import { GenericEvent, ServiceSlackMessage, SlackMessage } from '@/types';
 import { bolt } from '@api/slack';
 
-import { messageSlack } from './generic-notifier';
+import { handleServiceSlackMessage, messageSlack } from './generic-notifier';
 
 describe('generic messages webhook', function () {
   let fastify;
@@ -24,7 +26,10 @@ describe('generic messages webhook', function () {
     jest
       .spyOn(DATADOG_API_INSTANCE, 'createEvent')
       .mockImplementation(jest.fn());
-    const response = await createNotifierRequest(fastify, testPayload);
+    const response = await createNotifierRequest(
+      fastify,
+      testPayload as GenericEvent
+    );
 
     expect(response.statusCode).toBe(200);
   });
@@ -65,7 +70,7 @@ describe('generic messages webhook', function () {
 
     it('writes to slack', async function () {
       const postMessageSpy = jest.spyOn(bolt.client.chat, 'postMessage');
-      await messageSlack(testPayload);
+      await messageSlack(testPayload.data[0] as SlackMessage);
       expect(postMessageSpy).toHaveBeenCalledTimes(1);
       const message = postMessageSpy.mock.calls[0][0];
       expect(message).toEqual({
@@ -75,18 +80,43 @@ describe('generic messages webhook', function () {
       });
     });
   });
+  describe('handleServiceSlackMessage tests', function () {
+    afterEach(function () {
+      jest.clearAllMocks();
+    });
+
+    it('writes to slack', async function () {
+      const postMessageSpy = jest.spyOn(bolt.client.chat, 'postMessage');
+      await handleServiceSlackMessage(
+        testServicePayload.data[0] as ServiceSlackMessage
+      );
+      expect(postMessageSpy).toHaveBeenCalledTimes(1);
+      const message = postMessageSpy.mock.calls[0][0];
+      expect(message).toEqual({
+        channel: 'feed-datdog',
+        text: 'Random text here',
+        unfurl_links: false,
+      });
+    });
+  });
 
   it('checks that slack msg is sent', async function () {
     const postMessageSpy = jest.spyOn(bolt.client.chat, 'postMessage');
-    const response = await createNotifierRequest(fastify, testPayload);
+    const response = await createNotifierRequest(
+      fastify,
+      testPayload as GenericEvent
+    );
 
-    expect(postMessageSpy).toHaveBeenCalledTimes(1);
+    expect(postMessageSpy).toHaveBeenCalledTimes(2);
 
     expect(response.statusCode).toBe(200);
   });
   it('checks that dd msg is sent', async function () {
     const ddMessageSpy = jest.spyOn(DATADOG_API_INSTANCE, 'createEvent');
-    const response = await createNotifierRequest(fastify, testPayload);
+    const response = await createNotifierRequest(
+      fastify,
+      testPayload as GenericEvent
+    );
 
     expect(ddMessageSpy).toHaveBeenCalledTimes(1);