ProjectLibertyLabs · JoeCap08055 · May 14, 2024
diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml
@@ -14,6 +14,8 @@ services:
 
   frequency:
     image: dsnp/instant-seal-node-with-deployed-schemas:latest
+    hostname: frequency
+    container_name: frequency
     # We need to specify the platform because it's the only image
     # built by Frequency at the moment, and auto-pull won't work otherwise
     platform: linux/amd64
@@ -36,7 +38,7 @@ services:
     container_name: webhook
     hostname: webhook
     build:
-      context: webhook-specification/mock-webhook-server
+      context: .
       dockerfile: webhook-specification/mock-webhook-server/Dockerfile
     ports:
       - 3001:3001

diff --git a/package.json b/package.json
@@ -19,6 +19,8 @@
     "lint:fix": "prettier --write ./src && eslint \"./src/**/*.ts\" --fix",
     "pretest": "cp env.template .env",
     "test": "jest --coverage --verbose",
+    "create-accounts": "set -a ; . ./setup/testing/.env ; tsx setup/testing/create-accounts.ts",
+    "clear-graphs": "set -a ; . ./setup/testing/.env ; tsx setup/testing/clear-graphs.ts",
     "chain-setup": "tsx setup/testing/index.ts",
     "reset-chain": "setup/testing/reset_chain"
   },

diff --git a/setup/testing/clear-graphs.ts b/setup/testing/clear-graphs.ts
@@ -0,0 +1,134 @@
+import { ExtrinsicHelper } from '@amplica-labs/frequency-scenario-template';
+import { createAddGraphKeyExtrinsic, createClearGraphExtrinsics, graphKeypair } from './graph';
+import { addExtrinsicToTrack, api, famousUser, followers, graph, initScenario, provider, submitAndTrackExtrinsics } from './utils';
+import { ProviderGraph, GraphKeyPair as ProviderGraphKeyPair } from 'reconnection-service/src/interfaces/provider-graph.interface';
+import fs from 'node:fs';
+import * as ScenarioTypes from './types';
+import * as ScenarioConstants from './constants';
+
+async function main() {
+  await initScenario();
+
+  const followersToProcess = [...followers.values()].filter((follower) => follower?.msaId);
+  if (famousUser?.msaId) {
+    followersToProcess.push(famousUser);
+  }
+
+  // Create graph extrinsics
+  await Promise.all(
+    followersToProcess.map(async (follower) => {
+      await createAddGraphKeyExtrinsic(api, graph, graphKeypair, follower);
+      await createClearGraphExtrinsics(api, follower);
+    }),
+  );
+  let graphKeysToUpdate = 0;
+  let graphsToClear = 0;
+
+  for (const follower of followers.values()) {
+    if (follower?.addGraphKey) {
+      addExtrinsicToTrack(follower.addGraphKey());
+      graphKeysToUpdate++;
+    }
+
+    if (follower?.resetGraph) {
+      follower.resetGraph.forEach((e) => addExtrinsicToTrack(e()));
+      graphsToClear++;
+    }
+  }
+
+  if (famousUser?.addGraphKey) {
+    addExtrinsicToTrack(famousUser.addGraphKey());
+    graphKeysToUpdate++;
+  }
+
+  if (famousUser?.resetGraph) {
+    famousUser.resetGraph.forEach((e) => addExtrinsicToTrack(e()));
+    graphsToClear++;
+  }
+
+  console.log(`
+Users requiring public graph key update: ${graphKeysToUpdate}
+Users requiring graph reset: ${graphsToClear}
+`);
+
+  await submitAndTrackExtrinsics(api, provider);
+
+  // Create JSON responses for each follower
+  if (!fs.existsSync('webhook-specification/mock-webhook-server/responses')) {
+    fs.mkdirSync('webhook-specification/mock-webhook-server/responses');
+  }
+  followers.forEach((follower) => {
+    const response: ScenarioTypes.ProviderResponse = {
+      dsnpId: follower.msaId!.toString(),
+      connections: {
+        data: [
+          {
+            dsnpId: famousUser.msaId!.toString(),
+            privacyType: 'Private',
+            direction: 'connectionTo',
+            connectionType: 'Follow',
+          },
+        ],
+      },
+      graphKeyPairs: [
+        {
+          keyType: 'X25519',
+          publicKey: ScenarioConstants.graphPublicKey,
+          privateKey: ScenarioConstants.graphPrivateKey,
+        } as ProviderGraphKeyPair,
+      ],
+    };
+
+    fs.writeFileSync(`webhook-specification/mock-webhook-server/responses/response.${follower.msaId?.toString()}.json`, JSON.stringify(response, undefined, 4));
+  });
+
+  const famousUserResponse = {
+    dsnpId: famousUser.msaId!.toString(),
+    graphKeyPairs: [
+      {
+        keyType: 'X25519',
+        publicKey: ScenarioConstants.graphPublicKey,
+        privateKey: ScenarioConstants.graphPrivateKey,
+      } as ProviderGraphKeyPair,
+    ],
+  };
+
+  const responses: ScenarioTypes.ProviderResponse[] = [];
+  const allFollowers = [...followers.values()];
+  while (allFollowers.length > 0) {
+    const followerSlice = allFollowers.splice(0, Math.min(allFollowers.length, 50));
+    const response = {
+      ...famousUserResponse,
+      connections: {
+        data: followerSlice.flatMap((follower) => [
+          {
+            dsnpId: follower.msaId!.toString(),
+            privacyType: 'Private',
+            direction: 'connectionFrom',
+            connectionType: 'Follow',
+          } as ProviderGraph,
+          {
+            dsnpId: follower.msaId!.toString(),
+            privacyType: 'Private',
+            direction: 'connectionTo',
+            connectionType: 'Follow',
+          } as ProviderGraph,
+        ]),
+      },
+    };
+    responses.push(response);
+  }
+
+  responses.forEach((response, index) => {
+    response.connections.pagination = {
+      pageNumber: index + 1,
+      pageCount: responses.length,
+      pageSize: response.connections.data.length,
+    };
+    fs.writeFileSync(`webhook-specification/mock-webhook-server/responses/response.${famousUser.msaId!.toString()}.${index + 1}.json`, JSON.stringify(response, undefined, 4));
+  });
+}
+
+main()
+  .catch((e) => console.error(e))
+  .finally(async () => ExtrinsicHelper.disconnect());
diff --git a/setup/testing/constants.ts b/setup/testing/constants.ts
@@ -0,0 +1,8 @@
+import { HexString } from '@polkadot/util/types';
+
+export const graphPublicKey: HexString = '0x0514f63edc89d414061bf451cc99b1f2b43fac920c351be60774559a31523c75';
+export const graphPrivateKey: HexString = '0x1c15b6d1af4716615a4eb83a2dfba3284e1c0a199603572e7b95c164f7ad90e3';
+export const CAPACITY_AMOUNT_TO_STAKE = 1_000_000_000_000_000n;
+export const DEFAULT_SCHEMAS = [5, 7, 8, 9, 10];
+export const NUM_FOLLOWERS = 10;
+export const MAX_EXTRINSICS_TO_SUBMIT = 100;
diff --git a/setup/testing/create-accounts.ts b/setup/testing/create-accounts.ts
@@ -0,0 +1,89 @@
+import { ApiPromise } from '@polkadot/api';
+import * as ScenarioTypes from './types';
+import { ExtrinsicHelper } from '@amplica-labs/frequency-scenario-template';
+import { getAddGraphKeyPayload, graphKeypair } from './graph';
+import { getAddProviderPayload, getCreateUserExtrinsic } from './msa';
+import { Graph } from '@dsnp/graph-sdk';
+import { addExtrinsicToTrack, api, famousUser, followers, initScenario, provider, submitAndTrackExtrinsics } from './utils';
+import { Block, Event } from '@polkadot/types/interfaces';
+import { GenericExtrinsic } from '@polkadot/types';
+import { AnyTuple } from '@polkadot/types/types';
+
+async function createUser(api: ApiPromise, graph: Graph, user: ScenarioTypes.ChainUser, provider: ScenarioTypes.ChainUser) {
+  if (!user.msaId) {
+    const { payload: addProviderPayload, proof } = await getAddProviderPayload(ExtrinsicHelper.apiPromise, user, provider);
+    user.create = () => api.tx.msa.createSponsoredAccountWithDelegation(user.keys!.publicKey, proof, addProviderPayload);
+
+    const graphKeyPayload = await getAddGraphKeyPayload(ExtrinsicHelper.apiPromise, graph, graphKeypair, user);
+    if (graphKeyPayload) {
+      const { payload: addGraphKeyPayload, proof: addGraphKeyProof } = graphKeyPayload;
+      user.addGraphKey = () => ExtrinsicHelper.apiPromise.tx.statefulStorage.applyItemActionsWithSignatureV2(user.keys!.publicKey, addGraphKeyProof, addGraphKeyPayload);
+    }
+  }
+}
+
+async function main() {
+  await initScenario();
+
+  let followersToCreate = 0;
+
+  // Create followers
+  await Promise.all([
+    [...followers.keys()].map((a) => {
+      const follower = followers.get(a)!;
+      return getCreateUserExtrinsic(api, follower, provider);
+    }),
+    getCreateUserExtrinsic(api, famousUser, provider),
+  ]);
+
+  for (const follower of followers.values()) {
+    if (follower?.create) {
+      addExtrinsicToTrack(follower.create());
+      followersToCreate++;
+    }
+  }
+
+  if (famousUser?.create) {
+    addExtrinsicToTrack(famousUser.create());
+    followersToCreate++;
+  }
+
+  console.log(`
+MSAs to create: ${followersToCreate}
+`);
+
+  await submitAndTrackExtrinsics(api, provider, (block: Block, _extrinsic: GenericExtrinsic<AnyTuple>, event: Event) => {
+    if (api.events.msa.MsaCreated.is(event)) {
+      const { msaId, key } = event.data;
+      const address = key.toString();
+      const follower = followers.get(address);
+      if (follower) {
+        follower.msaId = msaId;
+        follower.createdAtBlock = block.header.number.toNumber();
+        followers.set(address, follower);
+      } else if (address === famousUser.keys!.address) {
+        famousUser.msaId = msaId;
+        famousUser.createdAtBlock = block.header.number.toNumber();
+      } else {
+        console.error('Cannot find follower ', address);
+      }
+    }
+  });
+
+  //   console.dir(
+  //     [...followers.values()].map((follower) => follower.createdAtBlock),
+  //     { maxArrayLength: null },
+  //   );
+  const firstBlock = Math.min(...[...followers.values()].map((follower) => follower.createdAtBlock || 0));
+  if (firstBlock) {
+    console.log(`First user created at block ${firstBlock}`);
+  }
+
+  if (famousUser.createdAtBlock) {
+    console.log(`Last (famous) user ${famousUser.msaId!.toString()} created at block ${famousUser.createdAtBlock}`);
+  }
+}
+
+main()
+  .catch((e) => console.error(e))
+  .finally(async () => ExtrinsicHelper.disconnect());
diff --git a/setup/testing/graph.ts b/setup/testing/graph.ts
@@ -0,0 +1,120 @@
+import '@frequency-chain/api-augment';
+import { AddGraphKeyAction, AddKeyUpdate, DsnpKeys, Graph, GraphKeyPair, GraphKeyType, ImportBundleBuilder, KeyData } from '@dsnp/graph-sdk';
+import { ApiPromise } from '@polkadot/api';
+import { AnyNumber, ISubmittableResult } from '@polkadot/types/types';
+import { hexToU8a, u8aToHex } from '@polkadot/util';
+import * as ScenarioTypes from './types';
+import { HexString } from '@polkadot/util/types';
+import { ItemizedSignaturePayload, Sr25519Signature, signPayloadSr25519 } from '@amplica-labs/frequency-scenario-template';
+import { getExpiration, privateFollowSchemaId, publicGraphKeySchemaId } from './utils';
+import { ItemizedStoragePageResponse, PageHash } from '@frequency-chain/api-augment/interfaces';
+import { SubmittableExtrinsic } from '@polkadot/api-base/types';
+
+export const graphPublicKey: HexString = '0x0514f63edc89d414061bf451cc99b1f2b43fac920c351be60774559a31523c75';
+export const graphPrivateKey: HexString = '0x1c15b6d1af4716615a4eb83a2dfba3284e1c0a199603572e7b95c164f7ad90e3';
+export const graphKeypair: GraphKeyPair = {
+  keyType: GraphKeyType.X25519,
+  publicKey: hexToU8a(graphPublicKey),
+  secretKey: hexToU8a(graphPrivateKey),
+};
+
+export function createGraphKeyPair(): GraphKeyPair {
+  return Graph.generateKeyPair(GraphKeyType.X25519);
+}
+
+async function isGraphKeyCurrent(api: ApiPromise, graph: Graph, keypair: GraphKeyPair, msaId: AnyNumber): Promise<[boolean, PageHash]> {
+  const itemizedResponse: ItemizedStoragePageResponse = await api.rpc.statefulStorage.getItemizedStorage(msaId, publicGraphKeySchemaId);
+  const keyData: KeyData[] = itemizedResponse.items.toArray().map((publicKey) => ({
+    index: publicKey.index.toNumber(),
+    content: hexToU8a(publicKey.payload.toHex()),
+  }));
+  const dsnpKeys: DsnpKeys = {
+    dsnpUserId: msaId.toString(),
+    keysHash: itemizedResponse.content_hash.toNumber(),
+    keys: keyData,
+  };
+
+  const bundle = new ImportBundleBuilder().withDsnpUserId(msaId.toString()).withDsnpKeys(dsnpKeys).build();
+  graph.importUserData([bundle]);
+
+  const keys = graph.getPublicKeys(msaId.toString());
+  const latestKey = u8aToHex(keys.pop()?.key);
+  if (latestKey === u8aToHex(keypair.publicKey)) {
+    return [true, itemizedResponse.content_hash];
+  }
+
+  return [false, itemizedResponse.content_hash];
+}
+
+export async function getAddGraphKeyPayload(
+  api: ApiPromise,
+  graph: Graph,
+  keypair: GraphKeyPair,
+  user: ScenarioTypes.ChainUser,
+): Promise<{ payload: ItemizedSignaturePayload; proof: Sr25519Signature } | null> {
+  const [keyIsCurrent, targetHash] = await isGraphKeyCurrent(api, graph, keypair, user.msaId!);
+  if (keyIsCurrent) {
+    return null;
+  }
+
+  const actions = [
+    {
+      type: 'AddGraphKey',
+      ownerDsnpUserId: user.msaId?.toString(),
+      newPublicKey: hexToU8a(graphPublicKey),
+    } as AddGraphKeyAction,
+  ];
+
+  graph.applyActions(actions);
+  const keyExport = graph.exportUserGraphUpdates(user.msaId!.toString());
+
+  if (keyExport.length > 0) {
+    const bundle = keyExport[0] as AddKeyUpdate;
+
+    const addAction = [
+      {
+        Add: {
+          data: u8aToHex(bundle.payload),
+        },
+      },
+    ];
+
+    const graphKeyAction = {
+      targetHash: targetHash,
+      schemaId: publicGraphKeySchemaId,
+      actions: addAction,
+      expiration: await getExpiration(api),
+    };
+
+    const payloadBytes = api.registry.createType('PalletStatefulStorageItemizedSignaturePayloadV2', graphKeyAction);
+    const proof = signPayloadSr25519(user.keys!, payloadBytes);
+
+    return { payload: { ...graphKeyAction }, proof };
+  }
+
+  return null;
+}
+
+export async function createAddGraphKeyExtrinsic(api: ApiPromise, graph: Graph, keypair: GraphKeyPair, user: ScenarioTypes.ChainUser): Promise<void> {
+  const [keyIsCurrent] = await isGraphKeyCurrent(api, graph, keypair, user.msaId!);
+  if (!keyIsCurrent) {
+    const result = await getAddGraphKeyPayload(api, graph, keypair, user);
+    if (result) {
+      const { payload, proof } = result;
+      user.addGraphKey = () => api.tx.statefulStorage.applyItemActionsWithSignatureV2(user.keys!.publicKey, proof, payload);
+    }
+  }
+}
+
+export async function createClearGraphExtrinsics(api: ApiPromise, user: ScenarioTypes.ChainUser) {
+  const clearGraphExtrinsics: (() => SubmittableExtrinsic<'promise', ISubmittableResult>)[] = [];
+
+  const pages = await api.rpc.statefulStorage.getPaginatedStorage(user.msaId!, privateFollowSchemaId);
+  pages.toArray().forEach((page) => {
+    clearGraphExtrinsics.push(() => api.tx.statefulStorage.deletePage(user.msaId!, privateFollowSchemaId, page.page_id, page.content_hash));
+  });
+
+  if (clearGraphExtrinsics.length > 0) {
+    user.resetGraph = clearGraphExtrinsics;
+  }
+}