feat: dockerize blockfrost-ryo

ginnun · ginnun · commit 6f46246084f7 · 2024-09-17T15:06:04.000+03:00
diff --git a/compose/common.yml b/compose/common.yml
@@ -69,6 +69,13 @@ x-provider-server-environment: &provider-server-environment
   SUBMIT_API_URL: ${SUBMIT_API_URL:-http://cardano-submit-api:8090/}
   USE_BLOCKFROST: ${USE_BLOCKFROST:-false}
   USE_SUBMIT_API: ${USE_SUBMIT_API:-false}
+  ASSET_PROVIDER: ${ASSET_PROVIDER:-dbsync}
+  UTXO_PROVIDER: ${UTXO_PROVIDER:-dbsync}
+  CHAIN_HISTORY_PROVIDER: ${CHAIN_HISTORY_PROVIDER:-dbsync}
+  REWARDS_PROVIDER: ${REWARDS_PROVIDER:-dbsync}
+  NETWORK_INFO_PROVIDER: ${NETWORK_INFO_PROVIDER:-dbsync}
+  TX_SUBMIT_PROVIDER: ${TX_SUBMIT_PROVIDER:-submit-node}
+  STAKE_POOL_PROVIDER: ${STAKE_POOL_PROVIDER:-dbsync}
 
 x-sdk-environment: &sdk-environment
   LOGGER_MIN_SEVERITY: ${LOGGER_MIN_SEVERITY:-info}
@@ -110,6 +117,18 @@ x-sdk-environment: &sdk-environment
   WEB_SOCKET_API_URL: ws://ws-server:3000/ws
 
 services:
+  blockfrost-ryo:
+    build:
+      context: "https://github.com/ginnun/blockfrost-backend-ryo.git#feat/custom-network-support"
+      dockerfile: Dockerfile
+    environment:
+      BLOCKFROST_CONFIG_SERVER_LISTEN_ADDRESS: 0.0.0.0
+    depends_on:
+      cardano-db-sync:
+        condition: service_started
+    ports:
+      - "3015:3000"
+
   cardano-db-sync:
     <<:
       - *logging
diff --git a/packages/cardano-services/blockfrost/keys/.gitignore b/packages/cardano-services/blockfrost/keys/.gitignore
diff --git a/packages/cardano-services/blockfrost/keys/README.md b/packages/cardano-services/blockfrost/keys/README.md
diff --git a/packages/cardano-services/blockfrost/ryo/local-blockfrost-ryo.yaml b/packages/cardano-services/blockfrost/ryo/local-blockfrost-ryo.yaml
@@ -0,0 +1,11 @@
+server:
+  listenAddress: "0.0.0.0"
+  port: 3000
+  debug: false
+dbSync:
+  host: "postgres"
+  port: 5432
+  database: "cexplorer"
+  user: "postgres"
+  password: "doNoUseThisSecret!"
+  maxConnections: 5
diff --git a/packages/cardano-services/docker-compose.yml b/packages/cardano-services/docker-compose.yml
@@ -9,6 +9,18 @@ x-sdk-environment: &sdk-environment
   HANDLE_POLICY_IDS: ${HANDLE_POLICY_IDS:-f0ff48bbb7bbe9d59a40f1ce90e9e9d0ff5002ec48f232b49ca0fb9a}
 
 services:
+
+  blockfrost-ryo:
+    environment:
+      <<: *sdk-environment
+      NODE_ENV: local-blockfrost-ryo
+      BLOCKFROST_CONFIG_NETWORK: ${NETWORK:-mainnet}
+      BLOCKFROST_CONFIG_TOKEN_REGISTRY_URL: ${TOKEN_METADATA_SERVER_URL:-https://tokens.cardano.org}
+
+    volumes:
+      - ./blockfrost/ryo:/app/config
+    profiles: [ blockfrost-ryo ]
+
   cardano-node:
     volumes:
       - ./config/network/${NETWORK:-mainnet}/cardano-node:/config
@@ -74,7 +86,7 @@ services:
 
 secrets:
   blockfrost_key:
-    file: ./blockfrost-keys/${NETWORK:-mainnet}.key
+    file: ./blockfrost/keys/${NETWORK:-mainnet}.key
 
 volumes:
   postgres-data:
diff --git a/packages/cardano-services/package.json b/packages/cardano-services/package.json
@@ -29,7 +29,7 @@
     "cleanup": "rm -rf dist node_modules",
     "cli": "ts-node --transpile-only src/cli.ts",
     "compose:single:up": "yarn compose:up cardano-node ogmios postgres",
-    "compose:up": "__FIX_UMASK__=$(chmod -R a+r ../../compose/placeholder-secrets) docker compose --env-file environments/.env.$NETWORK -p cardano-services-$NETWORK -f docker-compose.yml -f ../../compose/common.yml -f ../../compose/pg-agent.yml -f ../../compose/$(uname -m).yml ${FILES:-} up",
+    "compose:up": "__FIX_UMASK__=$(chmod -R a+r ../../compose/placeholder-secrets) docker compose --env-file environments/.env.$NETWORK -p cardano-services-$NETWORK -f docker-compose.yml -f ../../compose/common.yml -f ../../compose/pg-agent.yml -f ../../compose/$(uname -m).yml ${FILES:-}  --profile ${DOCKER_COMPOSE_PROFILE:-none} up",
     "compose:down": "docker compose -p cardano-services-$NETWORK -f docker-compose.yml -f ../../compose/common.yml -f ../../compose/pg-agent.yml -f ../../compose/$(uname -m).yml down -t 120",
     "coverage": "yarn test --coverage || true",
     "circular-deps:check": "madge --circular dist/cjs",
@@ -38,17 +38,21 @@
     "generate-migration": "typeorm-ts-node-commonjs migration:generate src/Projection/migrations/migrations -d src/migrationDataSource.ts",
     "mainnet:single:up": "NETWORK=mainnet yarn compose:single:up",
     "mainnet:up": "NETWORK=mainnet SUBMIT_API_ARGS=--mainnet yarn compose:up",
+    "mainnet:blockfrost:up": "DOCKER_COMPOSE_PROFILE='blockfrost-ryo' yarn mainnet:up",
     "mainnet:down": "NETWORK=mainnet yarn compose:down",
     "prepack": "yarn build",
     "preprod:single:up": "NETWORK=preprod yarn compose:single:up",
     "preprod:up": "NETWORK=preprod SUBMIT_API_ARGS='--testnet-magic 1' yarn compose:up",
+    "preprod:blockfrost:up": "DOCKER_COMPOSE_PROFILE='blockfrost-ryo' yarn preprod:up",
     "preprod:down": "NETWORK=preprod yarn compose:down",
     "pretest": "yarn build",
     "preview:single:up": "NETWORK=preview yarn compose:single:up",
     "preview:up": "NETWORK=preview SUBMIT_API_ARGS='--testnet-magic 2' yarn compose:up",
+    "preview:blockfrost:up": "DOCKER_COMPOSE_PROFILE='blockfrost-ryo' yarn preview:up",
     "preview:down": "NETWORK=preview yarn compose:down",
     "sanchonet:single:up": "NETWORK=sanchonet yarn compose:single:up",
     "sanchonet:up": "NETWORK=sanchonet SUBMIT_API_ARGS='--testnet-magic 4' yarn compose:up",
+    "sanchonet:blockfrost:up": "DOCKER_COMPOSE_PROFILE='blockfrost-ryo' yarn sanchonet:up",
     "sanchonet:down": "NETWORK=sanchonet yarn compose:down",
     "test": "jest --runInBand -c ./jest.config.js --selectProjects unit",
     "test:build:verify": "tsc --build ./test",
diff --git a/packages/cardano-services/src/Program/options/providers.ts b/packages/cardano-services/src/Program/options/providers.ts
@@ -22,58 +22,52 @@ export const ProviderImplementationDescription = 'Select one of the available pr
 const argParser = (impl: string) => ProviderImplementation[impl.toUpperCase() as keyof typeof ProviderImplementation];
 export const providerSelectionOptions = [
   newOption(
-    '--asset-provider <implementation>',
+    '--asset-provider <assetProvider>',
     ProviderImplementationDescription,
     'ASSET_PROVIDER',
     argParser,
     ProviderImplementation.DBSYNC
-  )
-    .conflicts('useTypeormAssetProvider')
-    .choices([ProviderImplementation.BLOCKFROST, ProviderImplementation.DBSYNC, ProviderImplementation.TYPEORM]),
+  ).choices([ProviderImplementation.BLOCKFROST, ProviderImplementation.DBSYNC, ProviderImplementation.TYPEORM]),
   newOption(
-    '--stake-pool-provider <implementation>',
+    '--stake-pool-provider <stakePoolProvider>',
     ProviderImplementationDescription,
     'STAKE_POOL_PROVIDER',
     argParser,
     ProviderImplementation.DBSYNC
-  )
-    .conflicts('useTypeormStakePoolProvider')
-    .choices([ProviderImplementation.DBSYNC, ProviderImplementation.TYPEORM]),
+  ).choices([ProviderImplementation.DBSYNC, ProviderImplementation.TYPEORM]),
   newOption(
-    '--utxo-provider <implementation>',
+    '--utxo-provider <utxoProvider>',
     ProviderImplementationDescription,
     'UTXO_PROVIDER',
     argParser,
     ProviderImplementation.DBSYNC
   ).choices([ProviderImplementation.BLOCKFROST, ProviderImplementation.DBSYNC]),
   newOption(
-    '--chain-history-provider <implementation>',
+    '--chain-history-provider <chainHistoryProvider>',
     ProviderImplementationDescription,
     'CHAIN_HISTORY_PROVIDER',
     argParser,
     ProviderImplementation.DBSYNC
   ).choices([ProviderImplementation.BLOCKFROST, ProviderImplementation.DBSYNC]),
   newOption(
-    '--rewards-provider <implementation>',
+    '--rewards-provider <rewardsProvider>',
     ProviderImplementationDescription,
     'REWARDS_PROVIDER',
     argParser,
     ProviderImplementation.DBSYNC
   ).choices([ProviderImplementation.BLOCKFROST, ProviderImplementation.DBSYNC]),
   newOption(
-    '--network-info-provider <implementation>',
+    '--network-info-provider <networkInfoProvider>',
     ProviderImplementationDescription,
     'NETWORK_INFO_PROVIDER',
     argParser,
     ProviderImplementation.DBSYNC
   ).choices([ProviderImplementation.BLOCKFROST, ProviderImplementation.DBSYNC]),
   newOption(
-    '--tx-submit-provider <implementation>',
+    '--tx-submit-provider <txSubmitProvider>',
     ProviderImplementationDescription,
     'TX_SUBMIT_PROVIDER',
     argParser,
     ProviderImplementation.SUBMIT_NODE
-  )
-    .conflicts('useSubmitApi')
-    .choices([ProviderImplementation.BLOCKFROST, ProviderImplementation.SUBMIT_API, ProviderImplementation.SUBMIT_NODE])
+  ).choices([ProviderImplementation.BLOCKFROST, ProviderImplementation.SUBMIT_API, ProviderImplementation.SUBMIT_NODE])
 ];
diff --git a/packages/cardano-services/src/Program/programs/types.ts b/packages/cardano-services/src/Program/programs/types.ts
@@ -63,7 +63,8 @@ export enum ProviderServerOptionDescriptions {
   UseBlockfrost = 'Enables Blockfrost cached data DB',
   UseSubmitApi = 'Use cardano-submit-api provider',
   UseTypeormAssetProvider = 'Use the TypeORM Asset Provider (default is db-sync)',
-  UseWebSocketApi = 'Use WebSocket API'
+  UseWebSocketApi = 'Use WebSocket API',
+  BlockfrostCustomNetworkUrl = 'URL for custom hosted blockfrost-ryo'
 }
 
 export type ProviderServerArgs = CommonProgramOptions &
diff --git a/packages/cardano-services/src/cli.ts b/packages/cardano-services/src/cli.ts
@@ -379,6 +379,12 @@ addOptions(withOgmiosOptions(withHandlePolicyIdsOptions(providerServerWithCommon
     'USE_WEB_SOCKET_API',
     (val) => stringOptionToBoolean(val, Programs.ProviderServer, ProviderServerOptionDescriptions.UseWebSocketApi)
   ),
+  newOption(
+    '--blockfrost-custom-network-url <blockfrostCustomNetworkUrl>',
+    ProviderServerOptionDescriptions.BlockfrostCustomNetworkUrl,
+    'BLOCKFROST_CUSTOM_NETWORK_URL',
+    urlValidator(ProviderServerOptionDescriptions.BlockfrostCustomNetworkUrl, true)
+  ),
   ...providerSelectionOptions
 ]).action(async (serviceNames: ServiceNames[], args: ProviderServerArgs) =>
   runServer('Provider server', { args, serviceNames }, () =>
diff --git a/packages/cardano-services/src/util/BlockfrostProvider/BlockfrostClientFactory.ts b/packages/cardano-services/src/util/BlockfrostProvider/BlockfrostClientFactory.ts
@@ -11,8 +11,18 @@ let blockfrostApi: BlockFrostAPI | undefined;
 export const getBlockfrostApi = () => {
   if (blockfrostApi !== undefined) return blockfrostApi;
 
+  // custom hosted instance
+  if (process.env.BLOCKFROST_CUSTOM_BACKEND_URL && process.env.BLOCKFROST_CUSTOM_BACKEND_URL !== '') {
+    blockfrostApi = new BlockFrostAPI({
+      customBackend: process.env.BLOCKFROST_CUSTOM_BACKEND_URL
+    });
+
+    return blockfrostApi;
+  }
+
+  // instance hosted by Blockfrost
   if (process.env.BLOCKFROST_API_KEY === undefined || process.env.BLOCKFROST_API_KEY === '')
-    throw new Error('BLOCKFROST_API_KEY environment variable is required');
+    throw new Error('BLOCKFROST_API_KEY or BLOCKFROST_CUSTOM_BACKEND_URL environment variable is required');
 
   if (process.env.NETWORK === undefined) throw new Error('NETWORK environment variable is required');
 
diff --git a/packages/cardano-services/test/cli.test.ts b/packages/cardano-services/test/cli.test.ts
@@ -1,6 +1,6 @@
 /* eslint-disable sonarjs/cognitive-complexity */
 /* eslint-disable sonarjs/no-duplicate-string */
-import { DEFAULT_FUZZY_SEARCH_OPTIONS } from '../src';
+import { DEFAULT_FUZZY_SEARCH_OPTIONS, ProviderServerOptionDescriptions } from '../src';
 import { createLogger } from '@cardano-sdk/util-dev';
 import { fork } from 'child_process';
 import path from 'path';
@@ -1113,6 +1113,22 @@ describe('CLI', () => {
         }));
     });
 
+    describe('blockfrostCustomNetworkUrl', () => {
+      const blockfrostCustomNetworkUrl = 'https://test/';
+
+      testCli('accepts an URL', 'provider', {
+        args: ['--blockfrost-custom-network-url', blockfrostCustomNetworkUrl],
+        env: { BLOCKFROST_CUSTOM_NETWORK_URL: blockfrostCustomNetworkUrl },
+        expectedArgs: { args: { blockfrostCustomNetworkUrl } }
+      });
+
+      testCli('expects an URL', 'provider', {
+        args: ['--blockfrost-custom-network-url', 'invalid'],
+        env: { BLOCKFROST_CUSTOM_NETWORK_URL: 'invalid' },
+        expectedError: `${ProviderServerOptionDescriptions.BlockfrostCustomNetworkUrl} - "invalid" is not an URL`
+      });
+    });
+
     describe('provider implementation selection', () => {
       testCli('check defaults', 'provider', {
         expectedArgs: {
@@ -1142,11 +1158,6 @@ describe('CLI', () => {
           env: { ASSET_PROVIDER: 'not-valid' },
           expectedError: 'is invalid'
         });
-        testCli('conflicts', 'provider', {
-          args: ['--asset-provider', 'blockfrost', '--use-typeorm-asset-provider', 'true'],
-          env: { ASSET_PROVIDER: 'blockfrost', USE_TYPEORM_ASSET_PROVIDER: 'true' },
-          expectedError: 'cannot be used with'
-        });
       });
       describe('--stake-pool-provider', () => {
         testCli('set successful', 'provider', {
@@ -1163,11 +1174,6 @@ describe('CLI', () => {
           env: { STAKE_POOL_PROVIDER: 'not-valid' },
           expectedError: 'is invalid'
         });
-        testCli('conflicts', 'provider', {
-          args: ['--stake-pool-provider', 'dbsync', '--use-typeorm-stake-pool-provider', 'true'],
-          env: { STAKE_POOL_PROVIDER: 'typeorm', USE_TYPEORM_STAKE_POOL_PROVIDER: 'true' },
-          expectedError: 'cannot be used with'
-        });
       });
 
       describe('--utxo-provider', () => {
diff --git a/packages/cardano-services/test/util/BlockfrsotProvider/BlockfrostClientFactory.test.ts b/packages/cardano-services/test/util/BlockfrsotProvider/BlockfrostClientFactory.test.ts
@@ -9,7 +9,7 @@ describe('BlockfrostClientFactory', () => {
       delete process.env.NETWORK;
     });
 
-    it('gets correctly, first time', () => {
+    it('gets correctly', () => {
       process.env.BLOCKFROST_API_KEY = 'testing';
       process.env.NETWORK = 'preprod';
       const apiFirst = getBlockfrostApi();
@@ -24,11 +24,24 @@ describe('BlockfrostClientFactory', () => {
       expect(apiSecond).toBeInstanceOf(BlockFrostAPI);
     });
 
-    it('throws if env vars missing', () => {
-      expect(() => getBlockfrostApi()).toThrow(new Error('BLOCKFROST_API_KEY environment variable is required'));
+    it('gets custom backend correctly', () => {
+      process.env.BLOCKFROST_CUSTOM_BACKEND_URL = 'http://localhost';
+      const apiFirst = getBlockfrostApi();
+      expect(apiFirst).toBeDefined();
+      expect(apiFirst).toBeInstanceOf(BlockFrostAPI);
+
+      // deleting proves that we initialize only once
+      delete process.env.BLOCKFROST_CUSTOM_BACKEND_URL;
+      const apiSecond = getBlockfrostApi();
+      expect(apiSecond).toBeDefined();
+      expect(apiSecond).toBeInstanceOf(BlockFrostAPI);
     });
 
     it('throws if env vars missing', () => {
+      expect(() => getBlockfrostApi()).toThrow(
+        new Error('BLOCKFROST_API_KEY or BLOCKFROST_CUSTOM_BACKEND_URL environment variable is required')
+      );
+
       process.env.BLOCKFROST_API_KEY = 'testing';
       expect(() => getBlockfrostApi()).toThrow(new Error('NETWORK environment variable is required'));
     });
diff --git a/packages/e2e/docker-compose.yml b/packages/e2e/docker-compose.yml
@@ -6,6 +6,17 @@ x-logging: &logging
       max-file: '10'
 
 services:
+
+  blockfrost-ryo:
+    depends_on:
+      local-testnet:
+        condition: service_healthy
+    environment:
+      NODE_ENV: local-network
+    volumes:
+      - ./local-network/config/network/blockfrost-ryo:/app/config
+    profiles: [blockfrost-ryo]
+
   local-testnet:
     <<: *logging
     build:
diff --git a/packages/e2e/local-network/scripts/make-babbage.sh b/packages/e2e/local-network/scripts/make-babbage.sh
@@ -303,7 +303,7 @@ for NODE in ${SP_NODES}; do
   echo "${ROOT}/${NODE}.sh"
 done
 
-echo "Update start time in genesis files"
+echo "Update start time in genesis files with ${timeUnix}"
 sed_i -E "s/\"startTime\": [0-9]+/\"startTime\": ${timeUnix}/" ${ROOT}/genesis/byron/genesis.json
 sed_i -E "s/\"systemStart\": \".*\"/\"systemStart\": \"${timeISO}\"/" ${ROOT}/genesis/shelley/genesis.json
 
@@ -359,6 +359,8 @@ cp "${ROOT}"/genesis/shelley/genesis.conway.json ./config/network/genesis/conway
 
 mkdir -p "${ROOT}/run"
 
+echo "${timeUnix}" > "${ROOT}/run/system_start"
+
 echo "#!/usr/bin/env bash" >"${ROOT}/run/all.sh"
 echo "" >>"${ROOT}/run/all.sh"
 echo "" >>"${ROOT}/run/all.sh"
diff --git a/packages/e2e/local-network/scripts/prepare_blockfrost_ryo.sh b/packages/e2e/local-network/scripts/prepare_blockfrost_ryo.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+timeUnix=$(cat network-files/run/system_start)
+
+echo "Update start time in blockfrost genesis file with ${timeUnix}"
+
+mkdir -p ./config/network/blockfrost-ryo/
+
+cp ./templates/blockfrost-ryo/genesis.json ./config/network/blockfrost-ryo/genesis.json
+cp ./templates/blockfrost-ryo/byron_genesis.json ./config/network/blockfrost-ryo/byron_genesis.json
+cp ./templates/blockfrost-ryo/local-network.yaml ./config/network/blockfrost-ryo/local-network.yaml
+
+sed -i -E "s/\"system_start\": [0-9]+/\"system_start\": ${timeUnix}/" ./config/network/blockfrost-ryo/genesis.json
diff --git a/packages/e2e/local-network/scripts/start.sh b/packages/e2e/local-network/scripts/start.sh
@@ -26,6 +26,7 @@ trap 'kill 0' INT
 
 echo "Run"
 ./scripts/make-babbage.sh
+./scripts/prepare_blockfrost_ryo.sh
 ./network-files/run/all.sh &
 
 if [ -d /sdk-ipc ] ; then cp -a config/network /sdk-ipc/config ; fi
diff --git a/packages/e2e/local-network/templates/blockfrost-ryo/byron_genesis.json b/packages/e2e/local-network/templates/blockfrost-ryo/byron_genesis.json
@@ -0,0 +1,6 @@
+{
+  "epoch_length": 100,
+  "slot_length": 1,
+  "safe_zone": 864,
+  "end_epoch": 0
+}
diff --git a/packages/e2e/local-network/templates/blockfrost-ryo/genesis.json b/packages/e2e/local-network/templates/blockfrost-ryo/genesis.json
@@ -0,0 +1,12 @@
+{
+  "active_slots_coefficient": 0.05,
+  "update_quorum": 5,
+  "max_lovelace_supply": "45000000000000000",
+  "network_magic": 888,
+  "epoch_length": 1000,
+  "system_start": 1000000,
+  "slots_per_kes_period": 129600,
+  "slot_length": 0.2,
+  "max_kes_evolutions": 62,
+  "security_param": 10
+}
diff --git a/packages/e2e/local-network/templates/blockfrost-ryo/local-network.yaml b/packages/e2e/local-network/templates/blockfrost-ryo/local-network.yaml
@@ -0,0 +1,14 @@
+server:
+  listenAddress: "0.0.0.0"
+  port: 3000
+  debug: true
+dbSync:
+  host: "postgres"
+  port: 5432
+  database: "cexplorer"
+  user: "postgres"
+  password: "doNoUseThisSecret!"
+  maxConnections: 5
+network: "custom"
+genesisDataFolder: '/app/config'
+tokenRegistryUrl: "https://metadata.cardano-testnet.iohkdev.io"
diff --git a/packages/e2e/package.json b/packages/e2e/package.json

Original file line number	Diff line number	Diff line change
`@@ -63,7 +63,8 @@ export enum ProviderServerOptionDescriptions {`
`63`	`63`	`UseBlockfrost = 'Enables Blockfrost cached data DB',`
`64`	`64`	`UseSubmitApi = 'Use cardano-submit-api provider',`
`65`	`65`	`UseTypeormAssetProvider = 'Use the TypeORM Asset Provider (default is db-sync)',`
`66`		`- UseWebSocketApi = 'Use WebSocket API'`
	`66`	`+ UseWebSocketApi = 'Use WebSocket API',`
	`67`	`+ BlockfrostCustomNetworkUrl = 'URL for custom hosted blockfrost-ryo'`
`67`	`68`	`}`
`68`	`69`
`69`	`70`	`export type ProviderServerArgs = CommonProgramOptions &`