From 1c3ad848d9a4a96d39d30017fb7be1a42ebcf12b Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Mon, 9 Aug 2021 08:10:49 -0300 Subject: [PATCH 1/5] using rest API --- src/bitcoin-node-connection.js | 10 +++------- src/bitcoin-rpc.js | 5 +++++ test/bitcoin-node-connection-test.js | 8 +++++++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/bitcoin-node-connection.js b/src/bitcoin-node-connection.js index e0da16a..962da72 100644 --- a/src/bitcoin-node-connection.js +++ b/src/bitcoin-node-connection.js @@ -49,12 +49,8 @@ class BitcoinNodeConnection { return { reorg: true } } - if (blockData.size >= 0xf000000) { // Avoids create a string longer than the limit - return this._responsefromBlockData(blockData) - } - const block = this._parseBlock( - await this.rpc.getBlockByHeight(targetBlockHeight, false), + await this.rpc.getRawBlockByHash(blockData.hash), targetBlockHeight ) return this._buildBlockResponse(block, targetBlockHeight) @@ -90,8 +86,8 @@ class BitcoinNodeConnection { return response } - _parseBlock (rpcResponse, requestedHeight) { - const bsvBlock = new bsv.Block(Buffer.from(rpcResponse, 'hex')) + _parseBlock (blockBuffer, requestedHeight) { + const bsvBlock = new bsv.Block(blockBuffer) return { height: requestedHeight, diff --git a/src/bitcoin-rpc.js b/src/bitcoin-rpc.js index 2ad2eeb..8644076 100644 --- a/src/bitcoin-rpc.js +++ b/src/bitcoin-rpc.js @@ -47,6 +47,11 @@ class BitcoinRpc { return this._rpcCall('getblockbyheight', [targetHeight, verbose]) } + async getRawBlockByHash (targetHash) { + const response = this._httpPost(`${this.baseUrl}/rest/block/${targetHash}.bin`) + return Buffer.from(await response.arrayBuffer()) + } + async _rpcCall (method, params) { const response = await this._httpPost(this.baseUrl, { jsonrpc: '1.0', diff --git a/test/bitcoin-node-connection-test.js b/test/bitcoin-node-connection-test.js index a7fa731..6e40588 100644 --- a/test/bitcoin-node-connection-test.js +++ b/test/bitcoin-node-connection-test.js @@ -41,11 +41,17 @@ class TestBitcoinRpc { return { size: block.size || block.hex.length, previousblockhash: block.previousblockhash, - tx: block.txs.map(tx => tx.hash) + tx: block.txs.map(tx => tx.hash), + hash: block.hash } } } + async getRawBlockByHash (targetHash) { + const block = this.blocks.find(block => block.hash === targetHash) + return Buffer.from(block.hex, 'hex') + } + // Test registerConfirmedTx (txid, rawTx) { From c30d1ac5658d9835f24ef07b7337e7b2d6f55272 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Mon, 9 Aug 2021 08:10:56 -0300 Subject: [PATCH 2/5] bump deps --- package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4677ee3..849bf37 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5686,9 +5686,9 @@ } }, "node_modules/tar": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.0.tgz", - "integrity": "sha512-DUCttfhsnLCjwoDoFcI+B2iJgYa93vBnDUATYEeRx6sntCTdN01VnqsIuTlALXla/LWooNg0yEGeB+Y8WdFxGA==", + "version": "6.1.6", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.6.tgz", + "integrity": "sha512-oaWyu5dQbHaYcyZCTfyPpC+VmI62/OM2RTUYavTk1MDr1cwW5Boi3baeYQKiZbY2uSQJGr+iMOzb/JFxLrft+g==", "dependencies": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", @@ -10615,9 +10615,9 @@ } }, "tar": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.0.tgz", - "integrity": "sha512-DUCttfhsnLCjwoDoFcI+B2iJgYa93vBnDUATYEeRx6sntCTdN01VnqsIuTlALXla/LWooNg0yEGeB+Y8WdFxGA==", + "version": "6.1.6", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.6.tgz", + "integrity": "sha512-oaWyu5dQbHaYcyZCTfyPpC+VmI62/OM2RTUYavTk1MDr1cwW5Boi3baeYQKiZbY2uSQJGr+iMOzb/JFxLrft+g==", "requires": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", From d04a30497ccd186a24ebed84a73767521faa0e4e Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Mon, 9 Aug 2021 08:11:09 -0300 Subject: [PATCH 3/5] logging each tx as debug statement --- src/indexer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/indexer.js b/src/indexer.js index a334690..7e57fad 100644 --- a/src/indexer.js +++ b/src/indexer.js @@ -117,7 +117,7 @@ class Indexer { } _onAddTransaction (txid) { - this.logger.info('Added', txid) + this.logger.debug('Added', txid) } _onDeleteTransaction (txid) { From 3f805e71917e2463f93b1bf89be63b85dd75c821 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Mon, 9 Aug 2021 09:43:15 -0300 Subject: [PATCH 4/5] sing rest api --- src/bitcoin-node-connection.js | 22 +++++++++--- src/bitcoin-rpc.js | 8 ++++- test/bitcoin-node-connection-test.js | 50 +++++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 7 deletions(-) diff --git a/src/bitcoin-node-connection.js b/src/bitcoin-node-connection.js index 962da72..346a603 100644 --- a/src/bitcoin-node-connection.js +++ b/src/bitcoin-node-connection.js @@ -15,10 +15,12 @@ class BitcoinNodeConnection { constructor (zmq, rpc) { this.zmq = zmq this.rpc = rpc + this.isRestApiEnabled = false } async connect (_height, _network) { await this.zmq.connect() + this.isRestApiEnabled = await this.rpc.isRestApiEnabled() } async disconnect () { @@ -49,11 +51,21 @@ class BitcoinNodeConnection { return { reorg: true } } - const block = this._parseBlock( - await this.rpc.getRawBlockByHash(blockData.hash), - targetBlockHeight - ) - return this._buildBlockResponse(block, targetBlockHeight) + if (this.isRestApiEnabled) { + const block = this._parseBlock( + await this.rpc.getRawBlockByHash(blockData.hash), + targetBlockHeight + ) + return this._buildBlockResponse(block, targetBlockHeight) + } else if (blockData.size >= 0xf000000) { + return this._responsefromBlockData(blockData) + } else { + const block = this._parseBlock( + await this.rpc.getBlockByHeight(targetBlockHeight, false), + targetBlockHeight + ) + return this._buildBlockResponse(block, targetBlockHeight) + } } async listenForMempool (mempoolTxCallback) { diff --git a/src/bitcoin-rpc.js b/src/bitcoin-rpc.js index 8644076..f1a8163 100644 --- a/src/bitcoin-rpc.js +++ b/src/bitcoin-rpc.js @@ -1,3 +1,4 @@ +const { response } = require('express') const fetch = require('node-fetch') const httpPost = async (url, jsonBody) => { @@ -28,6 +29,11 @@ class BitcoinRpc { this.baseUrl = baseUrl } + async isRestApiEnabled () { + const response = fetch(`${this.baseUrl}/rest/tx/9834daa6d34690981888f7db4c1c36686ebb9b685d37115abc38e0e75f9cd98d.hex`) + return response.ok + } + /** * @param {String} txid */ @@ -48,7 +54,7 @@ class BitcoinRpc { } async getRawBlockByHash (targetHash) { - const response = this._httpPost(`${this.baseUrl}/rest/block/${targetHash}.bin`) + const response = await fetch(`${this.baseUrl}/rest/block/${targetHash}.bin`) return Buffer.from(await response.arrayBuffer()) } diff --git a/test/bitcoin-node-connection-test.js b/test/bitcoin-node-connection-test.js index 6e40588..26779fd 100644 --- a/test/bitcoin-node-connection-test.js +++ b/test/bitcoin-node-connection-test.js @@ -19,6 +19,11 @@ class TestBitcoinRpc { } ] this.nextBlockHeight = 1001 + this.isRestEnabled = true + } + + async isRestApiEnabled () { + return this.isRestEnabled } async getRawTransaction (txid, verbose = true) { @@ -48,6 +53,9 @@ class TestBitcoinRpc { } async getRawBlockByHash (targetHash) { + if (!this.isRestEnabled) { + throw new Error('rest api is not enabled') + } const block = this.blocks.find(block => block.hash === targetHash) return Buffer.from(block.hex, 'hex') } @@ -112,6 +120,10 @@ class TestZmq { this.handler = handler } + async connect () { + // nothing + } + // test publishTx (tx) { @@ -179,10 +191,11 @@ describe('BitcoinNodeConnection', () => { let instance = null let run = null - beforeEach(() => { + beforeEach(async () => { bitcoinZmq = new TestZmq() bitcoinRpc = new TestBitcoinRpc() instance = new BitcoinNodeConnection(bitcoinZmq, bitcoinRpc) + await instance.connect() run = new Run({ purse: { @@ -379,6 +392,41 @@ describe('BitcoinNodeConnection', () => { const nextBlock = await instance.getNextBlock(previousBlock.height, null) expect(nextBlock.txids).to.eql([randomRunTx.hash]) }) + + describe('when rest-api is not enabled and a giant block arrives', () => { + beforeEach(() => { + bitcoinRpc.isRestEnabled = false + }) + it ('does not get txs one by one', async () => { + const randomTx = buildRandomTx() + const randomRunTx = await buildRandomRunTx(run) + bitcoinRpc.registerUnconfirmedTx(randomTx.hash, randomTx.toBuffer().toString('hex')) + bitcoinRpc.registerUnconfirmedTx(randomRunTx.hash, randomRunTx.toBuffer().toString('hex')) + bitcoinRpc.closeBlock(0x1fffffe8 + 1) + const previousBlock = bitcoinRpc.blocks[bitcoinRpc.blocks.length - 2] + + expect(instance.getNextBlock(previousBlock.height, null)).not.to.eventually.throw() + }) + }) + + describe('when rest-api is enabled and a giant block arrives', () => { + beforeEach(() => { + bitcoinRpc.isRestEnabled = true + }) + it('does not call getRawTransaction', async () => { + const randomTx = buildRandomTx() + const randomRunTx = await buildRandomRunTx(run) + bitcoinRpc.registerUnconfirmedTx(randomTx.hash, randomTx.toBuffer().toString('hex')) + bitcoinRpc.registerUnconfirmedTx(randomRunTx.hash, randomRunTx.toBuffer().toString('hex')) + bitcoinRpc.closeBlock(0x1fffffe8 + 1) + + bitcoinRpc.getRawTransaction = () => { throw new Error('should not call') } + const previousBlock = bitcoinRpc.blocks[bitcoinRpc.blocks.length - 2] + + const nextBlock = await instance.getNextBlock(previousBlock.height, null) + expect(nextBlock.txhexs.length).to.eql(nextBlock.txids.length) + }) + }) }) describe('#listenForMempool', () => { From 10b8701caddd03d71d2784ed70d2f22769a8a0a5 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Mon, 9 Aug 2021 09:47:07 -0300 Subject: [PATCH 5/5] lint fixes. --- src/bitcoin-rpc.js | 1 - test/bitcoin-node-connection-test.js | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/bitcoin-rpc.js b/src/bitcoin-rpc.js index f1a8163..9d20f66 100644 --- a/src/bitcoin-rpc.js +++ b/src/bitcoin-rpc.js @@ -1,4 +1,3 @@ -const { response } = require('express') const fetch = require('node-fetch') const httpPost = async (url, jsonBody) => { diff --git a/test/bitcoin-node-connection-test.js b/test/bitcoin-node-connection-test.js index 26779fd..8afa2ce 100644 --- a/test/bitcoin-node-connection-test.js +++ b/test/bitcoin-node-connection-test.js @@ -397,14 +397,14 @@ describe('BitcoinNodeConnection', () => { beforeEach(() => { bitcoinRpc.isRestEnabled = false }) - it ('does not get txs one by one', async () => { + it('does not get txs one by one', async () => { const randomTx = buildRandomTx() const randomRunTx = await buildRandomRunTx(run) bitcoinRpc.registerUnconfirmedTx(randomTx.hash, randomTx.toBuffer().toString('hex')) bitcoinRpc.registerUnconfirmedTx(randomRunTx.hash, randomRunTx.toBuffer().toString('hex')) bitcoinRpc.closeBlock(0x1fffffe8 + 1) const previousBlock = bitcoinRpc.blocks[bitcoinRpc.blocks.length - 2] - + expect(instance.getNextBlock(previousBlock.height, null)).not.to.eventually.throw() }) }) @@ -420,9 +420,9 @@ describe('BitcoinNodeConnection', () => { bitcoinRpc.registerUnconfirmedTx(randomRunTx.hash, randomRunTx.toBuffer().toString('hex')) bitcoinRpc.closeBlock(0x1fffffe8 + 1) - bitcoinRpc.getRawTransaction = () => { throw new Error('should not call') } + bitcoinRpc.getRawTransaction = () => { throw new Error('should not call') } const previousBlock = bitcoinRpc.blocks[bitcoinRpc.blocks.length - 2] - + const nextBlock = await instance.getNextBlock(previousBlock.height, null) expect(nextBlock.txhexs.length).to.eql(nextBlock.txids.length) })