fix!: yield uint8arraylists instead of concatenating buffers (#391)

* fix: yield uint8arraylists instead of concatenating buffers

In order to avoid unnecessary buffer copies, update to the new libp2p
connection encrypter API that lets connection encrypters consume/yield
lists of buffers instead of requiring them to be concatenated
before/after encryption/decryption.

* chore: fix tcp version

* feat: use libp2p component logger

Refactors code to use the component logger from libp2p to allow more
flexible logging patterns.

Nb. adds a `NoiseComponents` interface separate from `NoiseInit` that
contains the `Metrics` instance - this is consistent with every other
libp2p module.

Refs: https://github.com/libp2p/js-libp2p/issue/2105
Refs: libp2p/js-libp2p#2198
Refs: https://github.com/libp2p/js-libp2p/issue/378

* chore: fix linter errors

---------

Co-authored-by: Cayman <[email protected]>

Author: achingbrain

benchmarks/benchmark.js

@@ -22,13 +22,13 @@ const bench = async function () {
   console.log('Init complete, running benchmark')
   const bench = new benchmark('handshake', {
     defer: true,
-    fn: async function (deffered) {
+    fn: async function (deferred) {
       const [inboundConnection, outboundConnection] = duplexPair()
       await Promise.all([
         initiator.secureOutbound(initiatorPeer, outboundConnection, responderPeer),
         responder.secureInbound(responderPeer, inboundConnection, initiatorPeer)
       ])
-      deffered.resolve()
+      deferred.resolve()
     }
   })
     .on('complete', function (stats) {

package.json

@@ -69,10 +69,9 @@
   "dependencies": {
     "@chainsafe/as-chacha20poly1305": "^0.1.0",
     "@chainsafe/as-sha256": "^0.4.1",
-    "@libp2p/crypto": "^2.0.0",
-    "@libp2p/interface": "^0.1.0",
-    "@libp2p/logger": "^3.0.0",
-    "@libp2p/peer-id": "^3.0.0",
+    "@libp2p/crypto": "^3.0.0",
+    "@libp2p/interface": "^1.0.0",
+    "@libp2p/peer-id": "^4.0.0",
     "@noble/ciphers": "^0.4.0",
     "@noble/curves": "^1.1.0",
     "@noble/hashes": "^1.3.1",
@@ -88,22 +87,22 @@
     "wherearewe": "^2.0.1"
   },
   "devDependencies": {
-    "@chainsafe/libp2p-yamux": "^5.0.0",
+    "@chainsafe/libp2p-yamux": "^6.0.0",
     "@libp2p/daemon-client": "^7.0.0",
     "@libp2p/daemon-server": "^6.0.0",
-    "@libp2p/interface-compliance-tests": "^4.0.0",
-    "@libp2p/interface-peer-id": "^2.0.2",
+    "@libp2p/interface-compliance-tests": "^5.0.0",
     "@libp2p/interop": "^9.0.0",
-    "@libp2p/peer-id-factory": "^3.0.0",
-    "@libp2p/tcp": "^8.0.0",
+    "@libp2p/logger": "^4.0.0",
+    "@libp2p/peer-id-factory": "^3.0.9",
+    "@libp2p/tcp": "^9.0.0",
     "@multiformats/multiaddr": "^12.1.0",
     "@types/sinon": "^17.0.1",
     "aegir": "^41.1.10",
     "benchmark": "^2.1.4",
     "execa": "^8.0.1",
     "go-libp2p": "^1.0.3",
     "iso-random-stream": "^2.0.2",
-    "libp2p": "^0.46.0",
+    "libp2p": "next",
     "mkdirp": "^3.0.0",
     "p-defer": "^4.0.0",
     "protons": "^7.0.0",

src/@types/handshake-interface.ts

@@ -1,12 +1,12 @@
-import type { bytes } from './basic.js'
 import type { NoiseSession } from './handshake.js'
 import type { NoiseExtensions } from '../proto/payload.js'
-import type { PeerId } from '@libp2p/interface/peer-id'
+import type { PeerId } from '@libp2p/interface'
+import type { Uint8ArrayList } from 'uint8arraylist'
 
 export interface IHandshake {
   session: NoiseSession
   remotePeer: PeerId
   remoteExtensions: NoiseExtensions
-  encrypt(plaintext: bytes, session: NoiseSession): bytes
-  decrypt(ciphertext: bytes, session: NoiseSession, dst?: Uint8Array): { plaintext: bytes, valid: boolean }
+  encrypt(plaintext: Uint8Array | Uint8ArrayList, session: NoiseSession): Uint8Array | Uint8ArrayList
+  decrypt(ciphertext: Uint8Array | Uint8ArrayList, session: NoiseSession, dst?: Uint8Array): { plaintext: Uint8Array | Uint8ArrayList, valid: boolean }
 }

src/@types/handshake.ts

@@ -1,13 +1,14 @@
 import type { bytes, bytes32, uint64 } from './basic.js'
 import type { KeyPair } from './libp2p.js'
 import type { Nonce } from '../nonce.js'
+import type { Uint8ArrayList } from 'uint8arraylist'
 
 export type Hkdf = [bytes, bytes, bytes]
 
 export interface MessageBuffer {
   ne: bytes32
-  ns: bytes
-  ciphertext: bytes
+  ns: Uint8Array | Uint8ArrayList
+  ciphertext: Uint8Array | Uint8ArrayList
 }
 
 export interface CipherState {
@@ -27,7 +28,7 @@ export interface HandshakeState {
   ss: SymmetricState
   s: KeyPair
   e?: KeyPair
-  rs: bytes32
+  rs: Uint8Array | Uint8ArrayList
   re: bytes32
   psk: bytes32
 }

src/@types/libp2p.ts

@@ -1,6 +1,6 @@
 import type { bytes32 } from './basic.js'
 import type { NoiseExtensions } from '../proto/payload.js'
-import type { ConnectionEncrypter } from '@libp2p/interface/connection-encrypter'
+import type { ConnectionEncrypter } from '@libp2p/interface'
 
 export interface KeyPair {
   publicKey: bytes32

src/crypto.ts

@@ -1,16 +1,17 @@
-import type { bytes32, bytes } from './@types/basic.js'
+import { type Uint8ArrayList } from 'uint8arraylist'
+import type { bytes32 } from './@types/basic.js'
 import type { Hkdf } from './@types/handshake.js'
 import type { KeyPair } from './@types/libp2p.js'
 
 export interface ICryptoInterface {
-  hashSHA256(data: Uint8Array): Uint8Array
+  hashSHA256(data: Uint8Array | Uint8ArrayList): Uint8Array
 
   getHKDF(ck: bytes32, ikm: Uint8Array): Hkdf
 
   generateX25519KeyPair(): KeyPair
   generateX25519KeyPairFromSeed(seed: Uint8Array): KeyPair
-  generateX25519SharedKey(privateKey: Uint8Array, publicKey: Uint8Array): Uint8Array
+  generateX25519SharedKey(privateKey: Uint8Array | Uint8ArrayList, publicKey: Uint8Array | Uint8ArrayList): Uint8Array
 
-  chaCha20Poly1305Encrypt(plaintext: Uint8Array, nonce: Uint8Array, ad: Uint8Array, k: bytes32): bytes
-  chaCha20Poly1305Decrypt(ciphertext: Uint8Array, nonce: Uint8Array, ad: Uint8Array, k: bytes32, dst?: Uint8Array): bytes | null
+  chaCha20Poly1305Encrypt(plaintext: Uint8Array | Uint8ArrayList, nonce: Uint8Array, ad: Uint8Array, k: bytes32): Uint8ArrayList | Uint8Array
+  chaCha20Poly1305Decrypt(ciphertext: Uint8Array | Uint8ArrayList, nonce: Uint8Array, ad: Uint8Array, k: bytes32, dst?: Uint8Array): Uint8ArrayList | Uint8Array | null
 }

src/crypto/index.ts

@@ -1,6 +1,7 @@
 import crypto from 'node:crypto'
 import { newInstance, ChaCha20Poly1305 } from '@chainsafe/as-chacha20poly1305'
 import { digest } from '@chainsafe/as-sha256'
+import { Uint8ArrayList } from 'uint8arraylist'
 import { isElectronMain } from 'wherearewe'
 import { pureJsCrypto } from './js.js'
 import type { KeyPair } from '../@types/libp2p.js'
@@ -13,50 +14,105 @@ const PKCS8_PREFIX = Buffer.from([0x30, 0x2e, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06
 const X25519_PREFIX = Buffer.from([0x30, 0x2a, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x6e, 0x03, 0x21, 0x00])
 const nodeCrypto: Pick<ICryptoInterface, 'hashSHA256' | 'chaCha20Poly1305Encrypt' | 'chaCha20Poly1305Decrypt'> = {
   hashSHA256 (data) {
-    return crypto.createHash('sha256').update(data).digest()
+    const hash = crypto.createHash('sha256')
+
+    if (data instanceof Uint8Array) {
+      return hash.update(data).digest()
+    }
+
+    for (const buf of data) {
+      hash.update(buf)
+    }
+
+    return hash.digest()
   },
 
   chaCha20Poly1305Encrypt (plaintext, nonce, ad, k) {
     const cipher = crypto.createCipheriv(CHACHA_POLY1305, k, nonce, {
       authTagLength: 16
     })
     cipher.setAAD(ad, { plaintextLength: plaintext.byteLength })
-    const updated = cipher.update(plaintext)
+
+    if (plaintext instanceof Uint8Array) {
+      const updated = cipher.update(plaintext)
+      const final = cipher.final()
+      const tag = cipher.getAuthTag()
+
+      return Buffer.concat([updated, tag, final], updated.byteLength + tag.byteLength + final.byteLength)
+    }
+
+    const output = new Uint8ArrayList()
+
+    for (const buf of plaintext) {
+      output.append(cipher.update(buf))
+    }
+
     const final = cipher.final()
-    const tag = cipher.getAuthTag()
 
-    const encrypted = Buffer.concat([updated, tag, final], updated.byteLength + tag.byteLength + final.byteLength)
-    return encrypted
+    if (final.byteLength > 0) {
+      output.append(final)
+    }
+
+    output.append(cipher.getAuthTag())
+
+    return output
   },
 
   chaCha20Poly1305Decrypt (ciphertext, nonce, ad, k, _dst) {
     const authTag = ciphertext.subarray(ciphertext.length - 16)
-    const text = ciphertext.subarray(0, ciphertext.length - 16)
     const decipher = crypto.createDecipheriv(CHACHA_POLY1305, k, nonce, {
       authTagLength: 16
     })
+
+    let text: Uint8Array | Uint8ArrayList
+
+    if (ciphertext instanceof Uint8Array) {
+      text = ciphertext.subarray(0, ciphertext.length - 16)
+    } else {
+      text = ciphertext.sublist(0, ciphertext.length - 16)
+    }
+
     decipher.setAAD(ad, {
       plaintextLength: text.byteLength
     })
     decipher.setAuthTag(authTag)
-    const updated = decipher.update(text)
+
+    if (text instanceof Uint8Array) {
+      const output = decipher.update(text)
+      const final = decipher.final()
+
+      if (final.byteLength > 0) {
+        return Buffer.concat([output, final], output.byteLength + final.byteLength)
+      }
+
+      return output
+    }
+
+    const output = new Uint8ArrayList()
+
+    for (const buf of text) {
+      output.append(decipher.update(buf))
+    }
+
     const final = decipher.final()
+
     if (final.byteLength > 0) {
-      return Buffer.concat([updated, final], updated.byteLength + final.byteLength)
+      output.append(final)
     }
-    return updated
+
+    return output
   }
 }
 
 const asCrypto: Pick<ICryptoInterface, 'hashSHA256' | 'chaCha20Poly1305Encrypt' | 'chaCha20Poly1305Decrypt'> = {
   hashSHA256 (data) {
-    return digest(data)
+    return digest(data.subarray())
   },
   chaCha20Poly1305Encrypt (plaintext, nonce, ad, k) {
-    return asImpl.seal(k, nonce, plaintext, ad)
+    return asImpl.seal(k, nonce, plaintext.subarray(), ad)
   },
   chaCha20Poly1305Decrypt (ciphertext, nonce, ad, k, dst) {
-    return asImpl.open(k, nonce, ciphertext, ad, dst)
+    return asImpl.open(k, nonce, ciphertext.subarray(), ad, dst)
   }
 }
 
@@ -69,13 +125,13 @@ export const defaultCrypto: ICryptoInterface = {
     return nodeCrypto.hashSHA256(data)
   },
   chaCha20Poly1305Encrypt (plaintext, nonce, ad, k) {
-    if (plaintext.length < 1200) {
+    if (plaintext.byteLength < 1200) {
       return asCrypto.chaCha20Poly1305Encrypt(plaintext, nonce, ad, k)
     }
     return nodeCrypto.chaCha20Poly1305Encrypt(plaintext, nonce, ad, k)
   },
   chaCha20Poly1305Decrypt (ciphertext, nonce, ad, k, dst) {
-    if (ciphertext.length < 1200) {
+    if (ciphertext.byteLength < 1200) {
       return asCrypto.chaCha20Poly1305Decrypt(ciphertext, nonce, ad, k, dst)
     }
     return nodeCrypto.chaCha20Poly1305Decrypt(ciphertext, nonce, ad, k, dst)
@@ -118,16 +174,26 @@ export const defaultCrypto: ICryptoInterface = {
       privateKey: seed
     }
   },
-  generateX25519SharedKey (privateKey: Uint8Array, publicKey: Uint8Array): Uint8Array {
-    publicKey = Buffer.concat([
-      X25519_PREFIX,
-      publicKey
-    ], X25519_PREFIX.byteLength + publicKey.byteLength)
-
-    privateKey = Buffer.concat([
-      PKCS8_PREFIX,
-      privateKey
-    ], PKCS8_PREFIX.byteLength + privateKey.byteLength)
+  generateX25519SharedKey (privateKey: Uint8Array | Uint8ArrayList, publicKey: Uint8Array | Uint8ArrayList): Uint8Array {
+    if (publicKey instanceof Uint8Array) {
+      publicKey = Buffer.concat([
+        X25519_PREFIX,
+        publicKey
+      ], X25519_PREFIX.byteLength + publicKey.byteLength)
+    } else {
+      publicKey.prepend(X25519_PREFIX)
+      publicKey = publicKey.subarray()
+    }
+
+    if (privateKey instanceof Uint8Array) {
+      privateKey = Buffer.concat([
+        PKCS8_PREFIX,
+        privateKey
+      ], PKCS8_PREFIX.byteLength + privateKey.byteLength)
+    } else {
+      privateKey.prepend(PKCS8_PREFIX)
+      privateKey = privateKey.subarray()
+    }
 
     return crypto.diffieHellman({
       publicKey: crypto.createPublicKey({

src/crypto/js.ts

@@ -2,14 +2,15 @@ import { chacha20poly1305 } from '@noble/ciphers/chacha'
 import { x25519 } from '@noble/curves/ed25519'
 import { extract, expand } from '@noble/hashes/hkdf'
 import { sha256 } from '@noble/hashes/sha256'
-import type { bytes, bytes32 } from '../@types/basic.js'
+import type { bytes32 } from '../@types/basic.js'
 import type { Hkdf } from '../@types/handshake.js'
 import type { KeyPair } from '../@types/libp2p.js'
 import type { ICryptoInterface } from '../crypto.js'
+import type { Uint8ArrayList } from 'uint8arraylist'
 
 export const pureJsCrypto: ICryptoInterface = {
-  hashSHA256 (data: Uint8Array): Uint8Array {
-    return sha256(data)
+  hashSHA256 (data: Uint8Array | Uint8ArrayList): Uint8Array {
+    return sha256(data.subarray())
   },
 
   getHKDF (ck: bytes32, ikm: Uint8Array): Hkdf {
@@ -43,15 +44,15 @@ export const pureJsCrypto: ICryptoInterface = {
     }
   },
 
-  generateX25519SharedKey (privateKey: Uint8Array, publicKey: Uint8Array): Uint8Array {
-    return x25519.getSharedSecret(privateKey, publicKey)
+  generateX25519SharedKey (privateKey: Uint8Array | Uint8ArrayList, publicKey: Uint8Array | Uint8ArrayList): Uint8Array {
+    return x25519.getSharedSecret(privateKey.subarray(), publicKey.subarray())
   },
 
-  chaCha20Poly1305Encrypt (plaintext: Uint8Array, nonce: Uint8Array, ad: Uint8Array, k: bytes32): bytes {
-    return chacha20poly1305(k, nonce, ad).encrypt(plaintext)
+  chaCha20Poly1305Encrypt (plaintext: Uint8Array | Uint8ArrayList, nonce: Uint8Array, ad: Uint8Array, k: bytes32): Uint8Array {
+    return chacha20poly1305(k, nonce, ad).encrypt(plaintext.subarray())
   },
 
-  chaCha20Poly1305Decrypt (ciphertext: Uint8Array, nonce: Uint8Array, ad: Uint8Array, k: bytes32, dst?: Uint8Array): bytes | null {
-    return chacha20poly1305(k, nonce, ad).decrypt(ciphertext, dst)
+  chaCha20Poly1305Decrypt (ciphertext: Uint8Array | Uint8ArrayList, nonce: Uint8Array, ad: Uint8Array, k: bytes32, dst?: Uint8Array): Uint8Array | null {
+    return chacha20poly1305(k, nonce, ad).decrypt(ciphertext.subarray(), dst)
   }
 }