Skip to content

Commit 9838008

Browse files
jasagredojasagredo
committed
Update to LSM
1 parent cbc6be6 commit 9838008

File tree

15 files changed

+294
-196
lines changed

15 files changed

+294
-196
lines changed

‎bench/plutus-scripts-bench/plutus-scripts-bench.cabal‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ library
8282
-- IOG dependencies
8383
--------------------------
8484
build-depends:
85-
, cardano-api ^>=10.18
85+
, cardano-api ^>=10.19
8686
, plutus-ledger-api ^>=1.53
8787
, plutus-tx ^>=1.53
8888
, plutus-tx-plugin ^>=1.53

‎bench/tx-generator/tx-generator.cabal‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,9 +111,9 @@ library
111111
, attoparsec-aeson
112112
, base16-bytestring
113113
, bytestring
114-
, cardano-api ^>= 10.18
114+
, cardano-api ^>= 10.19
115115
, cardano-binary
116-
, cardano-cli ^>= 10.12
116+
, cardano-cli ^>= 10.13
117117
, cardano-crypto-class
118118
, cardano-crypto-wrapper
119119
, cardano-data

‎cabal.project‎

Lines changed: 56 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ repository cardano-haskell-packages
1313
-- See CONTRIBUTING for information about these, including some Nix commands
1414
-- you need to run if you change them
1515
index-state:
16-
, hackage.haskell.org 2025-09-24T20:00:55Z
17-
, cardano-haskell-packages 2025-10-21T11:16:53Z
16+
, hackage.haskell.org 2025-10-23T13:39:53Z
17+
, cardano-haskell-packages 2025-10-24T10:49:53Z
1818

1919
packages:
2020
cardano-node
@@ -61,29 +61,71 @@ package plutus-scripts-bench
6161
allow-newer:
6262
, katip:Win32
6363

64-
if impl (ghc >= 9.12)
65-
allow-newer:
66-
-- https://github.com/kapralVV/Unique/issues/11
67-
, Unique:hashable
68-
69-
-- https://github.com/Gabriella439/Haskell-Pipes-Safe-Library/pull/70
70-
, pipes-safe:base
71-
7264
-- IMPORTANT
7365
-- Do NOT add more source-repository-package stanzas here unless they are strictly
7466
-- temporary! Please read the section in CONTRIBUTING about updating dependencies.
7567

7668
source-repository-package
7769
type: git
7870
location: https://github.com/intersectmbo/cardano-cli.git
79-
tag: 001597b206c3ae9269ff9f3b0e9ce8022b6adef2
80-
--sha256: sha256-HZferamexH4l7WpgfI+HYx5Hatv6UfLBis14c0PiGo0=
71+
tag: bf631b6c9f6674c9b96215ed1424320d32897c0f
72+
--sha256: sha256-0hELctv6C2pyW/fQD93StlKe2YmF4i3bA6ma2wjaxgg=
8173
subdir: cardano-cli
8274

8375
source-repository-package
8476
type: git
8577
location: https://github.com/IntersectMBO/cardano-api
86-
tag: a6be460ffe1a2d440edeabc4925c06162b9edb84
87-
--sha256: sha256-Lc+ke3B50c+YuLS14Mb/Z3LfcfT1oUoe1AiGyrywzEw=
78+
tag: dc4d610eb6aae7fd29747e5c2d803ac59134cc78
79+
--sha256: sha256-otSK6RYzQXr3lReuXspvUsB98QNrXypvID13Gq+GMjI=
8880
subdir:
8981
cardano-api
82+
83+
source-repository-package
84+
type: git
85+
location: https://github.com/IntersectMBO/ouroboros-consensus
86+
tag: f4cd17c9c1fb27256a3137f1f45c81ced9e3e005
87+
--sha256: sha256-JqouAFS2HFb74YNlr7AePryJvcItK6A1W6iZAQqXLnU=
88+
subdir:
89+
ouroboros-consensus
90+
ouroboros-consensus-cardano
91+
ouroboros-consensus-diffusion
92+
93+
source-repository-package
94+
type: git
95+
location: https://github.com/IntersectMBO/cardano-ledger
96+
tag: fb09078fa55015c881303a2ddb609c024cec258f
97+
--sha256: sha256-9Y9CRiyMn0AWD+C4aNVMaJgrj3FDAYfCX4VrLvtoMaI=
98+
subdir:
99+
eras/allegra/impl
100+
eras/alonzo/impl
101+
eras/alonzo/test-suite
102+
eras/babbage/impl
103+
eras/conway/impl
104+
eras/dijkstra/impl
105+
eras/mary/impl
106+
eras/shelley/impl
107+
eras/shelley/test-suite
108+
eras/shelley-ma/test-suite
109+
libs/cardano-ledger-api
110+
libs/cardano-ledger-core
111+
libs/cardano-ledger-binary
112+
libs/cardano-protocol-tpraos
113+
libs/non-integral
114+
libs/small-steps
115+
libs/cardano-data
116+
libs/set-algebra
117+
libs/vector-map
118+
eras/byron/chain/executable-spec
119+
eras/byron/ledger/executable-spec
120+
eras/byron/ledger/impl
121+
eras/byron/crypto
122+
123+
-- Backported version of https://github.com/IntersectMBO/ouroboros-network/pull/5161
124+
source-repository-package
125+
type: git
126+
location: https://github.com/IntersectMBO/ouroboros-network
127+
tag: 1385b53cefb81e79553b6b0252537455833ea9c4
128+
--sha256: sha256-zZ7WsMfRs1fG16bmvI5vIh4fhQ8RGyEvYGLSWlrxpg0=
129+
subdir:
130+
ouroboros-network-api
131+
ouroboros-network

‎cardano-node-chairman/cardano-node-chairman.cabal‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ executable cardano-node-chairman
4444
build-depends: cardano-api
4545
, cardano-crypto-class
4646
, cardano-git-rev ^>= 0.2.2
47-
, cardano-ledger-core ^>= 1.18
47+
, cardano-ledger-core ^>= 1.19
4848
, cardano-node ^>= 10.5
4949
, cardano-prelude
5050
, containers
@@ -88,5 +88,5 @@ test-suite chairman-tests
8888
ghc-options: -threaded -rtsopts "-with-rtsopts=-N -T"
8989

9090
build-tool-depends: cardano-node:cardano-node
91-
, cardano-cli:cardano-cli ^>= 10.12
91+
, cardano-cli:cardano-cli ^>= 10.13
9292
, cardano-node-chairman:cardano-node-chairman

‎cardano-node/cardano-node.cabal‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ library
141141
, async
142142
, base16-bytestring
143143
, bytestring
144-
, cardano-api ^>= 10.18
144+
, cardano-api ^>= 10.19
145145
, cardano-crypto-class
146146
, cardano-crypto-wrapper
147147
, cardano-git-rev ^>=0.2.2
@@ -188,7 +188,7 @@ library
188188
, network-mux >= 0.8
189189
, nothunks
190190
, optparse-applicative-fork >= 0.18.1
191-
, ouroboros-consensus ^>= 0.28
191+
, ouroboros-consensus:{ouroboros-consensus, ouroboros-consensus-lmdb, ouroboros-consensus-lsm} ^>= 0.28
192192
, ouroboros-consensus-cardano ^>= 0.26
193193
, ouroboros-consensus-diffusion ^>= 0.24
194194
, ouroboros-consensus-protocol

‎cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs‎

Lines changed: 43 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,35 @@
11
{-# LANGUAGE DataKinds #-}
22
{-# LANGUAGE DerivingStrategies #-}
3+
{-# LANGUAGE FlexibleContexts #-}
34
{-# LANGUAGE GeneralisedNewtypeDeriving #-}
5+
{-# LANGUAGE ScopedTypeVariables #-}
6+
{-# LANGUAGE TypeApplications #-}
47

58
{-# OPTIONS_GHC -Wno-orphans #-}
69

710
module Cardano.Node.Configuration.LedgerDB (
8-
DeprecatedOptions (..)
9-
, LedgerDbConfiguration (..)
10-
, LedgerDbSelectorFlag(..)
11-
, Gigabytes
12-
, noDeprecatedOptions
13-
, selectorToArgs
14-
) where
11+
DeprecatedOptions (..),
12+
LedgerDbConfiguration (..),
13+
LedgerDbSelectorFlag (..),
14+
Gigabytes,
15+
noDeprecatedOptions,
16+
selectorToArgs,
17+
) where
1518

19+
import Ouroboros.Consensus.Ledger.SupportsProtocol
20+
import Ouroboros.Consensus.Storage.LedgerDB.API
1621
import Ouroboros.Consensus.Storage.LedgerDB.Args
1722
import Ouroboros.Consensus.Storage.LedgerDB.Snapshots
1823
import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.Args as V1
19-
import Ouroboros.Consensus.Storage.LedgerDB.V1.BackingStore.Impl.LMDB (LMDBLimits (..))
20-
import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.Args as V2
21-
import Ouroboros.Consensus.Util.Args
24+
import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.BackingStore.Impl.LMDB as LMDB
25+
import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.InMemory as InMemory
26+
import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.LSM as LSM
2227

2328
import qualified Data.Aeson.Types as Aeson (FromJSON)
2429
import Data.Maybe (fromMaybe)
25-
import Data.SOP.Dict
30+
import Data.Proxy
31+
import System.FilePath
32+
import System.Random (StdGen)
2633

2734
-- | Choose the LedgerDB Backend
2835
--
@@ -34,21 +41,25 @@ import Data.SOP.Dict
3441
--
3542
-- - 'V1LMDB': uses less memory but is somewhat slower.
3643
--
37-
-- - 'V1InMemory': Not intended for production. It is an in-memory reproduction
38-
-- of the LMDB implementation.
44+
-- - 'V2LSM': Uses the LSM backend.
3945
data LedgerDbSelectorFlag =
4046
V1LMDB
4147
V1.FlushFrequency
4248
-- ^ The frequency at which changes are flushed to the disk.
4349
(Maybe FilePath)
44-
-- ^ Path for the live tables.
50+
-- ^ Path for the live tables. If not provided the default will be used
51+
-- (@<fast-storage>/lmdb@).
4552
(Maybe Gigabytes)
4653
-- ^ A map size can be specified, this is the maximum disk space the LMDB
4754
-- database can fill. If not provided, the default of 16GB will be used.
4855
(Maybe Int)
4956
-- ^ An override to the max number of readers.
50-
| V1InMemory V1.FlushFrequency
5157
| V2InMemory
58+
| V2LSM
59+
(Maybe FilePath)
60+
-- ^ Maybe a custom path to the LSM database. If not provided the default
61+
-- will be used (@<fast-storage>/lsm@).
62+
5263
deriving (Eq, Show)
5364

5465
-- | Some options that existed in the TopLevel were now moved to a
@@ -118,24 +129,23 @@ toBytes (Gigabytes x) = x * 1024 * 1024 * 1024
118129
-- * The @lmdb-simple@ and @haskell-lmdb@ forked repositories.
119130
-- * The official LMDB API documentation at
120131
-- <http://www.lmdb.tech/doc/group__mdb.html>.
121-
defaultLMDBLimits :: LMDBLimits
122-
defaultLMDBLimits = LMDBLimits {
123-
lmdbMapSize = 16 * 1024 * 1024 * 1024
124-
, lmdbMaxDatabases = 10
125-
, lmdbMaxReaders = 16
132+
defaultLMDBLimits :: LMDB.LMDBLimits
133+
defaultLMDBLimits = LMDB.LMDBLimits {
134+
LMDB.lmdbMapSize = 16 * 1024 * 1024 * 1024
135+
, LMDB.lmdbMaxDatabases = 10
136+
, LMDB.lmdbMaxReaders = 16
126137
}
127138

128-
defaultLMDBPath :: FilePath
129-
defaultLMDBPath = "mainnet/db/lmdb"
139+
defaultLMDBPath :: FilePath -> FilePath
140+
defaultLMDBPath = (</> "lmdb")
130141

131-
selectorToArgs :: LedgerDbSelectorFlag -> Complete LedgerDbFlavorArgs IO
132-
selectorToArgs (V1InMemory ff) = LedgerDbFlavorArgsV1 $ V1.V1Args ff V1.InMemoryBackingStoreArgs
133-
selectorToArgs V2InMemory = LedgerDbFlavorArgsV2 $ V2.V2Args V2.InMemoryHandleArgs
134-
selectorToArgs (V1LMDB ff fp l mxReaders) =
135-
LedgerDbFlavorArgsV1
136-
$ V1.V1Args ff
137-
$ V1.LMDBBackingStoreArgs
138-
(fromMaybe defaultLMDBPath fp)
139-
(maybe id (\overrideMaxReaders lim -> lim { lmdbMaxReaders = overrideMaxReaders }) mxReaders
140-
$ maybe id (\ll lim -> lim { lmdbMapSize = toBytes ll }) l defaultLMDBLimits)
141-
Dict
142+
selectorToArgs :: forall blk. (LedgerSupportsProtocol blk, LedgerSupportsLedgerDB blk) => LedgerDbSelectorFlag -> FilePath -> StdGen -> (LedgerDbBackendArgs IO blk, StdGen)
143+
selectorToArgs V2InMemory _ = InMemory.mkInMemoryArgs
144+
selectorToArgs (V1LMDB ff fp l mxReaders) fastStoragePath =
145+
LMDB.mkLMDBArgs
146+
ff
147+
(fromMaybe (defaultLMDBPath fastStoragePath) fp)
148+
( maybe id (\overrideMaxReaders lim -> lim{LMDB.lmdbMaxReaders = overrideMaxReaders}) mxReaders $
149+
maybe id (\ll lim -> lim{LMDB.lmdbMapSize = toBytes ll}) l defaultLMDBLimits
150+
)
151+
selectorToArgs (V2LSM fp) fastStoragePath = LSM.mkLSMArgs (Proxy @blk) (fromMaybe "lsm" fp) fastStoragePath

‎cardano-node/src/Cardano/Node/Configuration/POM.hs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -466,16 +466,16 @@ instance FromJSON PartialNodeConfiguration where
466466
qsize <- (fmap RequestedQueryBatchSize <$> o .:? "QueryBatchSize") .!= DefaultQueryBatchSize
467467
backend <- o .:? "Backend" .!= "V2InMemory"
468468
selector <- case backend of
469-
"V1InMemory" -> do
470-
flush <- (fmap RequestedFlushFrequency <$> o .:? "FlushFrequency") .!= DefaultFlushFrequency
471-
return $ V1InMemory flush
472469
"V1LMDB" -> do
473470
flush <- (fmap RequestedFlushFrequency <$> o .:? "FlushFrequency") .!= DefaultFlushFrequency
474471
mapSize :: Maybe Gigabytes <- o .:? "MapSize"
475472
lmdbPath :: Maybe FilePath <- o .:? "LiveTablesPath"
476473
mxReaders :: Maybe Int <- o .:? "MaxReaders"
477474
return $ V1LMDB flush lmdbPath mapSize mxReaders
478475
"V2InMemory" -> return V2InMemory
476+
"V2LSM" -> do
477+
lsmPath :: Maybe FilePath <- o .:? "LSMDatabasePath"
478+
pure $ V2LSM lsmPath
479479
_ -> fail $ "Malformed LedgerDB Backend: " <> backend
480480
pure $ Just $ LedgerDbConfiguration ldbSnapNum ldbSnapInterval qsize selector deprecatedOpts
481481

‎cardano-node/src/Cardano/Node/Run.hs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ import Cardano.Tracing.Tracers
6868
import qualified Ouroboros.Consensus.Config as Consensus
6969
import Ouroboros.Consensus.Config.SupportsNode (ConfigSupportsNode (..))
7070
import Ouroboros.Consensus.Node (SnapshotPolicyArgs (..),
71-
NodeDatabasePaths (..), RunNodeArgs (..), StdRunNodeArgs (..))
71+
NodeDatabasePaths (..), nonImmutableDbPath, RunNodeArgs (..), StdRunNodeArgs (..))
7272
import Ouroboros.Consensus.Protocol.Praos.AgentClient (KESAgentClientTrace)
7373
import Ouroboros.Consensus.Ledger.SupportsMempool (GenTxId)
7474
import Ouroboros.Consensus.Node (RunNodeArgs (..),
@@ -79,7 +79,6 @@ import Ouroboros.Consensus.Node.NetworkProtocolVersion
7979
import Ouroboros.Consensus.Node.ProtocolInfo
8080
import qualified Ouroboros.Consensus.Node.Tracers as Consensus
8181
import qualified Ouroboros.Consensus.Storage.LedgerDB.Args as LDBArgs
82-
import Ouroboros.Consensus.Storage.LedgerDB.V2.Args
8382
import Ouroboros.Consensus.Util.Args
8483
import Ouroboros.Consensus.Util.Orphans ()
8584

@@ -560,7 +559,7 @@ handleSimpleNode blockType runP tracers nc onKernel = do
560559
, srnChainSyncIdleTimeout = customizeChainSyncTimeout
561560
, srnSnapshotPolicyArgs = snapshotPolicyArgs
562561
, srnQueryBatchSize = queryBatchSize
563-
, srnLdbFlavorArgs = selectorToArgs ldbBackend
562+
, srnLedgerDbBackendArgs = selectorToArgs ldbBackend (nonImmutableDbPath dbPath)
564563
}
565564
where
566565
customizeChainSyncTimeout :: ChainSyncIdleTimeout

‎cardano-node/src/Cardano/Node/Tracing/Era/Shelley.hs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1073,6 +1073,8 @@ instance
10731073
, LogFormatting (PredicateFailure (Ledger.EraRule "CERTS" era))
10741074
) => LogFormatting (Conway.ConwayLedgerPredFailure era) where
10751075
forMachine v (Conway.ConwayUtxowFailure f) = forMachine v f
1076+
forMachine _verb (Conway.ConwayWithdrawalsMissingAccounts _) = mconcat [ "todo" .= String "todo"]
1077+
forMachine _verb (Conway.ConwayIncompleteWithdrawals _) = mconcat [ "todo" .= String "todo"]
10761078
forMachine _ (Conway.ConwayTxRefScriptsSizeTooBig Mismatch {mismatchSupplied, mismatchExpected}) =
10771079
mconcat [ "kind" .= String "ConwayTxRefScriptsSizeTooBig"
10781080
, "actual" .= mismatchSupplied

0 commit comments

Comments
 (0)