Skip to content

Commit 749c390

Browse files
jasagredojasagredo
committed
Implement non-native snapshots command
1 parent a462347 commit 749c390

File tree

6 files changed

+105
-11
lines changed

6 files changed

+105
-11
lines changed

‎cabal.project‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,12 +82,12 @@ source-repository-package
8282
source-repository-package
8383
type: git
8484
location: https://github.com/IntersectMBO/ouroboros-consensus
85-
tag: 41f32e52644aa168c4f4d4ef645ed6c3ec2ecfd6
86-
--sha256: sha256-4ukkuRURJ24nXN+sfbFrDr0aYc1SPFvQO0XCVJuQris=
85+
tag: 163895614992007a09675fd15980578ecd6e1e91
86+
--sha256: sha256-1rbaFlFzSiMNe6vzUMuIsO4COF16adUZYak/jfyw4IU=
8787
subdir:
8888
ouroboros-consensus
89-
ouroboros-consensus-cardano
9089
ouroboros-consensus-diffusion
90+
ouroboros-consensus-cardano
9191

9292
source-repository-package
9393
type: git

‎cardano-node/app/cardano-node.hs‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@ import Cardano.Git.Rev (gitRev)
88
import Cardano.Node.Configuration.POM (PartialNodeConfiguration(..))
99
import Cardano.Node.Handlers.TopLevel
1010
import Cardano.Node.Parsers (nodeCLIParser, parserHelpHeader, parserHelpOptions,
11-
renderHelpDoc)
11+
renderHelpDoc, parseSnapshotsCmd)
1212
import Cardano.Node.Run (runNode)
13+
import Cardano.Snapshots.Run (canonicalizeSnapshots, NodeDatabasePaths)
1314
import Cardano.Node.Tracing.Documentation (TraceDocumentationCmd (..),
1415
parseTraceDocumentationCmd, runTraceDocumentationCmd)
1516

@@ -37,6 +38,7 @@ main = do
3738
warnIfSet args pncMaybeMempoolCapacityOverride "mempool-capacity-override" "MempoolCapacityBytesOverride"
3839
runNode args
3940
TraceDocumentation tdc -> runTraceDocumentationCmd tdc
41+
CanonicalizeSnapshotsCmd cfg db -> canonicalizeSnapshots cfg db
4042
VersionCmd -> runVersionCommand
4143

4244
where
@@ -54,29 +56,32 @@ main = do
5456

5557
opts :: Opt.ParserInfo Command
5658
opts =
57-
Opt.info (fmap RunCmd nodeCLIParser
59+
let pp = fmap RunCmd nodeCLIParser
5860
<|> fmap TraceDocumentation parseTraceDocumentationCmd
5961
<|> parseVersionCmd
60-
<**> helperBrief "help" "Show this help text" nodeCliHelpMain)
62+
<|> fmap (uncurry CanonicalizeSnapshotsCmd) parseSnapshotsCmd
63+
in Opt.info (pp
64+
<**> helperBrief "help" "Show this help text" (nodeCliHelpMain pp))
6165

6266
( Opt.fullDesc <>
63-
Opt.progDesc "Start node of the Cardano blockchain."
67+
Opt.progDesc "The Cardano blockchain node"
6468
)
6569

6670
helperBrief :: String -> String -> String -> Parser (a -> a)
6771
helperBrief l d helpText = Opt.abortOption (Opt.InfoMsg helpText) $ mconcat
6872
[ Opt.long l
6973
, Opt.help d ]
7074

71-
nodeCliHelpMain :: String
72-
nodeCliHelpMain = renderHelpDoc 80 $
73-
parserHelpHeader "cardano-node" nodeCLIParser
75+
nodeCliHelpMain :: Parser a -> String
76+
nodeCliHelpMain pp = renderHelpDoc 80 $
77+
parserHelpHeader "cardano-node" pp
7478
<$$> ""
7579
<$$> parserHelpOptions nodeCLIParser
7680

7781

7882
data Command = RunCmd PartialNodeConfiguration
7983
| TraceDocumentation TraceDocumentationCmd
84+
| CanonicalizeSnapshotsCmd FilePath (Maybe NodeDatabasePaths)
8085
| VersionCmd
8186

8287
-- Yes! A --version flag or version command. Either guess is right!

‎cardano-node/cardano-node.cabal‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,7 @@ library
132132
Cardano.Tracing.Shutdown
133133
Cardano.Tracing.Startup
134134
Cardano.Tracing.Tracers
135+
Cardano.Snapshots.Run
135136

136137
other-modules: Paths_cardano_node
137138
autogen-modules: Paths_cardano_node
@@ -189,7 +190,7 @@ library
189190
, nothunks
190191
, optparse-applicative-fork >= 0.18.1
191192
, ouroboros-consensus:{ouroboros-consensus, ouroboros-consensus-lmdb, ouroboros-consensus-lsm} ^>= 0.28
192-
, ouroboros-consensus-cardano ^>= 0.26
193+
, ouroboros-consensus-cardano:{ouroboros-consensus-cardano, snapshot-conversion} ^>= 0.26
193194
, ouroboros-consensus-diffusion ^>= 0.24
194195
, ouroboros-consensus-protocol
195196
, ouroboros-network-api ^>= 0.16

‎cardano-node/src/Cardano/Node/Configuration/POM.hs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,8 @@ data PartialNodeConfiguration
269269
, pncGenesisConfigFlags :: !(Last GenesisConfigFlags)
270270

271271
, pncResponderCoreAffinityPolicy :: !(Last ResponderCoreAffinityPolicy)
272+
273+
, pncCanonicalSnapshotOutputPath :: !(Last FilePath)
272274
} deriving (Eq, Generic, Show)
273275

274276
instance AdjustFilePaths PartialNodeConfiguration where
@@ -381,6 +383,9 @@ instance FromJSON PartialNodeConfiguration where
381383
<$> v .:? "ResponderCoreAffinityPolicy"
382384
<*> v .:? "ForkPolicy" -- deprecated
383385

386+
pncCanonicalSnapshotOutputPath <-
387+
Last <$> v .:? "CanonicalSnapshotsOutputPath"
388+
384389
pure PartialNodeConfiguration {
385390
pncProtocolConfig
386391
, pncSocketConfig = Last . Just $ SocketConfig mempty mempty mempty pncSocketPath
@@ -425,6 +430,7 @@ instance FromJSON PartialNodeConfiguration where
425430
, pncPeerSharing
426431
, pncGenesisConfigFlags
427432
, pncResponderCoreAffinityPolicy
433+
, pncCanonicalSnapshotOutputPath
428434
}
429435
where
430436
parseMempoolCapacityBytesOverride v = parseNoOverride <|> parseOverride
@@ -687,6 +693,7 @@ defaultPartialNodeConfiguration =
687693
, pncGenesisConfigFlags = Last (Just defaultGenesisConfigFlags)
688694
-- https://ouroboros-consensus.cardano.intersectmbo.org/haddocks/ouroboros-consensus-diffusion/Ouroboros-Consensus-Node-Genesis.html#v:defaultGenesisConfigFlags
689695
, pncResponderCoreAffinityPolicy = Last $ Just NoResponderCoreAffinity
696+
, pncCanonicalSnapshotOutputPath = mempty
690697
}
691698

692699
lastOption :: Parser a -> Parser (Last a)

‎cardano-node/src/Cardano/Node/Parsers.hs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ module Cardano.Node.Parsers
1111
, parserHelpOptions
1212
, renderHelpDoc
1313
, parseHostPort
14+
, parseSnapshotsCmd
1415
) where
1516

1617
import Cardano.Logging.Types
@@ -140,6 +141,7 @@ nodeRunParser = do
140141
, pncPeerSharing = mempty
141142
, pncGenesisConfigFlags = mempty
142143
, pncResponderCoreAffinityPolicy = mempty
144+
, pncCanonicalSnapshotOutputPath = mempty
143145
}
144146

145147
parseSocketPath :: Text -> Parser SocketPath
@@ -434,3 +436,15 @@ parserHelpOptions = fromMaybe mempty . OptI.unChunk . OptI.fullDesc (Opt.prefs m
434436
renderHelpDoc :: Int -> OptI.Doc -> String
435437
renderHelpDoc cols =
436438
(`OptI.renderShowS` "") . OptI.layoutPretty (OptI.LayoutOptions (OptI.AvailablePerLine cols 1.0))
439+
440+
parseSnapshotsCmd :: Parser (FilePath, Maybe NodeDatabasePaths)
441+
parseSnapshotsCmd = subparser
442+
( commandGroup "Canonicalize snapshots"
443+
<> metavar "run"
444+
<> command "canonicalize-snapshots"
445+
(info (((,)
446+
<$> parseConfigFile
447+
<*> optional (parseDbPath <|> fmap OnePathForAllDbs parseImmutableDbPath)
448+
) <**> helper)
449+
(progDesc "Canonicalize all snapshots" ))
450+
)

‎cardano-node/src/Cardano/Snapshots/Run.hs‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
{-# LANGUAGE GADTs #-}
2+
{-# LANGUAGE TypeApplications #-}
3+
{-# LANGUAGE ViewPatterns #-}
4+
5+
module Cardano.Snapshots.Run (
6+
canonicalizeSnapshots,
7+
NodeDatabasePaths,
8+
) where
9+
10+
import qualified Cardano.Api.Consensus as Api
11+
import Cardano.Node.Configuration.LedgerDB
12+
import Cardano.Node.Configuration.POM
13+
import Cardano.Node.Protocol
14+
import Cardano.Node.Types (ConfigYamlFilePath (..))
15+
import Control.Exception
16+
import Control.Monad (forM_)
17+
import Control.Monad.Except
18+
import Data.Monoid (Last (..))
19+
import Ouroboros.Consensus.Cardano.SnapshotConversion
20+
import Ouroboros.Consensus.Node (NodeDatabasePaths (..), immutableDbPath)
21+
import System.Directory (doesFileExist, listDirectory)
22+
import System.FilePath ((</>))
23+
24+
canonicalizeSnapshots :: FilePath -> Maybe NodeDatabasePaths -> IO ()
25+
canonicalizeSnapshots cfg (Last -> db) = do
26+
configYamlPc <- parseNodeConfigurationFP $ Just $ ConfigYamlFilePath cfg
27+
28+
let cfgFromFile = defaultPartialNodeConfiguration <> configYamlPc
29+
30+
mOut = getLast (pncCanonicalSnapshotOutputPath cfgFromFile)
31+
32+
mOtherConfigs = do
33+
a <- getLast (pncDatabaseFile cfgFromFile <> db)
34+
b <- getLast (pncLedgerDbConfig cfgFromFile)
35+
c <- getLast (pncProtocolConfig cfgFromFile)
36+
d <- getLast (pncProtocolFiles cfgFromFile)
37+
pure (a, b, c, d)
38+
39+
case (mOut, mOtherConfigs) of
40+
(Nothing, _) -> pure ()
41+
(_, Nothing) -> error "Impossible, some arguments were missing yet there should be at least a default value for those"
42+
(Just out, Just (immutableDbPath -> dbPath, LedgerDbConfiguration _ _ _ selector _, pInfo, cfgFiles)) -> do
43+
snaps <- listDirectory (dbPath </> "ledger")
44+
someConsensusProto <-
45+
runThrowExceptT $
46+
mkConsensusProtocol
47+
pInfo
48+
(Just cfgFiles)
49+
case someConsensusProto of
50+
SomeConsensusProtocol Api.CardanoBlockType pInfoArgs -> do
51+
let inFmt = case selector of
52+
V1LMDB{} -> LMDB
53+
V2InMemory{} -> Mem
54+
V2LSM Nothing -> flip LSM (dbPath </> "lsm")
55+
V2LSM (Just lsmDb) -> flip LSM (dbPath </> lsmDb)
56+
forM_ snaps $ \snap -> do
57+
exists <- doesFileExist (out </> snap </> "meta")
58+
if exists
59+
then putStrLn $ "Snapshot at " <> dbPath </> "ledger" </> snap <> " already converted"
60+
else do
61+
putStrLn $ "Converting snapshot at " <> dbPath </> "ledger" </> snap
62+
runThrowExceptT $ convertSnapshot False (fst $ Api.protocolInfo @IO pInfoArgs) (inFmt (dbPath </> "ledger" </> snap)) (Mem $ out </> snap)
63+
putStrLn "Done"
64+
_ -> pure ()
65+
66+
runThrowExceptT :: (Exception e) => ExceptT e IO a -> IO a
67+
runThrowExceptT act = runExceptT act >>= either throwIO pure

0 commit comments

Comments
 (0)