Merge pull request #5732 from IntersectMBO/smelc/debug-native-testnet…

…-dreps

cardano-testnet: fix that genesis files generated by create-testnet-data were ignored

cardano-testnet/cardano-testnet.cabal

@@ -32,6 +32,7 @@ library
   import:               project-config
 
   build-depends:        aeson
+                      , aeson-pretty
                       , ansi-terminal
                       , bytestring
                       , cardano-api ^>= 8.42.0.0

cardano-testnet/src/Testnet/Components/Configuration.hs

@@ -7,7 +7,7 @@
 
 module Testnet.Components.Configuration
   ( anyEraToString
-  , createConfigYaml
+  , createConfigJson
   , createSPOGenesisAndFiles
   , eraToString
   , mkTopologyConfig
@@ -32,6 +32,7 @@ import           Control.Monad
 import           Control.Monad.Catch (MonadCatch)
 import           Data.Aeson
 import qualified Data.Aeson as Aeson
+import qualified Data.Aeson.Encode.Pretty as Aeson
 import qualified Data.Aeson.KeyMap as Aeson
 import qualified Data.Aeson.Lens as L
 import qualified Data.ByteString.Lazy as LBS
@@ -41,6 +42,7 @@ import qualified Data.Text as Text
 import           GHC.Stack (HasCallStack)
 import qualified GHC.Stack as GHC
 import           Lens.Micro
+import qualified System.Directory as System
 import           System.FilePath.Posix (takeDirectory, (</>))
 
 import           Testnet.Defaults
@@ -56,18 +58,18 @@ import qualified Hedgehog.Extras.Test.Base as H
 import qualified Hedgehog.Extras.Test.File as H
 
 -- | Returns JSON encoded hashes of the era, as well as the hard fork configuration toggle.
-createConfigYaml :: ()
+createConfigJson :: ()
   => (MonadTest m, MonadIO m, HasCallStack)
   => TmpAbsolutePath
   -> AnyCardanoEra -- ^ The era used for generating the hard fork configuration toggle
   -> m LBS.ByteString
-createConfigYaml (TmpAbsolutePath tempAbsPath) era = GHC.withFrozenCallStack $ do
+createConfigJson (TmpAbsolutePath tempAbsPath) era = GHC.withFrozenCallStack $ do
   byronGenesisHash <- getByronGenesisHash $ tempAbsPath </> "byron/genesis.json"
   shelleyGenesisHash <- getHash ShelleyEra "ShelleyGenesisHash"
   alonzoGenesisHash  <- getHash AlonzoEra  "AlonzoGenesisHash"
   conwayGenesisHash  <- getHash ConwayEra  "ConwayGenesisHash"
 
-  return . Aeson.encode . Aeson.Object
+  return . Aeson.encodePretty . Aeson.Object
     $ mconcat [ byronGenesisHash
               , shelleyGenesisHash
               , alonzoGenesisHash
@@ -95,34 +97,29 @@ createSPOGenesisAndFiles
   -> ConwayGenesis StandardCrypto -- ^ The conway genesis to use, for example 'Defaults.defaultConwayGenesis'.
   -> TmpAbsolutePath
   -> m FilePath -- ^ Shelley genesis directory
-createSPOGenesisAndFiles (NumPools numPoolNodes) era shelleyGenesis alonzoGenesis conwayGenesis (TmpAbsolutePath tempAbsPath) = do
-  let genesisShelleyFpAbs = tempAbsPath </> defaultGenesisFilepath ShelleyEra
-      genesisAlonzoFpAbs = tempAbsPath </> defaultGenesisFilepath AlonzoEra
-      genesisConwayFpAbs = tempAbsPath </> defaultGenesisFilepath ConwayEra
-      genesisShelleyDirAbs = takeDirectory genesisShelleyFpAbs
+createSPOGenesisAndFiles (NumPools numPoolNodes) era shelleyGenesis alonzoGenesis conwayGenesis (TmpAbsolutePath tempAbsPath) = GHC.withFrozenCallStack $ do
+  let inputGenesisShelleyFp = tempAbsPath </> genesisInputFilepath ShelleyEra
+      inputGenesisAlonzoFp  = tempAbsPath </> genesisInputFilepath AlonzoEra
+      inputGenesisConwayFp  = tempAbsPath </> genesisInputFilepath ConwayEra
+
+  -- We write the genesis files to disk, to pass them to create-testnet-data.
+  -- Then, create-testnet-data will output (possibly augmented/modified) versions
+  -- and we remove those input files (see below), to avoid confusion.
+  H.evalIO $ do
+    LBS.writeFile inputGenesisShelleyFp $ Aeson.encodePretty shelleyGenesis
+    LBS.writeFile inputGenesisAlonzoFp  $ Aeson.encodePretty alonzoGenesis
+    LBS.writeFile inputGenesisConwayFp  $ Aeson.encodePretty conwayGenesis
+
+  let genesisShelleyDirAbs = takeDirectory inputGenesisShelleyFp
   genesisShelleyDir <- H.createDirectoryIfMissing genesisShelleyDirAbs
   let testnetMagic = sgNetworkMagic shelleyGenesis
       numStakeDelegators = 3 :: Int
       startTime = sgSystemStart shelleyGenesis
 
-  -- We create the initial genesis file to avoid having to re-write the genesis file later
-  -- with the parameters we want. The user must provide genesis files or we will use a default.
-  -- We should *never* be modifying the genesis file after @cardanoTestnet@ is run because this
-  -- is sure to be a source of confusion if users provide genesis files and we are mutating them
-  -- without their knowledge.
-  H.evalIO $ do
-    LBS.writeFile genesisShelleyFpAbs $ encode shelleyGenesis
-    LBS.writeFile genesisAlonzoFpAbs  $ encode alonzoGenesis
-    LBS.writeFile genesisConwayFpAbs  $ encode conwayGenesis
-
-  -- TODO: Remove this rewrite.
+ -- TODO: Remove this rewrite.
  -- 50 second epochs
  -- Epoch length should be "10 * k / f" where "k = securityParam, f = activeSlotsCoeff"
-  H.rewriteJsonFile @Value genesisShelleyFpAbs $ \o -> o
-    -- TODO: remove rho and tau adjustment after https://github.com/IntersectMBO/cardano-api/pull/425 gets
-    -- integrated with newer cardano-api into node
-    & L.key "protocolParams" .  L.key "rho" . L._Number  .~ 0.1
-    & L.key "protocolParams" .  L.key "tau" . L._Number  .~ 0.1
+  H.rewriteJsonFile @Value inputGenesisShelleyFp $ \o -> o
     & L.key "securityParam" . L._Integer .~ 5
     & L.key "updateQuorum" . L._Integer .~ 2
 
@@ -135,9 +132,9 @@ createSPOGenesisAndFiles (NumPools numPoolNodes) era shelleyGenesis alonzoGenesi
 
   execCli_
     [ anyEraToString era, "genesis", "create-testnet-data"
-    , "--spec-shelley", genesisShelleyFpAbs
-    , "--spec-alonzo",  genesisAlonzoFpAbs
-    , "--spec-conway",  genesisConwayFpAbs
+    , "--spec-shelley", inputGenesisShelleyFp
+    , "--spec-alonzo",  inputGenesisAlonzoFp
+    , "--spec-conway",  inputGenesisConwayFp
     , "--testnet-magic", show testnetMagic
     , "--pools", show numPoolNodes
     , "--total-supply",     show @Int 2_000_000_000_000
@@ -149,35 +146,34 @@ createSPOGenesisAndFiles (NumPools numPoolNodes) era shelleyGenesis alonzoGenesi
     , "--out-dir", tempAbsPath
     ]
 
-  -- Here we move all of the keys etc generated by create-testnet-data
-  -- for the nodes to use
+  -- Remove the input files. We don't need them anymore, since create-testnet-data wrote new versions.
+  forM_ [inputGenesisShelleyFp, inputGenesisAlonzoFp, inputGenesisConwayFp] (liftIO . System.removeFile)
 
   -- Move all genesis related files
-
   genesisByronDir <- H.createDirectoryIfMissing $ tempAbsPath </> "byron"
 
   files <- H.listDirectory tempAbsPath
-  forM_ files $ \file -> do
-    H.note file
+  forM_ files H.note
 
   H.renameFile (tempAbsPath </> "byron-gen-command" </> "genesis.json") (genesisByronDir </> "genesis.json")
-  H.renameFile (tempAbsPath </> "shelley-genesis.json") (genesisShelleyDir </> "genesis.shelley.json")
 
   -- For some reason when setting "--total-supply 10E16" in create-testnet-data, we're getting negative
-  -- treasury. TODO: This should be fixed by https://github.com/IntersectMBO/cardano-cli/pull/644
-  -- So this can be removed when cardano-cli is upgraded above 8.20.3.0.
-  H.rewriteJsonFile @Value (genesisShelleyDir </> "genesis.shelley.json") $ \o -> o
+  -- treasury. TODO: This was supposed to be fixed by https://github.com/IntersectMBO/cardano-cli/pull/644,
+  -- but no, it's still there.
+  H.rewriteJsonFile @Value (tempAbsPath </> defaultGenesisFilepath ShelleyEra) $ \o -> o
     & L.key "maxLovelaceSupply" . L._Integer .~ 10_000_000_000_000_000
 
   return genesisShelleyDir
+  where
+    genesisInputFilepath e = "genesis-input." <> anyEraToString (AnyCardanoEra e) <> ".json"
 
 ifaceAddress :: String
 ifaceAddress = "127.0.0.1"
 
 -- TODO: Reconcile all other mkTopologyConfig functions. NB: We only intend
 -- to support current era on mainnet and the upcoming era.
 mkTopologyConfig :: Int -> [Int] -> Int -> Bool -> LBS.ByteString
-mkTopologyConfig numNodes allPorts port False = Aeson.encode topologyNonP2P
+mkTopologyConfig numNodes allPorts port False = Aeson.encodePretty topologyNonP2P
   where
     topologyNonP2P :: NonP2P.NetworkTopology
     topologyNonP2P =
@@ -187,7 +183,7 @@ mkTopologyConfig numNodes allPorts port False = Aeson.encode topologyNonP2P
                                (numNodes - 1)
         | peerPort <- allPorts List.\\ [port]
         ]
-mkTopologyConfig numNodes allPorts port True = Aeson.encode topologyP2P
+mkTopologyConfig numNodes allPorts port True = Aeson.encodePretty topologyP2P
   where
     rootConfig :: P2P.RootConfig
     rootConfig =

cardano-testnet/src/Testnet/Defaults.hs

@@ -14,6 +14,8 @@ module Testnet.Defaults
   , defaultByronProtocolParamsJsonValue
   , defaultYamlConfig
   , defaultConwayGenesis
+  , defaultDRepVkeyFp
+  , defaultDRepSkeyFp
   , defaultShelleyGenesis
   , defaultGenesisFilepath
   , defaultYamlHardforkViaConfig
@@ -64,7 +66,7 @@ import           Data.Word
 import           GHC.Stack
 import           Lens.Micro
 import           Numeric.Natural
-import           System.FilePath.Posix ((</>))
+import           System.FilePath ((</>))
 
 import           Test.Cardano.Ledger.Core.Rational
 import           Testnet.Start.Types
@@ -552,7 +554,21 @@ defaultMainnetTopology =
   in RealNodeTopology [single]
 
 defaultGenesisFilepath :: CardanoEra a -> FilePath
-defaultGenesisFilepath era = "shelley" </> ("genesis." <> eraToString era <> ".json")
+defaultGenesisFilepath era =
+  -- This path is actually generated by create-testnet-data. Don't change it.
+  eraToString era <> "-genesis.json"
+
+-- | The relative path to DRep keys in directories created by cardano-testnet
+defaultDRepVkeyFp
+  :: Int -- ^ The DRep's index (starts at 1)
+  -> FilePath
+defaultDRepVkeyFp n = "drep-keys" </> ("drep" <> show n) </> "drep.vkey"
+
+-- | The relative path to DRep secret keys in directories created by cardano-testnet
+defaultDRepSkeyFp
+  :: Int -- ^ The DRep's index (starts at 1)
+  -> FilePath
+defaultDRepSkeyFp n = "drep-keys" </> ("drep" <> show n) </> "drep.skey"
 
 -- TODO: We should not hardcode a script like this. We need to move
 -- plutus-example from plutus apps to cardano-node-testnet. This will

cardano-testnet/src/Testnet/Start/Cardano.hs

@@ -324,9 +324,9 @@ cardanoTestnet
         mkNodeName i = "pool" <> show i
 
     -- Add Byron, Shelley and Alonzo genesis hashes to node configuration
-    finalYamlConfig <- createConfigYaml (TmpAbsolutePath tmpAbsPath) era
+    config <- createConfigJson (TmpAbsolutePath tmpAbsPath) era
 
-    H.evalIO $ LBS.writeFile configurationFile finalYamlConfig
+    H.evalIO $ LBS.writeFile configurationFile config
 
     -- Byron related
     forM_ (zip [1..] portNumbers) $ \(i, portNumber) -> do

cardano-testnet/test/cardano-testnet-golden/files/golden/allegra_node_default_config.json

@@ -1,7 +1,7 @@
 {
-    "AlonzoGenesisFile": "shelley/genesis.alonzo.json",
+    "AlonzoGenesisFile": "alonzo-genesis.json",
     "ByronGenesisFile": "byron/genesis.json",
-    "ConwayGenesisFile": "shelley/genesis.conway.json",
+    "ConwayGenesisFile": "conway-genesis.json",
     "EnableLogMetrics": false,
     "EnableLogging": true,
     "EnableP2P": false,
@@ -15,7 +15,7 @@
     "PBftSignatureThreshold": 0.6,
     "Protocol": "Cardano",
     "RequiresNetworkMagic": "RequiresMagic",
-    "ShelleyGenesisFile": "shelley/genesis.shelley.json",
+    "ShelleyGenesisFile": "shelley-genesis.json",
     "SocketPath": "db/node.socket",
     "TestAllegraHardForkAtEpoch": 0,
     "TestShelleyHardForkAtEpoch": 0,

cardano-testnet/test/cardano-testnet-golden/files/golden/alonzo_node_default_config.json

@@ -1,7 +1,7 @@
 {
-    "AlonzoGenesisFile": "shelley/genesis.alonzo.json",
+    "AlonzoGenesisFile": "alonzo-genesis.json",
     "ByronGenesisFile": "byron/genesis.json",
-    "ConwayGenesisFile": "shelley/genesis.conway.json",
+    "ConwayGenesisFile": "conway-genesis.json",
     "EnableLogMetrics": false,
     "EnableLogging": true,
     "EnableP2P": false,
@@ -15,7 +15,7 @@
     "PBftSignatureThreshold": 0.6,
     "Protocol": "Cardano",
     "RequiresNetworkMagic": "RequiresMagic",
-    "ShelleyGenesisFile": "shelley/genesis.shelley.json",
+    "ShelleyGenesisFile": "shelley-genesis.json",
     "SocketPath": "db/node.socket",
     "TestAllegraHardForkAtEpoch": 0,
     "TestAlonzoHardForkAtEpoch": 0,

cardano-testnet/test/cardano-testnet-golden/files/golden/babbage_node_default_config.json

@@ -1,7 +1,7 @@
 {
-    "AlonzoGenesisFile": "shelley/genesis.alonzo.json",
+    "AlonzoGenesisFile": "alonzo-genesis.json",
     "ByronGenesisFile": "byron/genesis.json",
-    "ConwayGenesisFile": "shelley/genesis.conway.json",
+    "ConwayGenesisFile": "conway-genesis.json",
     "EnableLogMetrics": false,
     "EnableLogging": true,
     "EnableP2P": false,
@@ -15,7 +15,7 @@
     "PBftSignatureThreshold": 0.6,
     "Protocol": "Cardano",
     "RequiresNetworkMagic": "RequiresMagic",
-    "ShelleyGenesisFile": "shelley/genesis.shelley.json",
+    "ShelleyGenesisFile": "shelley-genesis.json",
     "SocketPath": "db/node.socket",
     "TestAllegraHardForkAtEpoch": 0,
     "TestAlonzoHardForkAtEpoch": 0,

cardano-testnet/test/cardano-testnet-golden/files/golden/byron_node_default_config.json

@@ -1,7 +1,7 @@
 {
-    "AlonzoGenesisFile": "shelley/genesis.alonzo.json",
+    "AlonzoGenesisFile": "alonzo-genesis.json",
     "ByronGenesisFile": "byron/genesis.json",
-    "ConwayGenesisFile": "shelley/genesis.conway.json",
+    "ConwayGenesisFile": "conway-genesis.json",
     "EnableLogMetrics": false,
     "EnableLogging": true,
     "EnableP2P": false,
@@ -13,7 +13,7 @@
     "PBftSignatureThreshold": 0.6,
     "Protocol": "Cardano",
     "RequiresNetworkMagic": "RequiresMagic",
-    "ShelleyGenesisFile": "shelley/genesis.shelley.json",
+    "ShelleyGenesisFile": "shelley-genesis.json",
     "SocketPath": "db/node.socket",
     "TraceBlockFetchClient": false,
     "TraceBlockFetchDecisions": false,

cardano-testnet/test/cardano-testnet-golden/files/golden/conway_node_default_config.json

@@ -1,7 +1,7 @@
 {
-    "AlonzoGenesisFile": "shelley/genesis.alonzo.json",
+    "AlonzoGenesisFile": "alonzo-genesis.json",
     "ByronGenesisFile": "byron/genesis.json",
-    "ConwayGenesisFile": "shelley/genesis.conway.json",
+    "ConwayGenesisFile": "conway-genesis.json",
     "EnableLogMetrics": false,
     "EnableLogging": true,
     "EnableP2P": false,
@@ -15,7 +15,7 @@
     "PBftSignatureThreshold": 0.6,
     "Protocol": "Cardano",
     "RequiresNetworkMagic": "RequiresMagic",
-    "ShelleyGenesisFile": "shelley/genesis.shelley.json",
+    "ShelleyGenesisFile": "shelley-genesis.json",
     "SocketPath": "db/node.socket",
     "TestAllegraHardForkAtEpoch": 0,
     "TestAlonzoHardForkAtEpoch": 0,

cardano-testnet/test/cardano-testnet-golden/files/golden/mary_node_default_config.json

@@ -1,7 +1,7 @@
 {
-    "AlonzoGenesisFile": "shelley/genesis.alonzo.json",
+    "AlonzoGenesisFile": "alonzo-genesis.json",
     "ByronGenesisFile": "byron/genesis.json",
-    "ConwayGenesisFile": "shelley/genesis.conway.json",
+    "ConwayGenesisFile": "conway-genesis.json",
     "EnableLogMetrics": false,
     "EnableLogging": true,
     "EnableP2P": false,
@@ -15,7 +15,7 @@
     "PBftSignatureThreshold": 0.6,
     "Protocol": "Cardano",
     "RequiresNetworkMagic": "RequiresMagic",
-    "ShelleyGenesisFile": "shelley/genesis.shelley.json",
+    "ShelleyGenesisFile": "shelley-genesis.json",
     "SocketPath": "db/node.socket",
     "TestAllegraHardForkAtEpoch": 0,
     "TestMaryHardForkAtEpoch": 0,

cardano-testnet/test/cardano-testnet-golden/files/golden/shelley_node_default_config.json

@@ -1,7 +1,7 @@
 {
-    "AlonzoGenesisFile": "shelley/genesis.alonzo.json",
+    "AlonzoGenesisFile": "alonzo-genesis.json",
     "ByronGenesisFile": "byron/genesis.json",
-    "ConwayGenesisFile": "shelley/genesis.conway.json",
+    "ConwayGenesisFile": "conway-genesis.json",
     "EnableLogMetrics": false,
     "EnableLogging": true,
     "EnableP2P": false,
@@ -15,7 +15,7 @@
     "PBftSignatureThreshold": 0.6,
     "Protocol": "Cardano",
     "RequiresNetworkMagic": "RequiresMagic",
-    "ShelleyGenesisFile": "shelley/genesis.shelley.json",
+    "ShelleyGenesisFile": "shelley-genesis.json",
     "SocketPath": "db/node.socket",
     "TestShelleyHardForkAtEpoch": 0,
     "TraceBlockFetchClient": false,

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/Babbage/LeadershipSchedule.hs

@@ -239,8 +239,8 @@ hprop_leadershipSchedule = H.integrationRetryWorkspace 2 "babbage-leadership-sch
       , "--out-file", testSpoOperationalCertFp
       ]
 
-  yamlBs <- createConfigYaml tempAbsPath (cardanoNodeEra cTestnetOptions)
-  H.lbsWriteFile configurationFile yamlBs
+  jsonBS <- createConfigJson tempAbsPath (cardanoNodeEra cTestnetOptions)
+  H.lbsWriteFile configurationFile jsonBS
   [newNodePort] <- requestAvailablePortNumbers 1
   eRuntime <- lift . lift . runExceptT $ startNode (TmpAbsolutePath work) "test-spo" "127.0.0.1" newNodePort testnetMagic
         [ "run"