Merge branch 'develop' into feature/fix-logic-for-rosetta-integration…

…-test

README.md

@@ -29,7 +29,7 @@ The Mina implementation of the Rosetta API offers a more stable and useful inter
 - [Mina Protocol](https://docs.minaprotocol.com/) documentation
 - [Installing and using a third-party wallet](https://docs.minaprotocol.com/using-mina/install-a-wallet)
 - [Sending a Payment using Mina's CLI](https://docs.minaprotocol.com/node-operators/sending-a-payment)
-- [Become a Node Operator](https://minaprotocol.com/docs/getting-started/)
+- [Become a Node Operator](https://docs.minaprotocol.com/node-operators/block-producer-node/getting-started)
 
 ### Technical Papers
 

automation/README.md

@@ -264,11 +264,11 @@ There are several public Grafana dashboards available here:
 
 The purpose of a public testnet is to allow end-users to try out the software and learn how to operate it. Thus, we accept sign-ups for stake to be allocated in the genesis, and commit those keys to the compiled genesis ledger.
 
-For context, these keys correspond to the "Fish Keys" in the QA Net deployments, and Online Fish Keys are ommitted in a Public Testnet deployment and "Offline Fish Keys" are instead delegated to the submitted User Keys.
+For context, these keys correspond to the "Fish Keys" in the QA Net deployments, and Online Fish Keys are omitted in a Public Testnet deployment and "Offline Fish Keys" are instead delegated to the submitted User Keys.
 
 ### Generate Genesis Ledger
 
-Once you have the keys for your deploymenet created, and the Staker Keys saved to a CSV, you can use them to generate a genesis ledger with the following command.
+Once you have the keys for your deployment created, and the Staker Keys saved to a CSV, you can use them to generate a genesis ledger with the following command.
 
 ```
 scripts/generate-keys-and-ledger.sh

buildkite/scripts/bench/run.sh

@@ -10,7 +10,7 @@ EXTRA_ARGS=""
 
 source buildkite/scripts/bench/install.sh
 
-MAINLINE_BRANCHES="-m develop -m compatible -m master -m dkijania/fix_benchmark_upload"
+MAINLINE_BRANCHES="-m develop -m compatible -m master -m dkijania/enhance_benchmarks"
 while [[ "$#" -gt 0 ]]; do case $1 in
   heap-usage) BENCHMARK="heap-usage"; ;;
   mina-base) BENCHMARK="mina-base"; ;;

buildkite/scripts/connect/connect-to-network.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+set -eo pipefail
+
+if [[ $# -ne 4 ]]; then
+    echo "Usage: $0 '<mina-debian-network>''<testnet-name>' '<wait-between-polling-graphql>''<wait-after-final-check>' "
+    exit 1
+fi
+
+MINA_DEBIAN_NETWORK=$1
+NETWORK_NAME=$2
+WAIT_BETWEEN_POLLING_GRAPHQL=$3
+WAIT_AFTER_FINAL_CHECK=$4
+
+# Don't prompt for answers during apt-get install
+export DEBIAN_FRONTEND=noninteractive
+
+sudo apt-get update
+sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl
+
+git config --global --add safe.directory /workdir
+
+source buildkite/scripts/export-git-env-vars.sh
+
+source buildkite/scripts/debian/install.sh "mina-${MINA_DEBIAN_NETWORK}" 1
+
+# Remove lockfile if present
+rm /home/opam/.mina-config/.mina-lock ||:
+
+mkdir -p /home/opam/libp2p-keys/
+# Pre-generated random password for this quick test
+export MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf
+mina libp2p generate-keypair --privkey-path /home/opam/libp2p-keys/key
+# Set permissions on the keypair so the daemon doesn't complain
+chmod -R 0700 /home/opam/libp2p-keys/
+
+# Restart in the background
+mina daemon \
+  --peer-list-url "https://storage.googleapis.com/seed-lists/${NETWORK_NAME}_seeds.txt" \
+  --libp2p-keypair "/home/opam/libp2p-keys/key" \
+& # -background
+
+# Attempt to connect to the GraphQL client every 10s for up to 8 minutes
+num_status_retries=24
+for ((i=1;i<=$num_status_retries;i++)); do
+  sleep $WAIT_BETWEEN_POLLING_GRAPHQL
+  set +e
+  sudo mina client status
+  status_exit_code=$?
+  set -e
+  if [ $status_exit_code -eq 0 ]; then
+    break
+  elif [ $i -eq $num_status_retries ]; then
+    exit $status_exit_code
+  fi
+done
+
+# Check that the daemon has connected to peers and is still up after 2 mins
+sleep "$WAIT_AFTER_FINAL_CHECK"
+mina client status
+if [ "$(mina advanced get-peers | wc -l)" -gt 0 ]; then
+    echo "Found some peers"
+else
+    echo "No peers found"
+    exit 1
+fi
+
+# Check network id
+NETWORK_ID=$(curl 'http://localhost:3085/graphql' \
+   -H 'accept: application/json' \
+   -H 'content-type: application/json' \
+   --data-raw '{"query":"query MyQuery {\n  networkID\n}\n","variables":null,"operationName":"MyQuery"}' \
+  | jq -r .data.networkID)
+
+EXPECTED_NETWORK=mina:$NETWORK_NAME
+
+if [[ "$NETWORK_ID" == "$EXPECTED_NETWORK" ]]; then
+    echo "Network id correct ($NETWORK_ID)"
+else
+    echo "Network id incorrect (expected: $EXPECTED_NETWORK got: $NETWORK_ID)"
+    exit 1
+fi

buildkite/scripts/run_promote_build_job.sh

@@ -46,7 +46,7 @@ function usage() {
   echo "  NEW_VERSION                 The new Debian version or new Docker tag"
   echo "  REMOVE_PROFILE_FROM_NAME    Should we remove profile suffix from debian name"
   echo "  PROFILE                     The Docker and Debian profile (Standard, Lightnet)"
-  echo "  NETWORK                     The Docker and Debian network (Devnet, Mainnet, Berkeley)"
+  echo "  NETWORK                     The Docker and Debian network (Devnet, Mainnet)"
   echo "  FROM_CHANNEL                Source debian channel"
   echo "  TO_CHANNEL                  Target debian channel"
   echo "  PUBLISH                     The Publish to docker.io flag. If defined, script will publish docker do docker.io. Otherwise it will still resides in gcr.io"

buildkite/scripts/run_verify_promoted_build_job.sh

@@ -45,7 +45,7 @@ function usage() {
   echo "  NEW_VERSION                 The new Debian version or new Docker tag"
   echo "  REMOVE_PROFILE_FROM_NAME    Should we remove profile suffix from debian name"
   echo "  PROFILE                     The Docker and Debian profile (Standard, Lightnet)"
-  echo "  NETWORK                     The Docker and Debian network (Devnet, Mainnet, Berkeley)"
+  echo "  NETWORK                     The Docker and Debian network (Devnet, Mainnet)"
   echo "  TO_CHANNEL                  Target debian channel"
   echo "  PUBLISH                     The Publish to docker.io flag. If defined, script will publish docker do docker.io. Otherwise it will still resides in gcr.io"
   echo ""

buildkite/src/Command/Bench/Base.dhall

@@ -1,3 +1,5 @@
+let B = ../../External/Buildkite.dhall
+
 let PipelineMode = ../../Pipeline/Mode.dhall
 
 let PipelineTag = ../../Pipeline/Tag.dhall
@@ -24,6 +26,8 @@ let Benchmarks = ../../Constants/Benchmarks.dhall
 
 let SelectFiles = ../../Lib/SelectFiles.dhall
 
+let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
+
 let Spec =
       { Type =
           { key : Text
@@ -67,6 +71,7 @@ let command
                 "Perf: ${spec.label} ${PipelineMode.capitalName spec.mode}"
             , key = spec.key
             , target = spec.size
+            , soft_fail = Some (B/SoftFail.Boolean True)
             , docker = None Docker.Type
             , depends_on = spec.dependsOn
             }

buildkite/src/Command/ConnectToTestnet.dhall → buildkite/src/Command/ConnectToNetwork.dhall

@@ -10,16 +10,17 @@ let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
 
 in  { step =
             \(dependsOn : List Command.TaggedKey.Type)
+        ->  \(mina_suffix : Text)
         ->  \(testnet : Text)
         ->  \(wait_between_graphql_poll : Text)
         ->  \(wait_before_final_check : Text)
         ->  \(soft_fail : B/SoftFail)
         ->  Command.build
               Command.Config::{
               , commands =
-                  RunInToolchain.runInToolchain
+                  RunInToolchain.runInToolchainBullseye
                     ([] : List Text)
-                    "./buildkite/scripts/connect-to-testnet.sh ${testnet} ${wait_between_graphql_poll} ${wait_before_final_check}"
+                    "./buildkite/scripts/connect/connect-to-network.sh ${mina_suffix} ${testnet} ${wait_between_graphql_poll} ${wait_before_final_check} "
               , label = "Connect to ${testnet}"
               , soft_fail = Some soft_fail
               , key = "connect-to-${testnet}"

buildkite/src/Constants/DebianVersions.dhall

@@ -75,6 +75,9 @@ let minimalDirtyWhen =
       , S.exactly "buildkite/src/Command/HardforkPackageGeneration" "dhall"
       , S.exactly "buildkite/src/Command/MinaArtifact" "dhall"
       , S.exactly "buildkite/src/Command/PatchArchiveTest" "dhall"
+      , S.exactly "buildkite/src/Command/Bench/Base" "dhall"
+      , S.strictlyStart (S.contains "scripts/benchmarks")
+      , S.strictlyStart (S.contains "buildkite/scripts/bench")
       , S.exactly "buildkite/src/Command/ReplayerTest" "dhall"
       , S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaArtifact")
       , S.strictlyStart (S.contains "dockerfiles/stages")
@@ -91,8 +94,7 @@ let bullseyeDirtyWhen =
         [ S.strictlyStart (S.contains "src")
         , S.strictlyStart (S.contains "automation")
         , S.strictly (S.contains "Makefile")
-        , S.exactly "buildkite/scripts/connect-to-testnet" "sh"
-        , S.exactly "buildkite/scripts/connect-to-mainnet-on-compatible" "sh"
+        , S.exactly "buildkite/scripts/connect/connect-to-network" "sh"
         , S.exactly "buildkite/scripts/rosetta-integration-tests" "sh"
         , S.exactly "buildkite/scripts/rosetta-integration-tests-full" "sh"
         , S.exactly "buildkite/scripts/rosetta-integration-tests-fast" "sh"

buildkite/src/Jobs/Test/ConnectToDevnet.dhall

@@ -10,39 +10,52 @@ let Pipeline = ../../Pipeline/Dsl.dhall
 
 let PipelineTag = ../../Pipeline/Tag.dhall
 
-let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall
+let PipelineMode = ../../Pipeline/Mode.dhall
 
-let Profiles = ../../Constants/Profiles.dhall
+let ConnectToNetwork = ../../Command/ConnectToNetwork.dhall
 
-let DebianVersions = ../../Constants/DebianVersions.dhall
+let Profiles = ../../Constants/Profiles.dhall
 
 let Network = ../../Constants/Network.dhall
 
+let Artifacts = ../../Constants/Artifacts.dhall
+
+let Dockers = ../../Constants/DockerVersions.dhall
+
+let network = Network.Type.Devnet
+
 let dependsOn =
-      DebianVersions.dependsOn
-        DebianVersions.DebVersion.Bullseye
-        Network.Type.Devnet
+      Dockers.dependsOn
+        Dockers.Type.Bullseye
+        network
         Profiles.Type.Standard
+        Artifacts.Type.Daemon
 
 in  Pipeline.build
       Pipeline.Config::{
       , spec = JobSpec::{
         , dirtyWhen =
           [ S.strictlyStart (S.contains "src")
-          , S.exactly "buildkite/scripts/connect-to-testnet" "sh"
+          , S.exactly "buildkite/scripts/connect/connect-to-network" "sh"
           , S.exactly "buildkite/src/Jobs/Test/ConnectToDevnet" "dhall"
-          , S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall"
+          , S.exactly "buildkite/src/Command/ConnectToNetwork" "dhall"
           ]
         , path = "Test"
         , name = "ConnectToDevnet"
-        , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ]
+        , mode = PipelineMode.Type.Stable
+        , tags =
+          [ PipelineTag.Type.Long
+          , PipelineTag.Type.Test
+          , PipelineTag.Type.Stable
+          ]
         }
       , steps =
-        [ ConnectToTestnet.step
+        [ ConnectToNetwork.step
             dependsOn
-            "${Network.lowerName Network.Type.Devnet}"
+            "${Network.lowerName network}"
+            "${Network.lowerName network}"
             "40s"
             "2m"
-            (B/SoftFail.Boolean True)
+            (B/SoftFail.Boolean False)
         ]
       }