Merge branch 'master' of github.com:cosmos/cosmos-sdk into aaronc/con…

…tainer-impl

CHANGELOG.md

@@ -103,9 +103,11 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 * [\#9695](https://github.com/cosmos/cosmos-sdk/pull/9695) `<app> keys migrate` CLI command now takes no arguments
 * [\#9246](https://github.com/cosmos/cosmos-sdk/pull/9246) Removed the CLI flag `--setup-config-only` from the `testnet` command and added the subcommand `init-files`.
+* [\#9780](https://github.com/cosmos/cosmos-sdk/pull/9780) Use sigs.k8s.io for yaml, which might lead to minor YAML output changes
 
 ### Improvements
 
+* [\#9780](https://github.com/cosmos/cosmos-sdk/pull/9780) Remove gogoproto `moretags` YAML annotations and add `sigs.k8s.io/yaml` for YAML marshalling.
 * (x/bank) [\#10134](https://github.com/cosmos/cosmos-sdk/pull/10134) Add `HasDenomMetadata` function to bank `Keeper` to check if a client coin denom metadata exists in state.
 * (store) [\#10026](https://github.com/cosmos/cosmos-sdk/pull/10026) Improve CacheKVStore datastructures / algorithms, to no longer take O(N^2) time when interleaving iterators and insertions.
 * (types) [\#10076](https://github.com/cosmos/cosmos-sdk/pull/10076) Significantly speedup and lower allocations for `Coins.String()`.

client/context.go

@@ -2,13 +2,12 @@ package client
 
 import (
 	"bufio"
-	"encoding/json"
 	"io"
 	"os"
 
 	"github.com/spf13/viper"
 
-	"gopkg.in/yaml.v2"
+	"sigs.k8s.io/yaml"
 
 	"github.com/gogo/protobuf/proto"
 	rpcclient "github.com/tendermint/tendermint/rpc/client"
@@ -279,16 +278,9 @@ func (ctx Context) PrintObjectLegacy(toPrint interface{}) error {
 }
 
 func (ctx Context) printOutput(out []byte) error {
+	var err error
 	if ctx.OutputFormat == "text" {
-		// handle text format by decoding and re-encoding JSON as YAML
-		var j interface{}
-
-		err := json.Unmarshal(out, &j)
-		if err != nil {
-			return err
-		}
-
-		out, err = yaml.Marshal(j)
+		out, err = yaml.JSONToYAML(out)
 		if err != nil {
 			return err
 		}
@@ -299,7 +291,7 @@ func (ctx Context) printOutput(out []byte) error {
 		writer = os.Stdout
 	}
 
-	_, err := writer.Write(out)
+	_, err = writer.Write(out)
 	if err != nil {
 		return err
 	}

client/keys/parse.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/tendermint/tendermint/libs/cli"
-	yaml "gopkg.in/yaml.v2"
+	"sigs.k8s.io/yaml"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/types/bech32"

client/keys/utils.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"io"
 
-	yaml "gopkg.in/yaml.v2"
+	"sigs.k8s.io/yaml"
 
 	cryptokeyring "github.com/cosmos/cosmos-sdk/crypto/keyring"
 )

codec/yaml.go

@@ -1,10 +1,8 @@
 package codec
 
 import (
-	"encoding/json"
-
 	"github.com/gogo/protobuf/proto"
-	"gopkg.in/yaml.v2"
+	"sigs.k8s.io/yaml"
 )
 
 // MarshalYAML marshals toPrint using JSONCodec to leverage specialized MarshalJSON methods
@@ -18,12 +16,5 @@ func MarshalYAML(cdc JSONCodec, toPrint proto.Message) ([]byte, error) {
 		return nil, err
 	}
 
-	// generate YAML by decoding JSON and re-encoding to YAML
-	var j interface{}
-	err = json.Unmarshal(bz, &j)
-	if err != nil {
-		return nil, err
-	}
-
-	return yaml.Marshal(j)
+	return yaml.JSONToYAML(bz)
 }

cosmovisor/CHANGELOG.md

@@ -41,7 +41,8 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 + [\#8590](https://github.com/cosmos/cosmos-sdk/pull/8590) File watcher for cosmovisor. Instead of parsing logs from stdin and stderr, we watch the `<DAEMON_HOME>/data/upgrade-info.json` file updates using polling mechanism.
 + [\#10128](https://github.com/cosmos/cosmos-sdk/pull/10128) Change default value of `DAEMON_RESTART_AFTER_UPGRADE` to `true`.
-+ [\#9999](https://github.com/cosmos/cosmos-sdk/issues/9999) Added `version` command that returns the cosmovisor version and the application version.
++ [\#9999](https://github.com/cosmos/cosmos-sdk/pull/10103) Added `version` command that returns the cosmovisor version and the application version.
++ [\#9973](https://github.com/cosmos/cosmos-sdk/pull/10056) Added support for pre-upgrade command in Cosmovisor to be called before the binary is upgraded. Added new environmental variable `DAEMON_PREUPGRADE_MAX_RETRIES` that holds the maximum number of times to reattempt pre-upgrade before failing.
 
 
 ### Improvements

cosmovisor/README.md

@@ -43,6 +43,8 @@ All arguments passed to `cosmovisor` will be passed to the application binary (a
 * `DAEMON_RESTART_AFTER_UPGRADE` (*optional*, default = `true`), if `true`, restarts the subprocess with the same command-line arguments and flags (but with the new binary) after a successful upgrade. Otherwise (`false`), `cosmovisor` stops running after an upgrade and requires the system administrator to manually restart it. Note restart is only after the upgrade and does not auto-restart the subprocess after an error occurs.
 * `DAEMON_POLL_INTERVAL` is the interval length in milliseconds for polling the upgrade plan file. Default: 300.
 * `UNSAFE_SKIP_BACKUP` (defaults to `false`), if set to `false`, backs up the data before trying the upgrade. Otherwise (`true`), upgrades directly without performing a backup. The default value of false is useful and recommended in case of failures and when a backup needed to rollback. We recommend using the default backup option `UNSAFE_SKIP_BACKUP=false`.
+* `DAEMON_PREUPGRADE_MAX_RETRIES` (defaults to `0`). The maximum number of times to call `pre-upgrade` in the application after exit status of `31`. After the maximum number of retries, cosmovisor fails the upgrade. 
+
 
 ### Folder Layout
 

cosmovisor/args.go

@@ -15,12 +15,13 @@ import (
 
 // environment variable names
 const (
-	envHome           = "DAEMON_HOME"
-	envName           = "DAEMON_NAME"
-	envDownloadBin    = "DAEMON_ALLOW_DOWNLOAD_BINARIES"
-	envRestartUpgrade = "DAEMON_RESTART_AFTER_UPGRADE"
-	envSkipBackup     = "UNSAFE_SKIP_BACKUP"
-	envInterval       = "DAEMON_POLL_INTERVAL"
+	envHome                 = "DAEMON_HOME"
+	envName                 = "DAEMON_NAME"
+	envDownloadBin          = "DAEMON_ALLOW_DOWNLOAD_BINARIES"
+	envRestartUpgrade       = "DAEMON_RESTART_AFTER_UPGRADE"
+	envSkipBackup           = "UNSAFE_SKIP_BACKUP"
+	envInterval             = "DAEMON_POLL_INTERVAL"
+	envPreupgradeMaxRetries = "DAEMON_PREUPGRADE_MAX_RETRIES"
 )
 
 const (
@@ -42,6 +43,7 @@ type Config struct {
 	RestartAfterUpgrade   bool
 	PollInterval          time.Duration
 	UnsafeSkipBackup      bool
+	PreupgradeMaxRetries  int
 
 	// currently running upgrade
 	currentUpgrade UpgradeInfo
@@ -146,6 +148,12 @@ func GetConfigFromEnv() (*Config, error) {
 	if err := cfg.validate(); err != nil {
 		return nil, err
 	}
+
+	envPreupgradeMaxRetriesVal := os.Getenv(envPreupgradeMaxRetries)
+	if cfg.PreupgradeMaxRetries, err = strconv.Atoi(envPreupgradeMaxRetriesVal); err != nil && envPreupgradeMaxRetriesVal != "" {
+		return nil, fmt.Errorf("%s could not be parsed to int: %w", envPreupgradeMaxRetries, err)
+	}
+
 	return cfg, nil
 }
 

cosmovisor/process.go

@@ -61,13 +61,13 @@ func (l Launcher) Run(args []string, stdout, stderr io.Writer) (bool, error) {
 	if err != nil || !needsUpdate {
 		return false, err
 	}
-	if err := doBackup(l.cfg); err != nil {
-		return false, err
-	}
 
-	if !SkipUpgrade(args, l.fw.currentInfo) {
-		err = doPreUpgrade(l.cfg)
-		if err != nil {
+	if !IsSkipUpgradeHeight(args, l.fw.currentInfo) {
+		if err := doBackup(l.cfg); err != nil {
+			return false, err
+		}
+
+		if err = doPreUpgrade(l.cfg); err != nil {
 			return false, err
 		}
 	}
@@ -152,32 +152,52 @@ func doBackup(cfg *Config) error {
 	return nil
 }
 
-// doPreUpgrade runs the pre-upgrade command defined by the application
+// doPreUpgrade runs the pre-upgrade command defined by the application and handles respective error codes
+// cfg contains the cosmovisor config from env var
 func doPreUpgrade(cfg *Config) error {
-	bin, err := cfg.CurrentBin()
-	preUpgradeCmd := exec.Command(bin, "pre-upgrade")
+	counter := 0
+	for {
+		if counter > cfg.PreupgradeMaxRetries {
+			return fmt.Errorf("pre-upgrade command failed. reached max attempt of retries - %d", cfg.PreupgradeMaxRetries)
+		}
 
-	_, err = preUpgradeCmd.Output()
+		err := executePreUpgradeCmd(cfg)
+		counter += 1
 
-	if err != nil {
-		if err.(*exec.ExitError).ProcessState.ExitCode() == 1 {
-			fmt.Println("pre-upgrade command does not exist. continuing the upgrade.")
-			return nil
-		}
-		if err.(*exec.ExitError).ProcessState.ExitCode() == 30 {
-			return fmt.Errorf("pre-upgrade command failed : %w", err)
-		}
-		if err.(*exec.ExitError).ProcessState.ExitCode() == 31 {
-			fmt.Println("pre-upgrade command failed. retrying.")
-			return doPreUpgrade(cfg)
+		if err != nil {
+			if err.(*exec.ExitError).ProcessState.ExitCode() == 1 {
+				fmt.Println("pre-upgrade command does not exist. continuing the upgrade.")
+				return nil
+			}
+			if err.(*exec.ExitError).ProcessState.ExitCode() == 30 {
+				return fmt.Errorf("pre-upgrade command failed : %w", err)
+			}
+			if err.(*exec.ExitError).ProcessState.ExitCode() == 31 {
+				fmt.Println("pre-upgrade command failed. retrying. attempt:", counter)
+				fmt.Println(err)
+				continue
+			}
 		}
+		fmt.Println("pre-upgrade successful. continuing the upgrade.")
+		return nil
 	}
-	fmt.Println("pre-upgrade successful. continuing the upgrade.")
-	return nil
 }
 
-// skipUpgrade checks if pre-upgrade script must be run. If the height in the upgrade plan matches any of the heights provided in --safe-skip-upgrade, the script is not run
-func SkipUpgrade(args []string, upgradeInfo UpgradeInfo) bool {
+// executePreUpgradeCmd runs the pre-upgrade command defined by the application
+// cfg contains the cosmosvisor config from the env vars
+func executePreUpgradeCmd(cfg *Config) error {
+	bin, err := cfg.CurrentBin()
+	if err != nil {
+		return err
+	}
+
+	preUpgradeCmd := exec.Command(bin, "pre-upgrade")
+	_, err = preUpgradeCmd.Output()
+	return err
+}
+
+// IsSkipUpgradeHeight checks if pre-upgrade script must be run. If the height in the upgrade plan matches any of the heights provided in --safe-skip-upgrade, the script is not run
+func IsSkipUpgradeHeight(args []string, upgradeInfo UpgradeInfo) bool {
 	skipUpgradeHeights := UpgradeSkipHeights(args)
 	for _, h := range skipUpgradeHeights {
 		if h == int(upgradeInfo.Height) {

cosmovisor/process_test.go

@@ -160,7 +160,7 @@ func TestSkipUpgrade(t *testing.T) {
 	for i := range cases {
 		tc := cases[i]
 		require := require.New(t)
-		h := cosmovisor.SkipUpgrade(tc.args, tc.upgradeInfo)
+		h := cosmovisor.IsSkipUpgradeHeight(tc.args, tc.upgradeInfo)
 		require.Equal(h, tc.expectRes)
 	}
 }

go.mod

@@ -54,7 +54,7 @@ require (
 	google.golang.org/grpc v1.40.0
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
-	gopkg.in/yaml.v2 v2.4.0
+	sigs.k8s.io/yaml v1.2.0
 )
 
 require (
@@ -118,6 +118,7 @@ require (
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	gopkg.in/ini.v1 v1.63.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	nhooyr.io/websocket v1.8.6 // indirect
 )

go.sum

@@ -1303,4 +1303,6 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=

proto/cosmos/auth/v1beta1/auth.proto

@@ -19,8 +19,8 @@ message BaseAccount {
 
   string              address = 1 [(cosmos_proto.scalar) = "cosmos.AddressString"];
   google.protobuf.Any pub_key = 2
-      [(gogoproto.jsontag) = "public_key,omitempty", (gogoproto.moretags) = "yaml:\"public_key\""];
-  uint64 account_number = 3 [(gogoproto.moretags) = "yaml:\"account_number\""];
+      [(gogoproto.jsontag) = "public_key,omitempty"];
+  uint64 account_number = 3;
   uint64 sequence       = 4;
 }
 
@@ -30,7 +30,7 @@ message ModuleAccount {
   option (gogoproto.goproto_stringer)        = false;
   option (cosmos_proto.implements_interface) = "ModuleAccountI";
 
-  BaseAccount     base_account = 1 [(gogoproto.embed) = true, (gogoproto.moretags) = "yaml:\"base_account\""];
+  BaseAccount     base_account = 1 [(gogoproto.embed) = true];
   string          name         = 2;
   repeated string permissions  = 3;
 }
@@ -40,11 +40,11 @@ message Params {
   option (gogoproto.equal)            = true;
   option (gogoproto.goproto_stringer) = false;
 
-  uint64 max_memo_characters     = 1 [(gogoproto.moretags) = "yaml:\"max_memo_characters\""];
-  uint64 tx_sig_limit            = 2 [(gogoproto.moretags) = "yaml:\"tx_sig_limit\""];
-  uint64 tx_size_cost_per_byte   = 3 [(gogoproto.moretags) = "yaml:\"tx_size_cost_per_byte\""];
+  uint64 max_memo_characters     = 1;
+  uint64 tx_sig_limit            = 2;
+  uint64 tx_size_cost_per_byte   = 3;
   uint64 sig_verify_cost_ed25519 = 4
-      [(gogoproto.customname) = "SigVerifyCostED25519", (gogoproto.moretags) = "yaml:\"sig_verify_cost_ed25519\""];
+      [(gogoproto.customname) = "SigVerifyCostED25519"];
   uint64 sig_verify_cost_secp256k1 = 5
-      [(gogoproto.customname) = "SigVerifyCostSecp256k1", (gogoproto.moretags) = "yaml:\"sig_verify_cost_secp256k1\""];
+      [(gogoproto.customname) = "SigVerifyCostSecp256k1"];
 }

proto/cosmos/bank/v1beta1/bank.proto

@@ -10,8 +10,8 @@ option go_package = "github.com/cosmos/cosmos-sdk/x/bank/types";
 // Params defines the parameters for the bank module.
 message Params {
   option (gogoproto.goproto_stringer)       = false;
-  repeated SendEnabled send_enabled         = 1 [(gogoproto.moretags) = "yaml:\"send_enabled,omitempty\""];
-  bool                 default_send_enabled = 2 [(gogoproto.moretags) = "yaml:\"default_send_enabled,omitempty\""];
+  repeated SendEnabled send_enabled         = 1;
+  bool                 default_send_enabled = 2;
 }
 
 // SendEnabled maps coin denom to a send_enabled status (whether a denom is

proto/cosmos/bank/v1beta1/genesis.proto

@@ -22,7 +22,7 @@ message GenesisState {
       [(gogoproto.castrepeated) = "github.com/cosmos/cosmos-sdk/types.Coins", (gogoproto.nullable) = false];
 
   // denom_metadata defines the metadata of the differents coins.
-  repeated Metadata denom_metadata = 4 [(gogoproto.moretags) = "yaml:\"denom_metadata\"", (gogoproto.nullable) = false];
+  repeated Metadata denom_metadata = 4 [(gogoproto.nullable) = false];
 }
 
 // Balance defines an account address and balance pair used in the bank module's