From 03062cfcee2651cbc1b390252602fd0ac779354b Mon Sep 17 00:00:00 2001 From: Jeeyong Um Date: Fri, 10 Sep 2021 16:45:12 +0900 Subject: [PATCH 1/3] perf: Fork bech32 from btcsuite/btcutil (#10082) --- types/bech32/bech32.go | 378 ++++++++++++++++++++++- types/bech32/bech32_test.go | 574 ++++++++++++++++++++++++++++++++++- types/bech32/convert.go | 32 ++ types/bech32/convert_test.go | 25 ++ types/bech32/error.go | 87 ++++++ 5 files changed, 1067 insertions(+), 29 deletions(-) create mode 100644 types/bech32/convert.go create mode 100644 types/bech32/convert_test.go create mode 100644 types/bech32/error.go diff --git a/types/bech32/bech32.go b/types/bech32/bech32.go index 22a177663eae..05d044d83244 100644 --- a/types/bech32/bech32.go +++ b/types/bech32/bech32.go @@ -1,32 +1,380 @@ +// Copyright (c) 2017 The btcsuite developers +// Copyright (c) 2019 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +// source: https://github.com/btcsuite/btcutil/blob/e2ba6805/bech32/bech32.go + package bech32 import ( - "fmt" - - "github.com/enigmampc/btcutil/bech32" + "strings" ) -// ConvertAndEncode converts from a base64 encoded byte string to base32 encoded byte string and then to bech32. -func ConvertAndEncode(hrp string, data []byte) (string, error) { - converted, err := bech32.ConvertBits(data, 8, 5, true) - if err != nil { - return "", fmt.Errorf("encoding bech32 failed: %w", err) +// charset is the set of characters used in the data section of bech32 strings. +// Note that this is ordered, such that for a given charset[i], i is the binary +// value of the character. +const charset = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" + +// gen encodes the generator polynomial for the bech32 BCH checksum. +var gen = []int{0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3} + +// toBytes converts each character in the string 'chars' to the value of the +// index of the correspoding character in 'charset'. +func toBytes(chars string) ([]byte, error) { + decoded := make([]byte, 0, len(chars)) + for i := 0; i < len(chars); i++ { + index := strings.IndexByte(charset, chars[i]) + if index < 0 { + return nil, ErrNonCharsetChar(chars[i]) + } + decoded = append(decoded, byte(index)) + } + return decoded, nil +} + +// bech32Polymod calculates the BCH checksum for a given hrp, values and +// checksum data. Checksum is optional, and if nil a 0 checksum is assumed. +// +// Values and checksum (if provided) MUST be encoded as 5 bits per element (base +// 32), otherwise the results are undefined. +// +// For more details on the polymod calculation, please refer to BIP 173. +func bech32Polymod(hrp string, values, checksum []byte) int { + chk := 1 + + // Account for the high bits of the HRP in the checksum. + for i := 0; i < len(hrp); i++ { + b := chk >> 25 + hiBits := int(hrp[i]) >> 5 + chk = (chk&0x1ffffff)<<5 ^ hiBits + for i := 0; i < 5; i++ { + if (b>>uint(i))&1 == 1 { + chk ^= gen[i] + } + } + } + + // Account for the separator (0) between high and low bits of the HRP. + // x^0 == x, so we eliminate the redundant xor used in the other rounds. + b := chk >> 25 + chk = (chk & 0x1ffffff) << 5 + for i := 0; i < 5; i++ { + if (b>>uint(i))&1 == 1 { + chk ^= gen[i] + } + } + + // Account for the low bits of the HRP. + for i := 0; i < len(hrp); i++ { + b := chk >> 25 + loBits := int(hrp[i]) & 31 + chk = (chk&0x1ffffff)<<5 ^ loBits + for i := 0; i < 5; i++ { + if (b>>uint(i))&1 == 1 { + chk ^= gen[i] + } + } + } + + // Account for the values. + for _, v := range values { + b := chk >> 25 + chk = (chk&0x1ffffff)<<5 ^ int(v) + for i := 0; i < 5; i++ { + if (b>>uint(i))&1 == 1 { + chk ^= gen[i] + } + } + } + + if checksum == nil { + // A nil checksum is used during encoding, so assume all bytes are zero. + // x^0 == x, so we eliminate the redundant xor used in the other rounds. + for v := 0; v < 6; v++ { + b := chk >> 25 + chk = (chk & 0x1ffffff) << 5 + for i := 0; i < 5; i++ { + if (b>>uint(i))&1 == 1 { + chk ^= gen[i] + } + } + } + } else { + // Checksum is provided during decoding, so use it. + for _, v := range checksum { + b := chk >> 25 + chk = (chk&0x1ffffff)<<5 ^ int(v) + for i := 0; i < 5; i++ { + if (b>>uint(i))&1 == 1 { + chk ^= gen[i] + } + } + } + } + + return chk +} + +// writeBech32Checksum calculates the checksum data expected for a string that +// will have the given hrp and payload data and writes it to the provided string +// builder. +// +// The payload data MUST be encoded as a base 32 (5 bits per element) byte slice +// and the hrp MUST only use the allowed character set (ascii chars between 33 +// and 126), otherwise the results are undefined. +// +// For more details on the checksum calculation, please refer to BIP 173. +func writeBech32Checksum(hrp string, data []byte, bldr *strings.Builder) { + polymod := bech32Polymod(hrp, data, nil) ^ 1 + for i := 0; i < 6; i++ { + b := byte((polymod >> uint(5*(5-i))) & 31) + + // This can't fail, given we explicitly cap the previous b byte by the + // first 31 bits. + c := charset[b] + bldr.WriteByte(c) } +} - return bech32.Encode(hrp, converted) +// bech32VerifyChecksum verifies whether the bech32 string specified by the +// provided hrp and payload data (encoded as 5 bits per element byte slice) has +// the correct checksum suffix. +// +// Data MUST have more than 6 elements, otherwise this function panics. +// +// For more details on the checksum verification, please refer to BIP 173. +func bech32VerifyChecksum(hrp string, data []byte) bool { + checksum := data[len(data)-6:] + values := data[:len(data)-6] + polymod := bech32Polymod(hrp, values, checksum) + return polymod == 1 } -// DecodeAndConvert decodes a bech32 encoded string and converts to base64 encoded bytes. -func DecodeAndConvert(bech string) (string, []byte, error) { - hrp, data, err := bech32.Decode(bech, 1023) +// DecodeNoLimit decodes a bech32 encoded string, returning the human-readable +// part and the data part excluding the checksum. This function does NOT +// validate against the BIP-173 maximum length allowed for bech32 strings and +// is meant for use in custom applications (such as lightning network payment +// requests), NOT on-chain addresses. +// +// Note that the returned data is 5-bit (base32) encoded and the human-readable +// part will be lowercase. +func DecodeNoLimit(bech string) (string, []byte, error) { + // The minimum allowed size of a bech32 string is 8 characters, since it + // needs a non-empty HRP, a separator, and a 6 character checksum. + if len(bech) < 8 { + return "", nil, ErrInvalidLength(len(bech)) + } + + // Only ASCII characters between 33 and 126 are allowed. + var hasLower, hasUpper bool + for i := 0; i < len(bech); i++ { + if bech[i] < 33 || bech[i] > 126 { + return "", nil, ErrInvalidCharacter(bech[i]) + } + + // The characters must be either all lowercase or all uppercase. Testing + // directly with ascii codes is safe here, given the previous test. + hasLower = hasLower || (bech[i] >= 97 && bech[i] <= 122) + hasUpper = hasUpper || (bech[i] >= 65 && bech[i] <= 90) + if hasLower && hasUpper { + return "", nil, ErrMixedCase{} + } + } + + // Bech32 standard uses only the lowercase for of strings for checksum + // calculation. + if hasUpper { + bech = strings.ToLower(bech) + } + + // The string is invalid if the last '1' is non-existent, it is the + // first character of the string (no human-readable part) or one of the + // last 6 characters of the string (since checksum cannot contain '1'). + one := strings.LastIndexByte(bech, '1') + if one < 1 || one+7 > len(bech) { + return "", nil, ErrInvalidSeparatorIndex(one) + } + + // The human-readable part is everything before the last '1'. + hrp := bech[:one] + data := bech[one+1:] + + // Each character corresponds to the byte with value of the index in + // 'charset'. + decoded, err := toBytes(data) if err != nil { - return "", nil, fmt.Errorf("decoding bech32 failed: %w", err) + return "", nil, err } - converted, err := bech32.ConvertBits(data, 5, 8, false) + // Verify if the checksum (stored inside decoded[:]) is valid, given the + // previously decoded hrp. + if !bech32VerifyChecksum(hrp, decoded) { + // Invalid checksum. Calculate what it should have been, so that the + // error contains this information. + + // Extract the payload bytes and actual checksum in the string. + actual := bech[len(bech)-6:] + payload := decoded[:len(decoded)-6] + + // Calculate the expected checksum, given the hrp and payload data. + var expectedBldr strings.Builder + expectedBldr.Grow(6) + writeBech32Checksum(hrp, payload, &expectedBldr) + expected := expectedBldr.String() + + err = ErrInvalidChecksum{ + Expected: expected, + Actual: actual, + } + return "", nil, err + } + + // We exclude the last 6 bytes, which is the checksum. + return hrp, decoded[:len(decoded)-6], nil +} + +// Decode decodes a bech32 encoded string, returning the human-readable part and +// the data part excluding the checksum. +// +// Note that the returned data is 5-bit (base32) encoded and the human-readable +// part will be lowercase. +func Decode(bech string) (string, []byte, error) { + // The maximum allowed length for a bech32 string is 90. + if len(bech) > 90 { + return "", nil, ErrInvalidLength(len(bech)) + } + + return DecodeNoLimit(bech) +} + +// Encode encodes a byte slice into a bech32 string with the given +// human-readable part (HRP). The HRP will be converted to lowercase if needed +// since mixed cased encodings are not permitted and lowercase is used for +// checksum purposes. Note that the bytes must each encode 5 bits (base32). +func Encode(hrp string, data []byte) (string, error) { + // The resulting bech32 string is the concatenation of the lowercase hrp, + // the separator 1, data and the 6-byte checksum. + hrp = strings.ToLower(hrp) + var bldr strings.Builder + bldr.Grow(len(hrp) + 1 + len(data) + 6) + bldr.WriteString(hrp) + bldr.WriteString("1") + + // Write the data part, using the bech32 charset. + for _, b := range data { + if int(b) >= len(charset) { + return "", ErrInvalidDataByte(b) + } + bldr.WriteByte(charset[b]) + } + + // Calculate and write the checksum of the data. + writeBech32Checksum(hrp, data, &bldr) + + return bldr.String(), nil +} + +// ConvertBits converts a byte slice where each byte is encoding fromBits bits, +// to a byte slice where each byte is encoding toBits bits. +func ConvertBits(data []byte, fromBits, toBits uint8, pad bool) ([]byte, error) { + if fromBits < 1 || fromBits > 8 || toBits < 1 || toBits > 8 { + return nil, ErrInvalidBitGroups{} + } + + // Determine the maximum size the resulting array can have after base + // conversion, so that we can size it a single time. This might be off + // by a byte depending on whether padding is used or not and if the input + // data is a multiple of both fromBits and toBits, but we ignore that and + // just size it to the maximum possible. + maxSize := len(data)*int(fromBits)/int(toBits) + 1 + + // The final bytes, each byte encoding toBits bits. + regrouped := make([]byte, 0, maxSize) + + // Keep track of the next byte we create and how many bits we have + // added to it out of the toBits goal. + nextByte := byte(0) + filledBits := uint8(0) + + for _, b := range data { + + // Discard unused bits. + b = b << (8 - fromBits) + + // How many bits remaining to extract from the input data. + remFromBits := fromBits + for remFromBits > 0 { + // How many bits remaining to be added to the next byte. + remToBits := toBits - filledBits + + // The number of bytes to next extract is the minimum of + // remFromBits and remToBits. + toExtract := remFromBits + if remToBits < toExtract { + toExtract = remToBits + } + + // Add the next bits to nextByte, shifting the already + // added bits to the left. + nextByte = (nextByte << toExtract) | (b >> (8 - toExtract)) + + // Discard the bits we just extracted and get ready for + // next iteration. + b = b << toExtract + remFromBits -= toExtract + filledBits += toExtract + + // If the nextByte is completely filled, we add it to + // our regrouped bytes and start on the next byte. + if filledBits == toBits { + regrouped = append(regrouped, nextByte) + filledBits = 0 + nextByte = 0 + } + } + } + + // We pad any unfinished group if specified. + if pad && filledBits > 0 { + nextByte = nextByte << (toBits - filledBits) + regrouped = append(regrouped, nextByte) + filledBits = 0 + nextByte = 0 + } + + // Any incomplete group must be <= 4 bits, and all zeroes. + if filledBits > 0 && (filledBits > 4 || nextByte != 0) { + return nil, ErrInvalidIncompleteGroup{} + } + + return regrouped, nil +} + +// EncodeFromBase256 converts a base256-encoded byte slice into a base32-encoded +// byte slice and then encodes it into a bech32 string with the given +// human-readable part (HRP). The HRP will be converted to lowercase if needed +// since mixed cased encodings are not permitted and lowercase is used for +// checksum purposes. +func EncodeFromBase256(hrp string, data []byte) (string, error) { + converted, err := ConvertBits(data, 8, 5, true) if err != nil { - return "", nil, fmt.Errorf("decoding bech32 failed: %w", err) + return "", err } + return Encode(hrp, converted) +} +// DecodeToBase256 decodes a bech32-encoded string into its associated +// human-readable part (HRP) and base32-encoded data, converts that data to a +// base256-encoded byte slice and returns it along with the lowercase HRP. +func DecodeToBase256(bech string) (string, []byte, error) { + hrp, data, err := Decode(bech) + if err != nil { + return "", nil, err + } + converted, err := ConvertBits(data, 5, 8, false) + if err != nil { + return "", nil, err + } return hrp, converted, nil } diff --git a/types/bech32/bech32_test.go b/types/bech32/bech32_test.go index ba2beb783220..9325dafdefe6 100644 --- a/types/bech32/bech32_test.go +++ b/types/bech32/bech32_test.go @@ -1,25 +1,571 @@ -package bech32_test +// Copyright (c) 2017-2020 The btcsuite developers +// Copyright (c) 2019 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +// source: https://github.com/btcsuite/btcutil/blob/e2ba6805/bech32/bech32_test.go + +package bech32 import ( "bytes" - "crypto/sha256" + "encoding/hex" + "fmt" + "strings" "testing" +) - "github.com/stretchr/testify/require" +// TestBech32 tests whether decoding and re-encoding the valid BIP-173 test +// vectors works and if decoding invalid test vectors fails for the correct +// reason. +func TestBech32(t *testing.T) { + tests := []struct { + str string + expectedError error + }{ + {"A12UEL5L", nil}, + {"an83characterlonghumanreadablepartthatcontainsthenumber1andtheexcludedcharactersbio1tt5tgs", nil}, + {"abcdef1qpzry9x8gf2tvdw0s3jn54khce6mua7lmqqqxw", nil}, + {"11qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqc8247j", nil}, + {"split1checkupstagehandshakeupstreamerranterredcaperred2y9e3w", nil}, + {"split1checkupstagehandshakeupstreamerranterredcaperred2y9e2w", ErrInvalidChecksum{"2y9e3w", "2y9e2w"}}, // invalid checksum + {"s lit1checkupstagehandshakeupstreamerranterredcaperredp8hs2p", ErrInvalidCharacter(' ')}, // invalid character (space) in hrp + {"spl\x7Ft1checkupstagehandshakeupstreamerranterredcaperred2y9e3w", ErrInvalidCharacter(127)}, // invalid character (DEL) in hrp + {"split1cheo2y9e2w", ErrNonCharsetChar('o')}, // invalid character (o) in data part + {"split1a2y9w", ErrInvalidSeparatorIndex(5)}, // too short data part + {"1checkupstagehandshakeupstreamerranterredcaperred2y9e3w", ErrInvalidSeparatorIndex(0)}, // empty hrp + {"11qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqsqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqc8247j", ErrInvalidLength(91)}, // too long - "github.com/cosmos/cosmos-sdk/types/bech32" -) + // Additional test vectors used in bitcoin core + {" 1nwldj5", ErrInvalidCharacter(' ')}, + {"\x7f" + "1axkwrx", ErrInvalidCharacter(0x7f)}, + {"\x801eym55h", ErrInvalidCharacter(0x80)}, + {"an84characterslonghumanreadablepartthatcontainsthenumber1andtheexcludedcharactersbio1569pvx", ErrInvalidLength(91)}, + {"pzry9x0s0muk", ErrInvalidSeparatorIndex(-1)}, + {"1pzry9x0s0muk", ErrInvalidSeparatorIndex(0)}, + {"x1b4n0q5v", ErrNonCharsetChar(98)}, + {"li1dgmt3", ErrInvalidSeparatorIndex(2)}, + {"de1lg7wt\xff", ErrInvalidCharacter(0xff)}, + {"A1G7SGD8", ErrInvalidChecksum{"2uel5l", "g7sgd8"}}, + {"10a06t8", ErrInvalidLength(7)}, + {"1qzzfhee", ErrInvalidSeparatorIndex(0)}, + {"a12UEL5L", ErrMixedCase{}}, + {"A12uEL5L", ErrMixedCase{}}, + } + + for i, test := range tests { + str := test.str + hrp, decoded, err := Decode(str) + if test.expectedError != err { + t.Errorf("%d: expected decoding error %v "+ + "instead got %v", i, test.expectedError, err) + continue + } + + if err != nil { + // End test case here if a decoding error was expected. + continue + } + + // Check that it encodes to the same string + encoded, err := Encode(hrp, decoded) + if err != nil { + t.Errorf("encoding failed: %v", err) + } + + if encoded != strings.ToLower(str) { + t.Errorf("expected data to encode to %v, but got %v", + str, encoded) + } + + // Flip a bit in the string an make sure it is caught. + pos := strings.LastIndexAny(str, "1") + flipped := str[:pos+1] + string((str[pos+1] ^ 1)) + str[pos+2:] + _, _, err = Decode(flipped) + if err == nil { + t.Error("expected decoding to fail") + } + } +} + +// TestMixedCaseEncode ensures mixed case HRPs are converted to lowercase as +// expected when encoding and that decoding the produced encoding when converted +// to all uppercase produces the lowercase HRP and original data. +func TestMixedCaseEncode(t *testing.T) { + tests := []struct { + name string + hrp string + data string + encoded string + }{{ + name: "all uppercase HRP with no data", + hrp: "A", + data: "", + encoded: "a12uel5l", + }, { + name: "all uppercase HRP with data", + hrp: "UPPERCASE", + data: "787878", + encoded: "uppercase10pu8sss7kmp", + }, { + name: "mixed case HRP even offsets uppercase", + hrp: "AbCdEf", + data: "00443214c74254b635cf84653a56d7c675be77df", + encoded: "abcdef1qpzry9x8gf2tvdw0s3jn54khce6mua7lmqqqxw", + }, { + name: "mixed case HRP odd offsets uppercase ", + hrp: "aBcDeF", + data: "00443214c74254b635cf84653a56d7c675be77df", + encoded: "abcdef1qpzry9x8gf2tvdw0s3jn54khce6mua7lmqqqxw", + }, { + name: "all lowercase HRP", + hrp: "abcdef", + data: "00443214c74254b635cf84653a56d7c675be77df", + encoded: "abcdef1qpzry9x8gf2tvdw0s3jn54khce6mua7lmqqqxw", + }} + + for _, test := range tests { + // Convert the text hex to bytes, convert those bytes from base256 to + // base32, then ensure the encoded result with the HRP provided in the + // test data is as expected. + data, err := hex.DecodeString(test.data) + if err != nil { + t.Errorf("%q: invalid hex %q: %v", test.name, test.data, err) + continue + } + convertedData, err := ConvertBits(data, 8, 5, true) + if err != nil { + t.Errorf("%q: unexpected convert bits error: %v", test.name, + err) + continue + } + gotEncoded, err := Encode(test.hrp, convertedData) + if err != nil { + t.Errorf("%q: unexpected encode error: %v", test.name, err) + continue + } + if gotEncoded != test.encoded { + t.Errorf("%q: mismatched encoding -- got %q, want %q", test.name, + gotEncoded, test.encoded) + continue + } + + // Ensure the decoding the expected lowercase encoding converted to all + // uppercase produces the lowercase HRP and original data. + gotHRP, gotData, err := Decode(strings.ToUpper(test.encoded)) + if err != nil { + t.Errorf("%q: unexpected decode error: %v", test.name, err) + continue + } + wantHRP := strings.ToLower(test.hrp) + if gotHRP != wantHRP { + t.Errorf("%q: mismatched decoded HRP -- got %q, want %q", test.name, + gotHRP, wantHRP) + continue + } + convertedGotData, err := ConvertBits(gotData, 5, 8, false) + if err != nil { + t.Errorf("%q: unexpected convert bits error: %v", test.name, + err) + continue + } + if !bytes.Equal(convertedGotData, data) { + t.Errorf("%q: mismatched data -- got %x, want %x", test.name, + convertedGotData, data) + continue + } + } +} + +// TestCanDecodeUnlimtedBech32 tests whether decoding a large bech32 string works +// when using the DecodeNoLimit version +func TestCanDecodeUnlimtedBech32(t *testing.T) { + input := "11qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqsqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq5kx0yd" + + // Sanity check that an input of this length errors on regular Decode() + _, _, err := Decode(input) + if err == nil { + t.Fatalf("Test vector not appropriate") + } + + // Try and decode it. + hrp, data, err := DecodeNoLimit(input) + if err != nil { + t.Fatalf("Expected decoding of large string to work. Got error: %v", err) + } + + // Verify data for correctness. + if hrp != "1" { + t.Fatalf("Unexpected hrp: %v", hrp) + } + decodedHex := fmt.Sprintf("%x", data) + expected := "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000" + if decodedHex != expected { + t.Fatalf("Unexpected decoded data: %s", decodedHex) + } +} + +// TestBech32Base256 ensures decoding and encoding various bech32, HRPs, and +// data produces the expected results when using EncodeFromBase256 and +// DecodeToBase256. It includes tests for proper handling of case +// manipulations. +func TestBech32Base256(t *testing.T) { + tests := []struct { + name string // test name + encoded string // bech32 string to decode + hrp string // expected human-readable part + data string // expected hex-encoded data + err error // expected error + }{{ + name: "all uppercase, no data", + encoded: "A12UEL5L", + hrp: "a", + data: "", + }, { + name: "long hrp with separator and excluded chars, no data", + encoded: "an83characterlonghumanreadablepartthatcontainsthenumber1andtheexcludedcharactersbio1tt5tgs", + hrp: "an83characterlonghumanreadablepartthatcontainsthenumber1andtheexcludedcharactersbio", + data: "", + }, { + name: "6 char hrp with data with leading zero", + encoded: "abcdef1qpzry9x8gf2tvdw0s3jn54khce6mua7lmqqqxw", + hrp: "abcdef", + data: "00443214c74254b635cf84653a56d7c675be77df", + }, { + name: "hrp same as separator and max length encoded string", + encoded: "11qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqc8247j", + hrp: "1", + data: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + }, { + name: "5 char hrp with data chosen to produce human-readable data part", + encoded: "split1checkupstagehandshakeupstreamerranterredcaperred2y9e3w", + hrp: "split", + data: "c5f38b70305f519bf66d85fb6cf03058f3dde463ecd7918f2dc743918f2d", + }, { + name: "same as previous but with checksum invalidated", + encoded: "split1checkupstagehandshakeupstreamerranterredcaperred2y9e2w", + err: ErrInvalidChecksum{"2y9e3w", "2y9e2w"}, + }, { + name: "hrp with invalid character (space)", + encoded: "s lit1checkupstagehandshakeupstreamerranterredcaperredp8hs2p", + err: ErrInvalidCharacter(' '), + }, { + name: "hrp with invalid character (DEL)", + encoded: "spl\x7ft1checkupstagehandshakeupstreamerranterredcaperred2y9e3w", + err: ErrInvalidCharacter(127), + }, { + name: "data part with invalid character (o)", + encoded: "split1cheo2y9e2w", + err: ErrNonCharsetChar('o'), + }, { + name: "data part too short", + encoded: "split1a2y9w", + err: ErrInvalidSeparatorIndex(5), + }, { + name: "empty hrp", + encoded: "1checkupstagehandshakeupstreamerranterredcaperred2y9e3w", + err: ErrInvalidSeparatorIndex(0), + }, { + name: "no separator", + encoded: "pzry9x0s0muk", + err: ErrInvalidSeparatorIndex(-1), + }, { + name: "too long by one char", + encoded: "11qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqsqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqc8247j", + err: ErrInvalidLength(91), + }, { + name: "invalid due to mixed case in hrp", + encoded: "aBcdef1qpzry9x8gf2tvdw0s3jn54khce6mua7lmqqqxw", + err: ErrMixedCase{}, + }, { + name: "invalid due to mixed case in data part", + encoded: "abcdef1Qpzry9x8gf2tvdw0s3jn54khce6mua7lmqqqxw", + err: ErrMixedCase{}, + }} + + for _, test := range tests { + // Ensure the decode either produces an error or not as expected. + str := test.encoded + gotHRP, gotData, err := DecodeToBase256(str) + if test.err != err { + t.Errorf("%q: unexpected decode error -- got %v, want %v", + test.name, err, test.err) + continue + } + if err != nil { + // End test case here if a decoding error was expected. + continue + } + + // Ensure the expected HRP and original data are as expected. + if gotHRP != test.hrp { + t.Errorf("%q: mismatched decoded HRP -- got %q, want %q", test.name, + gotHRP, test.hrp) + continue + } + data, err := hex.DecodeString(test.data) + if err != nil { + t.Errorf("%q: invalid hex %q: %v", test.name, test.data, err) + continue + } + if !bytes.Equal(gotData, data) { + t.Errorf("%q: mismatched data -- got %x, want %x", test.name, + gotData, data) + continue + } + + // Encode the same data with the HRP converted to all uppercase and + // ensure the result is the lowercase version of the original encoded + // bech32 string. + gotEncoded, err := EncodeFromBase256(strings.ToUpper(test.hrp), data) + if err != nil { + t.Errorf("%q: unexpected uppercase HRP encode error: %v", test.name, + err) + } + wantEncoded := strings.ToLower(str) + if gotEncoded != wantEncoded { + t.Errorf("%q: mismatched encoding -- got %q, want %q", test.name, + gotEncoded, wantEncoded) + } + + // Encode the same data with the HRP converted to all lowercase and + // ensure the result is the lowercase version of the original encoded + // bech32 string. + gotEncoded, err = EncodeFromBase256(strings.ToLower(test.hrp), data) + if err != nil { + t.Errorf("%q: unexpected lowercase HRP encode error: %v", test.name, + err) + } + if gotEncoded != wantEncoded { + t.Errorf("%q: mismatched encoding -- got %q, want %q", test.name, + gotEncoded, wantEncoded) + } + + // Encode the same data with the HRP converted to mixed upper and + // lowercase and ensure the result is the lowercase version of the + // original encoded bech32 string. + var mixedHRPBuilder strings.Builder + for i, r := range test.hrp { + if i%2 == 0 { + mixedHRPBuilder.WriteString(strings.ToUpper(string(r))) + continue + } + mixedHRPBuilder.WriteRune(r) + } + gotEncoded, err = EncodeFromBase256(mixedHRPBuilder.String(), data) + if err != nil { + t.Errorf("%q: unexpected lowercase HRP encode error: %v", test.name, + err) + } + if gotEncoded != wantEncoded { + t.Errorf("%q: mismatched encoding -- got %q, want %q", test.name, + gotEncoded, wantEncoded) + } + + // Ensure a bit flip in the string is caught. + pos := strings.LastIndexAny(test.encoded, "1") + flipped := str[:pos+1] + string((str[pos+1] ^ 1)) + str[pos+2:] + _, _, err = DecodeToBase256(flipped) + if err == nil { + t.Error("expected decoding to fail") + } + } +} + +// BenchmarkEncodeDecodeCycle performs a benchmark for a full encode/decode +// cycle of a bech32 string. It also reports the allocation count, which we +// expect to be 2 for a fully optimized cycle. +func BenchmarkEncodeDecodeCycle(b *testing.B) { + // Use a fixed, 49-byte raw data for testing. + inputData, err := hex.DecodeString("cbe6365ddbcda9a9915422c3f091c13f8c7b2f263b8d34067bd12c274408473fa764871c9dd51b1bb34873b3473b633ed1") + if err != nil { + b.Fatalf("failed to initialize input data: %v", err) + } + + // Convert this into a 79-byte, base 32 byte slice. + base32Input, err := ConvertBits(inputData, 8, 5, true) + if err != nil { + b.Fatalf("failed to convert input to 32 bits-per-element: %v", err) + } + + // Use a fixed hrp for the tests. This should generate an encoded bech32 + // string of size 90 (the maximum allowed by BIP-173). + hrp := "bc" + + // Begin the benchmark. Given that we test one roundtrip per iteration + // (that is, one Encode() and one Decode() operation), we expect at most + // 2 allocations per reported test op. + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + str, err := Encode(hrp, base32Input) + if err != nil { + b.Fatalf("failed to encode input: %v", err) + } + + _, _, err = Decode(str) + if err != nil { + b.Fatalf("failed to decode string: %v", err) + } + } +} + +// TestConvertBits tests whether base conversion works using TestConvertBits(). +func TestConvertBits(t *testing.T) { + tests := []struct { + input string + output string + fromBits uint8 + toBits uint8 + pad bool + }{ + // Trivial empty conversions. + {"", "", 8, 5, false}, + {"", "", 8, 5, true}, + {"", "", 5, 8, false}, + {"", "", 5, 8, true}, + + // Conversions of 0 value with/without padding. + {"00", "00", 8, 5, false}, + {"00", "0000", 8, 5, true}, + {"0000", "00", 5, 8, false}, + {"0000", "0000", 5, 8, true}, + + // Testing when conversion ends exactly at the byte edge. This makes + // both padded and unpadded versions the same. + {"0000000000", "0000000000000000", 8, 5, false}, + {"0000000000", "0000000000000000", 8, 5, true}, + {"0000000000000000", "0000000000", 5, 8, false}, + {"0000000000000000", "0000000000", 5, 8, true}, + + // Conversions of full byte sequences. + {"ffffff", "1f1f1f1f1e", 8, 5, true}, + {"1f1f1f1f1e", "ffffff", 5, 8, false}, + {"1f1f1f1f1e", "ffffff00", 5, 8, true}, + + // Sample random conversions. + {"c9ca", "190705", 8, 5, false}, + {"c9ca", "19070500", 8, 5, true}, + {"19070500", "c9ca", 5, 8, false}, + {"19070500", "c9ca00", 5, 8, true}, + + // Test cases tested on TestConvertBitsFailures with their corresponding + // fixes. + {"ff", "1f1c", 8, 5, true}, + {"1f1c10", "ff20", 5, 8, true}, -func TestEncodeAndDecode(t *testing.T) { - sum := sha256.Sum256([]byte("hello world\n")) - ss := "shasum" + // Large conversions. + { + "cbe6365ddbcda9a9915422c3f091c13f8c7b2f263b8d34067bd12c274408473fa764871c9dd51b1bb34873b3473b633ed1", + "190f13030c170e1b1916141a13040a14040b011f01040e01071e0607160b1906070e06130801131b1a0416020e110008081c1f1a0e19040703120e1d0a06181b160d0407070c1a07070d11131d1408", + 8, 5, true, + }, + { + "190f13030c170e1b1916141a13040a14040b011f01040e01071e0607160b1906070e06130801131b1a0416020e110008081c1f1a0e19040703120e1d0a06181b160d0407070c1a07070d11131d1408", + "cbe6365ddbcda9a9915422c3f091c13f8c7b2f263b8d34067bd12c274408473fa764871c9dd51b1bb34873b3473b633ed100", + 5, 8, true, + }, + } - bech, err := bech32.ConvertAndEncode(ss, sum[:]) - require.NoError(t, err) + for i, tc := range tests { + input, err := hex.DecodeString(tc.input) + if err != nil { + t.Fatalf("invalid test input data: %v", err) + } + + expected, err := hex.DecodeString(tc.output) + if err != nil { + t.Fatalf("invalid test output data: %v", err) + } + + actual, err := ConvertBits(input, tc.fromBits, tc.toBits, tc.pad) + if err != nil { + t.Fatalf("test case %d failed: %v", i, err) + } + + if !bytes.Equal(actual, expected) { + t.Fatalf("test case %d has wrong output; expected=%x actual=%x", + i, expected, actual) + } + } +} + +// TestConvertBitsFailures tests for the expected conversion failures of +// ConvertBits(). +func TestConvertBitsFailures(t *testing.T) { + tests := []struct { + input string + fromBits uint8 + toBits uint8 + pad bool + err error + }{ + // Not enough output bytes when not using padding. + {"ff", 8, 5, false, ErrInvalidIncompleteGroup{}}, + {"1f1c10", 5, 8, false, ErrInvalidIncompleteGroup{}}, + + // Unsupported bit conversions. + {"", 0, 5, false, ErrInvalidBitGroups{}}, + {"", 10, 5, false, ErrInvalidBitGroups{}}, + {"", 5, 0, false, ErrInvalidBitGroups{}}, + {"", 5, 10, false, ErrInvalidBitGroups{}}, + } + + for i, tc := range tests { + input, err := hex.DecodeString(tc.input) + if err != nil { + t.Fatalf("invalid test input data: %v", err) + } + + _, err = ConvertBits(input, tc.fromBits, tc.toBits, tc.pad) + if err != tc.err { + t.Fatalf("test case %d failure: expected '%v' got '%v'", i, + tc.err, err) + } + } + +} + +// BenchmarkConvertBitsDown benchmarks the speed and memory allocation behavior +// of ConvertBits when converting from a higher base into a lower base (e.g. 8 +// => 5). +// +// Only a single allocation is expected, which is used for the output array. +func BenchmarkConvertBitsDown(b *testing.B) { + // Use a fixed, 49-byte raw data for testing. + inputData, err := hex.DecodeString("cbe6365ddbcda9a9915422c3f091c13f8c7b2f263b8d34067bd12c274408473fa764871c9dd51b1bb34873b3473b633ed1") + if err != nil { + b.Fatalf("failed to initialize input data: %v", err) + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := ConvertBits(inputData, 8, 5, true) + if err != nil { + b.Fatalf("error converting bits: %v", err) + } + } +} - hrp, data, err := bech32.DecodeAndConvert(bech) - require.NoError(t, err) +// BenchmarkConvertBitsDown benchmarks the speed and memory allocation behavior +// of ConvertBits when converting from a lower base into a higher base (e.g. 5 +// => 8). +// +// Only a single allocation is expected, which is used for the output array. +func BenchmarkConvertBitsUp(b *testing.B) { + // Use a fixed, 79-byte raw data for testing. + inputData, err := hex.DecodeString("190f13030c170e1b1916141a13040a14040b011f01040e01071e0607160b1906070e06130801131b1a0416020e110008081c1f1a0e19040703120e1d0a06181b160d0407070c1a07070d11131d1408") + if err != nil { + b.Fatalf("failed to initialize input data: %v", err) + } - require.Equal(t, hrp, ss, "Invalid hrp") - require.True(t, bytes.Equal(data, sum[:]), "Invalid decode") + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := ConvertBits(inputData, 8, 5, true) + if err != nil { + b.Fatalf("error converting bits: %v", err) + } + } } diff --git a/types/bech32/convert.go b/types/bech32/convert.go new file mode 100644 index 000000000000..22a177663eae --- /dev/null +++ b/types/bech32/convert.go @@ -0,0 +1,32 @@ +package bech32 + +import ( + "fmt" + + "github.com/enigmampc/btcutil/bech32" +) + +// ConvertAndEncode converts from a base64 encoded byte string to base32 encoded byte string and then to bech32. +func ConvertAndEncode(hrp string, data []byte) (string, error) { + converted, err := bech32.ConvertBits(data, 8, 5, true) + if err != nil { + return "", fmt.Errorf("encoding bech32 failed: %w", err) + } + + return bech32.Encode(hrp, converted) +} + +// DecodeAndConvert decodes a bech32 encoded string and converts to base64 encoded bytes. +func DecodeAndConvert(bech string) (string, []byte, error) { + hrp, data, err := bech32.Decode(bech, 1023) + if err != nil { + return "", nil, fmt.Errorf("decoding bech32 failed: %w", err) + } + + converted, err := bech32.ConvertBits(data, 5, 8, false) + if err != nil { + return "", nil, fmt.Errorf("decoding bech32 failed: %w", err) + } + + return hrp, converted, nil +} diff --git a/types/bech32/convert_test.go b/types/bech32/convert_test.go new file mode 100644 index 000000000000..ba2beb783220 --- /dev/null +++ b/types/bech32/convert_test.go @@ -0,0 +1,25 @@ +package bech32_test + +import ( + "bytes" + "crypto/sha256" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cosmos/cosmos-sdk/types/bech32" +) + +func TestEncodeAndDecode(t *testing.T) { + sum := sha256.Sum256([]byte("hello world\n")) + ss := "shasum" + + bech, err := bech32.ConvertAndEncode(ss, sum[:]) + require.NoError(t, err) + + hrp, data, err := bech32.DecodeAndConvert(bech) + require.NoError(t, err) + + require.Equal(t, hrp, ss, "Invalid hrp") + require.True(t, bytes.Equal(data, sum[:]), "Invalid decode") +} diff --git a/types/bech32/error.go b/types/bech32/error.go new file mode 100644 index 000000000000..7940296a904a --- /dev/null +++ b/types/bech32/error.go @@ -0,0 +1,87 @@ +// Copyright (c) 2019 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +// source: https://github.com/btcsuite/btcutil/blob/e2ba6805/bech32/error.go + +package bech32 + +import ( + "fmt" +) + +// ErrMixedCase is returned when the bech32 string has both lower and uppercase +// characters. +type ErrMixedCase struct{} + +func (e ErrMixedCase) Error() string { + return "string not all lowercase or all uppercase" +} + +// ErrInvalidBitGroups is returned when conversion is attempted between byte +// slices using bit-per-element of unsupported value. +type ErrInvalidBitGroups struct{} + +func (e ErrInvalidBitGroups) Error() string { + return "only bit groups between 1 and 8 allowed" +} + +// ErrInvalidIncompleteGroup is returned when then byte slice used as input has +// data of wrong length. +type ErrInvalidIncompleteGroup struct{} + +func (e ErrInvalidIncompleteGroup) Error() string { + return "invalid incomplete group" +} + +// ErrInvalidLength is returned when the bech32 string has an invalid length +// given the BIP-173 defined restrictions. +type ErrInvalidLength int + +func (e ErrInvalidLength) Error() string { + return fmt.Sprintf("invalid bech32 string length %d", int(e)) +} + +// ErrInvalidCharacter is returned when the bech32 string has a character +// outside the range of the supported charset. +type ErrInvalidCharacter rune + +func (e ErrInvalidCharacter) Error() string { + return fmt.Sprintf("invalid character in string: '%c'", rune(e)) +} + +// ErrInvalidSeparatorIndex is returned when the separator character '1' is +// in an invalid position in the bech32 string. +type ErrInvalidSeparatorIndex int + +func (e ErrInvalidSeparatorIndex) Error() string { + return fmt.Sprintf("invalid separator index %d", int(e)) +} + +// ErrNonCharsetChar is returned when a character outside of the specific +// bech32 charset is used in the string. +type ErrNonCharsetChar rune + +func (e ErrNonCharsetChar) Error() string { + return fmt.Sprintf("invalid character not part of charset: %v", int(e)) +} + +// ErrInvalidChecksum is returned when the extracted checksum of the string +// is different than what was expected. +type ErrInvalidChecksum struct { + Expected string + Actual string +} + +func (e ErrInvalidChecksum) Error() string { + return fmt.Sprintf("invalid checksum (expected %v got %v)", + e.Expected, e.Actual) +} + +// ErrInvalidDataByte is returned when a byte outside the range required for +// conversion into a string was found. +type ErrInvalidDataByte byte + +func (e ErrInvalidDataByte) Error() string { + return fmt.Sprintf("invalid data byte: %v", byte(e)) +} From 4f289b22569beeb7ca44e08aab263857801ab93e Mon Sep 17 00:00:00 2001 From: Jeeyong Um Date: Fri, 10 Sep 2021 17:01:48 +0900 Subject: [PATCH 2/3] fix: Make the length limitation configurable on bech32 --- types/bech32/bech32.go | 12 ++++++++---- types/bech32/bech32_test.go | 10 +++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/types/bech32/bech32.go b/types/bech32/bech32.go index 05d044d83244..dd183ad905c6 100644 --- a/types/bech32/bech32.go +++ b/types/bech32/bech32.go @@ -11,6 +11,10 @@ import ( "strings" ) +// LengthLimitBIP173 is the maximum length of the bech32-encoded address +// defined by BIP-173. +const LengthLimitBIP173 = 90 + // charset is the set of characters used in the data section of bech32 strings. // Note that this is ordered, such that for a given charset[i], i is the binary // value of the character. @@ -239,9 +243,9 @@ func DecodeNoLimit(bech string) (string, []byte, error) { // // Note that the returned data is 5-bit (base32) encoded and the human-readable // part will be lowercase. -func Decode(bech string) (string, []byte, error) { - // The maximum allowed length for a bech32 string is 90. - if len(bech) > 90 { +func Decode(bech string, limit int) (string, []byte, error) { + // The length of the given string should not exceed the limit. + if len(bech) > limit { return "", nil, ErrInvalidLength(len(bech)) } @@ -368,7 +372,7 @@ func EncodeFromBase256(hrp string, data []byte) (string, error) { // human-readable part (HRP) and base32-encoded data, converts that data to a // base256-encoded byte slice and returns it along with the lowercase HRP. func DecodeToBase256(bech string) (string, []byte, error) { - hrp, data, err := Decode(bech) + hrp, data, err := Decode(bech, LengthLimitBIP173) if err != nil { return "", nil, err } diff --git a/types/bech32/bech32_test.go b/types/bech32/bech32_test.go index 9325dafdefe6..7462131f8b35 100644 --- a/types/bech32/bech32_test.go +++ b/types/bech32/bech32_test.go @@ -55,7 +55,7 @@ func TestBech32(t *testing.T) { for i, test := range tests { str := test.str - hrp, decoded, err := Decode(str) + hrp, decoded, err := Decode(str, LengthLimitBIP173) if test.expectedError != err { t.Errorf("%d: expected decoding error %v "+ "instead got %v", i, test.expectedError, err) @@ -81,7 +81,7 @@ func TestBech32(t *testing.T) { // Flip a bit in the string an make sure it is caught. pos := strings.LastIndexAny(str, "1") flipped := str[:pos+1] + string((str[pos+1] ^ 1)) + str[pos+2:] - _, _, err = Decode(flipped) + _, _, err = Decode(flipped, LengthLimitBIP173) if err == nil { t.Error("expected decoding to fail") } @@ -152,7 +152,7 @@ func TestMixedCaseEncode(t *testing.T) { // Ensure the decoding the expected lowercase encoding converted to all // uppercase produces the lowercase HRP and original data. - gotHRP, gotData, err := Decode(strings.ToUpper(test.encoded)) + gotHRP, gotData, err := Decode(strings.ToUpper(test.encoded), LengthLimitBIP173) if err != nil { t.Errorf("%q: unexpected decode error: %v", test.name, err) continue @@ -183,7 +183,7 @@ func TestCanDecodeUnlimtedBech32(t *testing.T) { input := "11qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqsqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq5kx0yd" // Sanity check that an input of this length errors on regular Decode() - _, _, err := Decode(input) + _, _, err := Decode(input, LengthLimitBIP173) if err == nil { t.Fatalf("Test vector not appropriate") } @@ -403,7 +403,7 @@ func BenchmarkEncodeDecodeCycle(b *testing.B) { b.Fatalf("failed to encode input: %v", err) } - _, _, err = Decode(str) + _, _, err = Decode(str, LengthLimitBIP173) if err != nil { b.Fatalf("failed to decode string: %v", err) } From 356f91847f0744014cfd299cc9ea1a8e17bf6c14 Mon Sep 17 00:00:00 2001 From: Jeeyong Um Date: Fri, 10 Sep 2021 19:21:35 +0900 Subject: [PATCH 3/3] fix: Fix lint errors in types/bech32 --- types/bech32/bech32.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/types/bech32/bech32.go b/types/bech32/bech32.go index dd183ad905c6..a1bde09e0bb8 100644 --- a/types/bech32/bech32.go +++ b/types/bech32/bech32.go @@ -304,7 +304,7 @@ func ConvertBits(data []byte, fromBits, toBits uint8, pad bool) ([]byte, error) for _, b := range data { // Discard unused bits. - b = b << (8 - fromBits) + b <<= (8 - fromBits) // How many bits remaining to extract from the input data. remFromBits := fromBits @@ -325,7 +325,7 @@ func ConvertBits(data []byte, fromBits, toBits uint8, pad bool) ([]byte, error) // Discard the bits we just extracted and get ready for // next iteration. - b = b << toExtract + b <<= toExtract remFromBits -= toExtract filledBits += toExtract @@ -341,7 +341,7 @@ func ConvertBits(data []byte, fromBits, toBits uint8, pad bool) ([]byte, error) // We pad any unfinished group if specified. if pad && filledBits > 0 { - nextByte = nextByte << (toBits - filledBits) + nextByte <<= (toBits - filledBits) regrouped = append(regrouped, nextByte) filledBits = 0 nextByte = 0