per CR discussion: genesis is always genesis.json

conduit/plugins/exporters/filewriter/README.md

@@ -2,6 +2,8 @@
 
 Write block data to files. This plugin works with the file rerader plugin to create a simple file-based pipeine.
 
+The genesis is always exported to a plain JSON file named `genesis.json` regardless of the `FilenamePattern`.
+
 ## Configuration
 
 ```yml @sample.yaml

conduit/plugins/exporters/filewriter/file_exporter.go

@@ -21,6 +21,8 @@ const (
 
 	// FilePattern is used to name the output files.
 	FilePattern = "%[1]d_block.msgp.gz"
+
+	GenesisFilename = "genesis.json"
 )
 
 type fileExporter struct {
@@ -72,15 +74,11 @@ func (exp *fileExporter) Init(_ context.Context, initProvider data.InitProvider,
 
 	exp.round = uint64(initProvider.NextDBRound())
 
-	// export the genesis as well in the same format
 	genesis := initProvider.GetGenesis()
-	genesisFile, err := GenesisFilename(exp.format, exp.gzip)
-	if err != nil {
-		return fmt.Errorf("Init() error: %w", err)
-	}
+	genesisPath := path.Join(exp.cfg.BlocksDir, GenesisFilename)
 
-	genesisPath := path.Join(exp.cfg.BlocksDir, genesisFile)
-	err = EncodeToFile(genesisPath, genesis, exp.format, exp.gzip)
+	// the genesis is always exported as plain JSON:
+	err = EncodeToFile(genesisPath, genesis, JSONFormat, false)
 	if err != nil {
 		return fmt.Errorf("Init() error sending to genesisPath=%s: %w", genesisPath, err)
 	}

conduit/plugins/exporters/filewriter/util.go

@@ -61,26 +61,6 @@ func ParseFilenamePattern(pattern string) (EncodingFormat, bool, error) {
 	return blockFormat, gzip, nil
 }
 
-// GenesisFilename returns the filename for a genesis file using a given format and possible gzip compression.
-func GenesisFilename(format EncodingFormat, isGzip bool) (string, error) {
-	var ext string
-
-	switch format {
-	case JSONFormat:
-		ext = ".json"
-	case MessagepackFormat:
-		ext = ".msgp"
-	default:
-		return "", fmt.Errorf("GenesisFilename(): unhandled format %d", format)
-	}
-
-	if isGzip {
-		ext += ".gz"
-	}
-
-	return fmt.Sprintf("genesis%s", ext), nil
-}
-
 // EncodeToFile enocods an object to a file using a given format and possible gzip compression.
 func EncodeToFile(filename string, v interface{}, format EncodingFormat, isGzip bool) error {
 	file, err := os.Create(filename)

conduit/plugins/exporters/filewriter/util_test.go

@@ -104,59 +104,6 @@ func TestParseFilenameFormat(t *testing.T) {
 	}
 }
 
-func TestGenesisFilename(t *testing.T) {
-	testCases := []struct {
-		blockFormat EncodingFormat
-		gzip        bool
-		result      string
-		err         string
-	}{
-		{
-			blockFormat: MessagepackFormat,
-			gzip:        false,
-			result:      "genesis.msgp",
-			err:         "",
-		},
-		{
-			blockFormat: MessagepackFormat,
-			gzip:        true,
-			result:      "genesis.msgp.gz",
-			err:         "",
-		},
-		{
-			blockFormat: JSONFormat,
-			gzip:        false,
-			result:      "genesis.json",
-			err:         "",
-		},
-		{
-			blockFormat: JSONFormat,
-			gzip:        true,
-			result:      "genesis.json.gz",
-			err:         "",
-		},
-		{
-			result:      "error case",
-			blockFormat: EncodingFormat(42),
-			err:         "GenesisFilename(): unhandled format 42",
-		},
-	}
-	for _, tc := range testCases {
-		tc := tc
-		t.Run(tc.result, func(t *testing.T) {
-			t.Parallel()
-
-			filename, err := GenesisFilename(tc.blockFormat, tc.gzip)
-			if tc.err == "" {
-				require.NoError(t, err)
-				require.Equal(t, tc.result, filename)
-			} else {
-				require.ErrorContains(t, err, tc.err)
-			}
-		})
-	}
-}
-
 func TestEncodeToAndFromFile(t *testing.T) {
 	tempdir := t.TempDir()
 

conduit/plugins/importers/filereader/README.md

@@ -2,6 +2,8 @@
 
 Read files from a directory and import them as blocks. This plugin works with the file exporter plugin to create a simple file-based pipeline.
 
+The genesis must be a plain JSON file named `genesis.json` regardless of the `FilenamePattern`.
+
 ## Configuration
 
 ```yml @sample.yaml

conduit/plugins/importers/filereader/fileReadWrite_test.go

@@ -50,6 +50,40 @@ func numGzippedFiles(t *testing.T) uint64 {
 	return gzCount
 }
 
+func fileBytes(t *testing.T, path string) []byte {
+	file, err := os.Open(path)
+	require.NoError(t, err, "error opening file %s", path)
+	defer file.Close()
+
+	var buf bytes.Buffer
+	_, err = buf.ReadFrom(file)
+	require.NoError(t, err, "error reading file %s", path)
+
+	return buf.Bytes()
+}
+
+func identicalFiles(t *testing.T, path1, path2 string) {
+	var file1, file2 *os.File
+
+	defer func() {
+		if file1 != nil {
+			file1.Close()
+		}
+		if file2 != nil {
+			file2.Close()
+		}
+	}()
+
+	bytes1 := fileBytes(t, path1)
+	bytes2 := fileBytes(t, path2)
+	require.Equal(t, len(bytes1), len(bytes2), "files %s and %s have different lengths", path1, path2)
+
+	for i, b1 := range bytes1 {
+		b2 := bytes2[i]
+		require.Equal(t, b1, b2, "files %s and %s differ at byte %d (%s) v (%s)", path1, path2, i, string(b1), string(b2))
+	}
+}
+
 func uncompressBytes(t *testing.T, path string) []byte {
 	file, err := os.Open(path)
 	require.NoError(t, err, "error opening file %s", path)
@@ -65,7 +99,6 @@ func uncompressBytes(t *testing.T, path string) []byte {
 
 	return buf.Bytes()
 }
-
 func identicalFilesUncompressed(t *testing.T, path1, path2 string) {
 	var file1, file2 *os.File
 
@@ -137,15 +170,15 @@ func TestRoundTrip(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, "generated-network", impGenesis.Network)
 
+	genesisFile := filewriter.GenesisFilename
 	// it should be the same as unmarshalling it directly from the expected path
-	genesisFile, err := filewriter.GenesisFilename(filewriter.MessagepackFormat, true)
-	require.Equal(t, "genesis.msgp.gz", genesisFile)
+	require.Equal(t, "genesis.json", genesisFile)
 	require.NoError(t, err)
 
 	impGenesisPath := path.Join(importerBlockDir, genesisFile)
 	genesis := &sdk.Genesis{}
 
-	err = filewriter.DecodeFromFile(impGenesisPath, genesis, filewriter.MessagepackFormat, true)
+	err = filewriter.DecodeFromFile(impGenesisPath, genesis, filewriter.JSONFormat, false)
 	require.NoError(t, err)
 
 	require.Equal(t, impGenesis, genesis)
@@ -166,7 +199,7 @@ func TestRoundTrip(t *testing.T) {
 	// It should have persisted the genesis which ought to be identical
 	// to the importer's.
 	expGenesisPath := path.Join(exporterBlockDir, genesisFile)
-	identicalFilesUncompressed(t, impGenesisPath, expGenesisPath)
+	identicalFiles(t, impGenesisPath, expGenesisPath)
 
 	// Simulate the pipeline
 	require.Equal(t, sdk.Round(0), round)

conduit/plugins/importers/filereader/filereader.go

@@ -74,20 +74,14 @@ func (r *fileReader) Init(ctx context.Context, _ data.InitProvider, cfg plugins.
 	return nil
 }
 
-// GetGenesis returns the genesis. Is is assumed that
-// the genesis file is available __in addition to__ the round 0 block file.
-// This is because the encoding assumed for the genesis is different
-// from the encoding assumed for blocks.
-// TODO: handle the case of a multipurpose file that contains both encodings.
+// GetGenesis returns the genesis. Is is assumed that the genesis file is available as `genesis.json`
+// regardless of chosen encoding format and gzip flag.
+// It is also assumed that there is a seperate round 0 block file adhering to the expected filename pattern with encoding.
+// This is because genesis and round 0 block have different data and encodings,
+// and the official network genesis files are plain uncompressed JSON.
 func (r *fileReader) GetGenesis() (*sdk.Genesis, error) {
-	genesisFile, err := filewriter.GenesisFilename(r.format, r.gzip)
-	if err != nil {
-		return nil, fmt.Errorf("GetGenesis(): failed to get genesis filename: %w", err)
-	}
-	genesisFile = path.Join(r.cfg.BlocksDir, genesisFile)
-
 	var genesis sdk.Genesis
-	err = filewriter.DecodeFromFile(genesisFile, &genesis, r.format, r.gzip)
+	err := filewriter.DecodeFromFile(path.Join(r.cfg.BlocksDir, filewriter.GenesisFilename), &genesis, filewriter.JSONFormat, false)
 	if err != nil {
 		return nil, fmt.Errorf("GetGenesis(): failed to process genesis file: %w", err)
 	}

conduit/plugins/importers/filereader/filereader_test.go

@@ -64,10 +64,9 @@ func initializeTestData(t *testing.T, dir string, numRounds int) sdk.Genesis {
 		Timestamp:   1234,
 	}
 
-	genesisFilename, err := filewriter.GenesisFilename(defaultEncodingFormat, defaultIsGzip)
-	require.NoError(t, err)
+	genesisFilename := filewriter.GenesisFilename
 
-	err = filewriter.EncodeToFile(path.Join(dir, genesisFilename), genesisA, defaultEncodingFormat, defaultIsGzip)
+	err := filewriter.EncodeToFile(path.Join(dir, genesisFilename), genesisA, filewriter.JSONFormat, false)
 	require.NoError(t, err)
 
 	for i := 0; i < numRounds; i++ {