add a light client database lock (#447)

Add a lock to prevent multiple processes from attempting to access the light client database at the same time. This typically resulted in unnecessary errors or even panics

relayer/tm-light-client.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sync"
 	"time"
 
 	retry "github.com/avast/retry-go"
@@ -23,11 +24,17 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
-// NOTE: currently we are discarding the very noisy light client logs
-// it would be nice if we could add a setting the chain or otherwise
-// that allowed users to enable light client logging. (maybe as a hidden prop
-// on the Chain struct that users could pass in the config??)
-var logger = light.Logger(log.NewTMLogger(log.NewSyncWriter(ioutil.Discard)))
+var (
+	// NOTE: currently we are discarding the very noisy light client logs
+	// it would be nice if we could add a setting the chain or otherwise
+	// that allowed users to enable light client logging. (maybe as a hidden prop
+	// on the Chain struct that users could pass in the config??)
+	logger = light.Logger(log.NewTMLogger(log.NewSyncWriter(ioutil.Discard)))
+
+	// a lock to prevent two processes from trying to access the light client
+	// database at the same time resulting in errors and panics.
+	lightDBMutex sync.Mutex
+)
 
 func lightError(err error) error { return fmt.Errorf("light client: %w", err) }
 
@@ -171,18 +178,19 @@ func (c *Chain) LightClient(db dbm.DB) (*light.Client, error) {
 // NewLightDB returns a new instance of the lightclient database connection
 // CONTRACT: must close the database connection when done with it (defer df())
 func (c *Chain) NewLightDB() (db *dbm.GoLevelDB, df func(), err error) {
-	if err := retry.Do(func() error {
-		db, err = dbm.NewGoLevelDB(c.ChainID, lightDir(c.HomePath))
-		if err != nil {
-			return fmt.Errorf("can't open light client database: %w", err)
-		}
-		return nil
-	}, rtyAtt, rtyDel, rtyErr); err != nil {
-		return nil, nil, err
+	// a lock is used to prevent error messages or panics from two processes
+	// trying to simultanenously use the light client
+	lightDBMutex.Lock()
+
+	db, err = dbm.NewGoLevelDB(c.ChainID, lightDir(c.HomePath))
+	if err != nil {
+		lightDBMutex.Unlock()
+		return nil, nil, fmt.Errorf("can't open light client database: %w", err)
 	}
 
 	df = func() {
 		err := db.Close()
+		lightDBMutex.Unlock()
 		if err != nil {
 			panic(err)
 		}