upload files to GCS

CHANGELOG.md

@@ -9,6 +9,8 @@
 * Ensure jobs option is greater than or equal to 1
 * Upload files to AWS S3
 * Upload files to a remote host with SFTP
+* Upload files to Google Cloud Storage (GCS)
+
 
 ## pg_back 2.0.1
 
@@ -18,6 +20,7 @@
 * Allow postgresql URIs as connection strings
 * Tell pg_dump and pg_dumpall never to prompt for a password
 
+
 ## pg_back 2.0.0
 
 * Full rewrite in Go

README.md

@@ -164,8 +164,8 @@ All files produced by a run can be uploaded to a remote location by setting the
 `s3`, `sftp` or `none`.
 
 When set to `s3`, files are uploaded to AWS S3. The `--s3-*` family of options
-can be used to tweak the access to the bucket. The `--s3-profile` option only reads
-credentials and basic configuration, s3 specific options are not used.
+can be used to tweak the access to the bucket. The `--s3-profile` option only
+reads credentials and basic configuration, s3 specific options are not used.
 
 When set to `sftp`, files are uploaded to a remote host using SFTP. The
 `--sftp-*` family of options can be used to setup the access to the host. The
@@ -176,6 +176,10 @@ is used to decrypt it and the password authentication method is not tried with
 the server. The only SSH authentication methods used are password and
 publickey. If an SSH agent is available, it is always used.
 
+When set to `gcs`, files are uploaded to Google Cloud Storage. The `--gcs-*`
+family of options can be used to setup access to the bucket. When `--gcs-keyfile`
+is empty, `GOOGLE_APPLICATION_CREDENTIALS` environment is used.
+
 The `--purge-remote` option can be set to `yes` to apply the same purge policy
 on the remote location as the local directory.
 

config.go

@@ -76,7 +76,7 @@ type options struct {
 	CipherPassphrase string
 	Decrypt          bool
 
-	Upload       string // values are none, s3, sftp
+	Upload       string // values are none, s3, sftp, gcs
 	PurgeRemote  bool
 	S3Region     string
 	S3Bucket     string
@@ -94,6 +94,10 @@ type options struct {
 	SFTPDirectory        string
 	SFTPIdentityFile     string // path to private key
 	SFTPIgnoreKnownHosts bool
+
+	GCSBucket          string
+	GCSEndPoint        string
+	GCSCredentialsFile string
 }
 
 func defaultOptions() options {
@@ -266,6 +270,10 @@ func parseCli(args []string) (options, []string, error) {
 	pflag.StringVar(&opts.SFTPIdentityFile, "sftp-identity", "", "Path to a private key")
 	SFTPIgnoreHostKey := pflag.String("sftp-ignore-hostkey", "no", "Check the target host key against local known hosts")
 
+	pflag.StringVar(&opts.GCSBucket, "gcs-bucket", "", "GCS bucket name")
+	pflag.StringVar(&opts.GCSEndPoint, "gcs-endpoint", "", "GCS endpoint URL")
+	pflag.StringVar(&opts.GCSCredentialsFile, "gcs-keyfile", "", "path to the GCS credentials file")
+
 	pflag.StringVarP(&opts.Host, "host", "h", "", "database server host or socket directory")
 	pflag.IntVarP(&opts.Port, "port", "p", 0, "database server port number")
 	pflag.StringVarP(&opts.Username, "username", "U", "", "connect as specified database user")
@@ -393,7 +401,7 @@ func parseCli(args []string) (options, []string, error) {
 	}
 
 	// Validate upload option
-	stores := []string{"none", "s3", "sftp"}
+	stores := []string{"none", "s3", "sftp", "gcs"}
 	if err := validateEnum(opts.Upload, stores); err != nil {
 		return opts, changed, fmt.Errorf("invalid value for --upload: %s", err)
 	}
@@ -486,6 +494,10 @@ func loadConfigurationFile(path string) (options, error) {
 	opts.SFTPIdentityFile = s.Key("sftp_identity").MustString("")
 	opts.SFTPIgnoreKnownHosts = s.Key("sftp_ignore_hostkey").MustBool(false)
 
+	opts.GCSBucket = s.Key("gcs_bucket").MustString("")
+	opts.GCSEndPoint = s.Key("gcs_endpoint").MustString("")
+	opts.GCSCredentialsFile = s.Key("gcs_keyfile").MustString("")
+
 	// Validate purge keep and time limit
 	keep, err := validatePurgeKeepValue(purgeKeep)
 	if err != nil {
@@ -517,7 +529,7 @@ func loadConfigurationFile(path string) (options, error) {
 	}
 
 	// Validate upload option
-	stores := []string{"none", "s3", "sftp"}
+	stores := []string{"none", "s3", "sftp", "gcs"}
 	if err := validateEnum(opts.Upload, stores); err != nil {
 		return opts, fmt.Errorf("invalid value for upload: %s", err)
 	}
@@ -728,6 +740,13 @@ func mergeCliAndConfigOptions(cliOpts options, configOpts options, onCli []strin
 		case "sftp-ignore-hostkey":
 			opts.SFTPIgnoreKnownHosts = cliOpts.SFTPIgnoreKnownHosts
 
+		case "gcs-bucket":
+			opts.GCSBucket = cliOpts.GCSBucket
+		case "gcs-endpoint":
+			opts.GCSEndPoint = cliOpts.GCSEndPoint
+		case "gcs-keyfile":
+			opts.GCSCredentialsFile = cliOpts.GCSCredentialsFile
+
 		case "host":
 			opts.Host = cliOpts.Host
 		case "port":

config_test.go

@@ -375,7 +375,7 @@ func TestParseCli(t *testing.T) {
 				},
 				false,
 				false,
-				"invalid value for --upload: value not found in [none s3 sftp]",
+				"invalid value for --upload: value not found in [none s3 sftp gcs]",
 				"",
 			},
 		}

go.mod

@@ -3,15 +3,17 @@ module github.com/orgrim/pg_back
 go 1.16
 
 require (
+	cloud.google.com/go/storage v1.18.2 // indirect
 	filippo.io/age v1.0.0 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
 	github.com/aws/aws-sdk-go v1.42.12 // indirect
-	github.com/google/go-cmp v0.5.4
+	github.com/google/go-cmp v0.5.6
 	github.com/jackc/pgconn v1.8.1 // indirect
 	github.com/jackc/pgtype v1.7.0
 	github.com/jackc/pgx/v4 v4.11.0
 	github.com/pkg/sftp v1.13.4 // indirect
 	github.com/spf13/pflag v1.0.5
 	golang.org/x/crypto v0.0.0-20211202192323-5770296d904e // indirect
+	google.golang.org/api v0.63.0 // indirect
 	gopkg.in/ini.v1 v1.62.0
 )