Skip to content

Commit

Permalink
Add an option to download from remote locations
Browse files Browse the repository at this point in the history 
The --download option takes the same values as upload and downloads the
files from the remote repository matching the globs given on the command
line. Dumps are not run, only decryption is performed, after download is
successful, if --decrypt is also used.
  • Loading branch information
@orgrim
orgrim committed May 6, 2024
1 parent c4c53ac commit bf909b7
Show file tree
Hide file tree
Showing 5 changed files with 291 additions and 69 deletions.
16 changes: 15 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ options of the PostgreSQL instance.
* Purge based on age and number of dumps to keep
* Dump from a hot standby by pausing replication replay
* Encrypt and decrypt dumps and other files
* Upload dumps to S3, GCS, Azure or a remote host with SFTP
* Upload and download dumps to S3, GCS, Azure or a remote host with SFTP

## Install

Expand Down Expand Up @@ -213,6 +213,20 @@ on the remote location as the local directory.
When files are encrypted and their unencrypted source is kept, only encrypted
files are uploaded.

### Downloading from remote locations

Previously uploaded files can be downloaded using the `--download` option with
a value different than `none`, similarly to `--upload`. The options to setup
the remote access are the same as `--upload`.

When downloading files, dumps are not performed. Arguments on the commandline
(database names when dumping) are used as shell globs to choose which files to
the backup directory.

If `--download` is used at the same time as `--decrypt`, files are downloaded
first, then files matching globs are decrypted.


## Restoring files

The following files are created:
Expand Down
45 changes: 28 additions & 17 deletions config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ type options struct {
DumpOnly bool

Upload string // values are none, s3, sftp, gcs
Download string // values are none, s3, sftp, gcs
PurgeRemote bool
S3Region string
S3Bucket string
Expand Down Expand Up @@ -132,6 +133,7 @@ func defaultOptions() options {
TimeFormat: timeFormat,
WithRolePasswords: true,
Upload: "none",
Download: "none",
AzureEndpoint: "blob.core.windows.net",
}
}
Expand Down Expand Up @@ -284,6 +286,7 @@ func parseCli(args []string) (options, []string, error) {
pflag.StringVar(&opts.CipherPrivateKey, "cipher-private-key", "", "AGE private key for decryption; in Bech32 encoding starting with 'AGE-SECRET-KEY-1'\n")

pflag.StringVar(&opts.Upload, "upload", "none", "upload produced files to target (s3, gcs,..) use \"none\" to override\nconfiguration file and disable upload")
pflag.StringVar(&opts.Download, "download", "none", "download files from target (s3, gcs,..) instead of dumping. DBNAMEs become\nglobs to select files")
purgeRemote := pflag.String("purge-remote", "no", "purge the file on remote location after upload, with the same rules\nas the local directory")

pflag.StringVar(&opts.S3Region, "s3-region", "", "S3 region")
Expand Down Expand Up @@ -438,35 +441,41 @@ func parseCli(args []string) (options, []string, error) {
}
}

// Validate upload option
// Validate upload and download options
stores := []string{"none", "s3", "sftp", "gcs", "azure"}
if err := validateEnum(opts.Upload, stores); err != nil {
return opts, changed, fmt.Errorf("invalid value for --upload: %s", err)
}

if err := validateEnum(opts.Download, stores); err != nil {
return opts, changed, fmt.Errorf("invalid value for --download: %s", err)
}

opts.PurgeRemote, err = validateYesNoOption(*purgeRemote)
if err != nil {
return opts, changed, fmt.Errorf("invalid value for --purge-remote: %s", err)
}

switch opts.Upload {
case "s3":
// Validate S3 options
opts.S3ForcePath, err = validateYesNoOption(*S3ForcePath)
if err != nil {
return opts, changed, fmt.Errorf("invalid value for --s3-force-path: %s", err)
}
for _, o := range []string{opts.Upload, opts.Download} {
switch o {
case "s3":
// Validate S3 options
opts.S3ForcePath, err = validateYesNoOption(*S3ForcePath)
if err != nil {
return opts, changed, fmt.Errorf("invalid value for --s3-force-path: %s", err)
}

S3WithTLS, err := validateYesNoOption(*S3UseTLS)
if err != nil {
return opts, changed, fmt.Errorf("invalid value for --s3-tls: %s", err)
}
opts.S3DisableTLS = !S3WithTLS
S3WithTLS, err := validateYesNoOption(*S3UseTLS)
if err != nil {
return opts, changed, fmt.Errorf("invalid value for --s3-tls: %s", err)
}
opts.S3DisableTLS = !S3WithTLS

case "sftp":
opts.SFTPIgnoreKnownHosts, err = validateYesNoOption(*SFTPIgnoreHostKey)
if err != nil {
return opts, changed, fmt.Errorf("invalid value for --sftp-ignore-hostkey: %s", err)
case "sftp":
opts.SFTPIgnoreKnownHosts, err = validateYesNoOption(*SFTPIgnoreHostKey)
if err != nil {
return opts, changed, fmt.Errorf("invalid value for --sftp-ignore-hostkey: %s", err)
}
}
}

Expand Down Expand Up @@ -816,6 +825,8 @@ func mergeCliAndConfigOptions(cliOpts options, configOpts options, onCli []strin

case "upload":
opts.Upload = cliOpts.Upload
case "download":
opts.Download = cliOpts.Download
case "purge-remote":
opts.PurgeRemote = cliOpts.PurgeRemote

Expand Down
40 changes: 40 additions & 0 deletions config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ func TestDefaultOptions(t *testing.T) {
TimeFormat: timeFormat,
WithRolePasswords: true,
Upload: "none",
Download: "none",
AzureEndpoint: "blob.core.windows.net",
}

Expand Down Expand Up @@ -238,6 +239,7 @@ func TestParseCli(t *testing.T) {
TimeFormat: timeFormat,
WithRolePasswords: true,
Upload: "none",
Download: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand All @@ -262,6 +264,7 @@ func TestParseCli(t *testing.T) {
TimeFormat: timeFormat,
WithRolePasswords: true,
Upload: "none",
Download: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand Down Expand Up @@ -311,13 +314,40 @@ func TestParseCli(t *testing.T) {
CipherPassphrase: "testpass",
WithRolePasswords: true,
Upload: "wrong",
Download: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
false,
"invalid value for --upload: value not found in [none s3 sftp gcs azure]",
"",
},
{
[]string{"--download", "wrong"},
options{
Directory: "/var/backups/postgresql",
Format: 'c',
DirJobs: 1,
CompressLevel: -1,
Jobs: 1,
PauseTimeout: 3600,
PurgeInterval: -30 * 24 * time.Hour,
PurgeKeep: 0,
SumAlgo: "none",
CfgFile: "/etc/pg_back/pg_back.conf",
TimeFormat: timeFormat,
Encrypt: true,
CipherPassphrase: "testpass",
WithRolePasswords: true,
Upload: "none",
Download: "wrong",
AzureEndpoint: "blob.core.windows.net",
},
false,
false,
"invalid value for --download: value not found in [none s3 sftp gcs azure]",
"",
},
{
[]string{"--decrypt", "--encrypt"},
defaults,
Expand All @@ -344,6 +374,7 @@ func TestParseCli(t *testing.T) {
CipherPassphrase: "mypass",
WithRolePasswords: true,
Upload: "none",
Download: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand All @@ -369,6 +400,7 @@ func TestParseCli(t *testing.T) {
CipherPrivateKey: "mykey",
WithRolePasswords: true,
Upload: "none",
Download: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand All @@ -394,6 +426,7 @@ func TestParseCli(t *testing.T) {
CipherPublicKey: "fakepubkey",
WithRolePasswords: true,
Upload: "none",
Download: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand Down Expand Up @@ -498,6 +531,7 @@ func TestLoadConfigurationFile(t *testing.T) {
TimeFormat: timeFormat,
WithRolePasswords: true,
Upload: "none",
Download: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand All @@ -519,6 +553,7 @@ func TestLoadConfigurationFile(t *testing.T) {
TimeFormat: timeFormat,
WithRolePasswords: true,
Upload: "none",
Download: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand All @@ -539,6 +574,7 @@ func TestLoadConfigurationFile(t *testing.T) {
TimeFormat: timeFormat,
WithRolePasswords: true,
Upload: "none",
Download: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand All @@ -559,6 +595,7 @@ func TestLoadConfigurationFile(t *testing.T) {
TimeFormat: "2006-01-02_15-04-05",
WithRolePasswords: true,
Upload: "none",
Download: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand Down Expand Up @@ -608,6 +645,7 @@ func TestLoadConfigurationFile(t *testing.T) {
}},
WithRolePasswords: true,
Upload: "none",
Download: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand Down Expand Up @@ -649,6 +687,7 @@ func TestLoadConfigurationFile(t *testing.T) {
}},
WithRolePasswords: false,
Upload: "none",
Download: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand Down Expand Up @@ -715,6 +754,7 @@ func TestMergeCliAndConfigoptions(t *testing.T) {
TimeFormat: timeFormat,
WithRolePasswords: true,
Upload: "none",
Download: "none",
AzureEndpoint: "blob.core.windows.net",
}

Expand Down
Loading

0 comments on commit bf909b7

Please sign in to comment.