Skip to content

Commit

Permalink
Add an option to list files from remote locations
Browse files Browse the repository at this point in the history 
The --list-remote option allow to list remote files, showing all files
by default. The output can be filtered using globs as extra arguments on
the command line. When given, databases are not dumped, and it takes
precedence over --download.
  • Loading branch information
@orgrim
orgrim committed May 6, 2024
1 parent bf909b7 commit cfeb54d
Show file tree
Hide file tree
Showing 4 changed files with 93 additions and 18 deletions.
9 changes: 6 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,9 +219,12 @@ Previously uploaded files can be downloaded using the `--download` option with
a value different than `none`, similarly to `--upload`. The options to setup
the remote access are the same as `--upload`.

When downloading files, dumps are not performed. Arguments on the commandline
(database names when dumping) are used as shell globs to choose which files to
the backup directory.
It is possible to only list remote files with `--list-remote` with a value
different than `none`, similarly to `--upload` and `--download`.

When listing or downloading files, dumps are not performed. Arguments on the
commandline (database names when dumping) are used as shell globs to
select/filter files.

If `--download` is used at the same time as `--decrypt`, files are downloaded
first, then files matching globs are decrypted.
Expand Down
11 changes: 10 additions & 1 deletion config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ type options struct {

Upload string // values are none, s3, sftp, gcs
Download string // values are none, s3, sftp, gcs
ListRemote string // values are none, s3, sftp, gcs
PurgeRemote bool
S3Region string
S3Bucket string
Expand Down Expand Up @@ -134,6 +135,7 @@ func defaultOptions() options {
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
}
}
Expand Down Expand Up @@ -287,6 +289,7 @@ func parseCli(args []string) (options, []string, error) {

pflag.StringVar(&opts.Upload, "upload", "none", "upload produced files to target (s3, gcs,..) use \"none\" to override\nconfiguration file and disable upload")
pflag.StringVar(&opts.Download, "download", "none", "download files from target (s3, gcs,..) instead of dumping. DBNAMEs become\nglobs to select files")
pflag.StringVar(&opts.ListRemote, "list-remote", "none", "list the remote files on s3, gcs, sftp, azure instead of dumping. DBNAMEs become\nglobs to select files")
purgeRemote := pflag.String("purge-remote", "no", "purge the file on remote location after upload, with the same rules\nas the local directory")

pflag.StringVar(&opts.S3Region, "s3-region", "", "S3 region")
Expand Down Expand Up @@ -451,12 +454,16 @@ func parseCli(args []string) (options, []string, error) {
return opts, changed, fmt.Errorf("invalid value for --download: %s", err)
}

if err := validateEnum(opts.ListRemote, stores); err != nil {
return opts, changed, fmt.Errorf("invalid value for --list-remote: %s", err)
}

opts.PurgeRemote, err = validateYesNoOption(*purgeRemote)
if err != nil {
return opts, changed, fmt.Errorf("invalid value for --purge-remote: %s", err)
}

for _, o := range []string{opts.Upload, opts.Download} {
for _, o := range []string{opts.Upload, opts.Download, opts.ListRemote} {
switch o {
case "s3":
// Validate S3 options
Expand Down Expand Up @@ -827,6 +834,8 @@ func mergeCliAndConfigOptions(cliOpts options, configOpts options, onCli []strin
opts.Upload = cliOpts.Upload
case "download":
opts.Download = cliOpts.Download
case "list-remote":
opts.ListRemote = cliOpts.ListRemote
case "purge-remote":
opts.PurgeRemote = cliOpts.PurgeRemote

Expand Down
15 changes: 15 additions & 0 deletions config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ func TestDefaultOptions(t *testing.T) {
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
}

Expand Down Expand Up @@ -240,6 +241,7 @@ func TestParseCli(t *testing.T) {
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand All @@ -265,6 +267,7 @@ func TestParseCli(t *testing.T) {
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand Down Expand Up @@ -315,6 +318,7 @@ func TestParseCli(t *testing.T) {
WithRolePasswords: true,
Upload: "wrong",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand All @@ -341,6 +345,7 @@ func TestParseCli(t *testing.T) {
WithRolePasswords: true,
Upload: "none",
Download: "wrong",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand Down Expand Up @@ -375,6 +380,7 @@ func TestParseCli(t *testing.T) {
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand All @@ -401,6 +407,7 @@ func TestParseCli(t *testing.T) {
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand All @@ -427,6 +434,7 @@ func TestParseCli(t *testing.T) {
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand Down Expand Up @@ -532,6 +540,7 @@ func TestLoadConfigurationFile(t *testing.T) {
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand All @@ -554,6 +563,7 @@ func TestLoadConfigurationFile(t *testing.T) {
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand All @@ -575,6 +585,7 @@ func TestLoadConfigurationFile(t *testing.T) {
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand All @@ -596,6 +607,7 @@ func TestLoadConfigurationFile(t *testing.T) {
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand Down Expand Up @@ -646,6 +658,7 @@ func TestLoadConfigurationFile(t *testing.T) {
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand Down Expand Up @@ -688,6 +701,7 @@ func TestLoadConfigurationFile(t *testing.T) {
WithRolePasswords: false,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand Down Expand Up @@ -755,6 +769,7 @@ func TestMergeCliAndConfigoptions(t *testing.T) {
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
}

Expand Down
76 changes: 62 additions & 14 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,32 +173,42 @@ func run() (retVal error) {
return fmt.Errorf("required cipher parameters not present: %w", err)
}

if (opts.Upload == "s3" || opts.Download == "s3") && opts.S3Bucket == "" {
if (opts.Upload == "s3" || opts.Download == "s3" || opts.ListRemote == "s3") && opts.S3Bucket == "" {
return fmt.Errorf("a bucket is mandatory with s3")
}

if (opts.Upload == "gcs" || opts.Download == "gcs") && opts.GCSBucket == "" {
if (opts.Upload == "gcs" || opts.Download == "gcs" || opts.ListRemote == "gcs") && opts.GCSBucket == "" {
return fmt.Errorf("a bucket is mandatory with gcs")
}

if (opts.Upload == "azure" || opts.Download == "azure") && opts.AzureContainer == "" {
if (opts.Upload == "azure" || opts.Download == "azure" || opts.ListRemote == "azure") && opts.AzureContainer == "" {
return fmt.Errorf("a container is mandatory with azure")
}

// Run actions that won't dump databases first, in that case the list
// of databases become file globs. Avoid getting wrong globs from the
// config file since we are using the remaining args from the command
// line that are usually as a list of databases to dump
globs := []string{}
for _, v := range cliOptList {
if v == "include-dbs" {
globs = opts.Dbnames
break
}
}

// Listing remote files take priority over the other options that won't dump databases
if opts.ListRemote != "none" {
if err := listRemoteFiles(opts.ListRemote, opts, globs); err != nil {
return err
}

return nil
}

// When asked to download or decrypt the backups, do it here and exit, we have all
// required input (passphrase and backup directory)
if opts.Decrypt || opts.Download != "none" {
// Avoid getting wrong globs from the config file since we are
// using the remaining args from the command line that are
// usually as a list of databases to dump
globs := []string{}
for _, v := range cliOptList {
if v == "include-dbs" {
globs = opts.Dbnames
break
}
}

if opts.Download != "none" {
if err := downloadFiles(opts.Download, opts, opts.Directory, globs); err != nil {
return err
Expand Down Expand Up @@ -1004,6 +1014,44 @@ func dumpConfigFiles(dir string, timeFormat string, db *pg, fc chan<- sumFileJob
return nil
}

func listRemoteFiles(repoName string, opts options, globs []string) error {
repo, err := NewRepo(repoName, opts)
if err != nil {
return err
}

remoteFiles, err := repo.List("")
if err != nil {
return fmt.Errorf("could not list contents of remote location: %w", err)
}

for _, i := range remoteFiles {
keep := false
if len(globs) == 0 {
keep = true
}

for _, glob := range globs {
keep, err = filepath.Match(glob, i.key)
if err != nil {
return fmt.Errorf("bad patern: %w", err)
}

if keep {
break
}
}

if !keep {
continue
}

fmt.Println(i.key)
}

return nil
}

func downloadFiles(repoName string, opts options, dir string, globs []string) error {
repo, err := NewRepo(repoName, opts)
if err != nil {
Expand Down

0 comments on commit cfeb54d

Please sign in to comment.