diff --git a/.gitignore b/.gitignore index 38ba5594..88c355a0 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,6 @@ dist/ !schema.json *.jsonl *.state + +# sundry junk used for testing and other fuckery +/tmp \ No newline at end of file diff --git a/cmd/slackdump/internal/dump/dump.go b/cmd/slackdump/internal/dump/dump.go index 185dc278..572e008e 100644 --- a/cmd/slackdump/internal/dump/dump.go +++ b/cmd/slackdump/internal/dump/dump.go @@ -7,6 +7,7 @@ import ( "errors" "flag" "fmt" + "os" "runtime/trace" "strings" "text/template" @@ -20,6 +21,7 @@ import ( "github.com/rusq/slackdump/v2/cmd/slackdump/internal/golang/base" "github.com/rusq/slackdump/v2/internal/app/config" "github.com/rusq/slackdump/v2/internal/app/nametmpl" + "github.com/rusq/slackdump/v2/internal/event/processor" "github.com/rusq/slackdump/v2/internal/structures" "github.com/rusq/slackdump/v2/types" ) @@ -108,6 +110,11 @@ func RunDump(ctx context.Context, cmd *base.Command, args []string) error { defer sess.Close() // Dump conversations. + + return dumpv3(ctx, sess, list, namer) +} + +func dumpv2(ctx context.Context, sess *slackdump.Session, list *structures.EntityList, namer namer) error { for _, link := range list.Include { conv, err := sess.Dump(ctx, link, opts.Oldest, opts.Latest) if err != nil { @@ -121,6 +128,32 @@ func RunDump(ctx context.Context, cmd *base.Command, args []string) error { return nil } +func dumpv3(ctx context.Context, sess *slackdump.Session, list *structures.EntityList, namer namer) error { + tmpdir, err := os.MkdirTemp("", "slackdump-*") + if err != nil { + return err + } + dlog.Printf("using %s as temporary directory", tmpdir) + f, err := os.CreateTemp(tmpdir, "events-*.jsonl") + if err != nil { + return err + } + defer f.Close() + + pr, err := processor.NewStandard(f, sess.Client(), tmpdir) + if err != nil { + return err + } + defer pr.Close() + + for _, link := range list.Include { + if err := sess.Stream(ctx, link, pr, opts.Oldest, opts.Latest); err != nil { + return err + } + } + return nil +} + // namer is a helper type to generate filenames for conversations. type namer struct { t *template.Template diff --git a/go.mod b/go.mod index 97d01af9..9ad8b0a9 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/joho/godotenv v1.4.0 github.com/kr/pty v1.1.8 github.com/playwright-community/playwright-go v0.2000.1 + github.com/rusq/asyncdl v0.1.1 github.com/rusq/chttp v1.0.1 github.com/rusq/dlog v1.3.3 github.com/rusq/encio v0.1.0 diff --git a/go.sum b/go.sum index bcf29d68..85b0e91a 100644 --- a/go.sum +++ b/go.sum @@ -113,6 +113,8 @@ github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.3.1 h1:SDPP7SHNl1L7KrEFCSJslJ/DM9DT02Nq2C61XrfHMmk= github.com/rivo/uniseg v0.3.1/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rusq/asyncdl v0.1.1 h1:DqatPlvpkcXtk8e4dtDP2ny5n9bdFuqEZqJcbTlIv7c= +github.com/rusq/asyncdl v0.1.1/go.mod h1:PDZURyzHbk71cd/WuMr7x1EeZw+n95/+xTpduUU46r8= github.com/rusq/chttp v1.0.1 h1:j3WE7+jQE9Rgw0E6mGMje8HxMCv09QRkKvR0oZ1R2vY= github.com/rusq/chttp v1.0.1/go.mod h1:9H/mMp/iUc4xDkSOY0rL6ecxd/YyaW7zE9GhR+mZfRg= github.com/rusq/dlog v1.3.3 h1:Q9fZW1H/YEnlDg3Ph1k/BRSBfi/q5ezI+8Metws9tTI= diff --git a/internal/event/processor/standard.go b/internal/event/processor/standard.go index 752cc1e3..e3dbba97 100644 --- a/internal/event/processor/standard.go +++ b/internal/event/processor/standard.go @@ -1,38 +1,50 @@ package processor import ( - "os" + "context" + "io" "github.com/rusq/fsadapter" + "github.com/rusq/slackdump/v2/downloader" "github.com/rusq/slackdump/v2/internal/event" "github.com/slack-go/slack" ) type Standard struct { *event.Recorder - channelID string - fs fsadapter.FS + dl *downloader.Client } -func NewStandard(channelID string, fs fsadapter.FS) (*Standard, error) { - f, err := os.CreateTemp("", "slackdump-"+channelID+"-*.jsonl") - if err != nil { - return nil, err - } - r := event.NewRecorder(f) +func NewStandard(w io.Writer, sess downloader.Downloader, dir string) (*Standard, error) { + r := event.NewRecorder(w) + dl := downloader.New(sess, fsadapter.NewDirectory(dir)) + dl.Start(context.Background()) return &Standard{ - Recorder: r, - channelID: channelID, - fs: fs, + Recorder: r, + dl: dl, }, nil } -func (s *Standard) Files(par *slack.Message, f []slack.File) error { +func (s *Standard) Files(channelID string, parent slack.Message, isThread bool, m []slack.File) error { // custom file processor, because we need to donwload those files - panic("implement me") + for i := range m { + if _, err := s.dl.DownloadFile(channelID, m[i]); err != nil { + return err + } + } + return nil +} + +func fileUrls(ff []slack.File) []string { + var urls = make([]string, 0, len(ff)) + for i := range ff { + urls = append(urls, ff[i].URLPrivate) + } + return urls } func (s *Standard) Close() error { // reconstruct the final json file - panic("implement me") + s.dl.Stop() + return nil }