about fucking time this shit finally does something

.gitignore

@@ -48,3 +48,6 @@ dist/
 !schema.json
 *.jsonl
 *.state
+
+# sundry junk used for testing and other fuckery
+/tmp

cmd/slackdump/internal/dump/dump.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"os"
 	"runtime/trace"
 	"strings"
 	"text/template"
@@ -20,6 +21,7 @@ import (
 	"github.com/rusq/slackdump/v2/cmd/slackdump/internal/golang/base"
 	"github.com/rusq/slackdump/v2/internal/app/config"
 	"github.com/rusq/slackdump/v2/internal/app/nametmpl"
+	"github.com/rusq/slackdump/v2/internal/event/processor"
 	"github.com/rusq/slackdump/v2/internal/structures"
 	"github.com/rusq/slackdump/v2/types"
 )
@@ -108,6 +110,11 @@ func RunDump(ctx context.Context, cmd *base.Command, args []string) error {
 	defer sess.Close()
 
 	// Dump conversations.
+
+	return dumpv3(ctx, sess, list, namer)
+}
+
+func dumpv2(ctx context.Context, sess *slackdump.Session, list *structures.EntityList, namer namer) error {
 	for _, link := range list.Include {
 		conv, err := sess.Dump(ctx, link, opts.Oldest, opts.Latest)
 		if err != nil {
@@ -121,6 +128,32 @@ func RunDump(ctx context.Context, cmd *base.Command, args []string) error {
 	return nil
 }
 
+func dumpv3(ctx context.Context, sess *slackdump.Session, list *structures.EntityList, namer namer) error {
+	tmpdir, err := os.MkdirTemp("", "slackdump-*")
+	if err != nil {
+		return err
+	}
+	dlog.Printf("using %s as temporary directory", tmpdir)
+	f, err := os.CreateTemp(tmpdir, "events-*.jsonl")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	pr, err := processor.NewStandard(f, sess.Client(), tmpdir)
+	if err != nil {
+		return err
+	}
+	defer pr.Close()
+
+	for _, link := range list.Include {
+		if err := sess.Stream(ctx, link, pr, opts.Oldest, opts.Latest); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // namer is a helper type to generate filenames for conversations.
 type namer struct {
 	t   *template.Template

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/joho/godotenv v1.4.0
 	github.com/kr/pty v1.1.8
 	github.com/playwright-community/playwright-go v0.2000.1
+	github.com/rusq/asyncdl v0.1.1
 	github.com/rusq/chttp v1.0.1
 	github.com/rusq/dlog v1.3.3
 	github.com/rusq/encio v0.1.0

go.sum

@@ -113,6 +113,8 @@ github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.3.1 h1:SDPP7SHNl1L7KrEFCSJslJ/DM9DT02Nq2C61XrfHMmk=
 github.com/rivo/uniseg v0.3.1/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rusq/asyncdl v0.1.1 h1:DqatPlvpkcXtk8e4dtDP2ny5n9bdFuqEZqJcbTlIv7c=
+github.com/rusq/asyncdl v0.1.1/go.mod h1:PDZURyzHbk71cd/WuMr7x1EeZw+n95/+xTpduUU46r8=
 github.com/rusq/chttp v1.0.1 h1:j3WE7+jQE9Rgw0E6mGMje8HxMCv09QRkKvR0oZ1R2vY=
 github.com/rusq/chttp v1.0.1/go.mod h1:9H/mMp/iUc4xDkSOY0rL6ecxd/YyaW7zE9GhR+mZfRg=
 github.com/rusq/dlog v1.3.3 h1:Q9fZW1H/YEnlDg3Ph1k/BRSBfi/q5ezI+8Metws9tTI=

internal/event/processor/standard.go

@@ -1,38 +1,50 @@
 package processor
 
 import (
-	"os"
+	"context"
+	"io"
 
 	"github.com/rusq/fsadapter"
+	"github.com/rusq/slackdump/v2/downloader"
 	"github.com/rusq/slackdump/v2/internal/event"
 	"github.com/slack-go/slack"
 )
 
 type Standard struct {
 	*event.Recorder
-	channelID string
-	fs        fsadapter.FS
+	dl *downloader.Client
 }
 
-func NewStandard(channelID string, fs fsadapter.FS) (*Standard, error) {
-	f, err := os.CreateTemp("", "slackdump-"+channelID+"-*.jsonl")
-	if err != nil {
-		return nil, err
-	}
-	r := event.NewRecorder(f)
+func NewStandard(w io.Writer, sess downloader.Downloader, dir string) (*Standard, error) {
+	r := event.NewRecorder(w)
+	dl := downloader.New(sess, fsadapter.NewDirectory(dir))
+	dl.Start(context.Background())
 	return &Standard{
-		Recorder:  r,
-		channelID: channelID,
-		fs:        fs,
+		Recorder: r,
+		dl:       dl,
 	}, nil
 }
 
-func (s *Standard) Files(par *slack.Message, f []slack.File) error {
+func (s *Standard) Files(channelID string, parent slack.Message, isThread bool, m []slack.File) error {
 	// custom file processor, because we need to donwload those files
-	panic("implement me")
+	for i := range m {
+		if _, err := s.dl.DownloadFile(channelID, m[i]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func fileUrls(ff []slack.File) []string {
+	var urls = make([]string, 0, len(ff))
+	for i := range ff {
+		urls = append(urls, ff[i].URLPrivate)
+	}
+	return urls
 }
 
 func (s *Standard) Close() error {
 	// reconstruct the final json file
-	panic("implement me")
+	s.dl.Stop()
+	return nil
 }