where did that file go?

.gitignore

@@ -50,4 +50,5 @@ dist/
 *.state
 
 # sundry junk used for testing and other fuckery
-/tmp
+/tmp
+*.dot

cmd/slackdump/internal/dump/dump.go

@@ -150,30 +150,6 @@ func dumpv2(ctx context.Context, sess *slackdump.Session, list *structures.Entit
 	return nil
 }
 
-// namer is a helper type to generate filenames for conversations.
-type namer struct {
-	t   *template.Template
-	ext string
-}
-
-// newNamer returns a new namer.  It must be called with a valid template.
-func newNamer(tmpl string, ext string) (namer, error) {
-	t, err := template.New("name").Parse(tmpl)
-	if err != nil {
-		return namer{}, err
-	}
-	return namer{t: t, ext: ext}, nil
-}
-
-// Filename returns the filename for the given conversation.
-func (n namer) Filename(conv *types.Conversation) string {
-	var buf strings.Builder
-	if err := n.t.Execute(&buf, conv); err != nil {
-		panic(err)
-	}
-	return buf.String() + "." + n.ext
-}
-
 func save(ctx context.Context, fs fsadapter.FS, filename string, conv *types.Conversation) error {
 	_, task := trace.NewTask(ctx, "saveData")
 	defer task.End()

cmd/slackdump/internal/dump/namer.go

@@ -0,0 +1,32 @@
+package dump
+
+import (
+	"strings"
+	"text/template"
+
+	"github.com/rusq/slackdump/v2/types"
+)
+
+// namer is a helper type to generate filenames for conversations.
+type namer struct {
+	t   *template.Template
+	ext string
+}
+
+// newNamer returns a new namer.  It must be called with a valid template.
+func newNamer(tmpl string, ext string) (namer, error) {
+	t, err := template.New("name").Parse(tmpl)
+	if err != nil {
+		return namer{}, err
+	}
+	return namer{t: t, ext: ext}, nil
+}
+
+// Filename returns the filename for the given conversation.
+func (n namer) Filename(conv *types.Conversation) string {
+	var buf strings.Builder
+	if err := n.t.Execute(&buf, conv); err != nil {
+		panic(err)
+	}
+	return buf.String() + "." + n.ext
+}

internal/chunk/player.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"io"
 	"path/filepath"
+	"strings"
 	"sync/atomic"
 
 	"github.com/slack-go/slack"
@@ -157,6 +158,7 @@ func (p *Player) HasMoreThreads(channelID string, threadTS string) bool {
 	return p.hasMore(threadID(channelID, threadTS))
 }
 
+// Reset resets the state of the Player.
 func (p *Player) Reset() error {
 	p.pointer = make(offsets)
 	_, err := p.rs.Seek(0, io.SeekStart)
@@ -193,7 +195,8 @@ type namer interface {
 	Name() string
 }
 
-// State returns the state of the player.
+// State generates and returns the state of the player.  It does not include
+// the path to the downloaded files.
 func (p *Player) State() (*state.State, error) {
 	var name string
 	if file, ok := p.rs.(namer); ok {
@@ -206,6 +209,8 @@ func (p *Player) State() (*state.State, error) {
 		}
 		if ev.Type == CFiles {
 			for _, f := range ev.Files {
+				// we are adding the files with the empty path as we
+				// have no way of knowing if the file was downloaded or not.
 				s.AddFile(ev.ChannelID, f.ID, "")
 			}
 		}
@@ -225,3 +230,44 @@ func (p *Player) State() (*state.State, error) {
 	}
 	return s, nil
 }
+
+// allMessagesForID returns all the messages for the given id. It will reset
+// the Player prior to execution.
+func (p *Player) allMessagesForID(id string) ([]slack.Message, error) {
+	if err := p.Reset(); err != nil {
+		return nil, err
+	}
+	var m []slack.Message
+	for {
+		chunk, err := p.tryGetChunk(id)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		m = append(m, chunk.Messages...)
+	}
+	return m, nil
+}
+
+// AllMessages returns all the messages for the given channel.
+func (p *Player) AllMessages(channelID string) ([]slack.Message, error) {
+	return p.allMessagesForID(channelID)
+}
+
+// AllThreadMessages returns all the messages for the given thread.
+func (p *Player) AllThreadMessages(channelID, threadTS string) ([]slack.Message, error) {
+	return p.allMessagesForID(threadID(channelID, threadTS))
+}
+
+// AllChannels returns all the channels in the chunkfile.
+func (p *Player) AllChannels() []string {
+	var ids []string
+	for id := range p.idx {
+		if !strings.Contains(id, ":") {
+			ids = append(ids, id)
+		}
+	}
+	return ids
+}

internal/chunk/state/chunk.go

@@ -0,0 +1,68 @@
+package state
+
+import (
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// OpenChunks attempts to open the chunk file linked in the State. If the
+// chunk is compressed, it will be decompressed and a temporary file will be
+// created. The temporary file will be removed when the OpenChunks is
+// closed.
+func (st *State) OpenChunks(basePath string) (io.ReadSeekCloser, error) {
+	f, err := os.Open(filepath.Join(basePath, st.Filename))
+	if err != nil {
+		return nil, err
+	}
+	if st.IsCompressed {
+		tf, err := uncompress(f)
+		if err != nil {
+			return nil, err
+		}
+		return removeOnClose(tf.Name(), tf), nil
+	}
+	return f, nil
+}
+
+func removeOnClose(name string, r io.ReadSeekCloser) io.ReadSeekCloser {
+	return removeWrapper{filename: name, ReadSeekCloser: r}
+}
+
+type removeWrapper struct {
+	io.ReadSeekCloser
+
+	filename string
+}
+
+func (r removeWrapper) Close() error {
+	err := r.ReadSeekCloser.Close()
+	if err != nil {
+		return err
+	}
+	return os.Remove(r.filename)
+}
+
+// uncompress decompresses a gzip file and returns a temporary file handler.
+// it must be removed after use.
+func uncompress(r io.Reader) (*os.File, error) {
+	gr, err := gzip.NewReader(r)
+	if err != nil {
+		return nil, err
+	}
+	defer gr.Close()
+	f, err := os.CreateTemp("", "fsadapter-*")
+	if err != nil {
+		return nil, err
+	}
+	_, err = io.Copy(f, gr)
+	if err != nil {
+		return nil, err
+	}
+	// reset temporary file position to prepare it for reading.
+	if _, err := f.Seek(0, io.SeekStart); err != nil {
+		return nil, err
+	}
+	return f, nil
+}

internal/chunk/state/state.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"os"
 	"strconv"
+	"strings"
 	"sync"
 )
 
@@ -100,6 +101,28 @@ func (s *State) AddFile(channelID, fileID string, path string) {
 	s.Files[channelID+":"+fileID] = path
 }
 
+// AllFiles returns all saved files for the given channel.
+func (s *State) AllFiles(channelID string) []string {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	var files []string
+	for fileChanID, path := range s.Files {
+		id, _, _ := strings.Cut(fileChanID, ":")
+		if id == channelID {
+			files = append(files, path)
+		}
+	}
+	return files
+}
+
+func (s *State) FilePath(channelID, fileID string) string {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	return s.Files[channelID+":"+fileID]
+}
+
 // tsUpdate updates the map with the given ID and value if the value is greater.
 func tsUpdate(m map[string]int64, id string, val string) {
 	currVal, err := ts2int(val)

internal/osext/move.go

@@ -0,0 +1,54 @@
+package osext
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/rusq/fsadapter"
+)
+
+// MoveFile moves a file from src to dst.  If dst already exists, it will be
+// overwritten.
+//
+// Adopted solution from https://stackoverflow.com/questions/50740902/move-a-file-to-a-different-drive-with-go
+// TODO: This is a temporary solution.  We should use os.Rename() instead, but
+// that doesn't work across filesystems, see the above link.
+func MoveFile(src string, fs fsadapter.FS, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("unable to open source file: %s", err)
+	}
+
+	out, err := fs.Create(dst)
+	if err != nil {
+		in.Close()
+		return fmt.Errorf("unable to open destination file: %s", err)
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, in)
+	in.Close()
+	if err != nil {
+		return fmt.Errorf("error writing output: %s", err)
+	}
+
+	// sync is not supported by fsadapter.
+	// if err := out.Sync(); err != nil {
+	// 	return fmt.Errorf("sync: %s", err)
+	// }
+
+	if _, err := os.Stat(src); err != nil {
+		return fmt.Errorf("stat: %s", err)
+	} else {
+		// Chmod not yet supported.
+		// if err := fs.Chmod(dst, si.Mode()); err != nil {
+		// 	return fmt.Errorf("chmod: %s", err)
+		// }
+	}
+
+	if err := os.Remove(src); err != nil {
+		return fmt.Errorf("failed removing source: %s", err)
+	}
+	return nil
+}

internal/osext/osext.go

@@ -0,0 +1,2 @@
+// Package osext provides some extended functionality for the os package.
+package osext