From a5fbef2aab7dd5840118a868954db8e123069948 Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Sun, 24 Nov 2024 19:56:45 +1000 Subject: [PATCH 1/2] fix mock generation and update tests --- .../internal/emoji/emojidl/emoji_mock_test.go | 28 +++--- .../internal/emoji/emojidl/emoji_test.go | 12 +-- .../workspaceui/test_mock_manager.go | 85 +++++++++++++++++++ .../workspace/workspaceui/workspaceui.go | 2 +- 4 files changed, 106 insertions(+), 21 deletions(-) create mode 100644 cmd/slackdump/internal/workspace/workspaceui/test_mock_manager.go diff --git a/cmd/slackdump/internal/emoji/emojidl/emoji_mock_test.go b/cmd/slackdump/internal/emoji/emojidl/emoji_mock_test.go index 17f964d9..1939897d 100644 --- a/cmd/slackdump/internal/emoji/emojidl/emoji_mock_test.go +++ b/cmd/slackdump/internal/emoji/emojidl/emoji_mock_test.go @@ -16,32 +16,32 @@ import ( gomock "go.uber.org/mock/gomock" ) -// Mockemojidumper is a mock of emojidumper interface. -type Mockemojidumper struct { +// MockEmojiDumper is a mock of EmojiDumper interface. +type MockEmojiDumper struct { ctrl *gomock.Controller - recorder *MockemojidumperMockRecorder + recorder *MockEmojiDumperMockRecorder isgomock struct{} } -// MockemojidumperMockRecorder is the mock recorder for Mockemojidumper. -type MockemojidumperMockRecorder struct { - mock *Mockemojidumper +// MockEmojiDumperMockRecorder is the mock recorder for MockEmojiDumper. +type MockEmojiDumperMockRecorder struct { + mock *MockEmojiDumper } -// NewMockemojidumper creates a new mock instance. -func NewMockemojidumper(ctrl *gomock.Controller) *Mockemojidumper { - mock := &Mockemojidumper{ctrl: ctrl} - mock.recorder = &MockemojidumperMockRecorder{mock} +// NewMockEmojiDumper creates a new mock instance. +func NewMockEmojiDumper(ctrl *gomock.Controller) *MockEmojiDumper { + mock := &MockEmojiDumper{ctrl: ctrl} + mock.recorder = &MockEmojiDumperMockRecorder{mock} return mock } // EXPECT returns an object that allows the caller to indicate expected use. -func (m *Mockemojidumper) EXPECT() *MockemojidumperMockRecorder { +func (m *MockEmojiDumper) EXPECT() *MockEmojiDumperMockRecorder { return m.recorder } // DumpEmojis mocks base method. -func (m *Mockemojidumper) DumpEmojis(ctx context.Context) (map[string]string, error) { +func (m *MockEmojiDumper) DumpEmojis(ctx context.Context) (map[string]string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DumpEmojis", ctx) ret0, _ := ret[0].(map[string]string) @@ -50,7 +50,7 @@ func (m *Mockemojidumper) DumpEmojis(ctx context.Context) (map[string]string, er } // DumpEmojis indicates an expected call of DumpEmojis. -func (mr *MockemojidumperMockRecorder) DumpEmojis(ctx any) *gomock.Call { +func (mr *MockEmojiDumperMockRecorder) DumpEmojis(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DumpEmojis", reflect.TypeOf((*Mockemojidumper)(nil).DumpEmojis), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DumpEmojis", reflect.TypeOf((*MockEmojiDumper)(nil).DumpEmojis), ctx) } diff --git a/cmd/slackdump/internal/emoji/emojidl/emoji_test.go b/cmd/slackdump/internal/emoji/emojidl/emoji_test.go index 7468a34b..c361e951 100644 --- a/cmd/slackdump/internal/emoji/emojidl/emoji_test.go +++ b/cmd/slackdump/internal/emoji/emojidl/emoji_test.go @@ -255,7 +255,7 @@ func Test_download(t *testing.T) { name string args args fetchFn fetchFunc - expect func(m *Mockemojidumper) + expect func(m *MockEmojiDumper) wantErr bool }{ { @@ -266,7 +266,7 @@ func Test_download(t *testing.T) { failFast: true, }, emptyFetchFn, - func(m *Mockemojidumper) { + func(m *MockEmojiDumper) { m.EXPECT(). DumpEmojis(gomock.Any()). Return(map[string]string{ @@ -283,7 +283,7 @@ func Test_download(t *testing.T) { failFast: true, }, emptyFetchFn, - func(m *Mockemojidumper) { + func(m *MockEmojiDumper) { m.EXPECT(). DumpEmojis(gomock.Any()). Return(map[string]string{ @@ -300,7 +300,7 @@ func Test_download(t *testing.T) { failFast: true, }, errorFetchFn, - func(m *Mockemojidumper) { + func(m *MockEmojiDumper) { m.EXPECT(). DumpEmojis(gomock.Any()). Return(map[string]string{ @@ -317,7 +317,7 @@ func Test_download(t *testing.T) { failFast: false, }, errorFetchFn, - func(m *Mockemojidumper) { + func(m *MockEmojiDumper) { m.EXPECT(). DumpEmojis(gomock.Any()). Return(nil, errors.New("no emojis for you, it's 1991.")) @@ -328,7 +328,7 @@ func Test_download(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { setGlobalFetchFn(tt.fetchFn) - sess := NewMockemojidumper(gomock.NewController(t)) + sess := NewMockEmojiDumper(gomock.NewController(t)) tt.expect(sess) fs, err := fsadapter.New(tt.args.output) if err != nil { diff --git a/cmd/slackdump/internal/workspace/workspaceui/test_mock_manager.go b/cmd/slackdump/internal/workspace/workspaceui/test_mock_manager.go new file mode 100644 index 00000000..b8b0ff6c --- /dev/null +++ b/cmd/slackdump/internal/workspace/workspaceui/test_mock_manager.go @@ -0,0 +1,85 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: workspaceui.go +// +// Generated by this command: +// +// mockgen -package workspaceui -destination=test_mock_manager.go -source workspaceui.go manager +// + +// Package workspaceui is a generated GoMock package. +package workspaceui + +import ( + context "context" + reflect "reflect" + + auth "github.com/rusq/slackdump/v3/auth" + gomock "go.uber.org/mock/gomock" +) + +// Mockmanager is a mock of manager interface. +type Mockmanager struct { + ctrl *gomock.Controller + recorder *MockmanagerMockRecorder + isgomock struct{} +} + +// MockmanagerMockRecorder is the mock recorder for Mockmanager. +type MockmanagerMockRecorder struct { + mock *Mockmanager +} + +// NewMockmanager creates a new mock instance. +func NewMockmanager(ctrl *gomock.Controller) *Mockmanager { + mock := &Mockmanager{ctrl: ctrl} + mock.recorder = &MockmanagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *Mockmanager) EXPECT() *MockmanagerMockRecorder { + return m.recorder +} + +// CreateAndSelect mocks base method. +func (m *Mockmanager) CreateAndSelect(ctx context.Context, p auth.Provider) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateAndSelect", ctx, p) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateAndSelect indicates an expected call of CreateAndSelect. +func (mr *MockmanagerMockRecorder) CreateAndSelect(ctx, p any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateAndSelect", reflect.TypeOf((*Mockmanager)(nil).CreateAndSelect), ctx, p) +} + +// Delete mocks base method. +func (m *Mockmanager) Delete(name string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", name) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockmanagerMockRecorder) Delete(name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*Mockmanager)(nil).Delete), name) +} + +// Select mocks base method. +func (m *Mockmanager) Select(name string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Select", name) + ret0, _ := ret[0].(error) + return ret0 +} + +// Select indicates an expected call of Select. +func (mr *MockmanagerMockRecorder) Select(name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Select", reflect.TypeOf((*Mockmanager)(nil).Select), name) +} diff --git a/cmd/slackdump/internal/workspace/workspaceui/workspaceui.go b/cmd/slackdump/internal/workspace/workspaceui/workspaceui.go index fa7c36b8..8d0f86e7 100644 --- a/cmd/slackdump/internal/workspace/workspaceui/workspaceui.go +++ b/cmd/slackdump/internal/workspace/workspaceui/workspaceui.go @@ -16,7 +16,7 @@ import ( "github.com/rusq/slackdump/v3/internal/cache" ) -//go:generate mockgen -package workspaceui -destination=test_mock_manager.go -source api.go manager +//go:generate mockgen -package workspaceui -destination=test_mock_manager.go -source workspaceui.go manager type manager interface { CreateAndSelect(ctx context.Context, p auth.Provider) (string, error) Select(name string) error From c66524c05e1505dc26b61fc8139e41be8146f878 Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Sun, 24 Nov 2024 20:37:20 +1000 Subject: [PATCH 2/2] implement #220 in v3 --- channels.go | 27 +++++++++++ channels_test.go | 94 ++++++++++++++++++++++++++++++++++++++ export/export.go | 49 ++++++++++++++++++-- export/future.go | 3 ++ export/mock_dumper_test.go | 15 ++++++ 5 files changed, 185 insertions(+), 3 deletions(-) diff --git a/channels.go b/channels.go index e7d01bcc..939ad7f7 100644 --- a/channels.go +++ b/channels.go @@ -97,3 +97,30 @@ func (s *Session) getChannels(ctx context.Context, chanTypes []string, cb func(t } return nil } + +// GetChannelMembers returns a list of all members in a channel. +func (sd *Session) GetChannelMembers(ctx context.Context, channelID string) ([]string, error) { + var ids []string + var cursor string + for { + var uu []string + var next string + if err := network.WithRetry(ctx, sd.limiter(network.Tier4), sd.cfg.limits.Tier4.Retries, func() error { + var err error + uu, next, err = sd.client.GetUsersInConversationContext(ctx, &slack.GetUsersInConversationParameters{ + ChannelID: channelID, + Cursor: cursor, + }) + return err + }); err != nil { + return nil, err + } + ids = append(ids, uu...) + + if next == "" { + break + } + cursor = next + } + return ids, nil +} diff --git a/channels_test.go b/channels_test.go index 4e092845..86ae1994 100644 --- a/channels_test.go +++ b/channels_test.go @@ -7,11 +7,13 @@ import ( "reflect" "testing" + "github.com/rusq/fsadapter" "github.com/rusq/slack" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" "github.com/rusq/slackdump/v3/internal/network" + "github.com/rusq/slackdump/v3/internal/structures" "github.com/rusq/slackdump/v3/types" ) @@ -136,3 +138,95 @@ func TestSession_GetChannels(t *testing.T) { }) } } + +func TestSession_GetChannelMembers(t *testing.T) { + type fields struct { + wspInfo *slack.AuthTestResponse + fs fsadapter.FS + Users types.Users + UserIndex structures.UserIndex + cfg config + } + type args struct { + ctx context.Context + channelID string + } + tests := []struct { + name string + fields fields + args args + expect func(mc *mockClienter) + want []string + wantErr bool + }{ + { + "ok, single call", + fields{cfg: defConfig}, + args{ + context.Background(), + "chanID", + }, + func(mc *mockClienter) { + mc.EXPECT().GetUsersInConversationContext(gomock.Any(), &slack.GetUsersInConversationParameters{ + ChannelID: "chanID", + }).Return([]string{"user1", "user2"}, "", nil) + }, + []string{"user1", "user2"}, + false, + }, + { + "ok, two calls", + fields{cfg: defConfig}, + args{ + context.Background(), + "chanID", + }, + func(mc *mockClienter) { + first := mc.EXPECT().GetUsersInConversationContext(gomock.Any(), &slack.GetUsersInConversationParameters{ + ChannelID: "chanID", + }).Return([]string{"user1", "user2"}, "cursor", nil).Times(1) + _ = mc.EXPECT().GetUsersInConversationContext(gomock.Any(), &slack.GetUsersInConversationParameters{ + ChannelID: "chanID", + Cursor: "cursor", + }).Return([]string{"user3"}, "", nil).After(first).Times(1) + }, + []string{"user1", "user2", "user3"}, + false, + }, + { + "error", + fields{cfg: defConfig}, + args{ + context.Background(), + "chanID", + }, + func(mc *mockClienter) { + mc.EXPECT().GetUsersInConversationContext(gomock.Any(), &slack.GetUsersInConversationParameters{ + ChannelID: "chanID", + }).Return([]string{}, "", errors.New("error fornicating corrugations")) + }, + nil, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mc := NewmockClienter(gomock.NewController(t)) + tt.expect(mc) + sd := &Session{ + client: mc, + wspInfo: tt.fields.wspInfo, + fs: tt.fields.fs, + cfg: tt.fields.cfg, + } + got, err := sd.GetChannelMembers(tt.args.ctx, tt.args.channelID) + if (err != nil) != tt.wantErr { + t.Errorf("Session.GetChannelMembers() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Session.GetChannelMembers() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/export/export.go b/export/export.go index cf561363..599ee5db 100644 --- a/export/export.go +++ b/export/export.go @@ -11,6 +11,7 @@ import ( "runtime/trace" "github.com/rusq/slack" + "golang.org/x/sync/errgroup" "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v3" @@ -128,10 +129,33 @@ func (se *Export) exclusiveExport(ctx context.Context, uidx structures.UserIndex se.lg.InfoContext(ctx, "skipping (excluded)", "channel_id", ch.ID) return nil } - if err := se.exportConversation(ctx, uidx, ch); err != nil { + + var eg errgroup.Group + + // 1. get members + var members []string + eg.Go(func() error { + var err error + members, err = se.sd.GetChannelMembers(ctx, ch.ID) + if err != nil { + return fmt.Errorf("error getting info for %s: %w", ch.ID, err) + } + return nil + }) + + // 2. export conversation + eg.Go(func() error { + if err := se.exportConversation(ctx, uidx, ch); err != nil { + return fmt.Errorf("error exporting conversation %s: %w", ch.ID, err) + } + return nil + }) + + // wait for both to finish + if err := eg.Wait(); err != nil { return err } - + ch.Members = members chans = append(chans, ch) return nil @@ -174,10 +198,29 @@ func (se *Export) inclusiveExport(ctx context.Context, uidx structures.UserIndex return nil, fmt.Errorf("error getting info for %s: %w", sl, err) } - if err := se.exportConversation(ctx, uidx, *ch); err != nil { + var eg errgroup.Group + var members []string + eg.Go(func() error { + var err error + members, err = se.sd.GetChannelMembers(ctx, ch.ID) + if err != nil { + return fmt.Errorf("error getting members for %s: %w", sl, err) + } + return nil + }) + + eg.Go(func() error { + if err := se.exportConversation(ctx, uidx, *ch); err != nil { + return fmt.Errorf("error exporting convesation %s: %w", ch.ID, err) + } + return nil + }) + + if err := eg.Wait(); err != nil { return nil, err } + ch.Members = members chans = append(chans, *ch) } diff --git a/export/future.go b/export/future.go index d222ba0d..7e155262 100644 --- a/export/future.go +++ b/export/future.go @@ -32,4 +32,7 @@ type dumper interface { // DumpRaw gets data from the Slack API and returns a Conversation object. DumpRaw(ctx context.Context, link string, oldest time.Time, latest time.Time, processFn ...slackdump.ProcessFunc) (*types.Conversation, error) + + // GetChannelMembers gets the list of members for a channel. + GetChannelMembers(ctx context.Context, channelID string) ([]string, error) } diff --git a/export/mock_dumper_test.go b/export/mock_dumper_test.go index 3082dd0e..5077ca99 100644 --- a/export/mock_dumper_test.go +++ b/export/mock_dumper_test.go @@ -92,6 +92,21 @@ func (mr *MockdumperMockRecorder) DumpRaw(ctx, link, oldest, latest any, process return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DumpRaw", reflect.TypeOf((*Mockdumper)(nil).DumpRaw), varargs...) } +// GetChannelMembers mocks base method. +func (m *Mockdumper) GetChannelMembers(ctx context.Context, channelID string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetChannelMembers", ctx, channelID) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetChannelMembers indicates an expected call of GetChannelMembers. +func (mr *MockdumperMockRecorder) GetChannelMembers(ctx, channelID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChannelMembers", reflect.TypeOf((*Mockdumper)(nil).GetChannelMembers), ctx, channelID) +} + // GetUsers mocks base method. func (m *Mockdumper) GetUsers(ctx context.Context) (types.Users, error) { m.ctrl.T.Helper()