From 2aacd2e4fc276385a71c45d0b125051b9d0ef7a4 Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Wed, 20 Mar 2024 22:22:49 +1000 Subject: [PATCH] add DMs, but nothing works --- auth/auth.go | 13 +++-- cmd/slackdump/internal/diag/edge.go | 33 +++++------ internal/cache/auth.go | 2 +- internal/cache/auth_test.go | 7 ++- internal/chunk/chunktest/auth.go | 19 +++--- internal/edge/api.go | 7 +++ internal/edge/dms.go | 89 +++++++++++++++++++++++++++++ internal/edge/edge.go | 13 ++++- internal/edge/search.go | 51 +++++++++-------- 9 files changed, 171 insertions(+), 63 deletions(-) create mode 100644 internal/edge/dms.go diff --git a/auth/auth.go b/auth/auth.go index 4eeb8a6c..8caf382c 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -31,7 +31,7 @@ type Provider interface { // retrieved. Validate() error // Test tests if credentials are valid. - Test(ctx context.Context) error + Test(ctx context.Context) (*slack.AuthTestResponse, error) // Client returns an authenticated HTTP client HTTPClient() (*http.Client, error) } @@ -107,22 +107,23 @@ func IsClientToken(tok string) bool { // TestAuth attempts to authenticate with the given provider. It will return // AuthError if failed. -func (s simpleProvider) Test(ctx context.Context) error { +func (s simpleProvider) Test(ctx context.Context) (*slack.AuthTestResponse, error) { ctx, task := trace.NewTask(ctx, "TestAuth") defer task.End() httpCl, err := s.HTTPClient() if err != nil { - return &Error{Err: err} + return nil, &Error{Err: err} } cl := slack.New(s.Token, slack.OptionHTTPClient(httpCl)) region := trace.StartRegion(ctx, "simpleProvider.Test") defer region.End() - if _, err := cl.AuthTestContext(ctx); err != nil { - return &Error{Err: err} + ai, err := cl.AuthTestContext(ctx) + if err != nil { + return ai, &Error{Err: err} } - return nil + return ai, nil } func (s simpleProvider) HTTPClient() (*http.Client, error) { diff --git a/cmd/slackdump/internal/diag/edge.go b/cmd/slackdump/internal/diag/edge.go index 06351adc..0249bdf4 100644 --- a/cmd/slackdump/internal/diag/edge.go +++ b/cmd/slackdump/internal/diag/edge.go @@ -3,7 +3,6 @@ package diag import ( "context" "encoding/json" - "fmt" "os" "github.com/rusq/slackdump/v3" @@ -43,6 +42,11 @@ func runEdge(ctx context.Context, cmd *base.Command, args []string) error { base.SetExitStatus(base.SAuthError) return err } + ai, err := prov.Test(ctx) + if err != nil { + return err + } + lg.Printf("auth test: %+v", ai) lg.Print("connected") sd, err := slackdump.New(ctx, prov) @@ -50,28 +54,12 @@ func runEdge(ctx context.Context, cmd *base.Command, args []string) error { base.SetExitStatus(base.SAuthError) return err } - cl, err := edge.New(cfg.Workspace, sd.Info().TeamID, prov.SlackToken(), prov.Cookies()) + + cl, err := edge.NewWithProvider(cfg.Workspace, sd.Info().TeamID, prov) if err != nil { base.SetExitStatus(base.SApplicationError) return err } - req := edge.UsersListRequest{ - Channels: []string{edgeParams.channel}, - Filter: "everyone AND NOT bots AND NOT apps", - Count: 20, - } - resp, err := cl.Post(ctx, "/users/list", &req) - if err != nil { - return err - } - var ur edge.UsersListResponse - if err := cl.ParseResponse(&ur, resp); err != nil { - return err - } - if !ur.Ok { - return fmt.Errorf("error: %s", ur.Error) - } - lg.Printf("%+v\n", ur) lg.Printf("*** Search for Channels test ***") channels, err := cl.SearchChannels(ctx, "") @@ -82,5 +70,12 @@ func runEdge(ctx context.Context, cmd *base.Command, args []string) error { enc.SetIndent("", " ") enc.Encode(channels) + lg.Printf("*** DMs test ***") + dms, err := cl.DMs(ctx) + if err != nil { + return err + } + enc.Encode(dms) + return nil } diff --git a/internal/cache/auth.go b/internal/cache/auth.go index 942a5290..ca08a569 100644 --- a/internal/cache/auth.go +++ b/internal/cache/auth.go @@ -193,7 +193,7 @@ func tryLoad(ctx context.Context, filename string) (auth.Provider, error) { return nil, err } // test the loaded credentials - if err := authTester(prov, ctx); err != nil { + if _, err := authTester(prov, ctx); err != nil { return nil, err } return prov, nil diff --git a/internal/cache/auth_test.go b/internal/cache/auth_test.go index e7e864ae..be81a0d5 100644 --- a/internal/cache/auth_test.go +++ b/internal/cache/auth_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" + "github.com/rusq/slack" "github.com/rusq/slackdump/v3/auth" "github.com/rusq/slackdump/v3/internal/fixtures" "github.com/rusq/slackdump/v3/internal/mocks/mock_appauth" @@ -240,9 +241,9 @@ func TestInitProvider(t *testing.T) { } } -func fakeAuthTester(err error) func(_ auth.Provider, ctx context.Context) error { - return func(_ auth.Provider, ctx context.Context) error { - return err +func fakeAuthTester(err error) func(_ auth.Provider, ctx context.Context) (*slack.AuthTestResponse, error) { + return func(_ auth.Provider, ctx context.Context) (*slack.AuthTestResponse, error) { + return nil, err } } diff --git a/internal/chunk/chunktest/auth.go b/internal/chunk/chunktest/auth.go index 90974cce..8d36b90a 100644 --- a/internal/chunk/chunktest/auth.go +++ b/internal/chunk/chunktest/auth.go @@ -3,16 +3,19 @@ package chunktest import ( "context" "net/http" + + "github.com/rusq/slack" ) // TestAuth to use with the chunktest server. type TestAuth struct { - FakeToken string - FakeCookies []*http.Cookie - WantValidateError error - WantTestError error - WantHTTPClient *http.Client - WantHTTPClientErr error + FakeToken string + FakeCookies []*http.Cookie + WantValidateError error + WantTestError error + WantHTTPClient *http.Client + WantHTTPClientErr error + WantAuthTestResponse *slack.AuthTestResponse } // SlackToken should return the Slack Token value. @@ -33,8 +36,8 @@ func (a *TestAuth) Validate() error { } // Test tests if credentials are valid. -func (a *TestAuth) Test(ctx context.Context) error { - return a.WantTestError +func (a *TestAuth) Test(ctx context.Context) (*slack.AuthTestResponse, error) { + return a.WantAuthTestResponse, a.WantTestError } // HTTPClient returns an authenticated HTTP client diff --git a/internal/edge/api.go b/internal/edge/api.go index 1ccc5e3d..1e5b0684 100644 --- a/internal/edge/api.go +++ b/internal/edge/api.go @@ -110,6 +110,13 @@ type UserMembershipResponse struct { BaseResponse } +type WebClientFields struct { + XReason string `json:"_x_reason"` + XMode string `json:"_x_mode"` + XSonic bool `json:"_x_sonic"` + XAppName string `json:"_x_app_name"` +} + var ErrNotOK = errors.New("server returned NOT OK") // GetUsers returns users from the slack edge api for the channel. User IDs diff --git a/internal/edge/dms.go b/internal/edge/dms.go new file mode 100644 index 00000000..5d56b00d --- /dev/null +++ b/internal/edge/dms.go @@ -0,0 +1,89 @@ +package edge + +import ( + "context" + "log/slog" + "net/url" + "time" +) + +type dmsForm struct { + Token string `json:"token"` + Count int `json:"count"` + IncludeClosed bool `json:"include_closed"` + IncludeChannel bool `json:"include_channel"` + ExcludeBots bool `json:"exclude_bots"` + Cursor string `json:"cursor,omitempty"` + WebClientFields +} + +func (d dmsForm) Values() url.Values { + return values(d, true) +} + +type dmsResponse struct { + BaseResponse + IMs []DM `json:"ims,omitempty"` + MPIMs []DM `json:"mpims,omitempty"` //TODO + +} + +type DM struct { + ID string `json:"id"` + // Message slack.Message `json:"message,omitempty"` + Channel Channel `json:"channel,omitempty"` + Latest string `json:"latest,omitempty"` // i.e. "1710632873.037269" +} + +type Channel struct { + ID string `json:"id"` + Created int64 `json:"created"` + IsFrozen bool `json:"is_frozen"` + IsArchived bool `json:"is_archived"` + IsIM bool `json:"is_im"` + IsOrgShared bool `json:"is_org_shared"` + ContextTeamID string `json:"context_team_id"` + Updated int64 `json:"updated"` + User string `json:"user"` + LastRead string `json:"last_read"` + Latest string `json:"latest"` + IsOpen bool `json:"is_open"` +} + +func (cl *Client) DMs(ctx context.Context) ([]DM, error) { + form := dmsForm{ + Token: cl.token, + Count: 250, + IncludeClosed: true, + IncludeChannel: true, + ExcludeBots: false, + Cursor: "", + WebClientFields: WebClientFields{ + XReason: "dms-tab-populate", + XMode: "online", + XSonic: true, + XAppName: "client", + }, + } + + var IMs []DM + var url = cl.webapiURL("client.dms") + slog.Info("url", "url", url) + for range 3 { + resp, err := cl.PostFormRaw(ctx, url, form.Values()) + if err != nil { + return nil, err + } + r := dmsResponse{} + if err := cl.ParseResponse(&r, resp); err != nil { + return nil, err + } + if r.ResponseMetadata.NextCursor == "" { + break + } + IMs = append(IMs, r.IMs...) + time.Sleep(300 * time.Millisecond) //TODO: hax + form.Cursor = r.ResponseMetadata.NextCursor + } + return IMs, nil +} diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 35944cc4..8e3721b0 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -7,6 +7,8 @@ import ( "context" "encoding/json" "fmt" + "io" + "log/slog" "net/http" "net/url" "strings" @@ -36,6 +38,7 @@ func NewWithClient(workspaceName string, teamID string, token string, cl *http.C workspaceName: workspaceName, cl: cl, token: token, + teamID: teamID, apiPath: fmt.Sprintf("https://edgeapi.slack.com/cache/%s/", teamID), }, nil } @@ -71,7 +74,8 @@ type BaseResponse struct { } type ResponseMetadata struct { - Messages []string `json:"messages,omitempty"` + Messages []string `json:"messages,omitempty"` + NextCursor string `json:"next_cursor,omitempty"` } func (r *BaseRequest) SetToken(token string) { @@ -105,7 +109,12 @@ func (cl *Client) Post(ctx context.Context, path string, req PostRequest) (*http func (cl *Client) ParseResponse(req any, resp *http.Response) error { defer resp.Body.Close() - dec := json.NewDecoder(resp.Body) + data, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + slog.Info("response", "body", string(data)) + dec := json.NewDecoder(bytes.NewReader(data)) return dec.Decode(req) } diff --git a/internal/edge/search.go b/internal/edge/search.go index 82635dfb..65413c50 100644 --- a/internal/edge/search.go +++ b/internal/edge/search.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "net/url" - "strconv" "time" "github.com/google/uuid" @@ -55,10 +54,7 @@ type searchForm struct { ExcludeMyChannels int `json:"exclude_my_channels"` SearchOnlyMyChannels bool `json:"search_only_my_channels"` RecommendSource string `json:"recommend_source"` - XReason string `json:"_x_reason"` - XMode string `json:"_x_mode"` - XSonic bool `json:"_x_sonic"` - XAppName string `json:"_x_app_name"` + WebClientFields } type searchChannelType string @@ -110,25 +106,15 @@ func (cl *Client) SearchChannels(ctx context.Context, query string) ([]slack.Cha ExcludeMyChannels: 0, SearchOnlyMyChannels: false, RecommendSource: "channel-browser", - XReason: "browser-query", - XMode: "online", - XSonic: true, - XAppName: "client", - } - var sup searchURLParams = searchURLParams{ - XID: "0b5495de-" + strconv.FormatInt(time.Now().Unix(), 10) + ".000", - SlackRoute: cl.teamID, - XVersionTS: time.Now().Unix(), - XFrontendBuild: "current", - XDesktopIA: 4, - XGrantry: true, - FP: 1, - } - var url = fmt.Sprintf("https://%s.slack.com/api/search.modules.channels", cl.workspaceName) - if uv := sup.Values(); len(uv) > 0 { - url += "?" + uv.Encode() + WebClientFields: WebClientFields{ + XReason: "browser-query", + XMode: "online", + XSonic: true, + XAppName: "client", + }, } + var url = cl.webapiURL("search.modules.channels") var cc []slack.Channel for { resp, err := cl.PostFormRaw(ctx, url, form.Values()) @@ -149,7 +135,24 @@ func (cl *Client) SearchChannels(ctx context.Context, query string) ([]slack.Cha return cc, nil } -type searchURLParams struct { +func (cl *Client) webapiURL(endpoint string) string { + // var sup webURLParam = webURLParam{ + // XID: "0b5495de-" + strconv.FormatInt(time.Now().Unix(), 10) + ".000", + // SlackRoute: cl.teamID, + // XVersionTS: time.Now().Unix(), + // XFrontendBuild: "current", + // XDesktopIA: 4, + // XGrantry: true, + // FP: 1, + // } + url := fmt.Sprintf("https://%s.slack.com/api/%s", cl.workspaceName, endpoint) + // if uv := sup.Values(); len(uv) > 0 { + // url += "?" + uv.Encode() + // } + return url +} + +type webURLParam struct { XID string `json:"_x_id,omitempty"` XCSID string `json:"_x_csid,omitempty"` SlackRoute string `json:"slack_route,omitempty"` @@ -160,6 +163,6 @@ type searchURLParams struct { FP int `json:"fp,omitempty"` } -func (sup searchURLParams) Values() url.Values { +func (sup webURLParam) Values() url.Values { return values(sup, true) }