Merge pull request #17 from rusq/url-export

URL parsing functions and limiter fine controls

.gitignore

@@ -30,6 +30,7 @@ experiments/*
 dist/*
 secrets.*
 cmd/slackdump/slackdump
+/slackdump
 cmd/sdconv/sdconv
 *~
 .env

README.rst

@@ -170,8 +170,20 @@ Q: **I'm getting ``invalid_auth``**
 
 A: Go get the new Cookie from the browser.
 
+Bulletin Board
+--------------
+
+Messages that were conveyed with the donations:
+
+- 25/01/2022: Stay away from `TheSignChef.com`_, ya hear, they don't pay what
+  they owe to their employees. 
 
 .. _Application: https://stackoverflow.com/questions/12908881/how-to-copy-cookies-in-google-chrome
 .. _`Buy me a cup of tea`: https://www.paypal.com/donate/?hosted_button_id=GUHCLSM7E54ZW
 .. _`Join discussion`: https://t.me/slackdump
 .. _`A set up guide on Medium.com`: https://medium.com/@gilyazov/downloading-your-private-slack-conversations-52e50428b3c2
+
+..
+  bulletin board links
+
+.. _`TheSignChef.com`: https://www.glassdoor.com.au/Reviews/TheSignChef-com-Reviews-E793259.htm

channels.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"runtime/trace"
 	"strings"
 	"text/tabwriter"
 
@@ -21,7 +22,10 @@ type Channels struct {
 // the type of messages to fetch.  See github.com/rusq/slack docs for possible
 // values
 func (sd *SlackDumper) getChannels(ctx context.Context, chanTypes []string) (*Channels, error) {
-	limiter := newLimiter(tier2)
+	ctx, task := trace.NewTask(ctx, "getChannels")
+	defer task.End()
+
+	limiter := newLimiter(tier2, sd.options.limiterBurst, int(sd.options.limiterBoost))
 
 	if chanTypes == nil {
 		chanTypes = allChanTypes
@@ -30,8 +34,18 @@ func (sd *SlackDumper) getChannels(ctx context.Context, chanTypes []string) (*Ch
 	params := &slack.GetConversationsParameters{Types: chanTypes}
 	allChannels := make([]slack.Channel, 0, 50)
 	for {
-		chans, nextcur, err := sd.client.GetConversations(params)
-		if err != nil {
+		var (
+			chans   []slack.Channel
+			nextcur string
+		)
+		if err := withRetry(ctx, limiter, sd.options.conversationRetries, func() error {
+			var err error
+			trace.WithRegion(ctx, "GetConversations", func() {
+				chans, nextcur, err = sd.client.GetConversations(params)
+			})
+			return err
+
+		}); err != nil {
 			return nil, err
 		}
 		allChannels = append(allChannels, chans...)
@@ -92,20 +106,15 @@ func (sd *SlackDumper) whoThisChannelFor(channel *slack.Channel) (who string) {
 	return who
 }
 
-// IsChannel checks if such a channel exists, returns true if it does
-func (sd *SlackDumper) IsChannel(c string) bool {
-	if c == "" {
-		return false
-	}
-	for i := range sd.Channels {
-		if sd.Channels[i].ID == c {
-			return true
-		}
-	}
-	return false
-}
-
+// username tries to resolve the username by ID. If the internal users map is not
+// initialised, it will return the ID, otherwise, if the user is not found in
+// cache, it will assume that the user is external, and return the ID with
+// "external" prefix.
 func (sd *SlackDumper) username(id string) string {
+	if sd.UserForID == nil {
+		// no user cache, use the IDs.
+		return id
+	}
 	user, ok := sd.UserForID[id]
 	if !ok {
 		return "<external>:" + id