Merge pull request #3 from twiny/dev

code clean
twiny · Aug 15, 2022 · cf728da · cf728da
2 parents f31f292 + 3dec287
commit cf728da
Show file tree

Hide file tree

Showing 6 changed files with 546 additions and 580 deletions.
diff --git a/README.md b/README.md
@@ -14,6 +14,28 @@ go build -o bin/spidy -v cmd/spidy/main.go
 ./bin/spidy -c config/config.yaml -u https://github.com
 ```
 
+## Usage
+
+```sh
+NAME:
+   Spidy - Domain name scraper
+
+USAGE:
+   spidy [global options] command [command options] [arguments...]
+
+VERSION:
+   2.0.0
+
+COMMANDS:
+   help, h  Shows a list of commands or help for one command
+
+GLOBAL OPTIONS:
+   --config path, -c path  path to config file
+   --help, -h              show help (default: false)
+   --urls urls, -u urls    urls of page to scrape  (accepts multiple inputs)
+   --version, -v           print the version (default: false)
+```
+
 ## Configuration
 
 ```yaml

diff --git a/cmd/spidy/api/spider.go b/cmd/spidy/api/spider.go
@@ -13,7 +13,6 @@ import (
 	"syscall"
 
 	//
-
 	"github.com/twiny/spidy/v2/internal/pkg/spider/v1"
 	"github.com/twiny/spidy/v2/internal/service/cache"
 	"github.com/twiny/spidy/v2/internal/service/writer"

diff --git a/internal/pkg/spider/v1/setting.go b/internal/pkg/spider/v1/setting.go
@@ -45,7 +45,7 @@ var defaultSetting = &Setting{
 			Interval: time.Second,
 		},
 		MaxBodySize: 10 * 1024 * 1024, // 10 MB
-		UserAgents:  []string{`Spidy v2 - Agent.`},
+		UserAgents:  []string{`Spidy/2.1; +https://github.com/twiny/spidy`},
 		Proxies:     []string{},
 	},
 	Log: struct {