Skip to content

Commit

Permalink
refactor: 重构项目
Browse files Browse the repository at this point in the history
- 将 db 相关功能移至根目录;
- 删除了 Region 接口;
- 删除了原来的 Versions 对象;
  • Loading branch information
caixw committed Apr 29, 2024
1 parent 0a9138d commit 61a1dd4
Show file tree
Hide file tree
Showing 19 changed files with 736 additions and 896 deletions.
6 changes: 3 additions & 3 deletions cmd/fetch/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ import (
"strconv"
"strings"

"github.com/issue9/cnregion/db"
"github.com/issue9/cnregion"
"github.com/issue9/cnregion/version"
)

func build(dataDir, output string, years ...int) error {
d := db.New()
d := cnregion.NewDB()

if len(years) == 0 {
years = version.All()
Expand All @@ -32,7 +32,7 @@ func build(dataDir, output string, years ...int) error {
return d.Dump(output, true)
}

func buildYear(d *db.DB, dataDir string, year int) error {
func buildYear(d *cnregion.DB, dataDir string, year int) error {
fmt.Printf("\n添加 %d 的数据\n", year)
if !d.AddVersion(year) {
fmt.Printf("已经存在该年份 %d 的数据\n\n", year)
Expand Down
10 changes: 5 additions & 5 deletions cmd/fetch/go.mod
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
module github.com/issue9/cnregion/cmd/fetch

go 1.22
go 1.22.0

require (
github.com/gocolly/colly/v2 v2.1.0
github.com/issue9/cmdopt v0.13.1
github.com/issue9/cnregion v1.2023.0
github.com/issue9/cnregion v1.2023.2
github.com/issue9/errwrap v0.3.2
github.com/issue9/sliceutil v0.15.1
github.com/issue9/term/v3 v3.2.6
github.com/issue9/sliceutil v0.16.1
github.com/issue9/term/v3 v3.2.8
)

require (
Expand All @@ -24,7 +24,7 @@ require (
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
github.com/temoto/robotstxt v1.1.2 // indirect
golang.org/x/net v0.5.0 // indirect
golang.org/x/sys v0.17.0 // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/text v0.6.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.28.1 // indirect
Expand Down
16 changes: 8 additions & 8 deletions cmd/fetch/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -53,16 +53,16 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/issue9/assert/v4 v4.1.1 h1:OhPE8SB8n/qZCNGLQa+6MQtr/B3oON0JAVj68k8jJlc=
github.com/issue9/assert/v4 v4.1.1/go.mod h1:v7qDRXi7AsaZZNh8eAK2rkLJg5/clztqQGA1DRv9Lv4=
github.com/issue9/assert/v4 v4.3.0 h1:W3XDKmttsfzihYGxJ9rJoL2ViJgWERB9IxfHcxjv65U=
github.com/issue9/assert/v4 v4.3.0/go.mod h1:v7qDRXi7AsaZZNh8eAK2rkLJg5/clztqQGA1DRv9Lv4=
github.com/issue9/cmdopt v0.13.1 h1:VA/Hgd92NBbZyHjZx1xcRCMhoc+XjI1LWhiuZkOZ0VU=
github.com/issue9/cmdopt v0.13.1/go.mod h1:7lnF45Ush0boi4/nDjeSDYV5lZfoOzp3jm+biJr0f4g=
github.com/issue9/errwrap v0.3.2 h1:7KEme9Pfe75M+sIMcPCn/DV90wjnOcRbO4DXVAHj3Fw=
github.com/issue9/errwrap v0.3.2/go.mod h1:KcCLuUGiffjooLCUjL89r1cyO8/HT/VRcQrneO53N3A=
github.com/issue9/sliceutil v0.15.1 h1:mV1VlQSO5E8sh2ab6pypLg7TdgGZlDb30E8quli01xY=
github.com/issue9/sliceutil v0.15.1/go.mod h1:ldun6sT4/bOJxuMtOXhtc6P7GCwE7L+avV86HNks7qk=
github.com/issue9/term/v3 v3.2.6 h1:+8sdVdT2GM3OoebNMd5wdt0bswJ1TBukK6HloKgoW3o=
github.com/issue9/term/v3 v3.2.6/go.mod h1:OCUgzwgKUbcHrlRyxQATPaYfNtNbWs17yzkogMpPIjA=
github.com/issue9/sliceutil v0.16.1 h1:NVZ2feN014oUmMpZvZUrKkKMGhAfzT8WNzJF81krebM=
github.com/issue9/sliceutil v0.16.1/go.mod h1:ldun6sT4/bOJxuMtOXhtc6P7GCwE7L+avV86HNks7qk=
github.com/issue9/term/v3 v3.2.8 h1:vAtsr9FLwrDQRyU7S8AOUI3f+QrYsXVHT/kee9MuMkk=
github.com/issue9/term/v3 v3.2.8/go.mod h1:S/xLgjEXJNr7C6UhcGTEOV5k94mzHRcYMmtvhLJoX6A=
github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
Expand Down Expand Up @@ -119,8 +119,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
Expand Down
78 changes: 78 additions & 0 deletions cnregion.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,81 @@
// [数据规则]: http://www.stats.gov.cn/tjsj/tjbz/200911/t20091125_8667.html
// [数据来源]: http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/
package cnregion

import (
"bytes"
"compress/gzip"
"io"
"io/fs"
"os"
)

// LoadFS 从数据文件加载数据
func LoadFS(f fs.FS, file, separator string, compress bool, version ...int) (*DB, error) {
data, err := fs.ReadFile(f, file)
if err != nil {
return nil, err
}
return Load(data, separator, compress, version...)
}

// Load 将数据内容加载至 DB 对象
//
// version 仅加载指定年份的数据,如果为空,则加载所有数据;
func Load(data []byte, separator string, compress bool, version ...int) (*DB, error) {
if compress {
rd, err := gzip.NewReader(bytes.NewReader(data))
if err != nil {
return nil, err
}

data, err = io.ReadAll(rd)
if err != nil {
return nil, err
}
}

db := &DB{
fullNameSeparator: separator,
filters: version,
}
if err := db.unmarshal(data); err != nil {
return nil, err
}

db.initDistricts()

return db, nil
}

// LoadFile 从数据文件加载数据
func LoadFile(file, separator string, compress bool, version ...int) (*DB, error) {
data, err := os.ReadFile(file)
if err != nil {
return nil, err
}
return Load(data, separator, compress, version...)
}

// Dump 输出到文件
func (db *DB) Dump(file string, compress bool) error {
data, err := db.marshal()
if err != nil {
return err
}

if compress {
buf := new(bytes.Buffer)
w := gzip.NewWriter(buf)
if _, err = w.Write(data); err != nil {
return err
}
if err = w.Close(); err != nil {
return err
}

data = buf.Bytes()
}

return os.WriteFile(file, data, os.ModePerm)
}
73 changes: 73 additions & 0 deletions cnregion_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// SPDX-FileCopyrightText: 2021-2024 caixw
//
// SPDX-License-Identifier: MIT

package cnregion

import (
"os"
"path/filepath"
"testing"

"github.com/issue9/assert/v4"

"github.com/issue9/cnregion/id"
"github.com/issue9/cnregion/version"
)

func TestLoad(t *testing.T) {
a := assert.New(t, false)

o1, err := Load(data, "-", false)
a.NotError(err).
Equal(o1.fullNameSeparator, obj.fullNameSeparator).
Equal(o1.versions, obj.versions).
Equal(len(o1.root.items), len(obj.root.items)).
Equal(o1.root.items[0].id, obj.root.items[0].id).
Equal(o1.root.items[0].fullID, obj.root.items[0].fullID).
Equal(o1.root.items[0].items[0].id, obj.root.items[0].items[0].id).
Equal(o1.root.items[1].items[0].id, obj.root.items[1].items[0].id).
Equal(o1.root.items[1].items[0].fullID, obj.root.items[1].items[0].fullID).
Equal(o1.root.items[1].items[1].fullID, obj.root.items[1].items[1].fullID).
NotEqual(o1.root.items[1].items[1].fullID, obj.root.items[1].items[0].fullID)

d1, err := obj.marshal()
a.NotError(err).NotNil(d1)
a.Equal(string(d1), string(data))

_, err = Load([]byte("100:[2020]:::1:0{}"), "-", false)
a.Equal(err, ErrIncompatible)

o1, err = Load(data, "-", false, 2019)
a.NotError(err).
Equal(0, len(o1.root.items))
}

func TestDB_LoadDump(t *testing.T) {
a := assert.New(t, false)

path := filepath.Join(os.TempDir(), "cnregion_db.dict")
a.NotError(obj.Dump(path, false))
d, err := LoadFile(path, "-", false)
a.NotError(err).NotNil(d)

path = filepath.Join(os.TempDir(), "cnregion_db_compress.dict")
a.NotError(obj.Dump(path, true))
d, err = LoadFile(path, "-", true)
a.NotError(err).NotNil(d)
}

func TestLoadFS(t *testing.T) {
a := assert.New(t, false)

obj, err := LoadFS(os.DirFS("./data"), "regions.db", "-", true)
a.NotError(err).NotNil(obj)
a.Equal(obj.versions, version.All()).
Equal(obj.fullNameSeparator, "-").
True(len(obj.root.items) > 0).
Equal(obj.root.items[0].level, id.Province).
Equal(obj.root.items[0].items[0].level, id.City).
Equal(obj.root.items[0].items[0].items[0].level, id.County).
Equal(obj.root.items[1].level, id.Province).
Equal(obj.root.items[2].items[0].level, id.City)
}
4 changes: 2 additions & 2 deletions data/embed.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,6 @@ var data embed.FS
// Embed 将 regions.db 的内容嵌入到程序中
//
// 这样可以让程序不依赖外部文件,但同时也会增加编译后程序的大小。
func Embed(separator string, version ...int) (*cnregion.Version, error) {
return cnregion.LoadFS(data, "regions.db", separator, version...)
func Embed(separator string, version ...int) (*cnregion.DB, error) {
return cnregion.LoadFS(data, "regions.db", separator, true, version...)
}
Loading

0 comments on commit 61a1dd4

Please sign in to comment.