diff --git a/go.mod b/go.mod index 0b3bd57..2e7f09b 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.19 require ( github.com/BurntSushi/toml v0.3.1 + github.com/asjdf/gorm-cache v1.2.3 github.com/caarlos0/env/v10 v10.0.0 github.com/caarlos0/env/v6 v6.10.1 github.com/gin-gonic/gin v1.8.1 @@ -11,8 +12,8 @@ require ( github.com/go-sql-driver/mysql v1.8.1 github.com/mattn/go-isatty v0.0.17 github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2 - github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.12.1 + github.com/redis/go-redis/v9 v9.5.1 github.com/stretchr/testify v1.8.3 github.com/uber-go/tally v3.5.0+incompatible github.com/vmihailenco/msgpack/v5 v5.3.5 @@ -31,11 +32,13 @@ require ( require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/bluele/gcache v0.0.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.2.0 // indirect github.com/coreos/go-systemd/v22 v22.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/glebarez/go-sqlite v1.21.2 // indirect @@ -54,6 +57,8 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.9.5 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgx/v5 v5.4.3 // indirect @@ -61,6 +66,7 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/jonboulle/clockwork v0.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/karlseguin/ccache/v3 v3.0.3 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/m3db/prometheus_client_golang v1.12.8 // indirect github.com/m3db/prometheus_client_model v0.2.1 // indirect @@ -69,6 +75,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.34.0 // indirect diff --git a/go.sum b/go.sum index ecf855f..9b9d2ab 100644 --- a/go.sum +++ b/go.sum @@ -40,19 +40,26 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/asjdf/gorm-cache v1.2.3 h1:h7GAMITzk6DdpOlAGlF0dUt25N8fK4R6zeQyO0pMqlA= +github.com/asjdf/gorm-cache v1.2.3/go.mod h1:PJjTYOCVblDX+GLbEndUqQKPxW+QibsIDO95EfPovBk= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= +github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/caarlos0/env/v10 v10.0.0 h1:yIHUBZGsyqCnpTkbjk8asUlx6RFhhEs+h7TOBdgdzXA= github.com/caarlos0/env/v10 v10.0.0/go.mod h1:ZfulV76NvVPw3tm591U4SwL3Xx9ldzBP9aGxzeN7G18= github.com/caarlos0/env/v6 v6.10.1 h1:t1mPSxNpei6M5yAeu1qtRdPAK29Nbcf/n3G7x+b3/II= github.com/caarlos0/env/v6 v6.10.1/go.mod h1:hvp/ryKXKipEkcuYjs9mI4bBCg+UI0Yhgm5Zu0ddvwc= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -69,6 +76,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -192,6 +201,10 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -219,6 +232,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/karlseguin/ccache/v3 v3.0.3 h1:cz+3tSdTrovp00xHPP3Y6ca/YuSl5kchhYG83wUPYN0= +github.com/karlseguin/ccache/v3 v3.0.3/go.mod h1:qxC372+Qn+IBj8Pe3KvGjHPj0sWwEF7AeZVhsNPZ6uY= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -301,6 +316,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= +github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= diff --git a/pkg/config/global.go b/pkg/config/global.go index b6dad92..f8a8640 100644 --- a/pkg/config/global.go +++ b/pkg/config/global.go @@ -33,7 +33,7 @@ func init() { GlobalCfg.GinConfig = GlobalCfg.GinConfig.DefaultConfig() GlobalCfg.RateLimit = GlobalCfg.RateLimit.DefaultConfig() GlobalCfg.Temporary = GlobalCfg.Temporary.DefaultConfig() - GlobalCfg.DBConfg = GlobalCfg.DBConfg.DefaultConfig() + GlobalCfg.DBConfig = GlobalCfg.DBConfig.DefaultConfig() } @@ -56,7 +56,7 @@ type Global struct { GinConfig `toml:"gin"` RateLimit `toml:"ratelimit"` Temporary `toml:"temporary"` - DBConfg `toml:"database"` + DBConfig `toml:"database"` } func (g Global) TOML() string { @@ -67,6 +67,6 @@ func (g Global) TOML() string { g.GinConfig.TOML(), g.RateLimit.TOML(), g.Temporary.TOML(), - g.DBConfg.TOML(), + g.DBConfig.TOML(), ) } diff --git a/pkg/config/global_test.go b/pkg/config/global_test.go index 0f98d23..6a79c12 100644 --- a/pkg/config/global_test.go +++ b/pkg/config/global_test.go @@ -134,6 +134,28 @@ func Test_global(t *testing.T) { port = "" ## 数据库名 database = "" - ## ca cert证书 - ca_cert = ""`) + ## args 参数 + args = "" +[dbcache] + ## orm_cache_type: redis, memory. 默认值为memory + orm_cache_type: memory + ## orm_cache_model 模式: CacheDisable, CacheOnlyPrimary, CacheOnlySearch 和 CacheAll. 默认值为CacheOnlySearch + orm_cache_model = CacheOnlySearch + ## orm_invalidate_when_update: 如果用户在数据库中更新/删除/创建某些内容，我们将使所有缓存数据无效以确保一致性. 如果true将会清理cache. 默认值为true + orm_invalidate_when_update = true + ## orm_async_write: 如果 true，那么我们将以异步模式写入缓存. 默认值为true + orm_async_write = true + ## orm_cache_ttl: CacheTTL 缓存 ttl，单位为 ms，其中 0 代表永远. 默认值为5000 + orm_cache_ttl = 5000 + ## orm_disable_cache_penetration_protect: 如果为 true，那么我们不会缓存 nil 结果, 默认值为false + orm_disable_cache_penetration_protect = false +[dbcachecfg] + ## cache addr + orm_cache_addr = + ## cache username + orm_cache_username = + ## cache password + orm_cache_password = + ## cache db id + orm_cache_dbid = 0`) } diff --git a/pkg/config/orm.go b/pkg/config/orm.go index 94d04a6..badbf80 100644 --- a/pkg/config/orm.go +++ b/pkg/config/orm.go @@ -21,25 +21,93 @@ import ( ) type DBTYPE string +type CACHETYPE string const ( MYSQL DBTYPE = "mysql" SQLITE3 DBTYPE = "sqlite3" POSTGRES DBTYPE = "postgres" + + CACHEREDIS CACHETYPE = "redis" + CACHEMEMORY CACHETYPE = "memory" +) + +type CacheModel string + +const ( + ORMCacheDisable CacheModel = "CacheDisable" + ORMCacheOnlyPrimary CacheModel = "CacheOnlyPrimary" + ORMCacheOnlySearch CacheModel = "CacheOnlySearch" + ORMCacheAll CacheModel = "CacheAll" ) -// DBConfg represents a database configuration -type DBConfg struct { - DBType DBTYPE `toml:"db_type" json:"db_type" env:"ORM_DBTYPE"` // dbtype - User string `toml:"user" json:"user" env:"ORM_USER"` // user - Password string `toml:"password" json:"password" env:"ORM_PASSWORD"` // password - Host string `toml:"host" json:"host" env:"ORM_HOST"` // host - Port string `toml:"port" json:"port" env:"ORM_PORT"` // port - Database string `toml:"database" json:"database" env:"ORM_DATABASE"` // database - CaCert string `toml:"ca_cert" json:"ca_cert" env:"ORM_CACERT"` // cacert +func (c CacheModel) Number() int { + switch c { + case ORMCacheDisable: + return 0 + case ORMCacheOnlyPrimary: + return 1 + case ORMCacheOnlySearch: + return 2 + case ORMCacheAll: + return 3 + default: + return 3 + } +} + +type OrmCacheCfg struct { + // host:port address. + Addr string `toml:"orm_cache_addr" json:"orm_cache_addr" env:"ORM_CACHE_ADDR"` + // Use the specified Username to authenticate the current connection + // with one of the connections defined in the ACL list when connecting + // to a Redis 6.0 instance, or greater, that is using the Redis ACL system. + Username string `toml:"orm_cache_username" json:"orm_cache_username" env:"ORM_CACHE_USERNAME"` + // Optional password. Must match the password specified in the + // requirepass server configuration option (if connecting to a Redis 5.0 instance, or lower), + // or the User Password when connecting to a Redis 6.0 instance, or greater, + // that is using the Redis ACL system. + Password string `toml:"orm_cache_password" json:"orm_cache_password" env:"ORM_CACHE_PASSWORD"` + // Database to be selected after connecting to the server. + DB int `toml:"orm_cache_dbid" json:"orm_cache_dbid" env:"ORM_CACHE_DBID"` +} + +type OrmCache struct { + // cache type: redis, memory + CacheType CACHETYPE `toml:"orm_cache_type" json:"orm_cache_type" env:"ORM_CACHE_TYPE" envDefault:"memory"` + + // 4 kinds of cache model + CacheModel CacheModel `toml:"orm_cache_model" json:"orm_cache_model" env:"ORM_CACHE_MODEL" envDefault:"CacheOnlySearch"` + + // if user update/delete/create something in DB, we invalidate all cached data to ensure consistency, + // else we do nothing to outdated cache. + InvalidateWhenUpdate bool `toml:"orm_invalidate_when_update" json:"orm_invalidate_when_update" env:"ORM_INVALIDATE_WHEN_UPDATE" envDefault:"true"` + + // AsyncWrite if true, then we will write cache in async mode + AsyncWrite bool `toml:"orm_async_write" json:"orm_async_write" env:"ORM_ASYNC_WRITE" envDefault:"true"` + + // CacheTTL cache ttl in ms, where 0 represents forever + CacheTTL int64 `toml:"orm_cache_ttl" json:"orm_cache_ttl" env:"ORM_CACHE_TTL" envDefault:"5000"` + + // DisableCachePenetration if true, then we will not cache nil result + DisableCachePenetrationProtect bool `toml:"orm_disable_cache_penetration_protect" json:"orm_disable_cache_penetration_protect" env:"ORM_DISABLE_CACHE_PENETRATION_PROTECT" envDefault:"false"` + + CacheCfg OrmCacheCfg `toml:"dbcachecfg"` +} + +// DBConfig represents a database configuration +type DBConfig struct { + DBType DBTYPE `toml:"db_type" json:"db_type" env:"ORM_DBTYPE"` // dbtype + User string `toml:"user" json:"user" env:"ORM_USER"` // user + Password string `toml:"password" json:"password" env:"ORM_PASSWORD"` // password + Host string `toml:"host" json:"host" env:"ORM_HOST"` // host + Port string `toml:"port" json:"port" env:"ORM_PORT"` // port + Database string `toml:"database" json:"database" env:"ORM_DATABASE"` // database + Args string `toml:"orm_args" json:"orm_args" env:"ORM_ARGS"` // args + Cache OrmCache `toml:"dbcache"` } -func (db DBConfg) TOML() string { +func (db DBConfig) TOML() string { return fmt.Sprintf(` [database] ## dbtype 模式: mysql/sqlite3/postgres @@ -54,25 +122,71 @@ func (db DBConfg) TOML() string { port = "%s" ## 数据库名 database = "%s" - ## ca cert证书 - ca_cert = "%s"`, + ## args 参数 + args = "%s" +[dbcache] + ## orm_cache_type: redis, memory. 默认值为memory + orm_cache_type: %s + ## orm_cache_model 模式: CacheDisable, CacheOnlyPrimary, CacheOnlySearch 和 CacheAll. 默认值为CacheOnlySearch + orm_cache_model = %s + ## orm_invalidate_when_update: 如果用户在数据库中更新/删除/创建某些内容，我们将使所有缓存数据无效以确保一致性. 如果true将会清理cache. 默认值为true + orm_invalidate_when_update = %v + ## orm_async_write: 如果 true，那么我们将以异步模式写入缓存. 默认值为true + orm_async_write = %v + ## orm_cache_ttl: CacheTTL 缓存 ttl，单位为 ms，其中 0 代表永远. 默认值为5000 + orm_cache_ttl = %v + ## orm_disable_cache_penetration_protect: 如果为 true，那么我们不会缓存 nil 结果, 默认值为false + orm_disable_cache_penetration_protect = %v +[dbcachecfg] + ## cache addr + orm_cache_addr = %v + ## cache username + orm_cache_username = %v + ## cache password + orm_cache_password = %v + ## cache db id + orm_cache_dbid = %v`, string(db.DBType), db.User, db.Password, db.Host, db.Port, db.Database, - db.CaCert) + db.Args, + string(db.Cache.CacheType), + string(db.Cache.CacheModel), + db.Cache.InvalidateWhenUpdate, + db.Cache.AsyncWrite, + db.Cache.CacheTTL, + db.Cache.DisableCachePenetrationProtect, + db.Cache.CacheCfg.Addr, + db.Cache.CacheCfg.Username, + db.Cache.CacheCfg.Password, + db.Cache.CacheCfg.DB) } -func (db DBConfg) DefaultConfig() DBConfg { - db = DBConfg{ +func (db DBConfig) DefaultConfig() DBConfig { + db = DBConfig{ DBType: MYSQL, User: "root", Password: "root", Host: "localhost", Port: "3306", Database: "test", + Cache: OrmCache{ + CacheType: CACHETYPE("memory"), + CacheModel: ORMCacheOnlySearch, + InvalidateWhenUpdate: true, + CacheTTL: 5000, + AsyncWrite: true, + DisableCachePenetrationProtect: false, + CacheCfg: OrmCacheCfg{ + Addr: "", + Username: "", + Password: "", + DB: 0, + }, + }, } return db } diff --git a/pkg/config/orm_test.go b/pkg/config/orm_test.go index 4e98de0..e09118f 100644 --- a/pkg/config/orm_test.go +++ b/pkg/config/orm_test.go @@ -25,7 +25,7 @@ import ( func Test_DBConfig(t *testing.T) { assert := assert.New(t) - aa := GlobalCfg.DBConfg.DefaultConfig().TOML() + aa := GlobalCfg.DBConfig.DefaultConfig().TOML() assert.Equal(aa, ` [database] ## dbtype 模式: mysql/sqlite3/postgres @@ -40,6 +40,28 @@ func Test_DBConfig(t *testing.T) { port = "3306" ## 数据库名 database = "test" - ## ca cert证书 - ca_cert = ""`) + ## args 参数 + args = "" +[dbcache] + ## orm_cache_type: redis, memory. 默认值为memory + orm_cache_type: memory + ## orm_cache_model 模式: CacheDisable, CacheOnlyPrimary, CacheOnlySearch 和 CacheAll. 默认值为CacheOnlySearch + orm_cache_model = CacheOnlySearch + ## orm_invalidate_when_update: 如果用户在数据库中更新/删除/创建某些内容，我们将使所有缓存数据无效以确保一致性. 如果true将会清理cache. 默认值为true + orm_invalidate_when_update = true + ## orm_async_write: 如果 true，那么我们将以异步模式写入缓存. 默认值为true + orm_async_write = true + ## orm_cache_ttl: CacheTTL 缓存 ttl，单位为 ms，其中 0 代表永远. 默认值为5000 + orm_cache_ttl = 5000 + ## orm_disable_cache_penetration_protect: 如果为 true，那么我们不会缓存 nil 结果, 默认值为false + orm_disable_cache_penetration_protect = false +[dbcachecfg] + ## cache addr + orm_cache_addr = + ## cache username + orm_cache_username = + ## cache password + orm_cache_password = + ## cache db id + orm_cache_dbid = 0`) } diff --git a/pkg/orm/cache/memory.go b/pkg/orm/cache/memory.go new file mode 100644 index 0000000..78a95f7 --- /dev/null +++ b/pkg/orm/cache/memory.go @@ -0,0 +1,36 @@ +/* +Copyright 2024 The KubeService-Stack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + gcache "github.com/asjdf/gorm-cache/cache" + gconfig "github.com/asjdf/gorm-cache/config" + gstorage "github.com/asjdf/gorm-cache/storage" + "github.com/kubeservice-stack/common/pkg/config" +) + +func NewMemoryCache(cfg *config.OrmCache) (gcache.Cache, error) { + return gcache.NewGorm2Cache(&gconfig.CacheConfig{ + CacheLevel: gconfig.CacheLevel(cfg.CacheModel.Number()), + CacheStorage: gstorage.NewMem(), + // when you create/update/delete objects, invalidate cache + InvalidateWhenUpdate: cfg.InvalidateWhenUpdate, + CacheTTL: cfg.CacheTTL, + DisableCachePenetrationProtect: cfg.DisableCachePenetrationProtect, + AsyncWrite: cfg.AsyncWrite, + }) +} diff --git a/pkg/orm/cache/redis.go b/pkg/orm/cache/redis.go new file mode 100644 index 0000000..cd4ce94 --- /dev/null +++ b/pkg/orm/cache/redis.go @@ -0,0 +1,44 @@ +/* +Copyright 2024 The KubeService-Stack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + gcache "github.com/asjdf/gorm-cache/cache" + gconfig "github.com/asjdf/gorm-cache/config" + gstorage "github.com/asjdf/gorm-cache/storage" + "github.com/kubeservice-stack/common/pkg/config" + "github.com/redis/go-redis/v9" +) + +func NewRedisCache(cfg *config.OrmCache) (gcache.Cache, error) { + redisClient := redis.NewClient(&redis.Options{ + Addr: cfg.CacheCfg.Addr, + Username: cfg.CacheCfg.Username, + Password: cfg.CacheCfg.Password, + DB: cfg.CacheCfg.DB, + }) + + return gcache.NewGorm2Cache(&gconfig.CacheConfig{ + CacheLevel: gconfig.CacheLevel(cfg.CacheModel.Number()), + CacheStorage: gstorage.NewRedis(&gstorage.RedisStoreConfig{Client: redisClient}), + // when you create/update/delete objects, invalidate cache + InvalidateWhenUpdate: cfg.InvalidateWhenUpdate, + CacheTTL: cfg.CacheTTL, + DisableCachePenetrationProtect: cfg.DisableCachePenetrationProtect, + AsyncWrite: cfg.AsyncWrite, + }) +} diff --git a/pkg/orm/interface.go b/pkg/orm/interface.go index d52377d..780a4ef 100644 --- a/pkg/orm/interface.go +++ b/pkg/orm/interface.go @@ -20,6 +20,7 @@ import ( "errors" "github.com/kubeservice-stack/common/pkg/config" + "github.com/kubeservice-stack/common/pkg/orm/cache" "gorm.io/gorm" "gorm.io/plugin/opentelemetry/tracing" ) @@ -34,7 +35,7 @@ type DBConn struct { db *gorm.DB } -type Instance func(cfg config.DBConfg) gorm.Dialector +type Instance func(cfg config.DBConfig) gorm.Dialector var adapters = make(map[config.DBTYPE]Instance) @@ -48,7 +49,7 @@ func Register(name config.DBTYPE, adapter Instance) { adapters[name] = adapter } -func NewDBConn(cfg config.DBConfg) (*DBConn, error) { +func NewDBConn(cfg config.DBConfig) (*DBConn, error) { gcfg := &gorm.Config{ DisableAutomaticPing: false, PrepareStmt: false, @@ -69,6 +70,26 @@ func NewDBConn(cfg config.DBConfg) (*DBConn, error) { return nil, err } + if cfg.Cache.CacheType == config.CACHEREDIS { + c, err := cache.NewRedisCache(&cfg.Cache) + if err != nil { + return nil, err + } + err = conn.Use(c) + if err != nil { + return nil, err + } + } else if cfg.Cache.CacheType == config.CACHEMEMORY { + c, err := cache.NewMemoryCache(&cfg.Cache) + if err != nil { + return nil, err + } + err = conn.Use(c) + if err != nil { + return nil, err + } + } + return &DBConn{ config: gcfg, db: conn, diff --git a/pkg/orm/interface_test.go b/pkg/orm/interface_test.go index ea2629f..b4a2763 100644 --- a/pkg/orm/interface_test.go +++ b/pkg/orm/interface_test.go @@ -25,14 +25,14 @@ import ( func Test_NewDBConn(t *testing.T) { assert := assert.New(t) - r, err := NewDBConn(config.GlobalCfg.DBConfg) + r, err := NewDBConn(config.GlobalCfg.DBConfig) assert.Error(err) assert.Nil(r) } func Test_NewDBConnSuccess(t *testing.T) { assert := assert.New(t) - r, err := NewDBConn(config.DBConfg{ + r, err := NewDBConn(config.DBConfig{ DBType: config.SQLITE3, Database: "file::memory:?cache=shared", }) diff --git a/pkg/orm/mysql.go b/pkg/orm/mysql.go index 3837874..3ea6c13 100644 --- a/pkg/orm/mysql.go +++ b/pkg/orm/mysql.go @@ -17,6 +17,7 @@ limitations under the License. package orm import ( + "strings" "time" driver_mysql "github.com/go-sql-driver/mysql" @@ -25,7 +26,7 @@ import ( "gorm.io/gorm" ) -func NewMySQL(cfg config.DBConfg) gorm.Dialector { +func NewMySQL(cfg config.DBConfig) gorm.Dialector { loc, _ := time.LoadLocation("UTC") dcfg := driver_mysql.Config{ User: cfg.User, @@ -33,6 +34,7 @@ func NewMySQL(cfg config.DBConfg) gorm.Dialector { Net: "tcp", Addr: cfg.Host + ":" + cfg.Port, DBName: cfg.Database, + Params: String2Map(cfg.Args), Loc: loc, AllowNativePasswords: true, ParseTime: true, @@ -41,6 +43,20 @@ func NewMySQL(cfg config.DBConfg) gorm.Dialector { return mysql.Open(dcfg.FormatDSN()) } +// [?param1=value1&...¶mN=valueN] +func String2Map(str string) map[string]string { + str1 := strings.Trim(str, "?") + v := strings.Split(str1, "&") + ret := make(map[string]string) + for _, value := range v { + kv := strings.Split(strings.Trim(value, " "), "=") + if len(kv) == 2 { + ret[kv[0]] = kv[1] + } + } + return ret +} + func init() { Register(config.MYSQL, NewMySQL) } diff --git a/pkg/orm/mysql_test.go b/pkg/orm/mysql_test.go index 5abac31..d3330c2 100644 --- a/pkg/orm/mysql_test.go +++ b/pkg/orm/mysql_test.go @@ -25,6 +25,27 @@ import ( func Test_MySQL(t *testing.T) { assert := assert.New(t) - r := NewMySQL(config.GlobalCfg.DBConfg) + r := NewMySQL(config.GlobalCfg.DBConfig) assert.NotNil(r) } + +func TestString2Map(t *testing.T) { + assert := assert.New(t) + ret := String2Map("") + assert.Equal(ret, map[string]string{}) + + ret = String2Map(" asdfsdf ?") + assert.Equal(ret, map[string]string{}) + + ret = String2Map(" dd=dd ?") + assert.Equal(ret, map[string]string{"dd": "dd"}) + + ret = String2Map(" dd=dd=dd ?") + assert.Equal(ret, map[string]string{}) + + ret = String2Map(" dd=dd&dd=aa ?") + assert.Equal(ret, map[string]string{"dd": "aa"}) + + ret = String2Map(" dd=dd&dd=aa&bb=aa ?") + assert.Equal(ret, map[string]string{"dd": "aa", "bb": "aa"}) +} diff --git a/pkg/orm/postgres.go b/pkg/orm/postgres.go index 2554a2f..d1fb60a 100644 --- a/pkg/orm/postgres.go +++ b/pkg/orm/postgres.go @@ -24,7 +24,7 @@ import ( "gorm.io/gorm" ) -func NewPostgres(cfg config.DBConfg) gorm.Dialector { +func NewPostgres(cfg config.DBConfig) gorm.Dialector { return postgres.Open(fmt.Sprintf( "host=%s port=%s user=%s dbname=%s password=%s sslmode=disable", cfg.Host, cfg.Port, cfg.User, cfg.Database, cfg.Password)) diff --git a/pkg/orm/postgres_test.go b/pkg/orm/postgres_test.go index 34688d1..922f4e3 100644 --- a/pkg/orm/postgres_test.go +++ b/pkg/orm/postgres_test.go @@ -25,6 +25,6 @@ import ( func Test_postgres(t *testing.T) { assert := assert.New(t) - r := NewPostgres(config.GlobalCfg.DBConfg) + r := NewPostgres(config.GlobalCfg.DBConfig) assert.NotNil(r) } diff --git a/pkg/orm/sqlite3.go b/pkg/orm/sqlite3.go index f26c367..8af9244 100644 --- a/pkg/orm/sqlite3.go +++ b/pkg/orm/sqlite3.go @@ -22,7 +22,7 @@ import ( "gorm.io/gorm" ) -func NewSqlite3(cfg config.DBConfg) gorm.Dialector { +func NewSqlite3(cfg config.DBConfig) gorm.Dialector { return sqlite.Open(cfg.Database) } diff --git a/pkg/orm/sqlite_test.go b/pkg/orm/sqlite_test.go index dc4ee07..d429f0b 100644 --- a/pkg/orm/sqlite_test.go +++ b/pkg/orm/sqlite_test.go @@ -25,6 +25,6 @@ import ( func Test_sqlite(t *testing.T) { assert := assert.New(t) - r := NewSqlite3(config.GlobalCfg.DBConfg) + r := NewSqlite3(config.GlobalCfg.DBConfig) assert.NotNil(r) }