diff --git a/changelog.md b/changelog.md index 1268e0551c..c8d3e6b8c4 100644 --- a/changelog.md +++ b/changelog.md @@ -16,6 +16,7 @@ - [#4077](https://github.com/ignite/cli/pull/4077) Merge the swagger files manually instead use nodetime `swagger-combine` - [#4100](https://github.com/ignite/cli/pull/4100) Set the `proto-dir` flag only for the `scaffold chain` command and use the proto path from the config - [#4111](https://github.com/ignite/cli/pull/4111) Remove vuex generation +- [#4133](https://github.com/ignite/cli/pull/4133) Improve buf rate limit - [#4113](https://github.com/ignite/cli/pull/4113) Generate chain config documentation automatically - [#4131](https://github.com/ignite/cli/pull/4131) Support `bytes` as data type in the `scaffold` commands diff --git a/docs/docs/02-guide/06-ibc.md b/docs/docs/02-guide/06-ibc.md index faa01a1cae..abddcd9c14 100644 --- a/docs/docs/02-guide/06-ibc.md +++ b/docs/docs/02-guide/06-ibc.md @@ -432,9 +432,6 @@ version: 1 build: proto: path: proto - third_party_paths: - - third_party/proto - - proto_vendor accounts: - name: alice coins: @@ -463,9 +460,6 @@ version: 1 build: proto: path: proto - third_party_paths: - - third_party/proto - - proto_vendor accounts: - name: alice coins: diff --git a/docs/docs/02-guide/07-interchange/02-init.md b/docs/docs/02-guide/07-interchange/02-init.md index a8c96755f2..2f19beee72 100644 --- a/docs/docs/02-guide/07-interchange/02-init.md +++ b/docs/docs/02-guide/07-interchange/02-init.md @@ -129,9 +129,6 @@ version: 1 build: proto: path: proto - third_party_paths: - - third_party/proto - - proto_vendor accounts: - name: alice coins: @@ -164,9 +161,6 @@ version: 1 build: proto: path: proto - third_party_paths: - - third_party/proto - - proto_vendor accounts: - name: alice coins: diff --git a/docs/docs/08-references/02-config.md b/docs/docs/08-references/02-config.md index 34347b9565..d0228a4a18 100644 --- a/docs/docs/08-references/02-config.md +++ b/docs/docs/08-references/02-config.md @@ -203,17 +203,6 @@ build: path: "myproto" ``` -Ignite comes with required third-party proto out of the box. Ignite also looks -into `third_party/proto` and `proto_vendor` directories for extra proto files. -If your project keeps third-party proto files in a different directory, you -should tell Ignite about this: - -```yml -build: - proto: - third_party_paths: ["my_third_party/proto"] -``` - ## Faucet The faucet service sends tokens to addresses. diff --git a/docs/versioned_docs/version-v0.25/kb/03-config.md b/docs/versioned_docs/version-v0.25/kb/03-config.md index 278498af6b..8357afa66b 100644 --- a/docs/versioned_docs/version-v0.25/kb/03-config.md +++ b/docs/versioned_docs/version-v0.25/kb/03-config.md @@ -57,7 +57,6 @@ build: | Key | Required | Type | Description | | ----------------- | -------- | --------------- | ------------------------------------------------------------------------------------------ | | path | N | String | Path to protocol buffer files. Default: `"proto"`. | -| third_party_paths | N | List of Strings | Path to third-party protocol buffer files. Default: `["third_party/proto", "proto_vendor"]`. | ## client diff --git a/docs/versioned_docs/version-v0.25/kb/06-proto.md b/docs/versioned_docs/version-v0.25/kb/06-proto.md index ce2ecc6c2f..34b6018b93 100644 --- a/docs/versioned_docs/version-v0.25/kb/06-proto.md +++ b/docs/versioned_docs/version-v0.25/kb/06-proto.md @@ -20,11 +20,3 @@ Third-party proto files, including those of Cosmos SDK and Tendermint, are bundl ```protobuf import "cosmos/base/query/v1beta1/pagination.proto"; ``` - -You can also manually add third-party proto files. By default, Ignite CLI imports proto files from these directories: `third_party/proto` and `proto_vendor`. You can define third-party paths of the import directory in `config.yml`: - -```yaml -build: - proto: - third_party_paths: ["my_third_party_proto"] -``` diff --git a/ignite/cmd/cmd.go b/ignite/cmd/cmd.go index da86ec4aa2..034af6cb3a 100644 --- a/ignite/cmd/cmd.go +++ b/ignite/cmd/cmd.go @@ -17,6 +17,7 @@ import ( "github.com/ignite/cli/v29/ignite/pkg/cache" "github.com/ignite/cli/v29/ignite/pkg/cliui" uilog "github.com/ignite/cli/v29/ignite/pkg/cliui/log" + "github.com/ignite/cli/v29/ignite/pkg/dircache" "github.com/ignite/cli/v29/ignite/pkg/errors" "github.com/ignite/cli/v29/ignite/pkg/gitpod" "github.com/ignite/cli/v29/ignite/pkg/goenv" @@ -275,6 +276,9 @@ func newCache(cmd *cobra.Command) (cache.Storage, error) { if err := storage.Clear(); err != nil { return cache.Storage{}, err } + if err := dircache.ClearCache(); err != nil { + return cache.Storage{}, err + } } return storage, nil diff --git a/ignite/config/chain/base/config.go b/ignite/config/chain/base/config.go index 0473307e88..56e52601ad 100644 --- a/ignite/config/chain/base/config.go +++ b/ignite/config/chain/base/config.go @@ -31,10 +31,6 @@ type Build struct { type Proto struct { // Path is the relative path of where app's proto files are located at. Path string `yaml:"path" doc:"Relative path where the application's proto files are located."` - - // ThirdPartyPath is the relative path of where the third party proto files are - // located that used by the app. - ThirdPartyPaths []string `yaml:"third_party_paths" doc:"Relative paths to third-party proto files used by the application."` } // Client configures code generation for clients. @@ -174,8 +170,7 @@ func DefaultConfig() Config { return Config{ Build: Build{ Proto: Proto{ - Path: defaults.ProtoDir, - ThirdPartyPaths: []string{"third_party/proto", "proto_vendor"}, + Path: defaults.ProtoDir, }, }, Faucet: Faucet{ diff --git a/ignite/config/chain/v1/config_test.go b/ignite/config/chain/v1/config_test.go index cbf9b3058c..b97c65a64c 100644 --- a/ignite/config/chain/v1/config_test.go +++ b/ignite/config/chain/v1/config_test.go @@ -30,8 +30,7 @@ func TestConfigDecode(t *testing.T) { Build: base.Build{ Binary: "evmosd", Proto: base.Proto{ - Path: "proto", - ThirdPartyPaths: []string{"third_party/proto", "proto_vendor"}, + Path: "proto", }, }, Accounts: []base.Account{ diff --git a/ignite/config/chain/v1/testdata/config.yaml b/ignite/config/chain/v1/testdata/config.yaml index 6221affba7..efebd4a499 100644 --- a/ignite/config/chain/v1/testdata/config.yaml +++ b/ignite/config/chain/v1/testdata/config.yaml @@ -3,9 +3,6 @@ build: binary: evmosd proto: path: proto - third_party_paths: - - third_party/proto - - proto_vendor accounts: - name: alice coins: diff --git a/ignite/config/chain/v1/testdata/config2.yaml b/ignite/config/chain/v1/testdata/config2.yaml index 11afb27845..e611e287ff 100644 --- a/ignite/config/chain/v1/testdata/config2.yaml +++ b/ignite/config/chain/v1/testdata/config2.yaml @@ -3,9 +3,6 @@ build: binary: evmosd proto: path: proto - third_party_paths: - - third_party/proto - - proto_vendor accounts: - name: alice coins: diff --git a/ignite/internal/analytics/analytics.go b/ignite/internal/analytics/analytics.go index 8ad12669b5..08f85b9302 100644 --- a/ignite/internal/analytics/analytics.go +++ b/ignite/internal/analytics/analytics.go @@ -12,6 +12,7 @@ import ( "github.com/manifoldco/promptui" "github.com/spf13/cobra" + "github.com/ignite/cli/v29/ignite/config" "github.com/ignite/cli/v29/ignite/pkg/gacli" "github.com/ignite/cli/v29/ignite/pkg/gitpod" "github.com/ignite/cli/v29/ignite/pkg/randstr" @@ -23,7 +24,6 @@ const ( envDoNotTrack = "DO_NOT_TRACK" envCI = "CI" envGitHubActions = "GITHUB_ACTIONS" - igniteDir = ".ignite" igniteAnonIdentity = "anon_identity.json" ) @@ -79,14 +79,15 @@ func checkDNT() (anonIdentity, error) { return anonIdentity{DoNotTrack: true}, nil } - home, err := os.UserHomeDir() + globalPath, err := config.DirPath() if err != nil { return anonIdentity{}, err } - if err := os.Mkdir(filepath.Join(home, igniteDir), 0o700); err != nil && !os.IsExist(err) { + if err := os.Mkdir(globalPath, 0o700); err != nil && !os.IsExist(err) { return anonIdentity{}, err } - identityPath := filepath.Join(home, igniteDir, igniteAnonIdentity) + + identityPath := filepath.Join(globalPath, igniteAnonIdentity) data, err := os.ReadFile(identityPath) if err != nil && !os.IsNotExist(err) { return anonIdentity{}, err diff --git a/ignite/pkg/cosmosbuf/buf.go b/ignite/pkg/cosmosbuf/buf.go index 94dedeb047..a0470532ea 100644 --- a/ignite/pkg/cosmosbuf/buf.go +++ b/ignite/pkg/cosmosbuf/buf.go @@ -3,16 +3,16 @@ package cosmosbuf import ( "context" "fmt" - "os" "path/filepath" "strings" + "github.com/gobwas/glob" "golang.org/x/sync/errgroup" + "github.com/ignite/cli/v29/ignite/pkg/cache" "github.com/ignite/cli/v29/ignite/pkg/cmdrunner/exec" - "github.com/ignite/cli/v29/ignite/pkg/cosmosver" + "github.com/ignite/cli/v29/ignite/pkg/dircache" "github.com/ignite/cli/v29/ignite/pkg/errors" - "github.com/ignite/cli/v29/ignite/pkg/protoanalysis" "github.com/ignite/cli/v29/ignite/pkg/xexec" "github.com/ignite/cli/v29/ignite/pkg/xos" ) @@ -23,10 +23,18 @@ type ( // Buf represents the buf application structure. Buf struct { - path string - sdkProtoDir string - cache *protoanalysis.Cache + path string + cache dircache.Cache } + + // genOptions used to configure code generation. + genOptions struct { + excluded []glob.Glob + fileByFile bool + } + + // GenOption configures code generation. + GenOption func(*genOptions) ) const ( @@ -35,6 +43,7 @@ const ( flagOutput = "output" flagErrorFormat = "error-format" flagLogFormat = "log-format" + flagPath = "path" flagOnly = "only" fmtJSON = "json" @@ -42,6 +51,8 @@ const ( CMDGenerate Command = "generate" CMDExport Command = "export" CMDMod Command = "mod" + + specCacheNamespace = "generate.buf" ) var ( @@ -58,15 +69,38 @@ var ( ErrProtoFilesNotFound = errors.New("no proto files found") ) +// ExcludeFiles exclude file names from the generate command using glob. +func ExcludeFiles(patterns ...string) GenOption { + return func(o *genOptions) { + for _, pattern := range patterns { + o.excluded = append(o.excluded, glob.MustCompile(pattern)) + } + } +} + +// FileByFile runs the generate command for each proto file. +func FileByFile() GenOption { + return func(o *genOptions) { + o.fileByFile = true + } +} + // New creates a new Buf based on the installed binary. -func New() (Buf, error) { +func New(cacheStorage cache.Storage, goModPath string) (Buf, error) { path, err := xexec.ResolveAbsPath(binaryName) if err != nil { return Buf{}, err } + + bufCacheDir := filepath.Join("buf", goModPath) + c, err := dircache.New(cacheStorage, bufCacheDir, specCacheNamespace) + if err != nil { + return Buf{}, err + } + return Buf{ path: path, - cache: protoanalysis.NewCache(), + cache: c, }, nil } @@ -85,7 +119,7 @@ func (b Buf) Update(ctx context.Context, modDir string, dependencies ...string) } } - cmd, err := b.generateCommand(CMDMod, flags, "update", modDir) + cmd, err := b.command(CMDMod, flags, "update", modDir) if err != nil { return err } @@ -95,33 +129,7 @@ func (b Buf) Update(ctx context.Context, modDir string, dependencies ...string) // Export runs the buf Export command for the files in the proto directory. func (b Buf) Export(ctx context.Context, protoDir, output string) error { - // Check if the proto directory is the Cosmos SDK one - // TODO(@julienrbrt): this whole custom handling can be deleted - // after https://github.com/cosmos/cosmos-sdk/pull/18993 in v29. - if strings.Contains(protoDir, cosmosver.CosmosSDKRepoName) { - if b.sdkProtoDir == "" { - // Copy Cosmos SDK proto path without the Buf workspace. - // This is done because the workspace contains a reference to - // a "orm/internal" proto folder that is not present by default - // in the SDK repository. - d, err := copySDKProtoDir(protoDir) - if err != nil { - return err - } - - b.sdkProtoDir = d - } - - // Split absolute path into an absolute prefix and a relative suffix - paths := strings.Split(protoDir, "/proto") - if len(paths) < 2 { - return errors.Errorf("invalid Cosmos SDK mod path: %s", protoDir) - } - - // Use the SDK copy to resolve SDK proto files - protoDir = filepath.Join(b.sdkProtoDir, paths[1]) - } - specs, err := xos.FindFiles(protoDir, xos.ProtoFile) + specs, err := xos.FindFilesExtension(protoDir, xos.ProtoFile) if err != nil { return err } @@ -132,7 +140,7 @@ func (b Buf) Export(ctx context.Context, protoDir, output string) error { flagOutput: output, } - cmd, err := b.generateCommand(CMDExport, flags, protoDir) + cmd, err := b.command(CMDExport, flags, protoDir) if err != nil { return err } @@ -143,61 +151,70 @@ func (b Buf) Export(ctx context.Context, protoDir, output string) error { // Generate runs the buf Generate command for each file into the proto directory. func (b Buf) Generate( ctx context.Context, - protoDir, + protoPath, output, template string, - excludeFilename ...string, + options ...GenOption, ) (err error) { - var ( - excluded = make(map[string]struct{}) - flags = map[string]string{ - flagTemplate: template, - flagOutput: output, - flagErrorFormat: fmtJSON, - flagLogFormat: fmtJSON, - } - ) - for _, file := range excludeFilename { - excluded[file] = struct{}{} + opts := genOptions{} + for _, apply := range options { + apply(&opts) } - // TODO(@julienrbrt): this whole custom handling can be deleted - // after https://github.com/cosmos/cosmos-sdk/pull/18993 in v29. - if strings.Contains(protoDir, cosmosver.CosmosSDKRepoName) { - if b.sdkProtoDir == "" { - b.sdkProtoDir, err = copySDKProtoDir(protoDir) - if err != nil { - return err - } - } - dirs := strings.Split(protoDir, "/proto/") - if len(dirs) < 2 { - return errors.Errorf("invalid Cosmos SDK mod path: %s", dirs) - } - protoDir = filepath.Join(b.sdkProtoDir, dirs[1]) + // find all proto files into the path. + foundFiles, err := xos.FindFilesExtension(protoPath, xos.ProtoFile) + if err != nil || len(foundFiles) == 0 { + return err } - pkgs, err := protoanalysis.Parse(ctx, b.cache, protoDir) - if err != nil { + // check if already exist a cache for the template. + key, err := b.cache.CopyTo(protoPath, output, template) + if err != nil && !errors.Is(err, dircache.ErrCacheNotFound) { return err + } else if err == nil { + return nil } - g, ctx := errgroup.WithContext(ctx) - for _, pkg := range pkgs { - for _, file := range pkg.Files { - if _, ok := excluded[filepath.Base(file.Path)]; ok { - continue + // remove excluded and cached files. + protoFiles := make([]string, 0) + for _, file := range foundFiles { + okExclude := false + for _, g := range opts.excluded { + if g.Match(file) { + okExclude = true + break } + } + if !okExclude { + protoFiles = append(protoFiles, file) + } + } + if len(protoFiles) == 0 { + return nil + } - specs, err := xos.FindFiles(protoDir, xos.ProtoFile) - if err != nil { - return err - } - if len(specs) == 0 { - continue - } + flags := map[string]string{ + flagTemplate: template, + flagOutput: output, + flagErrorFormat: fmtJSON, + flagLogFormat: fmtJSON, + } - cmd, err := b.generateCommand(CMDGenerate, flags, file.Path) + if !opts.fileByFile { + cmd, err := b.command(CMDGenerate, flags, protoPath) + if err != nil { + return err + } + for _, file := range protoFiles { + cmd = append(cmd, fmt.Sprintf("--%s=%s", flagPath, file)) + } + if err := b.runCommand(ctx, cmd...); err != nil { + return err + } + } else { + g, ctx := errgroup.WithContext(ctx) + for _, file := range protoFiles { + cmd, err := b.command(CMDGenerate, flags, file) if err != nil { return err } @@ -207,16 +224,12 @@ func (b Buf) Generate( return b.runCommand(ctx, cmd...) }) } + if err := g.Wait(); err != nil { + return err + } } - return g.Wait() -} -// Cleanup deletes temporary files and directories. -func (b Buf) Cleanup() error { - if b.sdkProtoDir != "" { - return os.RemoveAll(b.sdkProtoDir) - } - return nil + return b.cache.Save(output, key) } // runCommand run the buf CLI command. @@ -227,8 +240,8 @@ func (b Buf) runCommand(ctx context.Context, cmd ...string) error { return exec.Exec(ctx, cmd, execOpts...) } -// generateCommand generate the buf CLI command. -func (b Buf) generateCommand( +// command generate the buf CLI command. +func (b Buf) command( c Command, flags map[string]string, args ...string, @@ -250,25 +263,3 @@ func (b Buf) generateCommand( } return command, nil } - -// findSDKProtoPath finds the Cosmos SDK proto folder path. -func findSDKProtoPath(protoDir string) string { - paths := strings.Split(protoDir, "@") - if len(paths) < 2 { - return protoDir - } - version := strings.Split(paths[1], "/")[0] - return fmt.Sprintf("%s@%s/proto", paths[0], version) -} - -// copySDKProtoDir copies the Cosmos SDK proto folder to a temporary directory. -// The temporary directory must be removed by the caller. -func copySDKProtoDir(protoDir string) (string, error) { - tmpDir, err := os.MkdirTemp("", "proto-sdk") - if err != nil { - return "", err - } - - srcPath := findSDKProtoPath(protoDir) - return tmpDir, xos.CopyFolder(srcPath, tmpDir) -} diff --git a/ignite/pkg/cosmosbuf/buf_test.go b/ignite/pkg/cosmosbuf/buf_test.go deleted file mode 100644 index 68187bf165..0000000000 --- a/ignite/pkg/cosmosbuf/buf_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package cosmosbuf - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestFindSDKPath(t *testing.T) { - testCases := []struct { - name string - protoDir string - want string - }{ - { - name: "full path", - protoDir: "/mod/github.com/cosmos/cosmos-sdk@v0.47.2/test/path/proto", - want: "/mod/github.com/cosmos/cosmos-sdk@v0.47.2/proto", - }, - { - name: "simple path", - protoDir: "myproto@v1/test/proto/animo/sdk", - want: "myproto@v1/proto", - }, - { - name: "only version", - protoDir: "test/myproto@v1", - want: "test/myproto@v1/proto", - }, - { - name: "semantic version", - protoDir: "test/myproto@v0.3.1/test/proto", - want: "test/myproto@v0.3.1/proto", - }, - { - name: "no version (local)", - protoDir: "test/myproto/test/proto", - want: "test/myproto/test/proto", - }, - } - - for _, tt := range testCases { - t.Run(tt.name, func(t *testing.T) { - got := findSDKProtoPath(tt.protoDir) - require.Equal(t, tt.want, got) - }) - } -} diff --git a/ignite/pkg/cosmosgen/cosmosgen.go b/ignite/pkg/cosmosgen/cosmosgen.go index 4ea01a58f1..6235d736d9 100644 --- a/ignite/pkg/cosmosgen/cosmosgen.go +++ b/ignite/pkg/cosmosgen/cosmosgen.go @@ -17,7 +17,6 @@ import ( // generateOptions used to configure code generation. type generateOptions struct { - includeDirs []string useCache bool updateBufModule bool ev events.Bus @@ -80,14 +79,6 @@ func WithOpenAPIGeneration(out string) Option { } } -// IncludeDirs configures the third party proto dirs that used by app's proto. -// relative to the projectPath. -func IncludeDirs(dirs []string) Option { - return func(o *generateOptions) { - o.includeDirs = dirs - } -} - // UpdateBufModule enables Buf config proto dependencies update. // This option updates app's Buf config when proto packages or // Buf modules are found within the Go dependencies. @@ -110,7 +101,7 @@ type generator struct { cacheStorage cache.Storage appPath string protoDir string - gomodPath string + goModPath string opts *generateOptions sdkImport string sdkDir string @@ -131,19 +122,17 @@ func (g *generator) cleanup() { // Generate generates code from protoDir of an SDK app residing at appPath with given options. // protoDir must be relative to the projectPath. -func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir, gomodPath string, options ...Option) error { - b, err := cosmosbuf.New() +func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir, goModPath string, options ...Option) error { + b, err := cosmosbuf.New(cacheStorage, goModPath) if err != nil { return err } - defer b.Cleanup() - g := &generator{ buf: b, appPath: appPath, protoDir: protoDir, - gomodPath: gomodPath, + goModPath: goModPath, opts: &generateOptions{}, thirdModules: make(map[string][]module.Module), thirdModuleIncludes: make(map[string]protoIncludes), diff --git a/ignite/pkg/cosmosgen/generate.go b/ignite/pkg/cosmosgen/generate.go index 81e1ec0543..fa4bfc7344 100644 --- a/ignite/pkg/cosmosgen/generate.go +++ b/ignite/pkg/cosmosgen/generate.go @@ -213,12 +213,7 @@ func (g *generator) setup(ctx context.Context) (err error) { } func (g *generator) getProtoIncludeFolders(modPath string) []string { - // Add default protoDir and default includeDirs - includePaths := []string{filepath.Join(modPath, g.protoDir)} - for _, dir := range g.opts.includeDirs { - includePaths = append(includePaths, filepath.Join(modPath, dir)) - } - return includePaths + return []string{filepath.Join(modPath, g.protoDir)} } func (g *generator) findBufPath(modpath string) (string, error) { diff --git a/ignite/pkg/cosmosgen/generate_go.go b/ignite/pkg/cosmosgen/generate_go.go index c8049c3134..2aeb52f6c9 100644 --- a/ignite/pkg/cosmosgen/generate_go.go +++ b/ignite/pkg/cosmosgen/generate_go.go @@ -7,6 +7,7 @@ import ( "github.com/otiai10/copy" + "github.com/ignite/cli/v29/ignite/pkg/cosmosbuf" "github.com/ignite/cli/v29/ignite/pkg/errors" ) @@ -18,41 +19,19 @@ func (g *generator) pulsarTemplate() string { return filepath.Join(g.appPath, g.protoDir, "buf.gen.pulsar.yaml") } -func (g *generator) generateGoGo(ctx context.Context) error { - // create a temporary dir to locate generated code under which later only some of them will be moved to the - // app's source code. this also prevents having leftover files in the app's source code or its parent dir - when - // command executed directly there - in case of an interrupt. - tmp, err := os.MkdirTemp("", "") - if err != nil { - return err - } - defer os.RemoveAll(tmp) - - protoPath := filepath.Join(g.appPath, g.protoDir) - - // code generate for each module. - err = g.buf.Generate(ctx, protoPath, tmp, g.gogoTemplate(), "module.proto") - if err != nil { - return err - } - - // move generated code for the app under the relative locations in its source code. - generatedPath := filepath.Join(tmp, g.gomodPath) - - _, err = os.Stat(generatedPath) - if err == nil { - err = copy.Copy(generatedPath, g.appPath) - if err != nil { - return errors.Wrap(err, "cannot copy path") - } - } else if !os.IsNotExist(err) { - return err - } +func (g *generator) protoPath() string { + return filepath.Join(g.appPath, g.protoDir) +} - return nil +func (g *generator) generateGoGo(ctx context.Context) error { + return g.generate(ctx, g.gogoTemplate(), g.goModPath, "*/module.proto") } func (g *generator) generatePulsar(ctx context.Context) error { + return g.generate(ctx, g.pulsarTemplate(), "") +} + +func (g *generator) generate(ctx context.Context, template, fromPath string, excluded ...string) error { // create a temporary dir to locate generated code under which later only some of them will be moved to the // app's source code. this also prevents having leftover files in the app's source code or its parent dir - when // command executed directly there - in case of an interrupt. @@ -62,18 +41,15 @@ func (g *generator) generatePulsar(ctx context.Context) error { } defer os.RemoveAll(tmp) - protoPath := filepath.Join(g.appPath, g.protoDir) - // code generate for each module. - err = g.buf.Generate(ctx, protoPath, tmp, g.pulsarTemplate()) - if err != nil { + if err := g.buf.Generate(ctx, g.protoPath(), tmp, template, cosmosbuf.ExcludeFiles(excluded...)); err != nil { return err } // move generated code for the app under the relative locations in its source code. - _, err = os.Stat(tmp) - if err == nil { - err = copy.Copy(tmp, g.appPath) + path := filepath.Join(tmp, fromPath) + if _, err := os.Stat(path); err == nil { + err = copy.Copy(path, g.appPath) if err != nil { return errors.Wrap(err, "cannot copy path") } diff --git a/ignite/pkg/cosmosgen/generate_openapi.go b/ignite/pkg/cosmosgen/generate_openapi.go index ebc029f6c3..28bf99a0a9 100644 --- a/ignite/pkg/cosmosgen/generate_openapi.go +++ b/ignite/pkg/cosmosgen/generate_openapi.go @@ -5,11 +5,14 @@ import ( "fmt" "os" "path/filepath" + "strings" + "github.com/blang/semver/v4" "github.com/iancoleman/strcase" "github.com/ignite/cli/v29/ignite/pkg/cache" "github.com/ignite/cli/v29/ignite/pkg/cosmosanalysis/module" + "github.com/ignite/cli/v29/ignite/pkg/cosmosbuf" "github.com/ignite/cli/v29/ignite/pkg/dirchange" "github.com/ignite/cli/v29/ignite/pkg/errors" swaggercombine "github.com/ignite/cli/v29/ignite/pkg/swagger-combine" @@ -32,7 +35,7 @@ func (g *generator) openAPITemplateForSTA() string { func (g *generator) generateOpenAPISpec(ctx context.Context) error { var ( specDirs []string - conf = swaggercombine.New("HTTP API Console", g.gomodPath) + conf = swaggercombine.New("HTTP API Console", g.goModPath) ) defer func() { for _, dir := range specDirs { @@ -46,14 +49,16 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { // gen generates a spec for a module where it's source code resides at src. // and adds needed swaggercombine configure for it. - gen := func(src string, m module.Module) (err error) { + gen := func(appPath, protoDir, name string) error { + name = strcase.ToCamel(name) + protoPath := filepath.Join(appPath, protoDir) + dir, err := os.MkdirTemp("", "gen-openapi-module-spec") if err != nil { return err } - checksumPaths := append([]string{m.Pkg.Path}, g.opts.includeDirs...) - checksum, err := dirchange.ChecksumFromPaths(src, checksumPaths...) + checksum, err := dirchange.ChecksumFromPaths(appPath, protoDir) if err != nil { return err } @@ -68,16 +73,31 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { if err := os.WriteFile(specPath, existingSpec, 0o644); err != nil { return err } - return conf.AddSpec(strcase.ToCamel(m.Pkg.Name), specPath, true) + return conf.AddSpec(name, specPath, true) } hasAnySpecChanged = true - err = g.buf.Generate(ctx, m.Pkg.Path, dir, g.openAPITemplate(), "module.proto") - if err != nil { - return err + if err = g.buf.Generate( + ctx, + protoPath, + dir, + g.openAPITemplate(), + cosmosbuf.ExcludeFiles( + "*/module.proto", + "*/testutil/*", + "*/testdata/*", + "*/cosmos/orm/*", + "*/cosmos/reflection/*", + "*/cosmos/app/v1alpha1/*", + "*/cosmos/tx/config/v1/config.proto", + "*/cosmos/msg/textual/v1/textual.proto", + ), + cosmosbuf.FileByFile(), + ); err != nil { + return errors.Wrapf(err, "failed to generate openapi spec %s, probally you need to exclude some proto files", protoPath) } - specs, err := xos.FindFiles(dir, xos.JSONFile) + specs, err := xos.FindFilesExtension(dir, xos.JSONFile) if err != nil { return err } @@ -90,7 +110,7 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { if err := specCache.Put(cacheKey, f); err != nil { return err } - if err := conf.AddSpec(strcase.ToCamel(m.Pkg.Name), spec, true); err != nil { + if err := conf.AddSpec(name, spec, true); err != nil { return err } } @@ -103,22 +123,27 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { // after add their path and config to swaggercombine.Config so we can combine them // into a single spec. - add := func(src string, modules []module.Module) error { - for _, m := range modules { - if err := gen(src, m); err != nil { - return err - } - } - return nil - } - // protoc openapi generator acts weird on concurrent run, so do not use goroutines here. - if err := add(g.appPath, g.appModules); err != nil { + if err := gen(g.appPath, g.protoDir, g.goModPath); err != nil { return err } - for src, modules := range g.thirdModules { - if err := add(src, modules); err != nil { + doneMods := make(map[string]struct{}) + for _, modules := range g.thirdModules { + if len(modules) == 0 { + continue + } + var ( + m = modules[0] + path = extractRootModulePath(m.Pkg.Path) + ) + + if _, ok := doneMods[path]; ok { + continue + } + doneMods[path] = struct{}{} + + if err := gen(path, "", m.Name); err != nil { return err } } @@ -151,7 +176,7 @@ func (g *generator) generateModuleOpenAPISpec(ctx context.Context, m module.Modu var ( specDirs []string title = "HTTP API Console " + m.Pkg.Name - conf = swaggercombine.New(title, g.gomodPath) + conf = swaggercombine.New(title, g.goModPath) ) defer func() { for _, dir := range specDirs { @@ -167,12 +192,12 @@ func (g *generator) generateModuleOpenAPISpec(ctx context.Context, m module.Modu return err } - err = g.buf.Generate(ctx, m.Pkg.Path, dir, g.openAPITemplateForSTA(), "module.proto") + err = g.buf.Generate(ctx, m.Pkg.Path, dir, g.openAPITemplateForSTA(), cosmosbuf.ExcludeFiles("*/module.proto")) if err != nil { return err } - specs, err := xos.FindFiles(dir, xos.JSONFile) + specs, err := xos.FindFilesExtension(dir, xos.JSONFile) if err != nil { return err } @@ -187,3 +212,21 @@ func (g *generator) generateModuleOpenAPISpec(ctx context.Context, m module.Modu // combine specs into one and save to out. return conf.Combine(out) } + +func extractRootModulePath(fullPath string) string { + var ( + segments = strings.Split(fullPath, "/") + modulePath = "/" + ) + + for _, segment := range segments { + modulePath = filepath.Join(modulePath, segment) + segmentName := strings.Split(segment, "@") + if len(segmentName) > 1 { + if _, err := semver.ParseTolerant(segmentName[1]); err == nil { + return modulePath + } + } + } + return fullPath +} diff --git a/ignite/pkg/cosmosgen/generate_openapi_test.go b/ignite/pkg/cosmosgen/generate_openapi_test.go new file mode 100644 index 0000000000..eca07b518d --- /dev/null +++ b/ignite/pkg/cosmosgen/generate_openapi_test.go @@ -0,0 +1,57 @@ +package cosmosgen + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_extractRootModulePath(t *testing.T) { + tests := []struct { + name string + path string + want string + }{ + { + name: "test cosmos-sdk path", + path: "/Users/ignite/Desktop/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.50.6/proto/cosmos/distribution/v1beta1", + want: "/Users/ignite/Desktop/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.50.6", + }, + { + name: "test cosmos-sdk module proto path", + path: "/Users/ignite/Desktop/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.50.6/x/bank", + want: "/Users/ignite/Desktop/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.50.6", + }, + { + name: "test ibc path", + path: "/Users/ignite/Desktop/go/pkg/mod/github.com/cosmos/ibc-go/v8@v8.2.0/proto/ibc/applications/interchain_accounts/controller/v1", + want: "/Users/ignite/Desktop/go/pkg/mod/github.com/cosmos/ibc-go/v8@v8.2.0", + }, + { + name: "test chain path", + path: "/Users/ignite/Desktop/go/src/github.com/ignite/venus", + want: "/Users/ignite/Desktop/go/src/github.com/ignite/venus", + }, + { + name: "test module path without version", + path: "/Users/ignite/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway/proto/applications", + want: "/Users/ignite/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway/proto/applications", + }, + { + name: "test module path with broken version", + path: "/Users/ignite/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway@v1.$/controller", + want: "/Users/ignite/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway@v1.$/controller", + }, + { + name: "test module path with v2 version", + path: "/Users/ignite/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway/v2@v2.19.1/proto/files", + want: "/Users/ignite/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway/v2@v2.19.1", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := extractRootModulePath(tt.path) + require.Equal(t, tt.want, got) + }) + } +} diff --git a/ignite/pkg/cosmosgen/generate_typescript.go b/ignite/pkg/cosmosgen/generate_typescript.go index 66d5f65179..47508c80aa 100644 --- a/ignite/pkg/cosmosgen/generate_typescript.go +++ b/ignite/pkg/cosmosgen/generate_typescript.go @@ -106,7 +106,7 @@ func (g *tsGenerator) generateModuleTemplates(ctx context.Context) error { gg.Go(func() error { cacheKey := m.Pkg.Path - paths := append([]string{m.Pkg.Path, g.g.opts.jsOut(m)}, g.g.opts.includeDirs...) + paths := []string{m.Pkg.Path, g.g.opts.jsOut(m)} // Always generate module templates by default unless cache is enabled, in which // case the module template is generated when one or more files were changed in diff --git a/ignite/pkg/dircache/cache.go b/ignite/pkg/dircache/cache.go new file mode 100644 index 0000000000..14efc36504 --- /dev/null +++ b/ignite/pkg/dircache/cache.go @@ -0,0 +1,111 @@ +package dircache + +import ( + "crypto/sha256" + "fmt" + "os" + "path/filepath" + + "github.com/otiai10/copy" + + "github.com/ignite/cli/v29/ignite/config" + "github.com/ignite/cli/v29/ignite/pkg/cache" + "github.com/ignite/cli/v29/ignite/pkg/dirchange" + "github.com/ignite/cli/v29/ignite/pkg/errors" +) + +var ErrCacheNotFound = errors.New("cache not found") + +type Cache struct { + path string + storageCache cache.Cache[string] +} + +// New creates a new Buf based on the installed binary. +func New(cacheStorage cache.Storage, dir, specNamespace string) (Cache, error) { + path, err := cachePath() + if err != nil { + return Cache{}, err + } + path = filepath.Join(path, dir) + if err := os.MkdirAll(path, 0o755); err != nil && !os.IsExist(err) { + return Cache{}, err + } + + return Cache{ + path: path, + storageCache: cache.New[string](cacheStorage, specNamespace), + }, nil +} + +// ClearCache remove the cache path. +func ClearCache() error { + path, err := cachePath() + if err != nil { + return err + } + return os.RemoveAll(path) +} + +// cachePath returns the cache path. +func cachePath() (string, error) { + globalPath, err := config.DirPath() + if err != nil { + return "", err + } + return filepath.Join(globalPath, "cache"), nil +} + +// cacheKey create the cache key. +func cacheKey(src string, keys ...string) (string, error) { + checksum, err := dirchange.ChecksumFromPaths(src, "") + if err != nil { + return "", err + } + + h := sha256.New() + if _, err := h.Write(checksum); err != nil { + return "", err + } + for _, key := range keys { + if _, err := h.Write([]byte(key)); err != nil { + return "", err + } + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +// CopyTo gets the cache folder based on the cache key from the storage and copies the folder to the output. +func (c Cache) CopyTo(src, output string, keys ...string) (string, error) { + key, err := cacheKey(src, keys...) + if err != nil { + return key, err + } + + cachedPath, err := c.storageCache.Get(key) + if errors.Is(err, cache.ErrorNotFound) { + return key, ErrCacheNotFound + } else if err != nil { + return key, err + } + + if err := copy.Copy(cachedPath, output); err != nil { + return "", errors.Wrapf(err, "get dir cache cannot copy path %s to %s", cachedPath, output) + } + return key, nil +} + +// Save copies the source to the cache folder and saves the path into the storage based on the key. +func (c Cache) Save(src, key string) error { + path := filepath.Join(c.path, key) + if err := os.Mkdir(path, 0o700); os.IsExist(err) { + return nil + } else if err != nil { + return err + } + + if err := copy.Copy(src, path); err != nil { + return errors.Wrapf(err, "save dir cache cannot copy path %s to %s", src, path) + } + return c.storageCache.Put(key, path) +} diff --git a/ignite/pkg/dircache/cache_test.go b/ignite/pkg/dircache/cache_test.go new file mode 100644 index 0000000000..8ba7d24953 --- /dev/null +++ b/ignite/pkg/dircache/cache_test.go @@ -0,0 +1,87 @@ +package dircache + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ignite/cli/v29/ignite/pkg/errors" +) + +func Test_cacheKey(t *testing.T) { + wd, err := os.Getwd() + require.NoError(t, err) + wd = filepath.Join(wd, "testdata") + + type args struct { + src string + keys []string + } + tests := []struct { + name string + args args + want string + err error + }{ + { + name: "no keys", + args: args{ + src: wd, + }, + want: "78f544d2184b8076ac527ba4728822de1a7fc77bf2d6a77e44d0193cb63ed26e", + }, + { + name: "one key", + args: args{ + src: wd, + keys: []string{"test"}, + }, + want: "5701099a1fcc67cd8b694295fbdecf537edcc8733bcc3adae0bdd7e65e28c8e5", + }, + { + name: "two keys", + args: args{ + src: wd, + keys: []string{"test1", "test2"}, + }, + want: "6299c9bd405a1c073fa711006f8aadf6420cf522ef446e36fc01586354726095", + }, + { + name: "duplicated keys", + args: args{ + src: wd, + keys: []string{"test", "test"}, + }, + want: "b9eb1b01931deccc44a354ab5aeb52337a465e5559069eb35b71ea0cbfe3c87f", + }, + { + name: "many keys", + args: args{ + src: wd, + keys: []string{"test1", "test2", "test3", "test4", "test5", "test6", "test6"}, + }, + want: "bbe74cfd33ba4d1244e8d0ea3e430081d06ed55be12c7772d345d3117a4dfc90", + }, + { + name: "invalid source", + args: args{ + src: "invalid_source", + }, + err: errors.New("no file in specified paths"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := cacheKey(tt.args.src, tt.args.keys...) + if tt.err != nil { + require.Error(t, err) + require.Equal(t, tt.err.Error(), err.Error()) + return + } + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} diff --git a/ignite/pkg/dircache/testdata/subdata/subfile b/ignite/pkg/dircache/testdata/subdata/subfile new file mode 100644 index 0000000000..41823eb5b1 --- /dev/null +++ b/ignite/pkg/dircache/testdata/subdata/subfile @@ -0,0 +1 @@ +subtest \ No newline at end of file diff --git a/ignite/pkg/dircache/testdata/testfile b/ignite/pkg/dircache/testdata/testfile new file mode 100644 index 0000000000..30d74d2584 --- /dev/null +++ b/ignite/pkg/dircache/testdata/testfile @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/ignite/pkg/xos/files.go b/ignite/pkg/xos/files.go index 60eb6e4a0e..0ef8ea661c 100644 --- a/ignite/pkg/xos/files.go +++ b/ignite/pkg/xos/files.go @@ -11,7 +11,21 @@ const ( ProtoFile = "proto" ) -func FindFiles(directory, extension string) ([]string, error) { +func FindFiles(directory string) ([]string, error) { + files := make([]string, 0) + return files, filepath.Walk(directory, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if !info.IsDir() { + files = append(files, path) + } + return nil + }) +} + +func FindFilesExtension(directory, extension string) ([]string, error) { files := make([]string, 0) return files, filepath.Walk(directory, func(path string, info os.FileInfo, err error) error { if err != nil { diff --git a/ignite/pkg/xos/files_test.go b/ignite/pkg/xos/files_test.go index dec88c6fc1..cbd5a34903 100644 --- a/ignite/pkg/xos/files_test.go +++ b/ignite/pkg/xos/files_test.go @@ -26,6 +26,13 @@ func TestFindFiles(t *testing.T) { want: []string{"file1.json", "file3.json", "file4.json"}, err: nil, }, + { + name: "test 3 json files with subfolder", + files: []string{"testdata/file1.json", "file2.txt", "foo/file3.json", "file4.json"}, + extension: "json", + want: []string{"testdata/file1.json", "foo/file3.json", "file4.json"}, + err: nil, + }, { name: "test 1 txt files", files: []string{"file1.json", "file2.txt", "file3.json", "file4.json"}, @@ -59,12 +66,13 @@ func TestFindFiles(t *testing.T) { for _, filename := range tt.files { filePath := filepath.Join(tempDir, filename) + require.NoError(t, os.MkdirAll(filepath.Dir(filePath), 0o755)) file, err := os.Create(filePath) require.NoError(t, err) require.NoError(t, file.Close()) } - gotFiles, err := xos.FindFiles(tempDir, tt.extension) + gotFiles, err := xos.FindFilesExtension(tempDir, tt.extension) if tt.err != nil { require.Error(t, err) require.ErrorIs(t, err, tt.err) @@ -76,7 +84,7 @@ func TestFindFiles(t *testing.T) { for i, filename := range tt.want { want[i] = filepath.Join(tempDir, filename) } - require.EqualValues(t, want, gotFiles) + require.ElementsMatch(t, want, gotFiles) }) } } diff --git a/ignite/services/chain/generate.go b/ignite/services/chain/generate.go index 2def391c22..c36854450b 100644 --- a/ignite/services/chain/generate.go +++ b/ignite/services/chain/generate.go @@ -145,10 +145,7 @@ func (c *Chain) Generate( c.ev.Send("Building proto...", events.ProgressUpdate()) - options := []cosmosgen.Option{ - cosmosgen.CollectEvents(c.ev), - cosmosgen.IncludeDirs(conf.Build.Proto.ThirdPartyPaths), - } + options := []cosmosgen.Option{cosmosgen.CollectEvents(c.ev)} if targetOptions.isGoEnabled { options = append(options, cosmosgen.WithGoGeneration()) diff --git a/ignite/services/scaffolder/scaffolder.go b/ignite/services/scaffolder/scaffolder.go index f50e55a2b1..71dfeb0ef5 100644 --- a/ignite/services/scaffolder/scaffolder.go +++ b/ignite/services/scaffolder/scaffolder.go @@ -124,7 +124,6 @@ func protoc(ctx context.Context, cacheStorage cache.Storage, projectPath, protoD options := []cosmosgen.Option{ cosmosgen.UpdateBufModule(), cosmosgen.WithGoGeneration(), - cosmosgen.IncludeDirs(conf.Build.Proto.ThirdPartyPaths), } // Generate Typescript client code if it's enabled diff --git a/ignite/templates/app/files/{{protoDir}}/buf.lock b/ignite/templates/app/files/{{protoDir}}/buf.lock index 77287f75ce..0a6a9564ce 100644 --- a/ignite/templates/app/files/{{protoDir}}/buf.lock +++ b/ignite/templates/app/files/{{protoDir}}/buf.lock @@ -8,24 +8,24 @@ deps: - remote: buf.build owner: cosmos repository: cosmos-proto - commit: 1935555c206d4afb9e94615dfd0fad31 + commit: 04467658e59e44bbb22fe568206e1f70 - remote: buf.build owner: cosmos repository: cosmos-sdk - commit: 954f7b05f38440fc8250134b15adec47 + commit: 05419252bcc241ea8023acf1ed4cadc5 - remote: buf.build owner: cosmos repository: gogo-proto - commit: 34d970b699f84aa382f3c29773a60836 + commit: 88ef6483f90f478fb938c37dde52ece3 - remote: buf.build owner: cosmos repository: ics23 - commit: 3c44d8daa8b44059ac744cd17d4a49d7 + commit: a9ee7c290ef34ee69d3f141b9b44dcee - remote: buf.build owner: googleapis repository: googleapis - commit: 75b4300737fb4efca0831636be94e517 + commit: 09703837a2ed48dbbbb3fdfbe6a84f5c - remote: buf.build owner: protocolbuffers repository: wellknowntypes - commit: 44e83bc050a4497fa7b36b34d95ca156 + commit: 3186086b2a8e44d9acdeeef2423c5de7 diff --git a/integration/doctor/testdata/config-need-migrate.txt b/integration/doctor/testdata/config-need-migrate.txt index e29d184340..388673c6f8 100644 --- a/integration/doctor/testdata/config-need-migrate.txt +++ b/integration/doctor/testdata/config-need-migrate.txt @@ -35,9 +35,6 @@ version: 1 build: proto: path: proto - third_party_paths: - - third_party/proto - - proto_vendor accounts: - name: alice coins: