feat(internal/postprocessor): detect and initialize new modules

internal/postprocessor/README.md

@@ -37,20 +37,17 @@ clients in the root directory.
 
 ### Run post-processor on all clients
 
-From the `google-cloud-go/internal/postprocessor` directory run:
-
+From the `google-cloud-go/internal/postprocessor` directory run: 
 ```bash
-go run main.go -stage-dir="../../owl-bot-staging/src/" -client-root="../.." -googleapis-dir="/path/to/local/googleapis"
+go run main.go -client-root="../.." -googleapis-dir="/path/to/local/googleapis"
 ```
 
 ### Run post-processor on select clients
 
-From the `google-cloud-go/internal/postprocessor` directory run the same
-command, but with an added `dirs` flag containing a comma-separated list of the
-names of the clients on which to run the post-processor. The example below shows
-the command for running the post-processor on the `accessapproval` and `asset`
-libraries:
-
+From the `google-cloud-go/internal/postprocessor` directory run the same command, but with an added `dirs` flag containing a comma-separated list of the names of the clients on which to run the post-processor. The example below shows the command for running the post-processor on the `accessapproval` and `asset` libraries:
 ```bash
-go run main.go -stage-dir="../../owl-bot-staging/src/" -client-root="../.." -googleapis-dir="/path/to/local/googleapis" -dirs="accessapproval,asset"
+go run main.go -client-root="../.." -googleapis-dir="/path/to/local/googleapis" -dirs="accessapproval,asset"
 ```
+
+### Initializing new modules
+To initialize the `internal/version.go`, `go.mod`, `README.md`, and `CHANGES.md` files in a new module, add the module to the slice in `modconfig.go`. The entry should correspond to the location where the `go.mod` file should be initialized minus the prefix "google-cloud-go/"

internal/postprocessor/_README.md.txt

@@ -0,0 +1,44 @@
+# {{.Name}}
+
+[![Go Reference](https://pkg.go.dev/badge/{{.ImportPath}}.svg)](https://pkg.go.dev/{{.ImportPath}})
+
+Go Client Library for {{.Name}}.
+
+## Install
+
+```bash
+go get {{.ImportPath}}
+```
+
+## Stability
+
+The stability of this module is indicated by SemVer.
+
+However, a `v1+` module may have breaking changes in two scenarios:
+
+* Packages with `alpha` or `beta` in the import path
+* The GoDoc has an explicit stability disclaimer (for example, for an experimental feature).
+
+## Google Cloud Samples
+
+To browse ready to use code samples check [Google Cloud Samples](https://cloud.google.com/docs/samples?l=go).
+
+## Go Version Support
+
+See the [Go Versions Supported](https://github.com/googleapis/google-cloud-go#go-versions-supported)
+section in the root directory's README.
+
+## Authorization
+
+See the [Authorization](https://github.com/googleapis/google-cloud-go#authorization)
+section in the root directory's README.
+
+## Contributing
+
+Contributions are welcome. Please, see the [CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md)
+document for details.
+
+Please note that this project is released with a Contributor Code of Conduct.
+By participating in this project you agree to abide by its terms. See
+[Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md#contributor-code-of-conduct)
+for more information.

internal/postprocessor/_internal_version.go.txt

@@ -0,0 +1,20 @@
+// Copyright {{.Year}} Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by gapicgen. DO NOT EDIT.
+
+package internal
+
+// Version is the current tagged release of the library.
+const Version = "0.0.0"

internal/postprocessor/_version.go.txt

@@ -0,0 +1,23 @@
+// Copyright {{.Year}} Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by gapicgen. DO NOT EDIT.
+
+package {{.Package}}
+
+import "{{.ModuleRootInternal}}"
+
+func init() {
+    versionClient = internal.Version
+}

internal/postprocessor/go.mod

@@ -33,7 +33,7 @@ require (
 	golang.org/x/sys v0.3.0 // indirect
 	golang.org/x/tools v0.4.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230112194545-e10362b5ecf9 // indirect
+	google.golang.org/genproto v0.0.0-20230202175211-008b39050e57 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect
 	gopkg.in/src-d/go-git.v4 v4.13.1 // indirect

internal/postprocessor/go.sum

@@ -103,8 +103,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20230112194545-e10362b5ecf9 h1:ru6tJGasqJpgjM4q3Qq2fS3FKQ6CPPSRqgolUVBc994=
-google.golang.org/genproto v0.0.0-20230112194545-e10362b5ecf9/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
+google.golang.org/genproto v0.0.0-20230202175211-008b39050e57 h1:vArvWooPH749rNHpBGgVl+U9B9dATjiEhJzcWGlovNs=
+google.golang.org/genproto v0.0.0-20230202175211-008b39050e57/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
 google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=

internal/postprocessor/main.go

@@ -16,16 +16,19 @@ package main
 
 import (
 	"context"
+	_ "embed"
 	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
+	"html/template"
 	"io/fs"
 	"log"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
+	"time"
 
 	"cloud.google.com/go/internal/gapicgen/generator"
 	"cloud.google.com/go/internal/gapicgen/git"
@@ -51,6 +54,15 @@ var (
 	secondPartTitlePattern = regexp.MustCompile(`.*\: *\` + apiNameOwlBotScope + ` *(?P<titleSecondPart>.*)`)
 )
 
+var (
+	//go:embed _README.md.txt
+	readmeTmpl string
+	//go:embed _version.go.txt
+	versionTmpl string
+	//go:embed _internal_version.go.txt
+	internalVersionTmpl string
+)
+
 func main() {
 	clientRoot := flag.String("client-root", "/workspace/google-cloud-go", "Path to clients.")
 	googleapisDir := flag.String("googleapis-dir", "", "Path to googleapis/googleapis repo.")
@@ -158,15 +170,17 @@ func (c *config) run(ctx context.Context) error {
 		log.Println("exiting post processing early")
 		return nil
 	}
-	// TODO(guadriana): Once the PR for initializing new modules is merged, we
-	// will need to add logic here that sets c.modules to modConfigs (a slice of
-	// all modules) if branchOverride != ""
-
-	// TODO(codyoss): In the future we may want to make it possible to be able
-	// to run this locally with a user defined remote branch.
+	manifest, err := c.Manifest(generator.MicrogenGapicConfigs)
+	if err != nil {
+		return err
+	}
+	if err := c.InitializeNewModules(manifest); err != nil {
+		return err
+	}
 	if err := c.SetScopesAndPRInfo(ctx); err != nil {
 		return err
 	}
+
 	if err := c.TidyAffectedMods(); err != nil {
 		return err
 	}
@@ -179,12 +193,143 @@ func (c *config) run(ctx context.Context) error {
 	if _, err := c.Manifest(generator.MicrogenGapicConfigs); err != nil {
 		return err
 	}
+	// TODO(codyoss): In the future we may want to make it possible to be able
+	// to run this locally with a user defined remote branch.
 	if err := c.WritePRInfoToFile(); err != nil {
 		return err
 	}
 	return nil
 }
 
+// InitializeNewModule detects new modules and clients and generates the required minimum files
+// For modules, the minimum required files are internal/version.go, README.md, CHANGES.md, and go.mod
+// For clients, the minimum required files are a version.go file
+func (c *config) InitializeNewModules(manifest map[string]generator.ManifestEntry) error {
+	log.Println("checking for new modules and clients")
+	for _, moduleName := range moduleConfigs {
+		modulePath := filepath.Join(c.googleCloudDir, moduleName)
+		importPath := filepath.Join("cloud.google.com/go", moduleName)
+
+		pathToModVersionFile := filepath.Join(modulePath, "internal/version.go")
+		// Check if <module>/internal/version.go file exists
+		if _, err := os.Stat(pathToModVersionFile); errors.Is(err, fs.ErrNotExist) {
+			var serviceImportPath string
+			for _, conf := range generator.MicrogenGapicConfigs {
+				if strings.Contains(conf.ImportPath, importPath) {
+					serviceImportPath = conf.ImportPath
+					break
+				}
+			}
+			if serviceImportPath == "" {
+				return fmt.Errorf("no corresponding config found for module %s. Cannot generate min required files", moduleName)
+			}
+			// serviceImportPath here should be a valid ImportPath from a MicrogenGapicConfigs
+			apiName := manifest[serviceImportPath].Description
+			if err := c.generateMinReqFilesNewMod(moduleName, modulePath, importPath, apiName); err != nil {
+				return err
+			}
+		}
+		// Check if version.go files exist for each client
+		filepath.WalkDir(modulePath, func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if !d.IsDir() {
+				return nil
+			}
+			splitPath := strings.Split(path, "/")
+			lastElement := splitPath[len(splitPath)-1]
+			if !strings.Contains(lastElement, "apiv") {
+				return nil
+			}
+			pathToClientVersionFile := filepath.Join(path, "version.go")
+			if _, err = os.Stat(pathToClientVersionFile); errors.Is(err, fs.ErrNotExist) {
+				log.Println("generating version.go file in", path)
+				if err := c.generateVersionFile(moduleName, path); err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+	}
+	return nil
+}
+
+func (c *config) generateMinReqFilesNewMod(moduleName, modulePath, importPath, apiName string) error {
+	log.Println("generating files for new module", apiName)
+	if err := generateReadmeAndChanges(modulePath, importPath, apiName); err != nil {
+		return err
+	}
+	if err := c.generateInternalVersionFile(moduleName); err != nil {
+		return err
+	}
+	if err := c.generateModule(modulePath, importPath); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *config) generateModule(modPath, importPath string) error {
+	if err := os.MkdirAll(modPath, os.ModePerm); err != nil {
+		return err
+	}
+	log.Printf("Creating %s/go.mod", modPath)
+	return gocmd.ModInit(modPath, importPath)
+}
+
+func (c *config) generateVersionFile(moduleName, modulePath string) error {
+	// These directories are not modules on purpose, don't generate a version
+	// file for them.
+	if strings.Contains(modulePath, "debugger/apiv2") {
+		return nil
+	}
+	rootPackage := filepath.Dir(modulePath)
+	rootModInternal := fmt.Sprintf("cloud.google.com/go/%s/internal", rootPackage)
+
+	f, err := os.Create(filepath.Join(modulePath, "version.go"))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	t := template.Must(template.New("version").Parse(versionTmpl))
+	versionData := struct {
+		Year               int
+		Package            string
+		ModuleRootInternal string
+	}{
+		Year:               time.Now().Year(),
+		Package:            moduleName,
+		ModuleRootInternal: rootModInternal,
+	}
+	if err := t.Execute(f, versionData); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *config) generateInternalVersionFile(apiName string) error {
+	rootModInternal := filepath.Join(apiName, "internal")
+	os.MkdirAll(filepath.Join(c.googleCloudDir, rootModInternal), os.ModePerm)
+
+	f, err := os.Create(filepath.Join(c.googleCloudDir, rootModInternal, "version.go"))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	t := template.Must(template.New("internal_version").Parse(internalVersionTmpl))
+	internalVersionData := struct {
+		Year int
+	}{
+		Year: time.Now().Year(),
+	}
+	if err := t.Execute(f, internalVersionData); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (c *config) getDirs() []string {
 	dirs := []string{}
 	for _, module := range c.modules {
@@ -203,6 +348,38 @@ func (c *config) TidyAffectedMods() error {
 	return nil
 }
 
+// Copied from generator package
+func generateReadmeAndChanges(path, importPath, apiName string) error {
+	readmePath := filepath.Join(path, "README.md")
+	log.Printf("Creating %q", readmePath)
+	readmeFile, err := os.Create(readmePath)
+	if err != nil {
+		return err
+	}
+	defer readmeFile.Close()
+	t := template.Must(template.New("readme").Parse(readmeTmpl))
+	readmeData := struct {
+		Name       string
+		ImportPath string
+	}{
+		Name:       apiName,
+		ImportPath: importPath,
+	}
+	if err := t.Execute(readmeFile, readmeData); err != nil {
+		return err
+	}
+
+	changesPath := filepath.Join(path, "CHANGES.md")
+	log.Printf("Creating %q", changesPath)
+	changesFile, err := os.Create(changesPath)
+	if err != nil {
+		return err
+	}
+	defer changesFile.Close()
+	_, err = changesFile.WriteString("# Changes\n")
+	return err
+}
+
 // RegenSnippets regenerates the snippets for all GAPICs configured to be generated.
 func (c *config) RegenSnippets() error {
 	log.Println("regenerating snippets")
@@ -394,7 +571,12 @@ func (c *config) processCommit(title, body string) (string, string, error) {
 		bodySlice[commitTitleIndex] = newCommitTitle
 	}
 	body = strings.Join(bodySlice, "\n")
-	c.modules = append(c.modules, modules...)
+	if c.branchOverride != "" {
+		c.modules = []string{}
+		c.modules = append(c.modules, moduleConfigs...)
+	} else {
+		c.modules = append(c.modules, modules...)
+	}
 	return newPRTitle, body, nil
 }