Merge pull request #8 from Fernando-Dourado/fernando/gitx

Added support for Git Experience folder convention and Inputsets

Readme.md

@@ -66,6 +66,13 @@ You can run the migration utility using the following commands:
 ```
 ./harness-remote-migrator -config /path/to/config.yaml -pipelines
 ```
+
+**Run ONLY for input sets:**
+
+```sh
+./harness-remote-migrator -config /path/to/config.yaml -inputsets
+```
+
 **Run ONLY for templates:**
 ```
 ./harness-remote-migrator -config /path/to/config.yaml -templates
@@ -110,7 +117,15 @@ You can run the migration utility using the following commands:
 - You can use the flag `custom-remote-path` to point where to save YAMLs inside the remote repository.
 - When using this argument you should avoid running multiple migrations at same time or all the files will be stored at the same path.
 
-### CLI Arguments
+### Git Experience
+
+Use the flag ```-gitx``` to enable support to move entities following the Git Experience folder path convention.
+
+```sh
+./harness-remote-migrator -config /path/to/config.yaml -gitx -environments
+```
+
+## CLI Arguments
 
 ***To be added***
 
@@ -140,6 +155,7 @@ This folder structure will be used to reference the files in your Services/Envir
 ## Supported Entities
 
 1. Pipelines
+1. InputSets
 1. Templates
 1. Services
 1. Environments

harness/api.go

@@ -78,6 +78,33 @@ func (api *APIRequest) GetAllPipelines(account, org, project string) (Pipelines,
 	return pipelines, nil
 }
 
+func (api *APIRequest) GetInputsets(account, org, project, pipeline string) ([]*InputsetContent, error) {
+
+	resp, err := api.Client.R().
+		SetHeader("x-api-key", api.APIKey).
+		SetHeader("Content-Type", "application/json").
+		SetQueryParams(map[string]string{
+			"routingId":          account,
+			"accountIdentifier":  account,
+			"orgIdentifier":      org,
+			"projectIdentifier":  project,
+			"pipelineIdentifier": pipeline,
+			"size":               "1000",
+		}).
+		Get(api.BaseURL + "/gateway/pipeline/api/inputSets")
+	if err != nil {
+		return nil, err
+	}
+
+	result := ListInputsetResponse{}
+	err = json.Unmarshal(resp.Body(), &result)
+	if err != nil {
+		return nil, err
+	}
+
+	return result.Data.Content, nil
+}
+
 func (api *APIRequest) GetAllTemplates(account, org, project string) (Templates, error) {
 	resp, err := api.Client.R().
 		SetHeader("x-api-key", api.APIKey).
@@ -240,6 +267,41 @@ func (e *EnvironmentClass) MoveEnvironmentToRemote(api *APIRequest, c Config) er
 	return err
 }
 
+func (is *InputsetContent) MoveInputsetToRemote(api *APIRequest, c Config, project, org string) error {
+	resp, err := api.Client.R().
+		SetHeader("x-api-key", api.APIKey).
+		SetPathParam("identifier", is.Identifier).
+		SetHeader("Content-Type", "application/json").
+		SetQueryParams(map[string]string{
+			"accountIdentifier":  c.AccountIdentifier,
+			"projectIdentifier":  project,
+			"orgIdentifier":      org,
+			"pipelineIdentifier": is.PipelineIdentifier,
+			"inputSetIdentifier": is.Identifier,
+			"connectorRef":       c.GitDetails.ConnectorRef,
+			"repoName":           c.GitDetails.RepoName,
+			"branch":             c.GitDetails.BranchName,
+			"isNewBranch":        "false",
+			"isHarnessCodeRepo":  "false",
+			"filePath":           c.GitDetails.FilePath,
+			"commitMsg":          c.GitDetails.CommitMessage,
+			"moveConfigType":     "INLINE_TO_REMOTE",
+		}).
+		Post(api.BaseURL + "/gateway/pipeline/api/inputSets/move-config/{identifier}")
+
+	if resp.StatusCode() != 200 {
+		ar := ApiResponse{}
+		err = json.Unmarshal(resp.Body(), &ar)
+		if err != nil {
+			return err
+		}
+		errMsg := fmt.Sprintf("CorrelationId: %s, ResponseMessages: %+v", ar.CorrelationID, ar.ResponseMessages)
+		return fmt.Errorf(errMsg)
+	}
+
+	return err
+}
+
 func (i *Infrastructure) MoveInfrastructureToRemote(api *APIRequest, c Config, envId string) error {
 
 	resp, err := api.Client.R().

harness/git_details.go

@@ -1,41 +1,71 @@
 package harness
 
-func GetPipelineFilePath(customGitDetailsFilePath string, p Project, pipeline PipelineContent) string {
+import (
+	"fmt"
+)
+
+func GetPipelineFilePath(gitX bool, customGitDetailsFilePath string, p Project, pipeline PipelineContent) string {
 	if len(customGitDetailsFilePath) == 0 {
+		if gitX {
+			return fmt.Sprintf(".harness/orgs/%s/projects/%s/pipelines/%s.yaml", string(p.OrgIdentifier), p.Identifier, pipeline.Identifier)
+		}
 		return "pipelines/" + string(p.OrgIdentifier) + "/" + p.Identifier + "/" + pipeline.Identifier + ".yaml"
 	} else {
 		return customGitDetailsFilePath + "/" + pipeline.Identifier + ".yaml"
 	}
 }
 
-func GetTemplateFilePath(customGitDetailsFilePath string, p Project, template Template) string {
+func GetTemplateFilePath(gitX bool, customGitDetailsFilePath string, p Project, template Template) string {
 	if len(customGitDetailsFilePath) == 0 {
+		if gitX {
+			return fmt.Sprintf(".harness/orgs/%s/projects/%s/templates/%s/%s.yaml", string(p.OrgIdentifier), p.Identifier, template.Identifier, template.VersionLabel)
+		}
 		return "templates/" + string(p.OrgIdentifier) + "/" + p.Identifier + "/" + template.Identifier + "-" + template.VersionLabel + ".yaml"
 	} else {
 		return customGitDetailsFilePath + "/" + template.Identifier + "-" + template.VersionLabel + ".yaml"
 	}
 }
 
-func GetServiceFilePath(customGitDetailsFilePath string, p Project, service ServiceClass) string {
+func GetServiceFilePath(gitX bool, customGitDetailsFilePath string, p Project, service ServiceClass) string {
 	if len(customGitDetailsFilePath) == 0 {
+		if gitX {
+			return fmt.Sprintf(".harness/orgs/%s/projects/%s/services/%s.yaml", string(p.OrgIdentifier), p.Identifier, service.Identifier)
+		}
 		return "services/" + string(p.OrgIdentifier) + "/" + p.Identifier + "/" + service.Identifier + ".yaml"
 	} else {
 		return customGitDetailsFilePath + "/" + service.Identifier + ".yaml"
 	}
 }
 
-func GetEnvironmentFilePath(customGitDetailsFilePath string, p Project, env EnvironmentClass) string {
+func GetEnvironmentFilePath(gitX bool, customGitDetailsFilePath string, p Project, env EnvironmentClass) string {
 	if len(customGitDetailsFilePath) == 0 {
+		if gitX {
+			return fmt.Sprintf(".harness/orgs/%s/projects/%s/environments/%s.yaml", string(p.OrgIdentifier), p.Identifier, env.Identifier)
+		}
 		return "environments/" + string(p.OrgIdentifier) + "/" + p.Identifier + "/" + env.Identifier + ".yaml"
 	} else {
 		return customGitDetailsFilePath + "/" + env.Identifier + ".yaml"
 	}
 }
 
-func GetInfrastructureFilePath(customGitDetailsFilePath string, p Project, env EnvironmentClass, infraDef Infrastructure) string {
+func GetInfrastructureFilePath(gitX bool, customGitDetailsFilePath string, p Project, env EnvironmentClass, infraDef Infrastructure) string {
 	if len(customGitDetailsFilePath) == 0 {
+		if gitX {
+			return fmt.Sprintf(".harness/orgs/%s/projects/%s/environments/%s/infrastructures/%s.yaml", string(p.OrgIdentifier), p.Identifier, env.Identifier, infraDef.Identifier)
+		}
 		return "environments/" + string(p.OrgIdentifier) + "/" + p.Identifier + "/" + env.Identifier + "-" + infraDef.Identifier + ".yaml"
 	} else {
 		return customGitDetailsFilePath + "/" + env.Identifier + "-" + infraDef.Identifier + ".yaml"
 	}
 }
+
+func GetInputsetFilePath(gitX bool, customGitDetailsFilePath string, p Project, is *InputsetContent) string {
+	if len(customGitDetailsFilePath) == 0 {
+		if gitX {
+			return fmt.Sprintf(".harness/orgs/%s/projects/%s/pipelines/%s/input_sets/%s.yaml", string(p.OrgIdentifier), p.Identifier, is.PipelineIdentifier, is.Identifier)
+		}
+		return "input_sets/" + string(p.OrgIdentifier) + "/" + p.Identifier + "/" + is.PipelineIdentifier + "/" + is.Identifier + ".yaml"
+	} else {
+		return customGitDetailsFilePath + "/" + is.PipelineIdentifier + "/" + is.Identifier + ".yaml"
+	}
+}

harness/input_sets.go

@@ -0,0 +1,26 @@
+package harness
+
+type ListInputsetResponse struct {
+	Status        string           `json:"status"`
+	Data          ListInputsetData `json:"data"`
+	CorrelationID string           `json:"correlationId"`
+}
+
+type ListInputsetData struct {
+	TotalPages    int64              `json:"totalPages"`
+	TotalItems    int64              `json:"totalItems"`
+	PageItemCount int64              `json:"pageItemCount"`
+	PageSize      int64              `json:"pageSize"`
+	Content       []*InputsetContent `json:"content"`
+	PageIndex     int64              `json:"pageIndex"`
+	Empty         bool               `json:"empty"`
+}
+
+type InputsetContent struct {
+	Identifier            string                `json:"identifier"`
+	Name                  string                `json:"name"`
+	PipelineIdentifier    string                `json:"pipelineIdentifier"`
+	InputSetType          string                `json:"inputSetType"`
+	EntityValidityDetails EntityValidityDetails `json:"entityValidityDetails"`
+	StoreType             string                `json:"storeType"`
+}

main.go

@@ -35,6 +35,7 @@ func main() {
 	targetProjects := flag.String("target-projects", "", "Provide a list of projects to target.")
 	allFlag := flag.Bool("all", false, "Migrate all entities.")
 	pipelinesFlag := flag.Bool("pipelines", false, "Migrate pipelines.")
+	inputsetsFlag := flag.Bool("inputsets", false, "Migrate inputsets.")
 	templatesFlag := flag.Bool("templates", false, "Migrate templates.")
 	servicesFlag := flag.Bool("services", false, "Migrate services")
 	envFlag := flag.Bool("environments", false, "Migrate environments")
@@ -47,11 +48,13 @@ func main() {
 	cgFolderStructure := flag.Bool("alt-path", false, "CG-like folder structure for Git")
 	prod3 := flag.Bool("prod3", false, "User Prod3 base URL for API calls")
 	customGitDetailsFilePath := flag.String("custom-remote-path", "", "A custom file path where to save remote manifests.")
+	gitX := flag.Bool("gitx", false, "Migrate entity following the Git Experience definitions")
 
 	flag.Parse()
 
 	type MigrationScope struct {
 		Pipelines            bool
+		Inputsets            bool
 		Templates            bool
 		Services             bool
 		Environments         bool
@@ -63,12 +66,14 @@ func main() {
 		UrlEncoding          bool
 		CGFolderStructure    bool
 		Prod3                bool
+		GitX                 bool
 	}
 	scope := MigrationScope{}
 
 	if *allFlag {
 		scope = MigrationScope{
 			Pipelines:            true,
+			Inputsets:            true,
 			Templates:            true,
 			Services:             true,
 			Environments:         true,
@@ -80,10 +85,12 @@ func main() {
 			UrlEncoding:          *urlEncoding,
 			CGFolderStructure:    false,
 			Prod3:                false,
+			GitX:                 *gitX,
 		}
 	} else {
 		scope = MigrationScope{
 			Pipelines:            *pipelinesFlag,
+			Inputsets:            *inputsetsFlag,
 			Templates:            *templatesFlag,
 			Services:             *servicesFlag,
 			Environments:         *envFlag,
@@ -95,6 +102,7 @@ func main() {
 			UrlEncoding:          *urlEncoding,
 			CGFolderStructure:    *cgFolderStructure,
 			Prod3:                *prod3,
+			GitX:                 *gitX,
 		}
 	}
 
@@ -130,7 +138,7 @@ func main() {
 		APIKey:  accountConfig.ApiKey,
 	}
 
-	if !scope.Pipelines && !scope.Templates && !scope.FileStore && !scope.Overrides && !scope.Services && !scope.Environments && !scope.InfraDef {
+	if !scope.Pipelines && !scope.Templates && !scope.FileStore && !scope.Overrides && !scope.Services && !scope.Environments && !scope.InfraDef && !scope.Inputsets {
 		log.Errorf(color.RedString("You need to specify at least one type of entity to migrate!"))
 		log.Errorf(color.RedString("Please use -pipelines, -templates, -services, -environments, -filestore or -overrides flags"))
 		log.Errorf(color.RedString("If you want to migrate all entities, use -all flag"))
@@ -233,7 +241,7 @@ func main() {
 						if scope.CGFolderStructure {
 							accountConfig.GitDetails.FilePath = "account/" + string(p.OrgIdentifier) + "/" + p.Identifier + "/pipelines/" + pipeline.Identifier + ".yaml"
 						} else {
-							accountConfig.GitDetails.FilePath = harness.GetPipelineFilePath(*customGitDetailsFilePath, p, pipeline)
+							accountConfig.GitDetails.FilePath = harness.GetPipelineFilePath(scope.GitX, *customGitDetailsFilePath, p, pipeline)
 						}
 					}
 					_, err := pipeline.MovePipelineToRemote(&api, accountConfig, string(p.OrgIdentifier), p.Identifier)
@@ -248,6 +256,39 @@ func main() {
 			}
 			pipelines = append(pipelines, projectPipelines.Data.Content...)
 		}
+		if scope.Inputsets {
+			log.Infof("Getting inputsets for project %s", p.Name)
+			projectPipelines, err := api.GetAllPipelines(accountConfig.AccountIdentifier, string(p.OrgIdentifier), p.Identifier)
+			if err != nil {
+				log.Errorf(color.RedString("Unable to get pipelines for inputsets - %s", err))
+				return
+			}
+
+			if len(projectPipelines.Data.Content) > 0 {
+				for _, pipeline := range projectPipelines.Data.Content {
+					if pipeline.StoreType != "REMOTE" {
+						continue
+					}
+
+					inputsets, err := api.GetInputsets(accountConfig.AccountIdentifier, string(p.OrgIdentifier), p.Identifier, pipeline.Identifier)
+					if err != nil {
+						log.Errorf(color.RedString("Unable to list inputsets from pipeline - %s", pipeline.Name))
+						continue
+					}
+
+					for _, is := range inputsets {
+						accountConfig.GitDetails.FilePath = harness.GetInputsetFilePath(scope.GitX, *customGitDetailsFilePath, p, is)
+
+						err := is.MoveInputsetToRemote(&api, accountConfig, p.Identifier, string(p.OrgIdentifier))
+						if err != nil {
+							log.Errorf(color.RedString("Unable to move inputsets [%s] for pipeline - %s", is.Name, pipeline.Name))
+							log.Errorf(color.RedString(err.Error()))
+
+						}
+					}
+				}
+			}
+		}
 
 		if scope.Templates {
 			// Get all templates for the project
@@ -271,7 +312,7 @@ func main() {
 							accountConfig.GitDetails.FilePath = "account/" + string(p.OrgIdentifier) + "/" + p.Identifier + "/templates/" + template.Identifier + "-" + template.VersionLabel + ".yaml"
 							template.GitDetails = accountConfig.GitDetails
 						} else {
-							accountConfig.GitDetails.FilePath = harness.GetTemplateFilePath(*customGitDetailsFilePath, p, template)
+							accountConfig.GitDetails.FilePath = harness.GetTemplateFilePath(scope.GitX, *customGitDetailsFilePath, p, template)
 							template.GitDetails = accountConfig.GitDetails
 						}
 					}
@@ -307,7 +348,7 @@ func main() {
 				servicesBar := pb.ProgressBarTemplate(servicesTmpl).Start(len(projectServices))
 
 				for _, service := range projectServices {
-					accountConfig.GitDetails.FilePath = harness.GetServiceFilePath(*customGitDetailsFilePath, p, *service)
+					accountConfig.GitDetails.FilePath = harness.GetServiceFilePath(scope.GitX, *customGitDetailsFilePath, p, *service)
 
 					if service.StoreType == "REMOTE" {
 						log.Infof("Service [%s] is already remote", service.Identifier)
@@ -344,7 +385,7 @@ func main() {
 				envBar := pb.ProgressBarTemplate(envTmpl).Start(len(projectEnvironments))
 
 				for _, environment := range projectEnvironments {
-					accountConfig.GitDetails.FilePath = harness.GetEnvironmentFilePath(*customGitDetailsFilePath, p, *environment)
+					accountConfig.GitDetails.FilePath = harness.GetEnvironmentFilePath(scope.GitX, *customGitDetailsFilePath, p, *environment)
 
 					if environment.StoreType == "REMOTE" {
 						log.Infof("Environment [%s] is already remote", environment.Identifier)
@@ -365,7 +406,7 @@ func main() {
 
 		// INFRA-DEF MOVE TO REMOTE
 		if scope.InfraDef {
-			err := processInfraDefScope(log, api, *customGitDetailsFilePath, accountConfig, project, p)
+			err := processInfraDefScope(log, api, *customGitDetailsFilePath, accountConfig, p, scope.GitX)
 			if err != nil {
 				log.Errorf(color.RedString("Unable to inline-to-remote infrastructure - %s", err))
 			}
@@ -879,7 +920,7 @@ func main() {
 	}
 }
 
-func processInfraDefScope(log *logrus.Logger, api harness.APIRequest, customGitDetailsFilePath string, accountConfig harness.Config, project harness.ProjectsContent, p harness.Project) error {
+func processInfraDefScope(log *logrus.Logger, api harness.APIRequest, customGitDetailsFilePath string, accountConfig harness.Config, p harness.Project, gitX bool) error {
 
 	projectEnvironments, err := api.GetEnvironments(accountConfig.AccountIdentifier, string(p.OrgIdentifier), p.Identifier)
 	if err != nil {
@@ -905,7 +946,7 @@ func processInfraDefScope(log *logrus.Logger, api harness.APIRequest, customGitD
 				if len(infras) > 0 {
 					for _, infraDef := range infras {
 						if infraDef.StoreType != "REMOTE" {
-							accountConfig.GitDetails.FilePath = harness.GetInfrastructureFilePath(customGitDetailsFilePath, p, *environment, *infraDef)
+							accountConfig.GitDetails.FilePath = harness.GetInfrastructureFilePath(gitX, customGitDetailsFilePath, p, *environment, *infraDef)
 
 							err = infraDef.MoveInfrastructureToRemote(&api, accountConfig, environment.Identifier)
 							if err != nil {