add handling function for remote url hostname

Author: Liujingfang1

pkg/loader/gitcloner.go

@@ -21,6 +21,7 @@ import (
 	"io/ioutil"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -44,7 +45,8 @@ func isRepoUrl(arg string) bool {
 			strings.HasPrefix(arg, "gh:") ||
 			strings.HasPrefix(arg, "github.com") ||
 			strings.HasPrefix(arg, "[email protected]:") ||
-			strings.Index(arg, "github.com/") > -1)
+			strings.Index(arg, "github.com/") > -1 ||
+			isAzureHost(arg) || isAWSHost(arg))
 }
 
 func makeTmpDir() (string, error) {
@@ -61,14 +63,14 @@ func simpleGitCloner(spec string) (
 	if err != nil {
 		return
 	}
-	repo, pathInCoDir, gitRef, err := parseGithubUrl(spec)
+	repo, pathInCoDir, gitRef, err := parseUrl(spec)
 	if err != nil {
 		return
 	}
 	cmd := exec.Command(
 		gitProgram,
 		"clone",
-		"https://github.com/"+repo+".git",
+		repo,
 		checkoutDir)
 	var out bytes.Buffer
 	cmd.Stdout = &out
@@ -90,27 +92,36 @@ func simpleGitCloner(spec string) (
 	return checkoutDir, pathInCoDir, nil
 }
 
+func parseUrl(n string) (
+	repo string, path string, gitRef string, err error) {
+	host, repo, path, gitRef, err := parseGithubUrl(n)
+	if err != nil {
+		return
+	}
+	if isAzureHost(host) || isAWSHost(host) {
+		repo = host + repo
+		return
+	}
+	repo = host + repo + ".git"
+	return
+}
+
 const refQuery = "?ref="
 
 // From strings like [email protected]:someOrg/someRepo.git or
 // https://github.com/someOrg/someRepo?ref=someHash, extract
 // the parts.
 func parseGithubUrl(n string) (
-	repo string, path string, gitRef string, err error) {
-	for _, p := range []string{
-		// Order matters here.
-		"git::", "gh:", "https://", "http://",
-		"git@", "github.com:", "github.com/", "gitlab.com/"} {
-		if strings.ToLower(n[:len(p)]) == p {
-			n = n[len(p):]
-		}
-	}
+	host string, repo string, path string, gitRef string, err error) {
+	host, n = parseHostSpec(n)
+	host = normalizeGitHostSpec(host)
+
 	if strings.HasSuffix(n, ".git") {
 		n = n[0 : len(n)-len(".git")]
 	}
 	i := strings.Index(n, string(filepath.Separator))
 	if i < 1 {
-		return "", "", "", errors.New("no separator")
+		return "", "", "", "", errors.New("no separator")
 	}
 	j := strings.Index(n[i+1:], string(filepath.Separator))
 	if j >= 0 {
@@ -131,3 +142,57 @@ func peelQuery(arg string) (string, string) {
 	}
 	return arg, ""
 }
+
+func parseHostSpec(n string) (string, string) {
+	var host string
+	for _, p := range []string{
+		// Order matters here.
+		"git::", "gh:", "ssh://", "https://", "http://",
+		"git@", "github.com:", "github.com/", "gitlab.com/"} {
+		if strings.ToLower(n[:len(p)]) == p {
+			n = n[len(p):]
+			host = host + p
+		}
+	}
+	for _, p := range []string{
+		"git-codecommit.[a-z0-9-]*.amazonaws.com/",
+		"dev.azure.com/",
+		".*visualstudio.com/"} {
+		index := regexp.MustCompile(p).FindStringIndex(n)
+		if len(index) > 0 {
+			host = host + n[0:index[len(index)-1]]
+			n = n[index[len(index)-1]:]
+		}
+	}
+	return host, n
+}
+
+func normalizeGitHostSpec(host string) string {
+	s := strings.ToLower(host)
+	if strings.Contains(s, "github.com") {
+		if strings.Contains(s, "git@") || strings.Contains(s, "ssh:") {
+			host = "[email protected]:"
+		} else {
+			host = "https://github.com/"
+		}
+	}
+	if strings.Contains(s, "gitlab") {
+		if strings.HasPrefix(s, "git::") {
+			host = strings.TrimLeft(s, "git::")
+		}
+	}
+	return host
+}
+
+// The format of Azure repo URL is documented
+// https://docs.microsoft.com/en-us/azure/devops/repos/git/clone?view=vsts&tabs=visual-studio#clone_url
+func isAzureHost(host string) bool {
+	return strings.Contains(host, "dev.azure.com") ||
+		strings.Contains(host, "visualstudio.com")
+}
+
+// The format of AWS repo URL is documented
+// https://docs.aws.amazon.com/codecommit/latest/userguide/regions.html
+func isAWSHost(host string) bool {
+	return strings.Contains(host, "amazonaws.com")
+}

pkg/loader/gitcloner_test.go

@@ -185,20 +185,20 @@ var paths = []string{"README.md", "foo/krusty.txt", ""}
 
 var hrefArgs = []string{"someBranch", ""}
 
-var extractFmts = []string{
-	"gh:%s",
-	"GH:%s",
-	"gitHub.com/%s",
-	"https://github.com/%s",
-	"hTTps://github.com/%s",
-	"git::https://gitlab.com/%s",
-	"github.com:%s",
+var extractFmts = map[string]string{
+	"gh:%s":                      "gh:",
+	"GH:%s":                      "gh:",
+	"gitHub.com/%s":              "https://github.com/",
+	"https://github.com/%s":      "https://github.com/",
+	"hTTps://github.com/%s":      "https://github.com/",
+	"git::https://gitlab.com/%s": "https://gitlab.com/",
+	"github.com:%s":              "https://github.com/",
 }
 
 func TestParseGithubUrl(t *testing.T) {
 	for _, repoName := range repoNames {
 		for _, pathName := range paths {
-			for _, extractFmt := range extractFmts {
+			for extractFmt, hostSpec := range extractFmts {
 				for _, hrefArg := range hrefArgs {
 					spec := repoName
 					if len(pathName) > 0 {
@@ -212,10 +212,16 @@ func TestParseGithubUrl(t *testing.T) {
 						t.Errorf("Should smell like github arg: %s\n", input)
 						continue
 					}
-					repo, path, gitRef, err := parseGithubUrl(input)
+					host, repo, path, gitRef, err := parseGithubUrl(input)
 					if err != nil {
 						t.Errorf("problem %v", err)
 					}
+					if host != hostSpec {
+						t.Errorf("\n"+
+							"         from %s\n"+
+							"  actual host %s\n"+
+							"expected host %s\n", input, host, hostSpec)
+					}
 					if repo != repoName {
 						t.Errorf("\n"+
 							"         from %s\n"+
@@ -239,3 +245,118 @@ func TestParseGithubUrl(t *testing.T) {
 		}
 	}
 }
+
+func TestParseUrl(t *testing.T) {
+	testcases := []struct {
+		input string
+		repo  string
+		path  string
+		ref   string
+	}{
+		{
+			input: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo/somedir",
+			repo:  "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo",
+			path:  "somedir",
+			ref:   "",
+		},
+		{
+			input: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo/somedir?ref=testbranch",
+			repo:  "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo",
+			path:  "somedir",
+			ref:   "testbranch",
+		},
+		{
+			input: "https://fabrikops2.visualstudio.com/someorg/somerepo?ref=master",
+			repo:  "https://fabrikops2.visualstudio.com/someorg/somerepo",
+			path:  "",
+			ref:   "master",
+		},
+		{
+			input: "http://github.com/someorg/somerepo/somedir",
+			repo:  "https://github.com/someorg/somerepo.git",
+			path:  "somedir",
+			ref:   "",
+		},
+		{
+			input: "[email protected]:someorg/somerepo/somedir",
+			repo:  "[email protected]:someorg/somerepo.git",
+			path:  "somedir",
+			ref:   "",
+		},
+	}
+	for _, testcase := range testcases {
+		repo, path, ref, err := parseUrl(testcase.input)
+		if err != nil {
+			t.Errorf("Unexpected error: %v", err)
+		}
+		if repo != testcase.repo {
+			t.Errorf("repo expected to be %v, but got %v on %s", testcase.repo, repo, testcase.input)
+		}
+		if path != testcase.path {
+			t.Errorf("path expected to be %v, but got %v on %s", testcase.path, path, testcase.input)
+		}
+		if ref != testcase.ref {
+			t.Errorf("ref expected to be %v, but got %v on %s", testcase.ref, ref, testcase.input)
+		}
+	}
+}
+
+func TestIsAzureHost(t *testing.T) {
+	testcases := []struct {
+		input  string
+		expect bool
+	}{
+		{
+			input:  "https://git-codecommit.us-east-2.amazonaws.com",
+			expect: false,
+		},
+		{
+			input:  "ssh://git-codecommit.us-east-2.amazonaws.com",
+			expect: false,
+		},
+		{
+			input:  "https://fabrikops2.visualstudio.com/",
+			expect: true,
+		},
+		{
+			input:  "https://dev.azure.com/myorg/myproject/",
+			expect: true,
+		},
+	}
+	for _, testcase := range testcases {
+		actual := isAzureHost(testcase.input)
+		if actual != testcase.expect {
+			t.Errorf("IsAzureHost: expected %v, but got %v on %s", testcase.expect, actual, testcase.input)
+		}
+	}
+}
+
+func TestIsAWSHost(t *testing.T) {
+	testcases := []struct {
+		input  string
+		expect bool
+	}{
+		{
+			input:  "https://git-codecommit.us-east-2.amazonaws.com",
+			expect: true,
+		},
+		{
+			input:  "ssh://git-codecommit.us-east-2.amazonaws.com",
+			expect: true,
+		},
+		{
+			input:  "[email protected]:",
+			expect: false,
+		},
+		{
+			input:  "http://github.com/",
+			expect: false,
+		},
+	}
+	for _, testcase := range testcases {
+		actual := isAWSHost(testcase.input)
+		if actual != testcase.expect {
+			t.Errorf("IsAWSHost: expected %v, but got %v on %s", testcase.expect, actual, testcase.input)
+		}
+	}
+}