feat: add support for authenticated git clone

Added support for authenticated git cloning using `gitToken` and `gitTokenKey`
parameters. Updated documentation to reflect the new parameters and their
usage. Modified resolver functions to handle authentication when cloning
repositories. Added tests to verify the functionality of authenticated git
cloning.

The differences between the two modes are:

- The `git clone` method support anonymous cloning and authenticated cloning.
- Depending of the Git provider `git clone` has a lower rate limit (if none)
  than the authenticated API.
- The authenticated API supports private repositories and fetches only the file
  at the specified path rather than doing a full clone.

Author: chmouel

docs/git-resolver.md

@@ -13,17 +13,19 @@ This Resolver responds to type `git`.
 
 ## Parameters
 
-| Param Name   | Description                                                                                                                                                                | Example Value                                               |
-|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------|
-| `url`        | URL of the repo to fetch and clone anonymously. Either `url`, or `repo` (with `org`) must be specified, but not both.                                                      | `https://github.com/tektoncd/catalog.git`                   |
-| `repo`       | The repository to find the resource in. Either `url`, or `repo` (with `org`) must be specified, but not both.                                                              | `pipeline`, `test-infra`                                    |
-| `org`        | The organization to find the repository in. Default can be set in [configuration](#configuration).                                                                         | `tektoncd`, `kubernetes`                                    |
-| `token`      | An optional secret name in the `PipelineRun` namespace to fetch the token from. Defaults to empty, meaning it will try to use the configuration from the global configmap. | `secret-name`, (empty)                                      |
-| `tokenKey`   | An optional key in the token secret name in the `PipelineRun` namespace to fetch the token from. Defaults to `token`.                                                      | `token`                                                     |
-| `revision`   | Git revision to checkout a file from. This can be commit SHA, branch or tag.                                                                                               | `aeb957601cf41c012be462827053a21a420befca` `main` `v0.38.2` |
-| `pathInRepo` | Where to find the file in the repo.                                                                                                                                        | `task/golang-build/0.3/golang-build.yaml`                   |
-| `serverURL`  | An optional server URL (that includes the https:// prefix) to connect for API operations                                                                                   | `https:/github.mycompany.com`                               |
-| `scmType`    | An optional SCM type to use for API operations                                                                                                                             | `github`, `gitlab`, `gitea`                                 |
+| Param Name    | Description                                                                                                                                                                | Example Value                                               |
+|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------|
+| `url`         | URL of the repo to fetch and clone anonymously. Either `url`, or `repo` (with `org`) must be specified, but not both.                                                      | `https://github.com/tektoncd/catalog.git`                   |
+| `repo`        | The repository to find the resource in. Either `url`, or `repo` (with `org`) must be specified, but not both.                                                              | `pipeline`, `test-infra`                                    |
+| `org`         | The organization to find the repository in. Default can be set in [configuration](#configuration).                                                                         | `tektoncd`, `kubernetes`                                    |
+| `token`       | An optional secret name in the `PipelineRun` namespace to fetch the token from. Defaults to empty, meaning it will try to use the configuration from the global configmap. | `secret-name`, (empty)                                      |
+| `tokenKey`    | An optional key in the token secret name in the `PipelineRun` namespace to fetch the token from. Defaults to `token`.                                                      | `token`                                                     |
+| `gitToken`       | An optional secret name in the `PipelineRun` namespace to fetch the token from when doing opration with the `git clone`. When empty it will use anonymous cloning.
+| `gitTokenKey` | An optional key in the token secret name in the `PipelineRun` namespace to fetch the token from when using the `git clone`. Defaults to `token`.                                                      | `token`                                                     |
+| `revision`    | Git revision to checkout a file from. This can be commit SHA, branch or tag.                                                                                               | `aeb957601cf41c012be462827053a21a420befca` `main` `v0.38.2` |
+| `pathInRepo`  | Where to find the file in the repo.                                                                                                                                        | `task/golang-build/0.3/golang-build.yaml`                   |
+| `serverURL`   | An optional server URL (that includes the https:// prefix) to connect for API operations                                                                                   | `https:/github.mycompany.com`                               |
+| `scmType`     | An optional SCM type to use for API operations                                                                                                                             | `github`, `gitlab`, `gitea`                                 |
 
 ## Requirements
 
@@ -55,11 +57,22 @@ for the name, namespace and defaults that the resolver ships with.
 
 ## Usage
 
-The `git` resolver has two modes: cloning a repository anonymously, or fetching individual files via an SCM provider's API using an API token.
+The `git` resolver has two modes: cloning a repository with `git clone` (with
+the `go-git` library), or fetching individual files via an SCM provider's API
+using an API token.
 
-### Anonymous Cloning
+The differences between the two modes are:
 
-Anonymous cloning is supported only for public repositories. This mode clones the full git repo.
+- The `git clone` method support anonymous cloning and authenticated cloning.
+- Depending of the Git provider `git clone` has a lower rate limit
+  than the authenticated API.
+- The authenticated API supports private repositories and fetches only the file
+at the specified path rather than doing a full clone.
+
+### Git Clone with git clone
+
+Git clone with `git clone` is supported for anonymous and
+authenticated cloning.
 
 #### Task Resolution
 
@@ -78,6 +91,11 @@ spec:
       value: main
     - name: pathInRepo
       value: task/git-clone/0.6/git-clone.yaml
+    # Uncomment the following lines to use a secret with a token
+    # - name: gitToken
+    #   value: "secret-with-token"
+    # - name: gitTokenKey (optional, defaults to "token")
+    #   value: "token"
 ```
 
 #### Pipeline resolution
@@ -97,6 +115,11 @@ spec:
       value: main
     - name: pathInRepo
       value: pipeline/simple/0.1/simple.yaml
+    # Uncomment the following lines to use a secret with a token
+    # - name: gitToken
+    #   value: "secret-with-token"
+    # - name: gitTokenKey (optional, defaults to "token")
+    #   value: "token"
   params:
   - name: name
     value: Ranni
@@ -193,11 +216,11 @@ spec:
 
 ### Specifying Configuration for Multiple Git Providers
 
-It is possible to specify configurations for multiple providers and even multiple configurations for same provider to use in 
+It is possible to specify configurations for multiple providers and even multiple configurations for same provider to use in
 different tekton resources. Firstly, details need to be added in configmap with the unique identifier key prefix.
-To use them in tekton resources, pass the unique key mentioned in configmap as an extra param to resolver with key 
-`configKey` and value will be the unique key. If no `configKey` param is passed, `default` will be used. Default 
-configuration to be used for git resolver can be specified in configmap by either mentioning no unique identifier or 
+To use them in tekton resources, pass the unique key mentioned in configmap as an extra param to resolver with key
+`configKey` and value will be the unique key. If no `configKey` param is passed, `default` will be used. Default
+configuration to be used for git resolver can be specified in configmap by either mentioning no unique identifier or
 using identifier `default`
 
 **Note**: `configKey` should not contain `.` while specifying configurations in configmap

pkg/remoteresolution/resolver/git/resolver.go

@@ -120,7 +120,7 @@ func (r *Resolver) Resolve(ctx context.Context, req *v1beta1.ResolutionRequestSp
 		}
 
 		if params[git.UrlParam] != "" {
-			return git.ResolveAnonymousGit(ctx, params)
+			return git.ResolveAnonymousGit(ctx, params, r.kubeClient, r.logger, r.cache, r.ttl)
 		}
 
 		return git.ResolveAPIGit(ctx, params, r.kubeClient, r.logger, r.cache, r.ttl, r.clientFunc)

pkg/resolution/resolver/git/config.go

@@ -62,6 +62,7 @@ type ScmConfig struct {
 	Org                string `json:"default-org"`
 	ServerURL          string `json:"server-url"`
 	SCMType            string `json:"scm-type"`
+	GitToken           string `json:"git-token"`
 	APISecretName      string `json:"api-token-secret-name"`
 	APISecretKey       string `json:"api-token-secret-key"`
 	APISecretNamespace string `json:"api-token-secret-namespace"`

pkg/resolution/resolver/git/params.go

@@ -33,6 +33,10 @@ const (
 	TokenParam string = "token"
 	// TokenKeyParam is an optional reference to a key in the TokenParam secret for SCM API authentication
 	TokenKeyParam string = "tokenKey"
+	// GitTokenParam is an optional reference to a secret name when using go-git for git authentication
+	GitTokenParam string = "gitToken"
+	// GitTokenParam is an optional reference to a secret name when using go-git for git authentication
+	GitTokenKeyParam string = "gitTokenKey"
 	// DefaultTokenKeyParam is the default key in the TokenParam secret for SCM API authentication
 	DefaultTokenKeyParam string = "token"
 	// scmTypeParam is an optional string overriding the scm-type configuration (ie: github, gitea, gitlab etc..)

pkg/resolution/resolver/git/resolver.go

@@ -31,6 +31,8 @@ import (
 	"github.com/go-git/go-git/v5"
 	gitcfg "github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
+	plumbTransport "github.com/go-git/go-git/v5/plumbing/transport"
+	"github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/go-git/go-git/v5/storage/memory"
 	"github.com/jenkins-x/go-scm/scm"
 	"github.com/jenkins-x/go-scm/scm/factory"
@@ -130,7 +132,7 @@ func (r *Resolver) Resolve(ctx context.Context, origParams []pipelinev1.Param) (
 	}
 
 	if params[UrlParam] != "" {
-		return ResolveAnonymousGit(ctx, params)
+		return ResolveAnonymousGit(ctx, params, r.kubeClient, r.logger, r.cache, r.ttl)
 	}
 
 	return ResolveAPIGit(ctx, params, r.kubeClient, r.logger, r.cache, r.ttl, r.clientFunc)
@@ -157,7 +159,7 @@ func validateRepoURL(url string) bool {
 	return re.MatchString(url)
 }
 
-func ResolveAnonymousGit(ctx context.Context, params map[string]string) (framework.ResolvedResource, error) {
+func ResolveAnonymousGit(ctx context.Context, params map[string]string, kubeclient kubernetes.Interface, logger *zap.SugaredLogger, cache *cache.LRUExpireCache, ttl time.Duration) (framework.ResolvedResource, error) {
 	conf, err := GetScmConfigForParamConfigKey(ctx, params)
 	if err != nil {
 		return nil, err
@@ -180,6 +182,33 @@ func ResolveAnonymousGit(ctx context.Context, params map[string]string) (framewo
 	cloneOpts := &git.CloneOptions{
 		URL: repo,
 	}
+
+	secretRef := &secretCacheKey{
+		name: params[GitTokenParam],
+		key:  params[GitTokenKeyParam],
+	}
+	if secretRef.name != "" {
+		if secretRef.key == "" {
+			secretRef.key = DefaultTokenKeyParam
+		}
+		secretRef.ns = common.RequestNamespace(ctx)
+	} else {
+		secretRef = nil
+	}
+
+	auth := plumbTransport.AuthMethod(nil)
+	if secretRef != nil {
+		gitToken, err := getAPIToken(ctx, secretRef, kubeclient, logger, cache, ttl, params, GitTokenKeyParam)
+		if err != nil {
+			return nil, err
+		}
+		auth = &http.BasicAuth{
+			Username: "git",
+			Password: string(gitToken),
+		}
+		cloneOpts.Auth = auth
+	}
+
 	filesystem := memfs.New()
 	repository, err := git.Clone(memory.NewStorage(), filesystem, cloneOpts)
 	if err != nil {
@@ -190,6 +219,7 @@ func ResolveAnonymousGit(ctx context.Context, params map[string]string) (framewo
 	refSpec := gitcfg.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/%s", revision, revision))
 	err = repository.Fetch(&git.FetchOptions{
 		RefSpecs: []gitcfg.RefSpec{refSpec},
+		Auth:     auth,
 	})
 	if err != nil {
 		var fetchErr git.NoMatchingRefSpecError
@@ -405,7 +435,7 @@ func ResolveAPIGit(ctx context.Context, params map[string]string, kubeclient kub
 	} else {
 		secretRef = nil
 	}
-	apiToken, err := getAPIToken(ctx, secretRef, kubeclient, logger, cache, ttl, params)
+	apiToken, err := getAPIToken(ctx, secretRef, kubeclient, logger, cache, ttl, params, APISecretNameKey)
 	if err != nil {
 		return nil, err
 	}
@@ -449,7 +479,7 @@ func ResolveAPIGit(ctx context.Context, params map[string]string, kubeclient kub
 	}, nil
 }
 
-func getAPIToken(ctx context.Context, apiSecret *secretCacheKey, kubeclient kubernetes.Interface, logger *zap.SugaredLogger, cache *cache.LRUExpireCache, ttl time.Duration, params map[string]string) ([]byte, error) {
+func getAPIToken(ctx context.Context, apiSecret *secretCacheKey, kubeclient kubernetes.Interface, logger *zap.SugaredLogger, cache *cache.LRUExpireCache, ttl time.Duration, params map[string]string, key string) ([]byte, error) {
 	conf, err := GetScmConfigForParamConfigKey(ctx, params)
 	if err != nil {
 		return nil, err
@@ -467,7 +497,7 @@ func getAPIToken(ctx context.Context, apiSecret *secretCacheKey, kubeclient kube
 	if apiSecret.name == "" {
 		apiSecret.name = conf.APISecretName
 		if apiSecret.name == "" {
-			err := fmt.Errorf("cannot get API token, required when specifying '%s' param, '%s' not specified in config", RepoParam, APISecretNameKey)
+			err := fmt.Errorf("cannot get API token, required when specifying '%s' param, '%s' not specified in config", RepoParam, key)
 			logger.Info(err)
 			return nil, err
 		}

pkg/resolution/resolver/git/resolver_test.go

@@ -280,17 +280,19 @@ func TestResolveNotEnabled(t *testing.T) {
 }
 
 type params struct {
-	url        string
-	revision   string
-	pathInRepo string
-	org        string
-	repo       string
-	token      string
-	tokenKey   string
-	namespace  string
-	serverURL  string
-	scmType    string
-	configKey  string
+	url         string
+	revision    string
+	pathInRepo  string
+	org         string
+	repo        string
+	token       string
+	tokenKey    string
+	namespace   string
+	serverURL   string
+	scmType     string
+	configKey   string
+	gitToken    string
+	gitTokenKey string
 }
 
 func TestResolve(t *testing.T) {
@@ -419,6 +421,26 @@ func TestResolve(t *testing.T) {
 			url:        anonFakeRepoURL,
 		},
 		expectedErr: createError(`error opening file "foo/non-exist": file does not exist`),
+	}, {
+		name: "clone: secret for git clone",
+		args: &params{
+			pathInRepo:  "./released",
+			url:         anonFakeRepoURL,
+			gitToken:    "token-secret",
+			gitTokenKey: "token",
+			namespace:   "foo",
+		},
+		expectedCommitSHA: commitSHAsInAnonRepo[2],
+		expectedStatus:    resolution.CreateResolutionRequestStatusWithData([]byte("released content in main branch and in tag v1")),
+	}, {
+		name: "clone: secret for git clone does not exist",
+		args: &params{
+			pathInRepo:  "./released",
+			url:         anonFakeRepoURL,
+			gitToken:    "non-existent",
+			gitTokenKey: "token",
+		},
+		expectedErr: createError(`cannot get API token, secret non-existent not found in namespace foo`),
 	}, {
 		name: "clone: revision does not exist",
 		args: &params{
@@ -721,9 +743,14 @@ func TestResolve(t *testing.T) {
 				if tc.config[tc.configIdentifer+APISecretNameKey] != "" && tc.config[tc.configIdentifer+APISecretNamespaceKey] != "" && tc.config[tc.configIdentifer+APISecretKeyKey] != "" && tc.apiToken != "" {
 					secretName, secretNameKey, secretNamespace = tc.config[tc.configIdentifer+APISecretNameKey], tc.config[tc.configIdentifer+APISecretKeyKey], tc.config[tc.configIdentifer+APISecretNamespaceKey]
 				}
+
 				if tc.args.token != "" && tc.args.namespace != "" && tc.args.tokenKey != "" {
 					secretName, secretNameKey, secretNamespace = tc.args.token, tc.args.tokenKey, tc.args.namespace
 				}
+
+				if tc.args.gitToken != "" && tc.args.gitTokenKey != "" && tc.args.namespace != "" {
+					secretName, secretNameKey, secretNamespace = tc.args.gitToken, tc.args.gitTokenKey, tc.args.namespace
+				}
 				if secretName == "" || secretNameKey == "" || secretNamespace == "" {
 					return
 				}
@@ -754,6 +781,9 @@ func createTestRepo(t *testing.T, commits []commitForRepo) (string, []string) {
 	tempDir := t.TempDir()
 
 	repo, err := git.PlainInit(tempDir, false)
+	if err != nil {
+		t.Fatalf("couldn't create test repo: %v", err)
+	}
 
 	worktree, err := repo.Worktree()
 	if err != nil {
@@ -922,6 +952,16 @@ func createRequest(args *params) *v1beta1.ResolutionRequest {
 			Name:  UrlParam,
 			Value: *pipelinev1.NewStructuredValues(args.url),
 		})
+		if args.gitToken != "" {
+			rr.Spec.Params = append(rr.Spec.Params, pipelinev1.Param{
+				Name:  GitTokenParam,
+				Value: *pipelinev1.NewStructuredValues(args.gitToken),
+			})
+			rr.Spec.Params = append(rr.Spec.Params, pipelinev1.Param{
+				Name:  GitTokenKeyParam,
+				Value: *pipelinev1.NewStructuredValues(args.gitTokenKey),
+			})
+		}
 	} else {
 		rr.Spec.Params = append(rr.Spec.Params, pipelinev1.Param{
 			Name:  RepoParam,