feat: add support for tag name for external CI

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/deckarep/golang-set v1.8.0
 	github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8
-	github.com/devtron-labs/common-lib v0.0.25-0.20240812113340-f14be466613d
+	github.com/devtron-labs/common-lib v0.16.1-0.20240903114838-d814f4cd60ca
 	github.com/devtron-labs/go-bitbucket v0.9.60-beta
 	github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80
 	github.com/evanphx/json-patch v5.7.0+incompatible
@@ -76,6 +76,7 @@ require (
 	go.uber.org/zap v1.21.0
 	golang.org/x/crypto v0.25.0
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
+	golang.org/x/mod v0.17.0
 	golang.org/x/oauth2 v0.21.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d
 	google.golang.org/grpc v1.59.0
@@ -244,7 +245,6 @@ require (
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/net v0.27.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/sys v0.22.0 // indirect

go.sum

@@ -187,8 +187,8 @@ github.com/devtron-labs/argo-workflows/v3 v3.5.10 h1:6rxQOesOzDz6SgQCMDQNHaehsKF
 github.com/devtron-labs/argo-workflows/v3 v3.5.10/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA=
 github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 h1:2+Q7Jdhpo/uMiaQiZZzAh+ZX7wEJIFuMFG6DEiMuo64=
 github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8/go.mod h1:702R6WIf5y9UzKGoCGxQ+x3l5Ws+l0fXg2xlCpSGFZI=
-github.com/devtron-labs/common-lib v0.0.25-0.20240812113340-f14be466613d h1:+iWXiVOyf9E0bcTia6x2sLFTM7xJc+9Z8q+BfbYr6eM=
-github.com/devtron-labs/common-lib v0.0.25-0.20240812113340-f14be466613d/go.mod h1:a7aCClaxYfnyYEENSe1RnkQCeW2AwmCAPYsuvgk0aW0=
+github.com/devtron-labs/common-lib v0.16.1-0.20240903114838-d814f4cd60ca h1:cS0cK+aAsMZ9GNcNK3mp9Bej5X/3cRmD/tDLc9/P1GM=
+github.com/devtron-labs/common-lib v0.16.1-0.20240903114838-d814f4cd60ca/go.mod h1:a7aCClaxYfnyYEENSe1RnkQCeW2AwmCAPYsuvgk0aW0=
 github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU=
 github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y=
 github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80 h1:xwbTeijNTf4/j1v+tSfwVqwLVnReas/NqEKeQHvSTys=

internal/sql/repository/CiArtifactRepository.go

@@ -105,6 +105,8 @@ type CiArtifactRepository interface {
 	Get(id int) (artifact *CiArtifact, err error)
 	GetArtifactParentCiAndWorkflowDetailsByIds(ids []int) ([]*CiArtifact, error)
 	GetByWfId(wfId int) (artifact *CiArtifact, err error)
+	IfArtifactExistByImage(imageName string, pipelineId int) (exist bool, err error)
+	IfArtifactExistByImageDigest(imageDigest string, imageName string, pipelineId int) (exist bool, err error)
 	GetArtifactsByCDPipeline(cdPipelineId, limit int, parentId int, parentType bean.WorkflowType) ([]*CiArtifact, error)
 	GetArtifactsByCDPipelineV3(listingFilterOpts *bean.ArtifactsListFilterOptions) ([]*CiArtifact, int, error)
 	GetLatestArtifactTimeByCiPipelineIds(ciPipelineIds []int) ([]*CiArtifact, error)
@@ -316,6 +318,29 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipeline(cdPipelineId, limi
 	return artifactsAll, err
 }
 
+func (impl CiArtifactRepositoryImpl) IfArtifactExistByImage(imageName string, pipelineId int) (exist bool, err error) {
+	count, err := impl.dbConnection.Model(&CiArtifact{}).
+		Where("image = ?", imageName).
+		Where("pipeline_id = ?", pipelineId).
+		Count()
+	if err != nil {
+		return false, err
+	}
+	return count > 0, nil
+}
+
+func (impl CiArtifactRepositoryImpl) IfArtifactExistByImageDigest(imageDigest string, imageName string, pipelineId int) (exist bool, err error) {
+	count, err := impl.dbConnection.Model(&CiArtifact{}).
+		Where("image_digest = ?", imageDigest).
+		Where("image = ?", imageName).
+		Where("pipeline_id = ?", pipelineId).
+		Count()
+	if err != nil {
+		return false, err
+	}
+	return count > 0, nil
+}
+
 func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipelineV3(listingFilterOpts *bean.ArtifactsListFilterOptions) ([]*CiArtifact, int, error) {
 
 	if listingFilterOpts.ParentStageType != bean.CI_WORKFLOW_TYPE && listingFilterOpts.ParentStageType != bean.WEBHOOK_WORKFLOW_TYPE {

pkg/pipeline/CiService.go

@@ -21,6 +21,8 @@ import (
 	"errors"
 	"fmt"
 	"github.com/caarlos0/env"
+	"github.com/devtron-labs/common-lib/utils"
+	bean3 "github.com/devtron-labs/common-lib/utils/bean"
 	"github.com/devtron-labs/devtron/pkg/infraConfig"
 	"github.com/devtron-labs/devtron/pkg/pipeline/adapter"
 	"github.com/devtron-labs/devtron/pkg/pipeline/bean/CiPipeline"
@@ -78,6 +80,7 @@ type CiServiceImpl struct {
 	eventClient                  client.EventClient
 	eventFactory                 client.EventFactory
 	ciPipelineRepository         pipelineConfig.CiPipelineRepository
+	ciArtifactRepository         repository5.CiArtifactRepository
 	pipelineStageService         PipelineStageService
 	userService                  user.UserService
 	ciTemplateService            CiTemplateService
@@ -99,6 +102,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService
 	ciWorkflowRepository pipelineConfig.CiWorkflowRepository, eventClient client.EventClient,
 	eventFactory client.EventFactory,
 	ciPipelineRepository pipelineConfig.CiPipelineRepository,
+	ciArtifactRepository repository5.CiArtifactRepository,
 	pipelineStageService PipelineStageService,
 	userService user.UserService,
 	ciTemplateService CiTemplateService, appCrudOperationService app.AppCrudOperationService, envRepository repository1.EnvironmentRepository, appRepository appRepository.AppRepository,
@@ -122,6 +126,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService
 		eventClient:                  eventClient,
 		eventFactory:                 eventFactory,
 		ciPipelineRepository:         ciPipelineRepository,
+		ciArtifactRepository:         ciArtifactRepository,
 		pipelineStageService:         pipelineStageService,
 		userService:                  userService,
 		ciTemplateService:            ciTemplateService,
@@ -158,15 +163,64 @@ func (impl *CiServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelin
 	}
 }
 
-func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial) error {
+func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, workflowRequest *types.WorkflowRequest) error {
+	// externalCi artifact is meant only for CI_JOB
+	if trigger.PipelineType != string(CiPipeline.CI_JOB) {
+		return nil
+	}
+
 	// checking if user has given run time parameters for externalCiArtifact, if given then sending git material to Ci-Runner
 	externalCiArtifact, exists := trigger.ExtraEnvironmentVariables[CiPipeline.ExtraEnvVarExternalCiArtifactKey]
 	// validate externalCiArtifact as docker image
 	if exists {
+		externalCiArtifact = strings.TrimSpace(externalCiArtifact)
 		if !strings.Contains(externalCiArtifact, ":") {
-			impl.Logger.Errorw("validation error", "externalCiArtifact", externalCiArtifact)
-			return fmt.Errorf("invalid image name given in externalCiArtifact")
+			if utils.IsValidDockerTagName(externalCiArtifact) {
+				fullImageUrl, err := utils.BuildDockerImagePath(bean3.DockerRegistryInfo{
+					DockerImageTag:     externalCiArtifact,
+					DockerRegistryId:   workflowRequest.DockerRegistryId,
+					DockerRegistryType: workflowRequest.DockerRegistryType,
+					DockerRegistryURL:  workflowRequest.DockerRegistryURL,
+					DockerRepository:   workflowRequest.DockerRepository,
+				})
+				if err != nil {
+					impl.Logger.Errorw("Error in building docker image", "err", err)
+					return err
+				}
+				externalCiArtifact = fullImageUrl
+			} else {
+				impl.Logger.Errorw("validation error", "externalCiArtifact", externalCiArtifact)
+				return fmt.Errorf("invalid image name or url given in externalCiArtifact")
+			}
+
+		}
+
+		trigger.ExtraEnvironmentVariables[CiPipeline.ExtraEnvVarExternalCiArtifactKey] = externalCiArtifact
+
+		var artifactExists bool
+		var err error
+		if trigger.ExtraEnvironmentVariables[CiPipeline.ExtraEnvVarImageDigestKey] == "" {
+			artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImage(externalCiArtifact, trigger.PipelineId)
+			if err != nil {
+				impl.Logger.Errorw("error in fetching ci artifact", "err", err)
+				return err
+			}
+			if artifactExists {
+				impl.Logger.Errorw("ci artifact already exists with same image name", "artifact", externalCiArtifact)
+				return fmt.Errorf("ci artifact already exists with same image name")
+			}
+		} else {
+			artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImageDigest(trigger.ExtraEnvironmentVariables[CiPipeline.ExtraEnvVarImageDigestKey], externalCiArtifact, trigger.PipelineId)
+			if err != nil {
+				impl.Logger.Errorw("error in fetching ci artifact", "err", err)
+				return err
+			}
+			if artifactExists {
+				impl.Logger.Errorw("ci artifact already exists  with same digest", "artifact", externalCiArtifact)
+				return fmt.Errorf("ci artifact already exists  with same digest")
+			}
 		}
+
 	}
 	if trigger.PipelineType == string(CiPipeline.CI_JOB) && len(ciMaterials) != 0 && !exists && externalCiArtifact == "" {
 		ciMaterials[0].GitMaterial = nil
@@ -181,10 +235,6 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error)
 	if err != nil {
 		return 0, err
 	}
-	err = impl.handleRuntimeParamsValidations(trigger, ciMaterials)
-	if err != nil {
-		return 0, err
-	}
 	ciPipelineScripts, err := impl.ciPipelineRepository.FindCiScriptsByCiPipelineId(trigger.PipelineId)
 	if err != nil && !util.IsErrNoRows(err) {
 		return 0, err
@@ -265,6 +315,17 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error)
 		impl.Logger.Errorw("make workflow req", "err", err)
 		return 0, err
 	}
+	err = impl.handleRuntimeParamsValidations(trigger, ciMaterials, workflowRequest)
+	if err != nil {
+		savedCiWf.Status = pipelineConfig.WorkflowAborted
+		savedCiWf.Message = err.Error()
+		err1 := impl.ciWorkflowRepository.UpdateWorkFlow(savedCiWf)
+		if err1 != nil {
+			impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag")
+		}
+		return 0, err
+	}
+
 	workflowRequest.Scope = scope
 	workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin
 	workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport

vendor/modules.txt

@@ -348,7 +348,7 @@ github.com/devtron-labs/authenticator/jwt
 github.com/devtron-labs/authenticator/middleware
 github.com/devtron-labs/authenticator/oidc
 github.com/devtron-labs/authenticator/password
-# github.com/devtron-labs/common-lib v0.0.25-0.20240812113340-f14be466613d
+# github.com/devtron-labs/common-lib v0.16.1-0.20240903114838-d814f4cd60ca
 ## explicit; go 1.21
 github.com/devtron-labs/common-lib/async
 github.com/devtron-labs/common-lib/blob-storage