fix: 5xx 4.0 iter (#4620)

* bugs revert

* 4th iter code changes

* delete dag exec

* initial poc (ongoing) of handling grpc errors

* grpc error handling at install helm chart and template services

* refactoring

* refactor

* code review changes with some refactoring

* removed unused files

* refactor

Author: prakash100198

api/helm-app/service/HelmAppService.go

@@ -11,7 +11,6 @@ import (
 	"github.com/devtron-labs/devtron/internal/constants"
 	repository2 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry"
 	"github.com/go-pg/pg"
-	"google.golang.org/grpc/codes"
 	"net/http"
 	"reflect"
 	"strconv"
@@ -389,7 +388,7 @@ func (impl *HelmAppServiceImpl) GetDeploymentHistory(ctx context.Context, app *A
 		ReleaseName:   app.ReleaseName,
 	}
 	history, err := impl.helmAppClient.GetDeploymentHistory(ctx, req)
-	if util.GetGRPCErrorDetailedMessage(err) == bean.ErrReleaseNotFound {
+	if util.GetClientErrorDetailedMessage(err) == bean.ErrReleaseNotFound {
 		err = &util.ApiError{
 			Code:            constants.HelmReleaseNotFound,
 			InternalMessage: bean.ErrReleaseNotFound,
@@ -552,16 +551,16 @@ func (impl *HelmAppServiceImpl) DeleteApplication(ctx context.Context, app *AppI
 
 	deleteApplicationResponse, err := impl.helmAppClient.DeleteApplication(ctx, req)
 	if err != nil {
-		code, message := util.GetGRPCDetailedError(err)
-		if code == codes.NotFound {
+		code, message := util.GetClientDetailedError(err)
+		if code.IsNotFoundCode() {
 			return nil, &util.ApiError{
-				Code:           "404",
-				HttpStatusCode: 200,
+				Code:           strconv.Itoa(http.StatusNotFound),
+				HttpStatusCode: 200, //need to revisit the status code
 				UserMessage:    message,
 			}
 		}
 		impl.logger.Errorw("error in deleting helm application", "err", err)
-		return nil, errors.New(util.GetGRPCErrorDetailedMessage(err))
+		return nil, errors.New(util.GetClientErrorDetailedMessage(err))
 	}
 
 	response := &openapi.UninstallReleaseResponse{
@@ -962,6 +961,10 @@ func (impl *HelmAppServiceImpl) TemplateChart(ctx context.Context, templateChart
 	templateChartResponse, err := impl.helmAppClient.TemplateChart(ctx, installReleaseRequest)
 	if err != nil {
 		impl.logger.Errorw("error in templating chart", "err", err)
+		clientErrCode, errMsg := util.GetClientDetailedError(err)
+		if clientErrCode.IsInvalidArgumentCode() {
+			return nil, &util.ApiError{HttpStatusCode: http.StatusConflict, Code: strconv.Itoa(http.StatusConflict), InternalMessage: errMsg, UserMessage: errMsg}
+		}
 		return nil, err
 	}
 

internal/errors/bean.go

@@ -0,0 +1,19 @@
+package errors
+
+import "google.golang.org/grpc/codes"
+
+type ClientStatusCode struct {
+	Code codes.Code
+}
+
+func (r *ClientStatusCode) IsInvalidArgumentCode() bool {
+	return r.Code == codes.InvalidArgument
+}
+
+func (r *ClientStatusCode) IsNotFoundCode() bool {
+	return r.Code == codes.NotFound
+}
+
+func (r *ClientStatusCode) IsFailedPreconditionCode() bool {
+	return r.Code == codes.FailedPrecondition
+}

internal/util/ErrorUtil.go

@@ -19,6 +19,7 @@ package util
 
 import (
 	"fmt"
+	"github.com/devtron-labs/devtron/internal/errors"
 	"github.com/go-pg/pg"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -50,16 +51,18 @@ func IsErrNoRows(err error) bool {
 	return pg.ErrNoRows == err
 }
 
-func GetGRPCErrorDetailedMessage(err error) string {
+func GetClientErrorDetailedMessage(err error) string {
 	if errStatus, ok := status.FromError(err); ok {
 		return errStatus.Message()
 	}
 	return err.Error()
 }
 
-func GetGRPCDetailedError(err error) (codes.Code, string) {
+func GetClientDetailedError(err error) (*errors.ClientStatusCode, string) {
+	grpcCode := &errors.ClientStatusCode{Code: codes.Unknown}
 	if errStatus, ok := status.FromError(err); ok {
-		return errStatus.Code(), errStatus.Message()
+		grpcCode.Code = errStatus.Code()
+		return grpcCode, errStatus.Message()
 	}
-	return codes.Unknown, err.Error()
+	return grpcCode, err.Error()
 }

pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go

@@ -13,6 +13,8 @@ import (
 	repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository"
 	commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean"
 	"github.com/go-pg/pg"
+	"net/http"
+	"strconv"
 	"strings"
 	"time"
 )
@@ -108,6 +110,10 @@ func (impl *FullModeDeploymentServiceImpl) UpdateAndSyncACDApps(installAppVersio
 	err = impl.argoClientWrapperService.SyncArgoCDApplicationIfNeededAndRefresh(ctx, acdAppName)
 	if err != nil {
 		impl.Logger.Errorw("error in getting argocd application with normal refresh", "err", err, "argoAppName", installAppVersionRequest.ACDAppName)
+		clientErrCode, errMsg := util.GetClientDetailedError(err)
+		if clientErrCode.IsFailedPreconditionCode() {
+			return &util.ApiError{HttpStatusCode: http.StatusPreconditionFailed, Code: strconv.Itoa(http.StatusPreconditionFailed), InternalMessage: errMsg, UserMessage: errMsg}
+		}
 		return err
 	}
 	if !impl.acdConfig.ArgoCDAutoSyncEnabled {

pkg/deployment/trigger/devtronApps/TriggerService.go

@@ -38,6 +38,7 @@ import (
 	bean5 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean"
 	"github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean"
 	"github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/helper"
+	clientErrors "github.com/devtron-labs/devtron/pkg/errors"
 	"github.com/devtron-labs/devtron/pkg/eventProcessor/out"
 	"github.com/devtron-labs/devtron/pkg/imageDigestPolicy"
 	"github.com/devtron-labs/devtron/pkg/pipeline"
@@ -992,7 +993,11 @@ func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(overrideRequest *bean
 			updateApplicationResponse, err := impl.helmAppClient.UpdateApplication(ctx, req)
 			if err != nil {
 				impl.logger.Errorw("error in updating helm application for cd pipeline", "err", err)
-				if util.GetGRPCErrorDetailedMessage(err) == context.Canceled.Error() {
+				apiError := clientErrors.ConvertToApiError(err)
+				if apiError != nil {
+					return false, apiError
+				}
+				if util.GetClientErrorDetailedMessage(err) == context.Canceled.Error() {
 					err = errors.New(pipelineConfig.NEW_DEPLOYMENT_INITIATED)
 				}
 				return false, err
@@ -1009,7 +1014,7 @@ func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(overrideRequest *bean
 				impl.logger.Errorw("error in helm install custom chart", "err", err)
 				return false, err
 			}
-			if util.GetGRPCErrorDetailedMessage(err) == context.Canceled.Error() {
+			if util.GetClientErrorDetailedMessage(err) == context.Canceled.Error() {
 				err = errors.New(pipelineConfig.NEW_DEPLOYMENT_INITIATED)
 			}
 
@@ -1020,10 +1025,13 @@ func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(overrideRequest *bean
 
 			if err != nil {
 				impl.logger.Errorw("error in helm install custom chart", "err", err)
-
 				if pgErr != nil {
 					impl.logger.Errorw("failed to update deployment app created flag in pipeline table", "err", err)
 				}
+				apiError := clientErrors.ConvertToApiError(err)
+				if apiError != nil {
+					return false, apiError
+				}
 				return false, err
 			}
 

pkg/errors/utils.go

@@ -0,0 +1,41 @@
+package errors
+
+import (
+	util2 "github.com/devtron-labs/devtron/internal/util"
+	"net/http"
+	"strconv"
+	"strings"
+)
+
+// extract out this custom error messages in kubelink and send custom error messages from kubelink
+const (
+	ClusterUnreachableErrorMsg  = "cluster unreachable"
+	CrdPreconditionErrorMsg     = "ensure CRDs are installed first"
+	ArrayStringMismatchErrorMsg = "got array expected string"
+	NamespaceNotFoundErrorMsg   = "namespace not found"
+	InvalidValueErrorMsg        = "invalid value in manifest"
+	OperationInProgressErrorMsg = "another operation (install/upgrade/rollback) is in progress"
+)
+
+var errorHttpStatusCodeMap = map[string]int{
+	ClusterUnreachableErrorMsg:  http.StatusUnprocessableEntity,
+	CrdPreconditionErrorMsg:     http.StatusPreconditionFailed,
+	NamespaceNotFoundErrorMsg:   http.StatusConflict,
+	ArrayStringMismatchErrorMsg: http.StatusFailedDependency,
+	InvalidValueErrorMsg:        http.StatusFailedDependency,
+	OperationInProgressErrorMsg: http.StatusConflict,
+}
+
+func ConvertToApiError(err error) *util2.ApiError {
+	for errMsg, statusCode := range errorHttpStatusCodeMap {
+		if strings.Contains(err.Error(), errMsg) {
+			return &util2.ApiError{
+				InternalMessage: err.Error(),
+				UserMessage:     err.Error(),
+				HttpStatusCode:  statusCode,
+				Code:            strconv.Itoa(statusCode),
+			}
+		}
+	}
+	return nil
+}

pkg/generateManifest/DeployementTemplateService.go

@@ -24,6 +24,7 @@ import (
 	util2 "github.com/devtron-labs/devtron/util"
 	"github.com/go-pg/pg"
 	"go.uber.org/zap"
+	"net/http"
 	"os"
 	"strconv"
 	"time"
@@ -364,6 +365,10 @@ func (impl DeploymentTemplateServiceImpl) GenerateManifest(ctx context.Context,
 	templateChartResponse, err := impl.helmAppClient.TemplateChart(ctx, installReleaseRequest)
 	if err != nil {
 		impl.Logger.Errorw("error in templating chart", "err", err)
+		clientErrCode, errMsg := util.GetClientDetailedError(err)
+		if clientErrCode.IsInvalidArgumentCode() {
+			return nil, &util.ApiError{HttpStatusCode: http.StatusConflict, Code: strconv.Itoa(http.StatusConflict), InternalMessage: errMsg, UserMessage: errMsg}
+		}
 		return nil, err
 	}
 	response := &openapi2.TemplateChartResponse{

pkg/workflow/cd/CdWorkflowCommonService.go

@@ -140,7 +140,7 @@ func (impl *CdWorkflowCommonServiceImpl) MarkCurrentDeploymentFailed(runner *pip
 	//update current WF with error status
 	impl.logger.Errorw("error in triggering cd WF, setting wf status as fail ", "wfId", runner.Id, "err", releaseErr)
 	runner.Status = pipelineConfig.WorkflowFailed
-	runner.Message = util.GetGRPCErrorDetailedMessage(releaseErr)
+	runner.Message = util.GetClientErrorDetailedMessage(releaseErr)
 	runner.FinishedOn = time.Now()
 	runner.UpdatedOn = time.Now()
 	runner.UpdatedBy = triggeredBy
@@ -239,7 +239,7 @@ func (impl *CdWorkflowCommonServiceImpl) UpdateCDWorkflowRunnerStatus(ctx contex
 }
 
 func extractTimelineFailedStatusDetails(err error) string {
-	errorString := util.GetGRPCErrorDetailedMessage(err)
+	errorString := util.GetClientErrorDetailedMessage(err)
 	switch errorString {
 	case pipelineConfig.FOUND_VULNERABILITY:
 		return pipelineConfig.TIMELINE_DESCRIPTION_VULNERABLE_IMAGE

pkg/workflow/dag/WorkflowDagExecutor.go

@@ -278,14 +278,14 @@ func (impl *WorkflowDagExecutorImpl) UpdateWorkflowRunnerStatusForDeployment(app
 	if err != nil {
 		impl.logger.Errorw("error in getting helm app release status", "appIdentifier", appIdentifier, "err", err)
 		// Handle release not found errors
-		if skipReleaseNotFound && util.GetGRPCErrorDetailedMessage(err) != bean6.ErrReleaseNotFound {
+		if skipReleaseNotFound && util.GetClientErrorDetailedMessage(err) != bean6.ErrReleaseNotFound {
 			// skip this error and continue for next workflow status
 			impl.logger.Warnw("found error, skipping helm apps status update for this trigger", "appIdentifier", appIdentifier, "err", err)
 			return false
 		}
 		// If release not found, mark the deployment as failure
 		wfr.Status = pipelineConfig.WorkflowFailed
-		wfr.Message = util.GetGRPCErrorDetailedMessage(err)
+		wfr.Message = util.GetClientErrorDetailedMessage(err)
 		wfr.FinishedOn = time.Now()
 		return true
 	}
@@ -325,7 +325,7 @@ func (impl *WorkflowDagExecutorImpl) UpdateWorkflowRunnerStatusForDeployment(app
 }
 
 func (impl *WorkflowDagExecutorImpl) handleAsyncTriggerReleaseError(releaseErr error, cdWfr *pipelineConfig.CdWorkflowRunner, overrideRequest *bean.ValuesOverrideRequest, appIdentifier *client2.AppIdentifier) {
-	releaseErrString := util.GetGRPCErrorDetailedMessage(releaseErr)
+	releaseErrString := util.GetClientErrorDetailedMessage(releaseErr)
 	switch releaseErrString {
 	case context.DeadlineExceeded.Error():
 		// if context deadline is exceeded fetch release status and UpdateWorkflowRunnerStatusForDeployment