feat(worker): add static-key to keep the same public url for serve static files (#3670)

close #3657

Author: bnjjj

docs/content/workflows/pipelines/actions/builtin/serve-static-files.md

@@ -12,11 +12,12 @@ Pay attention this action is only available if your objectstore is configured to
 ## Parameters
 * name: Name to display in CDS UI and identify your static files
 * path: Path where static files will be uploaded (example: mywebsite/*). If it's a file, the entrypoint would be set to this filename by default.
-* tag: Filename (and not path) for the entrypoint when serving static files (default: if empty it would be index.html).
+* entrypoint: Filename (and not path) for the entrypoint when serving static files (default: if empty it would be index.html).
+* static-key: Indicate a static-key which will be a reference to keep the same generated URL. Example: {{.git.branch}}.
 
 ### Example
 
-* In this example I created a website with script in bash and use action `Serve Static Files`.
+* In this example I created a website with script in bash and use action `Serve Static Files`. If you want to keep the same URL by .git.branch for example you can indicate in the advanced parameters a `static-key` equals to `{{.git.branch}}`.
 
 ![img](/images/workflows.pipelines.actions.builtin.serve-static-files-job.png)
 

engine/api/action/builtin.go

@@ -272,6 +272,12 @@ Semver used if fully compatible with https://semver.org/
 		Type:        sdk.StringParameter,
 		Value:       "",
 		Advanced:    true})
+	serveStaticAct.Parameter(sdk.Parameter{
+		Name:        "static-key",
+		Description: "Indicate a static-key which will be a reference to keep the same generated URL. Example: {{.git.branch}}",
+		Type:        sdk.StringParameter,
+		Value:       "",
+		Advanced:    true})
 
 	return checkBuiltinAction(db, serveStaticAct)
 }

engine/api/workflow_queue_staticfiles.go

@@ -55,6 +55,11 @@ func (api *API) postWorkflowJobStaticFilesHandler() service.Handler {
 			entrypoint = m.Value["entrypoint"][0]
 		}
 
+		var staticKey string
+		if len(m.Value["static_key"]) > 0 {
+			staticKey = m.Value["static_key"][0]
+		}
+
 		if fileName == "" {
 			return sdk.WrapError(sdk.ErrWrongRequest, "Content-Disposition header is not set")
 		}
@@ -64,13 +69,26 @@ func (api *API) postWorkflowJobStaticFilesHandler() service.Handler {
 			return sdk.WrapError(errJ, "Cannot load node job run")
 		}
 
+		nodeRun, errNr := workflow.LoadNodeRunByID(db, nodeJobRun.WorkflowNodeRunID, workflow.LoadRunOptions{DisableDetailledNodeRun: true})
+		if errNr != nil {
+			return sdk.WrapError(errNr, "Cannot load node run")
+		}
+
 		staticFile := sdk.StaticFiles{
 			Name:         name,
 			EntryPoint:   entrypoint,
+			StaticKey:    staticKey,
+			WorkflowID:   nodeRun.WorkflowID,
 			NodeRunID:    nodeJobRun.WorkflowNodeRunID,
 			NodeJobRunID: nodeJobRunID,
 		}
 
+		if staticFile.StaticKey != "" {
+			if err := objectstore.Delete(&staticFile); err != nil {
+				return sdk.WrapError(err, "Cannot delete existing static files")
+			}
+		}
+
 		files := m.File[fileName]
 		if len(files) == 1 {
 			file, err := files[0].Open()
@@ -119,13 +137,25 @@ func (api *API) postWorkflowJobStaticFilesWithTempURLHandler() service.Handler {
 			return sdk.WithStack(err)
 		}
 
+		if staticfile.StaticKey != "" {
+			if err := objectstore.Delete(&staticfile); err != nil {
+				return sdk.WrapError(err, "Cannot delete existing static files")
+			}
+		}
+
 		nodeJobRun, errJ := workflow.LoadNodeJobRun(api.mustDB(), api.Cache, id)
 		if errJ != nil {
 			return sdk.WrapError(errJ, "Cannot load node job run")
 		}
 		staticfile.NodeRunID = nodeJobRun.WorkflowNodeRunID
 		staticfile.Name = name
 
+		nodeRun, errNr := workflow.LoadNodeRunByID(api.mustDB(), nodeJobRun.WorkflowNodeRunID, workflow.LoadRunOptions{DisableDetailledNodeRun: true})
+		if errNr != nil {
+			return sdk.WrapError(errNr, "Cannot load node run")
+		}
+		staticfile.WorkflowID = nodeRun.WorkflowID
+
 		var retryURL = 10
 		var url, key string
 		var errorStoreURL error

engine/sql/141_static_files_key.sql

@@ -0,0 +1,9 @@
+-- +migrate Up
+
+INSERT into action_parameter (action_id, name, description, type, value, advanced) values((select id from action where name = 'Serve Static Files' and type = 'Builtin'), 'static-key', 'Indicate a static-key which will be a reference to keep the same generated URL. Example: {{.git.branch}}', 'string', '', true);
+ALTER TABLE workflow_node_run_static_files ADD COLUMN static_key TEXT DEFAULT '';
+
+-- +migrate Down
+
+DELETE from action_parameter where name = 'static-key' and action_id = (select id from action where name = 'Serve Static Files' and type = 'Builtin');
+ALTER TABLE workflow_node_run_static_files DROP COLUMN static_key;

engine/worker/builtin_serve_static_files.go

@@ -33,6 +33,7 @@ func runServeStaticFiles(w *currentWorker) BuiltInAction {
 			sendLog(res.Reason)
 			return res
 		}
+		staticKey := sdk.ParameterValue(a.Parameters, "static-key")
 
 		// Global all files matching filePath
 		filesPath, err := filepath.Glob(path)
@@ -65,7 +66,7 @@ func runServeStaticFiles(w *currentWorker) BuiltInAction {
 		}
 
 		sendLog(fmt.Sprintf(`Upload and serving files in progress... with entrypoint "%s"`, entrypoint.Value))
-		publicURL, _, _, err := w.client.QueueStaticFilesUpload(ctx, buildID, name.Value, entrypoint.Value, file)
+		publicURL, _, _, err := w.client.QueueStaticFilesUpload(ctx, buildID, name.Value, entrypoint.Value, staticKey, file)
 		if err != nil {
 			res.Status = sdk.StatusFail.String()
 			res.Reason = fmt.Sprintf("Cannot upload static files: %v", err)

sdk/cdsclient/client_queue.go

@@ -564,10 +564,11 @@ func (c *client) QueueServiceLogs(ctx context.Context, logs []sdk.ServiceLog) er
 
 //  STATIC FILES -----
 
-func (c *client) QueueStaticFilesUpload(ctx context.Context, nodeJobRunID int64, name, entrypoint string, tarContent io.Reader) (string, bool, time.Duration, error) {
+func (c *client) QueueStaticFilesUpload(ctx context.Context, nodeJobRunID int64, name, entrypoint, staticKey string, tarContent io.Reader) (string, bool, time.Duration, error) {
 	t0 := time.Now()
 	staticFile := sdk.StaticFiles{
 		EntryPoint:   entrypoint,
+		StaticKey:    staticKey,
 		Name:         name,
 		NodeJobRunID: nodeJobRunID,
 	}
@@ -699,6 +700,7 @@ func (c *client) queueDirectStaticFilesUpload(staticFile *sdk.StaticFiles, tarCo
 
 	_ = writer.WriteField("name", staticFile.Name)
 	_ = writer.WriteField("entrypoint", staticFile.EntryPoint)
+	_ = writer.WriteField("static_key", staticFile.StaticKey)
 
 	if errclose := writer.Close(); errclose != nil {
 		return "", errclose

sdk/cdsclient/interface.go

@@ -223,7 +223,7 @@ type QueueClient interface {
 	QueueJobSendSpawnInfo(ctx context.Context, isWorkflowJob bool, id int64, in []sdk.SpawnInfo) error
 	QueueSendResult(ctx context.Context, id int64, res sdk.Result) error
 	QueueArtifactUpload(ctx context.Context, id int64, tag, filePath string) (bool, time.Duration, error)
-	QueueStaticFilesUpload(ctx context.Context, nodeJobRunID int64, name, entrypoint string, tarContent io.Reader) (string, bool, time.Duration, error)
+	QueueStaticFilesUpload(ctx context.Context, nodeJobRunID int64, name, entrypoint, staticKey string, tarContent io.Reader) (string, bool, time.Duration, error)
 	QueueJobTag(ctx context.Context, jobID int64, tags []sdk.WorkflowRunTag) error
 	QueueJobIncAttempts(ctx context.Context, jobID int64) ([]int64, error)
 	QueueServiceLogs(ctx context.Context, logs []sdk.ServiceLog) error

sdk/static_files.go

@@ -11,9 +11,11 @@ import (
 type StaticFiles struct {
 	ID           int64     `json:"id" db:"id" cli:"id"`
 	Name         string    `json:"name" db:"name" cli:"name"`
+	WorkflowID   int64     `json:"workflow_id" db:"-"`
 	NodeRunID    int64     `json:"workflow_node_run_id" db:"workflow_node_run_id"`
 	NodeJobRunID int64     `json:"workflow_node_run_job_id,omitempty" db:"-"`
 	EntryPoint   string    `json:"entrypoint" db:"entrypoint"`
+	StaticKey    string    `json:"static_key" db:"static_key"`
 	PublicURL    string    `json:"public_url" db:"public_url" cli:"public_url"`
 	Created      time.Time `json:"created" db:"created" cli:"created"`
 
@@ -28,13 +30,19 @@ func (staticfile *StaticFiles) GetName() string {
 
 //GetPath returns the path of the artifact
 func (staticfile *StaticFiles) GetPath() string {
-	container := base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%d-%s", staticfile.NodeJobRunID, url.PathEscape(staticfile.Name))))
-	return container
+	var container string
+	if staticfile.StaticKey != "" {
+		container = fmt.Sprintf("%d-%s-%s", staticfile.WorkflowID, url.PathEscape(staticfile.StaticKey), url.PathEscape(staticfile.Name))
+	} else {
+		container = fmt.Sprintf("%d-%s", staticfile.NodeJobRunID, url.PathEscape(staticfile.Name))
+	}
+	return base64.RawURLEncoding.EncodeToString([]byte(container))
 }
 
 // Equal returns true if  StaticFiles are equal to another one
 func (staticfile StaticFiles) Equal(currStaticfile StaticFiles) bool {
 	return currStaticfile.NodeRunID == staticfile.NodeRunID &&
 		currStaticfile.Name == staticfile.Name &&
-		currStaticfile.EntryPoint == staticfile.EntryPoint
+		currStaticfile.EntryPoint == staticfile.EntryPoint &&
+		currStaticfile.StaticKey == staticfile.StaticKey
 }

ui/package.json

@@ -20,7 +20,7 @@
     "build:prod": "ng build --prod",
     "precommit": "lint-staged",
     "sentry:release": "sentry-cli releases -o ${SENTRY_ORG} -p ${SENTRY_PROJECT} new ${CDS_VERSION}",
-    "sentry:sourcemaps": "sentry-cli releases -o ${SENTRY_ORG} -p ${SENTRY_PROJECT} files ${CDS_VERSION} upload-sourcemaps"
+    "sentry:sourcemaps": "sentry-cli releases -o ${SENTRY_ORG} -p ${SENTRY_PROJECT} files ${CDS_VERSION} upload-sourcemaps --url-prefix=${SENTRY_CDS_PREFIX_URL}"
   },
   "lint-staged": {
     "linters": {