Skip to content

Commit 43c8a35

Browse files
YongxuanzhangYongxuanzhang
committed
[TEP-0076]Support Array Results substitution
This is part of work in TEP-0076. This commit provides the support to apply array results replacements. Previous this commit we support emitting array results so users can write array results to task level, but we cannot pass array results from tasks within one pipeline. This commit adds the support for this.
1 parent e24df83 commit 43c8a35

File tree

12 files changed

+211
-12
lines changed

12 files changed

+211
-12
lines changed

‎docs/variables.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ For instructions on using variable substitutions see the relevant section of [th
2222
| `tasks.<taskName>.results.<resultName>` | The value of the `Task's` result. Can alter `Task` execution order within a `Pipeline`.) |
2323
| `tasks.<taskName>.results['<resultName>']` | (see above)) |
2424
| `tasks.<taskName>.results["<resultName>"]` | (see above)) |
25+
| `tasks.<taskName>.results.<resultName>[*]` | The array value of the `Task's` result. Can alter `Task` execution order within a `Pipeline`.) |
26+
| `tasks.<taskName>.results['<resultName>'][*]` | (see above)) |
27+
| `tasks.<taskName>.results["<resultName>"][*]` | (see above)) |
2528
| `workspaces.<workspaceName>.bound` | Whether a `Workspace` has been bound or not. "false" if the `Workspace` declaration has `optional: true` and the Workspace binding was omitted by the PipelineRun. |
2629
| `context.pipelineRun.name` | The name of the `PipelineRun` that this `Pipeline` is running in. |
2730
| `context.pipelineRun.namespace` | The namespace of the `PipelineRun` that this `Pipeline` is running in. |

‎examples/v1beta1/pipelineruns/alpha/pipelinerun-array-results.yaml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
apiVersion: tekton.dev/v1beta1
2+
kind: PipelineRun
3+
metadata:
4+
name: pipelinerun-array-results
5+
spec:
6+
pipelineSpec:
7+
tasks:
8+
- name: task1
9+
taskSpec:
10+
results:
11+
- name: array-results
12+
type: array
13+
description: The array results
14+
steps:
15+
- name: write-array
16+
image: bash:latest
17+
script: |
18+
#!/usr/bin/env bash
19+
echo -n "[\"1\",\"2\",\"3\"]" | tee $(results.array-results.path)
20+
- name: task2
21+
params:
22+
- name: foo
23+
value: "$(tasks.task1.results.array-results[*])"
24+
taskSpec:
25+
params:
26+
- name: foo
27+
type: array
28+
default:
29+
- "defaultparam1"
30+
- "defaultparam2"
31+
steps:
32+
- name: print-param
33+
image: bash:latest
34+
args: [
35+
"echo",
36+
"$(params.foo[*])"
37+
]

‎pkg/apis/pipeline/v1beta1/result_types.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ limitations under the License.
1313

1414
package v1beta1
1515

16+
import "strings"
17+
1618
// TaskResult used to describe the results of a task
1719
type TaskResult struct {
1820
// Name the given name
@@ -60,3 +62,9 @@ const (
6062

6163
// AllResultsTypes can be used for ResultsTypes validation.
6264
var AllResultsTypes = []ResultsType{ResultsTypeString, ResultsTypeArray, ResultsTypeObject}
65+
66+
// ResultsArrayReference returns the reference of the result from array parameter reference
67+
// returns results.resultname[*] from $(results.resultname[*])
68+
func ResultsArrayReference(a string) string {
69+
return strings.TrimSuffix(strings.TrimPrefix(a, "$("), ")")
70+
}

‎pkg/apis/pipeline/v1beta1/resultref.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,18 @@ const (
3535
// ResultResultPart Constant used to define the "results" part of a pipeline result reference
3636
ResultResultPart = "results"
3737
// TODO(#2462) use one regex across all substitutions
38-
variableSubstitutionFormat = `\$\([_a-zA-Z0-9.-]+(\.[_a-zA-Z0-9.-]+)*\)`
38+
// variableSubstitutionFormat matches format like $result.resultname, $result.resultname[int] and $result.resultname[*]
39+
variableSubstitutionFormat = `\$\([_a-zA-Z0-9.-]+(\.[_a-zA-Z0-9.-]+)*(\[([0-9])*\*?\])?\)`
40+
// excludeArrayIndexing will replace all `[int]` and `[*]` for parseExpression to extract result name
41+
excludeArrayIndexing = `\[([0-9])*\*?\]`
3942
// ResultNameFormat Constant used to define the the regex Result.Name should follow
4043
ResultNameFormat = `^([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$`
4144
)
4245

43-
var variableSubstitutionRegex = regexp.MustCompile(variableSubstitutionFormat)
46+
// VariableSubstitutionRegex is a regex to find all matching substitutions
47+
var VariableSubstitutionRegex = regexp.MustCompile(variableSubstitutionFormat)
4448
var resultNameFormatRegex = regexp.MustCompile(ResultNameFormat)
49+
var excludeArrayIndexingRegex = regexp.MustCompile(excludeArrayIndexing)
4550

4651
// NewResultRefs extracts all ResultReferences from a param or a pipeline result.
4752
// If the ResultReference can be extracted, they are returned. Expressions which are not
@@ -107,7 +112,7 @@ func GetVarSubstitutionExpressionsForPipelineResult(result PipelineResult) ([]st
107112
}
108113

109114
func validateString(value string) []string {
110-
expressions := variableSubstitutionRegex.FindAllString(value, -1)
115+
expressions := VariableSubstitutionRegex.FindAllString(value, -1)
111116
if expressions == nil {
112117
return nil
113118
}
@@ -127,6 +132,7 @@ func parseExpression(substitutionExpression string) (string, string, error) {
127132
if len(subExpressions) != 4 || subExpressions[0] != ResultTaskPart || subExpressions[2] != ResultResultPart {
128133
return "", "", fmt.Errorf("Must be of the form %q", resultExpressionFormat)
129134
}
135+
subExpressions[3] = excludeArrayIndexingRegex.ReplaceAllString(subExpressions[3], "")
130136
return subExpressions[1], subExpressions[3], nil
131137
}
132138

‎pkg/apis/pipeline/v1beta1/when_types.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,9 +62,12 @@ func (we *WhenExpression) applyReplacements(replacements map[string]string, arra
6262
for _, val := range we.Values {
6363
// arrayReplacements holds a list of array parameters with a pattern - params.arrayParam1
6464
// array params are referenced using $(params.arrayParam1[*])
65+
// array results are referenced using $(results.resultname[*])
6566
// check if the param exist in the arrayReplacements to replace it with a list of values
6667
if _, ok := arrayReplacements[fmt.Sprintf("%s.%s", ParamsPrefix, ArrayReference(val))]; ok {
6768
replacedValues = append(replacedValues, substitution.ApplyArrayReplacements(val, replacements, arrayReplacements)...)
69+
} else if _, ok := arrayReplacements[fmt.Sprintf("%s", ResultsArrayReference(val))]; ok {
70+
replacedValues = append(replacedValues, substitution.ApplyArrayReplacements(val, replacements, arrayReplacements)...)
6871
} else {
6972
replacedValues = append(replacedValues, substitution.ApplyReplacements(val, replacements))
7073
}

‎pkg/apis/pipeline/v1beta1/when_types_test.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,25 @@ func TestApplyReplacements(t *testing.T) {
248248
Operator: selection.In,
249249
Values: []string{"barfoo"},
250250
},
251+
}, {
252+
name: "replace array results variables",
253+
original: &WhenExpression{
254+
Input: "$(tasks.foo.results.bar)",
255+
Operator: selection.In,
256+
Values: []string{"$(tasks.aTask.results.aResult[*])"},
257+
},
258+
replacements: map[string]string{
259+
"tasks.foo.results.bar": "foobar",
260+
"tasks.aTask.results.aResult[*]": "barfoo",
261+
},
262+
arrayReplacements: map[string][]string{
263+
"tasks.aTask.results.aResult[*]": {"dev", "stage"},
264+
},
265+
expected: &WhenExpression{
266+
Input: "foobar",
267+
Operator: selection.In,
268+
Values: []string{"dev", "stage"},
269+
},
251270
}, {
252271
name: "replace array params",
253272
original: &WhenExpression{

‎pkg/reconciler/pipeline/dag/dag_test.go‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,7 @@ func TestBuild_TaskParamsFromTaskResults(t *testing.T) {
371371
c := v1beta1.PipelineTask{Name: "c"}
372372
d := v1beta1.PipelineTask{Name: "d"}
373373
e := v1beta1.PipelineTask{Name: "e"}
374+
f := v1beta1.PipelineTask{Name: "f"}
374375
xDependsOnA := v1beta1.PipelineTask{
375376
Name: "x",
376377
Params: []v1beta1.Param{{
@@ -393,27 +394,38 @@ func TestBuild_TaskParamsFromTaskResults(t *testing.T) {
393394
Value: *v1beta1.NewArrayOrString("$(tasks.d.results.resultD) $(tasks.e.results.resultE)"),
394395
}},
395396
}
397+
wDependsOnF := v1beta1.PipelineTask{
398+
Name: "w",
399+
Params: []v1beta1.Param{{
400+
Name: "paramw",
401+
Value: *v1beta1.NewArrayOrString("$(tasks.f.results.resultF[*])"),
402+
}},
403+
}
396404

397-
// a b c d e
398-
// | \ / \ /
399-
// x y z
405+
// a b c d e f
406+
// | \ / \ / |
407+
// x y z w
400408
nodeA := &dag.Node{Task: a}
401409
nodeB := &dag.Node{Task: b}
402410
nodeC := &dag.Node{Task: c}
403411
nodeD := &dag.Node{Task: d}
404412
nodeE := &dag.Node{Task: e}
413+
nodeF := &dag.Node{Task: f}
405414
nodeX := &dag.Node{Task: xDependsOnA}
406415
nodeY := &dag.Node{Task: yDependsOnBRunsAfterC}
407416
nodeZ := &dag.Node{Task: zDependsOnDAndE}
417+
nodeW := &dag.Node{Task: wDependsOnF}
408418

409419
nodeA.Next = []*dag.Node{nodeX}
410420
nodeB.Next = []*dag.Node{nodeY}
411421
nodeC.Next = []*dag.Node{nodeY}
412422
nodeD.Next = []*dag.Node{nodeZ}
413423
nodeE.Next = []*dag.Node{nodeZ}
424+
nodeF.Next = []*dag.Node{nodeW}
414425
nodeX.Prev = []*dag.Node{nodeA}
415426
nodeY.Prev = []*dag.Node{nodeB, nodeC}
416427
nodeZ.Prev = []*dag.Node{nodeD, nodeE}
428+
nodeW.Prev = []*dag.Node{nodeF}
417429

418430
expectedDAG := &dag.Graph{
419431
Nodes: map[string]*dag.Node{
@@ -422,15 +434,17 @@ func TestBuild_TaskParamsFromTaskResults(t *testing.T) {
422434
"c": nodeC,
423435
"d": nodeD,
424436
"e": nodeE,
437+
"f": nodeF,
425438
"x": nodeX,
426439
"y": nodeY,
427440
"z": nodeZ,
441+
"w": nodeW,
428442
},
429443
}
430444
p := &v1beta1.Pipeline{
431445
ObjectMeta: metav1.ObjectMeta{Name: "pipeline"},
432446
Spec: v1beta1.PipelineSpec{
433-
Tasks: []v1beta1.PipelineTask{a, b, c, d, e, xDependsOnA, yDependsOnBRunsAfterC, zDependsOnDAndE},
447+
Tasks: []v1beta1.PipelineTask{a, b, c, d, e, f, xDependsOnA, yDependsOnBRunsAfterC, zDependsOnDAndE, wDependsOnF},
434448
},
435449
}
436450
tasks := v1beta1.PipelineTaskList(p.Spec.Tasks)

‎pkg/reconciler/pipelinerun/resources/apply.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,17 +99,18 @@ func ApplyPipelineTaskContexts(pt *v1beta1.PipelineTask) *v1beta1.PipelineTask {
9999
// ApplyTaskResults applies the ResolvedResultRef to each PipelineTask.Params and Pipeline.WhenExpressions in targets
100100
func ApplyTaskResults(targets PipelineRunState, resolvedResultRefs ResolvedResultRefs) {
101101
stringReplacements := resolvedResultRefs.getStringReplacements()
102+
arrayReplacements := resolvedResultRefs.getArrayReplacements()
102103
for _, resolvedPipelineRunTask := range targets {
103104
// also make substitution for resolved condition checks
104105
for _, resolvedConditionCheck := range resolvedPipelineRunTask.ResolvedConditionChecks {
105106
pipelineTaskCondition := resolvedConditionCheck.PipelineTaskCondition.DeepCopy()
106-
pipelineTaskCondition.Params = replaceParamValues(pipelineTaskCondition.Params, stringReplacements, nil)
107+
pipelineTaskCondition.Params = replaceParamValues(pipelineTaskCondition.Params, stringReplacements, arrayReplacements)
107108
resolvedConditionCheck.PipelineTaskCondition = pipelineTaskCondition
108109
}
109110
if resolvedPipelineRunTask.PipelineTask != nil {
110111
pipelineTask := resolvedPipelineRunTask.PipelineTask.DeepCopy()
111-
pipelineTask.Params = replaceParamValues(pipelineTask.Params, stringReplacements, nil)
112-
pipelineTask.WhenExpressions = pipelineTask.WhenExpressions.ReplaceWhenExpressionsVariables(stringReplacements, nil)
112+
pipelineTask.Params = replaceParamValues(pipelineTask.Params, stringReplacements, arrayReplacements)
113+
pipelineTask.WhenExpressions = pipelineTask.WhenExpressions.ReplaceWhenExpressionsVariables(stringReplacements, arrayReplacements)
113114
resolvedPipelineRunTask.PipelineTask = pipelineTask
114115
}
115116
}

‎pkg/reconciler/pipelinerun/resources/apply_test.go‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -401,6 +401,38 @@ func TestApplyTaskResults_MinimalExpression(t *testing.T) {
401401
}},
402402
},
403403
}},
404+
}, {
405+
name: "Test array result substitution on minimal variable substitution expression - params",
406+
resolvedResultRefs: ResolvedResultRefs{{
407+
Value: *v1beta1.NewArrayOrString("arrayResultValueOne", "arrayResultValueTwo"),
408+
ResultReference: v1beta1.ResultRef{
409+
PipelineTask: "aTask",
410+
Result: "a.Result",
411+
},
412+
FromTaskRun: "aTaskRun",
413+
}},
414+
targets: PipelineRunState{{
415+
PipelineTask: &v1beta1.PipelineTask{
416+
Name: "bTask",
417+
TaskRef: &v1beta1.TaskRef{Name: "bTask"},
418+
Params: []v1beta1.Param{{
419+
Name: "bParam",
420+
Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeArray,
421+
ArrayVal: []string{`$(tasks.aTask.results["a.Result"][*])`},
422+
},
423+
}},
424+
},
425+
}},
426+
want: PipelineRunState{{
427+
PipelineTask: &v1beta1.PipelineTask{
428+
Name: "bTask",
429+
TaskRef: &v1beta1.TaskRef{Name: "bTask"},
430+
Params: []v1beta1.Param{{
431+
Name: "bParam",
432+
Value: *v1beta1.NewArrayOrString("arrayResultValueOne", "arrayResultValueTwo"),
433+
}},
434+
},
435+
}},
404436
}, {
405437
name: "Test result substitution on minimal variable substitution expression - when expressions",
406438
resolvedResultRefs: ResolvedResultRefs{{
@@ -433,6 +465,39 @@ func TestApplyTaskResults_MinimalExpression(t *testing.T) {
433465
}},
434466
},
435467
}},
468+
}, {
469+
name: "Test array result substitution on minimal variable substitution expression - when expressions",
470+
resolvedResultRefs: ResolvedResultRefs{{
471+
Value: *v1beta1.NewArrayOrString("arrayResultValueOne", "arrayResultValueTwo"),
472+
ResultReference: v1beta1.ResultRef{
473+
PipelineTask: "aTask",
474+
Result: "aResult",
475+
},
476+
FromTaskRun: "aTaskRun",
477+
}},
478+
targets: PipelineRunState{{
479+
PipelineTask: &v1beta1.PipelineTask{
480+
Name: "bTask",
481+
TaskRef: &v1beta1.TaskRef{Name: "bTask"},
482+
WhenExpressions: []v1beta1.WhenExpression{{
483+
// Note that Input doesn't support array replacement.
484+
Input: "anInput",
485+
Operator: selection.In,
486+
Values: []string{"$(tasks.aTask.results.aResult[*])"},
487+
}},
488+
},
489+
}},
490+
want: PipelineRunState{{
491+
PipelineTask: &v1beta1.PipelineTask{
492+
Name: "bTask",
493+
TaskRef: &v1beta1.TaskRef{Name: "bTask"},
494+
WhenExpressions: []v1beta1.WhenExpression{{
495+
Input: "anInput",
496+
Operator: selection.In,
497+
Values: []string{"arrayResultValueOne", "arrayResultValueTwo"},
498+
}},
499+
},
500+
}},
436501
}} {
437502
t.Run(tt.name, func(t *testing.T) {
438503
ApplyTaskResults(tt.targets, tt.resolvedResultRefs)

‎pkg/reconciler/pipelinerun/resources/resultrefresolution.go‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -187,8 +187,22 @@ func findTaskResultForParam(taskRun *v1beta1.TaskRun, reference *v1beta1.ResultR
187187
func (rs ResolvedResultRefs) getStringReplacements() map[string]string {
188188
replacements := map[string]string{}
189189
for _, r := range rs {
190-
for _, target := range r.getReplaceTarget() {
191-
replacements[target] = r.Value.StringVal
190+
if r.Value.Type == v1beta1.ParamType(v1beta1.ResultsTypeString) {
191+
for _, target := range r.getReplaceTarget() {
192+
replacements[target] = r.Value.StringVal
193+
}
194+
}
195+
}
196+
return replacements
197+
}
198+
199+
func (rs ResolvedResultRefs) getArrayReplacements() map[string][]string {
200+
replacements := map[string][]string{}
201+
for _, r := range rs {
202+
if r.Value.Type == v1beta1.ParamType(v1beta1.ResultsTypeArray) {
203+
for _, target := range r.getArrayReplaceTarget() {
204+
replacements[target] = r.Value.ArrayVal
205+
}
192206
}
193207
}
194208
return replacements
@@ -201,3 +215,11 @@ func (r *ResolvedResultRef) getReplaceTarget() []string {
201215
fmt.Sprintf("%s.%s.%s['%s']", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),
202216
}
203217
}
218+
219+
func (r *ResolvedResultRef) getArrayReplaceTarget() []string {
220+
return []string{
221+
fmt.Sprintf("%s.%s.%s.%s[*]", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),
222+
fmt.Sprintf("%s.%s.%s[%q][*]", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),
223+
fmt.Sprintf("%s.%s.%s['%s'][*]", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),
224+
}
225+
}

0 commit comments

Comments
 (0)