Skip to content

Commit 2056ea5

Browse files
YongxuanzhangYongxuanzhang
committed
[TEP-0076]Support Array Results substitution
This is part of work in TEP-0076. This commit provides the support to apply array results replacements. Previous this commit we support emitting array results so users can write array results to task level, but we cannot pass array results from tasks within one pipeline. This commit adds the support for this.
1 parent e24df83 commit 2056ea5

File tree

11 files changed

+209
-13
lines changed

11 files changed

+209
-13
lines changed

‎examples/v1beta1/pipelineruns/alpha/pipelinerun-array-results.yaml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
apiVersion: tekton.dev/v1beta1
2+
kind: PipelineRun
3+
metadata:
4+
name: pipelinerun-array-results
5+
spec:
6+
pipelineSpec:
7+
tasks:
8+
- name: task1
9+
taskSpec:
10+
results:
11+
- name: array-results
12+
description: The current date in human readable format
13+
steps:
14+
- name: write-array
15+
image: bash:latest
16+
script: |
17+
#!/usr/bin/env bash
18+
echo -n "[\"1\",\"2\",\"3\"]" | tee $(results.array-results.path)
19+
- name: task2
20+
params:
21+
- name: foo
22+
value: "$(tasks.task1.results.array-results[*])"
23+
taskSpec:
24+
params:
25+
- name: foo
26+
type: array
27+
default:
28+
- "defaultparam1"
29+
- "defaultparam2"
30+
steps:
31+
- name: print-param
32+
image: bash:latest
33+
args: [
34+
"echo",
35+
"$(params.foo)"
36+
]
37+

‎pkg/apis/pipeline/v1beta1/result_types.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ limitations under the License.
1313

1414
package v1beta1
1515

16+
import "strings"
17+
1618
// TaskResult used to describe the results of a task
1719
type TaskResult struct {
1820
// Name the given name
@@ -60,3 +62,9 @@ const (
6062

6163
// AllResultsTypes can be used for ResultsTypes validation.
6264
var AllResultsTypes = []ResultsType{ResultsTypeString, ResultsTypeArray, ResultsTypeObject}
65+
66+
// ResultsArrayReference returns the reference of the result from array parameter reference
67+
// returns results.resultname[*] from $(results.resultname[*])
68+
func ResultsArrayReference(a string) string {
69+
return strings.TrimSuffix(strings.TrimPrefix(a, "$("), ")")
70+
}

‎pkg/apis/pipeline/v1beta1/resultref.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,17 @@ const (
3535
// ResultResultPart Constant used to define the "results" part of a pipeline result reference
3636
ResultResultPart = "results"
3737
// TODO(#2462) use one regex across all substitutions
38-
variableSubstitutionFormat = `\$\([_a-zA-Z0-9.-]+(\.[_a-zA-Z0-9.-]+)*\)`
38+
// variableSubstitutionFormat matches format like $result.resultname[int] and $result.resultname[*]
39+
variableSubstitutionFormat = `\$\([_a-zA-Z0-9.-]+(\.[_a-zA-Z0-9.-]+)*(\[([0-9])*\*?\])?\)`
40+
// excludeArrayIndexing will replace all `[int]` and `[*]` for parseExpression to extract result name
41+
excludeArrayIndexing = `\[([0-9])*\*?\]`
3942
// ResultNameFormat Constant used to define the the regex Result.Name should follow
4043
ResultNameFormat = `^([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$`
4144
)
4245

43-
var variableSubstitutionRegex = regexp.MustCompile(variableSubstitutionFormat)
46+
var VariableSubstitutionRegex = regexp.MustCompile(variableSubstitutionFormat)
4447
var resultNameFormatRegex = regexp.MustCompile(ResultNameFormat)
48+
var excludeArrayIndexingRegex = regexp.MustCompile(excludeArrayIndexing)
4549

4650
// NewResultRefs extracts all ResultReferences from a param or a pipeline result.
4751
// If the ResultReference can be extracted, they are returned. Expressions which are not
@@ -107,7 +111,7 @@ func GetVarSubstitutionExpressionsForPipelineResult(result PipelineResult) ([]st
107111
}
108112

109113
func validateString(value string) []string {
110-
expressions := variableSubstitutionRegex.FindAllString(value, -1)
114+
expressions := VariableSubstitutionRegex.FindAllString(value, -1)
111115
if expressions == nil {
112116
return nil
113117
}
@@ -127,6 +131,7 @@ func parseExpression(substitutionExpression string) (string, string, error) {
127131
if len(subExpressions) != 4 || subExpressions[0] != ResultTaskPart || subExpressions[2] != ResultResultPart {
128132
return "", "", fmt.Errorf("Must be of the form %q", resultExpressionFormat)
129133
}
134+
subExpressions[3] = excludeArrayIndexingRegex.ReplaceAllString(subExpressions[3], "")
130135
return subExpressions[1], subExpressions[3], nil
131136
}
132137

‎pkg/apis/pipeline/v1beta1/when_types.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,14 @@ func (we *WhenExpression) applyReplacements(replacements map[string]string, arra
6262
for _, val := range we.Values {
6363
// arrayReplacements holds a list of array parameters with a pattern - params.arrayParam1
6464
// array params are referenced using $(params.arrayParam1[*])
65+
// array results are referenced using $(results.resultname[*])
6566
// check if the param exist in the arrayReplacements to replace it with a list of values
67+
fmt.Println(fmt.Sprintf("%s", ResultsArrayReference(val)))
6668
if _, ok := arrayReplacements[fmt.Sprintf("%s.%s", ParamsPrefix, ArrayReference(val))]; ok {
6769
replacedValues = append(replacedValues, substitution.ApplyArrayReplacements(val, replacements, arrayReplacements)...)
68-
} else {
70+
} else if _, ok := arrayReplacements[fmt.Sprintf("%s", ResultsArrayReference(val))]; ok{
71+
replacedValues = append(replacedValues, substitution.ApplyArrayReplacements(val, replacements, arrayReplacements)...)
72+
}else{
6973
replacedValues = append(replacedValues, substitution.ApplyReplacements(val, replacements))
7074
}
7175
}

‎pkg/apis/pipeline/v1beta1/when_types_test.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,25 @@ func TestApplyReplacements(t *testing.T) {
248248
Operator: selection.In,
249249
Values: []string{"barfoo"},
250250
},
251+
}, {
252+
name: "replace array results variables",
253+
original: &WhenExpression{
254+
Input: "$(tasks.foo.results.bar)",
255+
Operator: selection.In,
256+
Values: []string{"$(tasks.aTask.results.aResult[*])"},
257+
},
258+
replacements: map[string]string{
259+
"tasks.foo.results.bar": "foobar",
260+
"tasks.aTask.results.aResult[*]": "barfoo",
261+
},
262+
arrayReplacements: map[string][]string{
263+
"tasks.aTask.results.aResult[*]": {"dev", "stage"},
264+
},
265+
expected: &WhenExpression{
266+
Input: "foobar",
267+
Operator: selection.In,
268+
Values: []string{"dev", "stage"},
269+
},
251270
}, {
252271
name: "replace array params",
253272
original: &WhenExpression{

‎pkg/reconciler/pipeline/dag/dag_test.go‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,7 @@ func TestBuild_TaskParamsFromTaskResults(t *testing.T) {
371371
c := v1beta1.PipelineTask{Name: "c"}
372372
d := v1beta1.PipelineTask{Name: "d"}
373373
e := v1beta1.PipelineTask{Name: "e"}
374+
f := v1beta1.PipelineTask{Name: "f"}
374375
xDependsOnA := v1beta1.PipelineTask{
375376
Name: "x",
376377
Params: []v1beta1.Param{{
@@ -393,27 +394,38 @@ func TestBuild_TaskParamsFromTaskResults(t *testing.T) {
393394
Value: *v1beta1.NewArrayOrString("$(tasks.d.results.resultD) $(tasks.e.results.resultE)"),
394395
}},
395396
}
397+
wDependsOnF := v1beta1.PipelineTask{
398+
Name: "w",
399+
Params: []v1beta1.Param{{
400+
Name: "paramw",
401+
Value: *v1beta1.NewArrayOrString("$(tasks.f.results.resultF[*])"),
402+
}},
403+
}
396404

397-
// a b c d e
398-
// | \ / \ /
399-
// x y z
405+
// a b c d e f
406+
// | \ / \ / |
407+
// x y z w
400408
nodeA := &dag.Node{Task: a}
401409
nodeB := &dag.Node{Task: b}
402410
nodeC := &dag.Node{Task: c}
403411
nodeD := &dag.Node{Task: d}
404412
nodeE := &dag.Node{Task: e}
413+
nodeF := &dag.Node{Task: f}
405414
nodeX := &dag.Node{Task: xDependsOnA}
406415
nodeY := &dag.Node{Task: yDependsOnBRunsAfterC}
407416
nodeZ := &dag.Node{Task: zDependsOnDAndE}
417+
nodeW := &dag.Node{Task: wDependsOnF}
408418

409419
nodeA.Next = []*dag.Node{nodeX}
410420
nodeB.Next = []*dag.Node{nodeY}
411421
nodeC.Next = []*dag.Node{nodeY}
412422
nodeD.Next = []*dag.Node{nodeZ}
413423
nodeE.Next = []*dag.Node{nodeZ}
424+
nodeF.Next = []*dag.Node{nodeW}
414425
nodeX.Prev = []*dag.Node{nodeA}
415426
nodeY.Prev = []*dag.Node{nodeB, nodeC}
416427
nodeZ.Prev = []*dag.Node{nodeD, nodeE}
428+
nodeW.Prev = []*dag.Node{nodeF}
417429

418430
expectedDAG := &dag.Graph{
419431
Nodes: map[string]*dag.Node{
@@ -422,15 +434,17 @@ func TestBuild_TaskParamsFromTaskResults(t *testing.T) {
422434
"c": nodeC,
423435
"d": nodeD,
424436
"e": nodeE,
437+
"f": nodeF,
425438
"x": nodeX,
426439
"y": nodeY,
427440
"z": nodeZ,
441+
"w": nodeW,
428442
},
429443
}
430444
p := &v1beta1.Pipeline{
431445
ObjectMeta: metav1.ObjectMeta{Name: "pipeline"},
432446
Spec: v1beta1.PipelineSpec{
433-
Tasks: []v1beta1.PipelineTask{a, b, c, d, e, xDependsOnA, yDependsOnBRunsAfterC, zDependsOnDAndE},
447+
Tasks: []v1beta1.PipelineTask{a, b, c, d, e, f, xDependsOnA, yDependsOnBRunsAfterC, zDependsOnDAndE, wDependsOnF},
434448
},
435449
}
436450
tasks := v1beta1.PipelineTaskList(p.Spec.Tasks)

‎pkg/reconciler/pipelinerun/resources/apply.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,17 +99,18 @@ func ApplyPipelineTaskContexts(pt *v1beta1.PipelineTask) *v1beta1.PipelineTask {
9999
// ApplyTaskResults applies the ResolvedResultRef to each PipelineTask.Params and Pipeline.WhenExpressions in targets
100100
func ApplyTaskResults(targets PipelineRunState, resolvedResultRefs ResolvedResultRefs) {
101101
stringReplacements := resolvedResultRefs.getStringReplacements()
102+
arrayReplacements := resolvedResultRefs.getArrayReplacements()
102103
for _, resolvedPipelineRunTask := range targets {
103104
// also make substitution for resolved condition checks
104105
for _, resolvedConditionCheck := range resolvedPipelineRunTask.ResolvedConditionChecks {
105106
pipelineTaskCondition := resolvedConditionCheck.PipelineTaskCondition.DeepCopy()
106-
pipelineTaskCondition.Params = replaceParamValues(pipelineTaskCondition.Params, stringReplacements, nil)
107+
pipelineTaskCondition.Params = replaceParamValues(pipelineTaskCondition.Params, stringReplacements, arrayReplacements)
107108
resolvedConditionCheck.PipelineTaskCondition = pipelineTaskCondition
108109
}
109110
if resolvedPipelineRunTask.PipelineTask != nil {
110111
pipelineTask := resolvedPipelineRunTask.PipelineTask.DeepCopy()
111-
pipelineTask.Params = replaceParamValues(pipelineTask.Params, stringReplacements, nil)
112-
pipelineTask.WhenExpressions = pipelineTask.WhenExpressions.ReplaceWhenExpressionsVariables(stringReplacements, nil)
112+
pipelineTask.Params = replaceParamValues(pipelineTask.Params, stringReplacements, arrayReplacements)
113+
pipelineTask.WhenExpressions = pipelineTask.WhenExpressions.ReplaceWhenExpressionsVariables(stringReplacements, arrayReplacements)
113114
resolvedPipelineRunTask.PipelineTask = pipelineTask
114115
}
115116
}

‎pkg/reconciler/pipelinerun/resources/apply_test.go‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -401,6 +401,38 @@ func TestApplyTaskResults_MinimalExpression(t *testing.T) {
401401
}},
402402
},
403403
}},
404+
}, {
405+
name: "Test array result substitution on minimal variable substitution expression - params",
406+
resolvedResultRefs: ResolvedResultRefs{{
407+
Value: *v1beta1.NewArrayOrString("arrayResultValueOne", "arrayResultValueTwo"),
408+
ResultReference: v1beta1.ResultRef{
409+
PipelineTask: "aTask",
410+
Result: "a.Result",
411+
},
412+
FromTaskRun: "aTaskRun",
413+
}},
414+
targets: PipelineRunState{{
415+
PipelineTask: &v1beta1.PipelineTask{
416+
Name: "bTask",
417+
TaskRef: &v1beta1.TaskRef{Name: "bTask"},
418+
Params: []v1beta1.Param{{
419+
Name: "bParam",
420+
Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeArray,
421+
ArrayVal: []string{`$(tasks.aTask.results["a.Result"][*])`},
422+
},
423+
}},
424+
},
425+
}},
426+
want: PipelineRunState{{
427+
PipelineTask: &v1beta1.PipelineTask{
428+
Name: "bTask",
429+
TaskRef: &v1beta1.TaskRef{Name: "bTask"},
430+
Params: []v1beta1.Param{{
431+
Name: "bParam",
432+
Value: *v1beta1.NewArrayOrString("arrayResultValueOne", "arrayResultValueTwo"),
433+
}},
434+
},
435+
}},
404436
}, {
405437
name: "Test result substitution on minimal variable substitution expression - when expressions",
406438
resolvedResultRefs: ResolvedResultRefs{{
@@ -433,6 +465,39 @@ func TestApplyTaskResults_MinimalExpression(t *testing.T) {
433465
}},
434466
},
435467
}},
468+
}, {
469+
name: "Test array result substitution on minimal variable substitution expression - when expressions",
470+
resolvedResultRefs: ResolvedResultRefs{{
471+
Value: *v1beta1.NewArrayOrString("arrayResultValueOne", "arrayResultValueTwo"),
472+
ResultReference: v1beta1.ResultRef{
473+
PipelineTask: "aTask",
474+
Result: "aResult",
475+
},
476+
FromTaskRun: "aTaskRun",
477+
}},
478+
targets: PipelineRunState{{
479+
PipelineTask: &v1beta1.PipelineTask{
480+
Name: "bTask",
481+
TaskRef: &v1beta1.TaskRef{Name: "bTask"},
482+
WhenExpressions: []v1beta1.WhenExpression{{
483+
// Note that Input doesn't support array replacement.
484+
Input: "anInput",
485+
Operator: selection.In,
486+
Values: []string{"$(tasks.aTask.results.aResult[*])"},
487+
}},
488+
},
489+
}},
490+
want: PipelineRunState{{
491+
PipelineTask: &v1beta1.PipelineTask{
492+
Name: "bTask",
493+
TaskRef: &v1beta1.TaskRef{Name: "bTask"},
494+
WhenExpressions: []v1beta1.WhenExpression{{
495+
Input: "anInput",
496+
Operator: selection.In,
497+
Values: []string{"arrayResultValueOne", "arrayResultValueTwo"},
498+
}},
499+
},
500+
}},
436501
}} {
437502
t.Run(tt.name, func(t *testing.T) {
438503
ApplyTaskResults(tt.targets, tt.resolvedResultRefs)

‎pkg/reconciler/pipelinerun/resources/resultrefresolution.go‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -187,8 +187,22 @@ func findTaskResultForParam(taskRun *v1beta1.TaskRun, reference *v1beta1.ResultR
187187
func (rs ResolvedResultRefs) getStringReplacements() map[string]string {
188188
replacements := map[string]string{}
189189
for _, r := range rs {
190-
for _, target := range r.getReplaceTarget() {
191-
replacements[target] = r.Value.StringVal
190+
if r.Value.Type == v1beta1.ParamType(v1beta1.ResultsTypeString) {
191+
for _, target := range r.getReplaceTarget() {
192+
replacements[target] = r.Value.StringVal
193+
}
194+
}
195+
}
196+
return replacements
197+
}
198+
199+
func (rs ResolvedResultRefs) getArrayReplacements() map[string][]string {
200+
replacements := map[string][]string{}
201+
for _, r := range rs {
202+
if r.Value.Type == v1beta1.ParamType(v1beta1.ResultsTypeArray) {
203+
for _, target := range r.getArrayReplaceTarget() {
204+
replacements[target] = r.Value.ArrayVal
205+
}
192206
}
193207
}
194208
return replacements
@@ -201,3 +215,11 @@ func (r *ResolvedResultRef) getReplaceTarget() []string {
201215
fmt.Sprintf("%s.%s.%s['%s']", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),
202216
}
203217
}
218+
219+
func (r *ResolvedResultRef) getArrayReplaceTarget() []string {
220+
return []string{
221+
fmt.Sprintf("%s.%s.%s.%s[*]", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),
222+
fmt.Sprintf("%s.%s.%s[%q][*]", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),
223+
fmt.Sprintf("%s.%s.%s['%s'][*]", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),
224+
}
225+
}

‎pkg/reconciler/taskrun/validate_resources.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,11 @@ func validateParams(ctx context.Context, paramSpecs []v1beta1.ParamSpec, params
7878
if extraParamsNames := extraParamsNames(ctx, neededParamsNames, providedParamsNames); len(extraParamsNames) != 0 {
7979
return fmt.Errorf("didn't need these params but they were provided anyway: %s", extraParamsNames)
8080
}
81+
// This is needed to support array replacements in params.Users want to use $(results.resultname[*])
82+
// to pass array result to array param, yet in yaml format this will be
83+
// unmarshalled to string for ArrayOrString. So we need to check and correct the param type.
84+
// Please check issue #4879 for more details and examples.
85+
correctAoStype(params, neededParamsTypes)
8186
if wrongTypeParamNames := wrongTypeParamsNames(params, matrix, neededParamsTypes); len(wrongTypeParamNames) != 0 {
8287
return fmt.Errorf("param types don't match the user-specified type: %s", wrongTypeParamNames)
8388
}
@@ -88,6 +93,16 @@ func validateParams(ctx context.Context, paramSpecs []v1beta1.ParamSpec, params
8893
return nil
8994
}
9095

96+
func correctAoStype(params []v1beta1.Param, neededParamsTypes map[string]v1beta1.ParamType) {
97+
for i := 0; i < len(params); i++ {
98+
if params[i].Value.Type == "string" && neededParamsTypes[params[i].Name] == "array" && v1beta1.VariableSubstitutionRegex.MatchString(params[i].Value.StringVal) {
99+
params[i].Value.Type = neededParamsTypes[params[i].Name]
100+
params[i].Value.ArrayVal = []string{params[i].Value.StringVal}
101+
params[i].Value.StringVal = ""
102+
}
103+
}
104+
}
105+
91106
func neededParamsNamesAndTypes(paramSpecs []v1beta1.ParamSpec) ([]string, map[string]v1beta1.ParamType) {
92107
var neededParamsNames []string
93108
neededParamsTypes := make(map[string]v1beta1.ParamType)

0 commit comments

Comments
 (0)