Skip to content

Commit f0a7635

Browse files
danielbdiasdanielbdias
authored
Update CLI e2e test engine to support CLI debug and add error handling for 404 errors on Get (#2573)
1 parent 77c8039 commit f0a7635

File tree

19 files changed

+380
-40
lines changed

19 files changed

+380
-40
lines changed

‎cli/cmd/config.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ func overrideConfig() {
114114
if err != nil {
115115
msg := fmt.Sprintf("cannot parse endpoint %s", overrideEndpoint)
116116
cliLogger.Error(msg, zap.Error(err))
117-
os.Exit(1)
117+
ExitCLI(1)
118118
}
119119
cliConfig.Scheme = scheme
120120
cliConfig.Endpoint = endpoint
@@ -129,7 +129,7 @@ func setupOutputFormat(cmd *cobra.Command) {
129129
o := formatters.Output(output)
130130
if !formatters.ValidOutput(o) {
131131
fmt.Fprintf(os.Stderr, "Invalid output format %s. Available formats are [%s]\n", output, outputFormatsString)
132-
os.Exit(1)
132+
ExitCLI(1)
133133
}
134134
formatters.SetOutput(o)
135135
}

‎cli/cmd/docgen_cmd.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,22 +26,22 @@ var docGenCmd = &cobra.Command{
2626
err := os.MkdirAll(docsOutputDir, os.ModePerm)
2727
if err != nil {
2828
fmt.Println(fmt.Errorf("could not create output dir: %w", err).Error())
29-
os.Exit(1)
29+
ExitCLI(1)
3030
}
3131

3232
err = doc.GenMarkdownTreeCustom(rootCmd, docsOutputDir, func(s string) string {
3333
return "# CLI Reference\n"
3434
}, func(s string) string { return s })
3535
if err != nil {
3636
fmt.Println(fmt.Errorf("could not generate documentation: %w", err).Error())
37-
os.Exit(1)
37+
ExitCLI(1)
3838
}
3939

4040
if docusaurusFolder != "" {
4141
err = generateDocusaurusSidebar(docsOutputDir, docusaurusFolder)
4242
if err != nil {
4343
fmt.Println(fmt.Errorf("could not create docusaurus sidebar file: %w", err).Error())
44-
os.Exit(1)
44+
ExitCLI(1)
4545
}
4646
}
4747
},

‎cli/cmd/get_cmd.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@ package cmd
22

33
import (
44
"context"
5+
"errors"
56
"fmt"
67

78
"github.com/kubeshop/tracetest/cli/analytics"
89
"github.com/kubeshop/tracetest/cli/formatters"
10+
"github.com/kubeshop/tracetest/cli/utils"
911
"github.com/spf13/cobra"
1012
)
1113

@@ -40,7 +42,9 @@ var getCmd = &cobra.Command{
4042
}
4143

4244
resource, err := resourceActions.Get(ctx, resourceID)
43-
if err != nil {
45+
if err != nil && errors.Is(err, utils.ResourceNotFound) {
46+
return fmt.Sprintf("Resource %s with ID %s not found", resourceType, resourceID), nil
47+
} else if err != nil {
4448
return "", err
4549
}
4650

‎cli/cmd/middleware.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package cmd
22

33
import (
44
"fmt"
5-
"os"
65

76
"github.com/spf13/cobra"
87
"go.uber.org/zap"
@@ -20,7 +19,7 @@ Version
2019
%s
2120

2221
An error ocurred when executing the command`, versionText), zap.Error(err))
23-
os.Exit(1)
22+
ExitCLI(1)
2423
return
2524
}
2625

‎cli/cmd/root.go‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ var (
1818
outputFormatsString = strings.Join(outputFormats, "|")
1919

2020
// overrides
21-
overrideEndpoint string
21+
overrideEndpoint string
22+
cliExitInterceptor func(code int)
2223
)
2324

2425
var rootCmd = &cobra.Command{
@@ -32,10 +33,23 @@ var rootCmd = &cobra.Command{
3233
func Execute() {
3334
if err := rootCmd.Execute(); err != nil {
3435
fmt.Fprintln(os.Stderr, err)
35-
os.Exit(1)
36+
ExitCLI(1)
3637
}
3738
}
3839

40+
func ExitCLI(errorCode int) {
41+
if cliExitInterceptor != nil {
42+
cliExitInterceptor(errorCode)
43+
return
44+
}
45+
46+
os.Exit(errorCode)
47+
}
48+
49+
func RegisterCLIExitInterceptor(interceptor func(int)) {
50+
cliExitInterceptor = interceptor
51+
}
52+
3953
var (
4054
cmdGroupConfig = &cobra.Group{
4155
ID: "configuration",

‎cli/utils/api.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,10 @@ func (resourceClient ResourceClient) Get(ctx context.Context, id string) (*file.
159159
}
160160

161161
defer resp.Body.Close()
162+
if resp.StatusCode == http.StatusNotFound {
163+
return nil, ResourceNotFound
164+
}
165+
162166
if resp.StatusCode != http.StatusOK {
163167
body, err := ioutil.ReadAll(resp.Body)
164168
if err != nil {

‎cli/utils/errors.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package utils
2+
3+
import "errors"
4+
5+
var (
6+
ResourceNotFound = errors.New("resource not found in API")
7+
)

‎docs/docs/deployment/kubernetes.mdx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ This is an example of a production-ready deployment, but real-world deployments
1414
This setup is ideal for CI/CD environments and QA teams working in shared environments. You can use a remote or local ([minikube](https://minikube.sigs.k8s.io/docs/start/), [kind](https://kind.sigs.k8s.io/), [k3d](https://k3d.io/), etc) cluster.
1515
:::
1616

17-
You have two options to install Tracetest on Kubernetes:
17+
You have two options to install Tracetest on Kubernetes:
1818

1919
- Using the [Tracetest CLI](../getting-started/installation) to guide your installation
2020
- Using the official [Helm chart](https://github.com/kubeshop/helm-charts/tree/main/charts/tracetest)

‎testing/cli-e2etest/README.md‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
# Tracetest CLI e2e tests
2+
3+
In this folder we have the End-to-end tests done on the CLI to guarantee that the CLI is working fine.
4+
The main idea is to test every CLI command against Tracetest server with different data stores and different Operating systems.
5+
6+
7+
## Implementation status
8+
9+
| Linux | Windows | MacOS |
10+
| ------------------ | ------- | ------ |
11+
| :white_check_mark: | :soon: | :soon: |
12+
13+
## Tracetest Data Store
14+
15+
| Jaeger | Tempo | OpenSearch | SignalFx | OTLP | ElasticAPM | New Relic | Lightstep | Datadog | AWS X-Ray | Honeycomb |
16+
| ------------------ | ------ | ---------- | -------- | ------ | ---------- | --------- | --------- | ------- | --------- | --------- |
17+
| :white_check_mark: | :soon: | :soon: | :soon: | :soon: | :soon: | :soon: | :soon: | :soon: | :soon: | :soon: |
18+
19+
## CLI Commands to Test
20+
21+
### Misc and Flags
22+
23+
| CLI Command | Tested | Test scenarios |
24+
| -------------- | ------------------ | ------------------------------------------------- |
25+
| `version` | :white_check_mark: | [VersionCommand](./testscenarios/version_test.go) |
26+
| `help` | :white_check_mark: | [HelpCommand](./testscenarios/help_test.go) |
27+
| `--help`, `-h` | :white_check_mark: | [HelpCommand](./testscenarios/help_test.go) |
28+
| `--config` | :white_check_mark: | All scenarios |
29+
30+
### Tests and Test Runner
31+
32+
| CLI Command | Tested | Test scenarios |
33+
| ------------------------------------------------------------------ | ------------------ | -------------- |
34+
| `test list` | :yellow_circle: | |
35+
| `test run -d [test-definition]` | :yellow_circle: | |
36+
| `test run -d [test-definition] -e [environment-id]` | :yellow_circle: | |
37+
| `test run -d [test-definition] -e [environment-definition]` | :yellow_circle: | |
38+
| `test run -d [transaction-definition]` | :yellow_circle: | |
39+
| `test run -d [transaction-definition] -e [environment-id]` | :yellow_circle: | |
40+
| `test run -d [transaction-definition] -e [environment-definition]` | :yellow_circle: | |
41+
42+
### Resources: Config
43+
44+
| CLI Command | Tested | Test scenarios |
45+
| ----------------------------------------------------- | ----------------| -------------- |
46+
| `apply config -f [config-file]` | :yellow_circle: | |
47+
| `delete config --id current` | :yellow_circle: | |
48+
| `export config --id current --file [config-file]` | :yellow_circle: | |
49+
| `get config --id current --output pretty` | :yellow_circle: | |
50+
| `get config --id current --output json` | :yellow_circle: | |
51+
| `get config --id current --output yaml` | :yellow_circle: | |
52+
| `list config --output pretty` | :yellow_circle: | |
53+
| `list config --output json` | :yellow_circle: | |
54+
| `list config --output yaml` | :yellow_circle: | |
55+
### Resources: Data Store
56+
57+
| CLI Command | Tested | Test scenarios |
58+
| -------------------------------------------------------- | ------------------ | -------------- |
59+
| `apply datastore -f [data-store-file]` | :white_check_mark: | [ApplyNewDatastore](./testscenarios/datastore/apply_new_datastore_test.go) |
60+
| `delete datastore --id current` | :white_check_mark: | [DeleteDatastore](./testscenarios/datastore/delete_datastore_test.go) |
61+
| `export datastore --id current --file [data-store-file]` | :yellow_circle: | |
62+
| `get datastore --id current --output pretty` | :white_check_mark: | [ApplyNewDatastore](./testscenarios/datastore/apply_new_datastore_test.go), [DeleteDatastore](./testscenarios/datastore/delete_datastore_test.go) |
63+
| `get datastore --id current --output json` | :yellow_circle: | |
64+
| `get datastore --id current --output yaml` | :yellow_circle: | |
65+
| `list datastore --output pretty` | :white_check_mark: | [ListDatastore](./testscenarios/datastore/list_datastore_test.go) |
66+
| `list datastore --output json` | :white_check_mark: | [ListDatastore](./testscenarios/datastore/list_datastore_test.go) |
67+
| `list datastore --output yaml` | :white_check_mark: | [ListDatastore](./testscenarios/datastore/list_datastore_test.go) |
68+
69+
### Resources: Demo
70+
71+
| CLI Command | Tested | Test scenarios |
72+
| ---------------------------------------------------- | ------------------ | -------------- |
73+
| `apply demo -f [new-demo-file]` | :yellow_circle: | |
74+
| `apply demo -f [existing-demo-file]` | :yellow_circle: | |
75+
| `delete demo --id [existing-id]` | :yellow_circle: | |
76+
| `delete demo --id [non-existing-id]` | :yellow_circle: | |
77+
| `export demo --id current --file [demo-file]` | :yellow_circle: | |
78+
| `get demo --id [non-existing-id]` | :yellow_circle: | |
79+
| `get demo --id [existing-id] --output pretty` | :yellow_circle: | |
80+
| `get demo --id [existing-id] --output json` | :yellow_circle: | |
81+
| `get demo --id [existing-id] --output yaml` | :yellow_circle: | |
82+
| `list demo --output pretty` | :yellow_circle: | |
83+
| `list demo --output json` | :yellow_circle: | |
84+
| `list demo --output yaml` | :yellow_circle: | |
85+
| `list demo --skip 1 --take 2` | :yellow_circle: | |
86+
| `list demo --sortBy name --sortDirection desc` | :yellow_circle: | |
87+
88+
### Resources: Environment
89+
90+
| CLI Command | Tested | Test scenarios |
91+
| ----------------------------------------------------------- | ------------------ | -------------- |
92+
| `apply environment -f [new-environment-file]` | :yellow_circle: | |
93+
| `apply environment -f [existing-environment-file]` | :yellow_circle: | |
94+
| `delete environment --id [existing-id]` | :yellow_circle: | |
95+
| `delete environment --id [non-existing-id]` | :yellow_circle: | |
96+
| `export environment --id current --file [environment-file]` | :yellow_circle: | |
97+
| `get environment --id [non-existing-id]` | :yellow_circle: | |
98+
| `get environment --id [existing-id] --output pretty` | :yellow_circle: | |
99+
| `get environment --id [existing-id] --output json` | :yellow_circle: | |
100+
| `get environment --id [existing-id] --output yaml` | :yellow_circle: | |
101+
| `list environment --output pretty` | :yellow_circle: | |
102+
| `list environment --output json` | :yellow_circle: | |
103+
| `list environment --output yaml` | :yellow_circle: | |
104+
| `list environment --skip 1 --take 2` | :yellow_circle: | |
105+
| `list environment --sortBy name --sortDirection desc` | :yellow_circle: | |
106+
107+
### Resources: PollingProfile
108+
109+
| CLI Command | Tested | Test scenarios |
110+
| --------------------------------------------------------------------- | --------------- | -------------- |
111+
| `apply pollingprofile -f [pollingprofile-file]` | :yellow_circle: | |
112+
| `delete pollingprofile --id current` | :yellow_circle: | |
113+
| `export pollingprofile --id current --file [pollingprofile-file]` | :yellow_circle: | |
114+
| `get pollingprofile --id current --output pretty` | :yellow_circle: | |
115+
| `get pollingprofile --id current --output json` | :yellow_circle: | |
116+
| `get pollingprofile --id current --output yaml` | :yellow_circle: | |
117+
| `list pollingprofile --output pretty` | :yellow_circle: | |
118+
| `list pollingprofile --output json` | :yellow_circle: | |
119+
| `list pollingprofile --output yaml` | :yellow_circle: | |
120+
121+
### Resources: Transactions
122+
123+
| CLI Command | Tested | Test scenarios |
124+
| ----------------------------------------------------------- | ------------------ | -------------- |
125+
| `apply transaction -f [new-transaction-file]` | :yellow_circle: | |
126+
| `apply transaction -f [existing-transaction-file]` | :yellow_circle: | |
127+
| `delete transaction --id [existing-id]` | :yellow_circle: | |
128+
| `delete transaction --id [non-existing-id]` | :yellow_circle: | |
129+
| `export transaction --id current --file [transaction-file]` | :yellow_circle: | |
130+
| `get transaction --id [non-existing-id]` | :yellow_circle: | |
131+
| `get transaction --id [existing-id] --output pretty` | :yellow_circle: | |
132+
| `get transaction --id [existing-id] --output json` | :yellow_circle: | |
133+
| `get transaction --id [existing-id] --output yaml` | :yellow_circle: | |
134+
| `list transaction --output pretty` | :yellow_circle: | |
135+
| `list transaction --output json` | :yellow_circle: | |
136+
| `list transaction --output yaml` | :yellow_circle: | |
137+
| `list transaction --skip 1 --take 2` | :yellow_circle: | |
138+
| `list transaction --sortBy name --sortDirection desc` | :yellow_circle: | |

‎testing/cli-e2etest/config/config.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
package config
2+
3+
import "os"
4+
5+
type EnvironmentVars struct {
6+
EnableCLIDebug bool
7+
TracetestCommand string
8+
TestEnvironment string
9+
}
10+
11+
var instance *EnvironmentVars
12+
13+
func GetConfigAsEnvVars() *EnvironmentVars {
14+
if instance != nil {
15+
return instance
16+
}
17+
18+
enableCLIDebug := (os.Getenv("ENABLE_CLI_DEBUG") == "true")
19+
20+
tracetestCommand := os.Getenv("TRACETEST_COMMAND")
21+
if tracetestCommand == "" {
22+
tracetestCommand = "tracetest"
23+
}
24+
25+
testEnvironment := os.Getenv("TEST_ENVIRONMENT")
26+
if testEnvironment == "" {
27+
testEnvironment = "jaeger"
28+
}
29+
30+
return &EnvironmentVars{
31+
EnableCLIDebug: enableCLIDebug,
32+
TracetestCommand: tracetestCommand,
33+
TestEnvironment: testEnvironment,
34+
}
35+
}

0 commit comments

Comments
 (0)