Skip to content

Commit ed66a7f

Browse files
LalatenduMohantyLalatenduMohanty
authored
Merge branch 'main' into remove-monitor-prometheues
2 parents 1d0c654 + 4a007b0 commit ed66a7f

File tree

18 files changed

+119
-208
lines changed

18 files changed

+119
-208
lines changed

‎.github/workflows/catalogd-e2e.yaml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,7 @@ jobs:
1717
with:
1818
go-version-file: "go.mod"
1919
- name: Run E2e
20-
working-directory: catalogd
21-
run: make e2e
20+
run: make test-catalogd-e2e
2221
upgrade-e2e:
2322
runs-on: ubuntu-latest
2423
steps:
@@ -27,5 +26,4 @@ jobs:
2726
with:
2827
go-version-file: "go.mod"
2928
- name: Run the upgrade e2e test
30-
working-directory: catalogd
31-
run: make test-upgrade-e2e
29+
run: make test-catalogd-upgrade-e2e

‎Makefile‎

Lines changed: 44 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ endif
2525
export IMAGE_TAG
2626

2727
IMG := $(IMAGE_REPO):$(IMAGE_TAG)
28+
CATALOGD_IMG := $(CATALOG_IMAGE_REPO):$(IMAGE_TAG)
2829

2930
# Define dependency versions (use go.mod if we also use Go code from dependency)
3031
export CERT_MGR_VERSION := v1.15.3
@@ -64,12 +65,15 @@ $(warning Could not find docker or podman in path! This may result in targets re
6465
endif
6566

6667
KUSTOMIZE_BUILD_DIR := config/overlays/cert-manager
68+
CATALOGD_KUSTOMIZE_BUILD_DIR := catalogd/config/overlays/cert-manager
6769

6870
# Disable -j flag for make
6971
.NOTPARALLEL:
7072

7173
.DEFAULT_GOAL := build
7274

75+
GINKGO := go run github.com/onsi/ginkgo/v2/ginkgo
76+
7377
#SECTION General
7478

7579
# The help target prints out all targets with their descriptions organized
@@ -209,6 +213,39 @@ test-e2e: KUSTOMIZE_BUILD_DIR := config/overlays/e2e
209213
test-e2e: GO_BUILD_FLAGS := -cover
210214
test-e2e: run image-registry e2e e2e-coverage kind-clean #HELP Run e2e test suite on local kind cluster
211215

216+
# Catalogd e2e tests
217+
FOCUS := $(if $(TEST),-v -focus "$(TEST)")
218+
ifeq ($(origin E2E_FLAGS), undefined)
219+
E2E_FLAGS :=
220+
endif
221+
test-catalogd-e2e: ## Run the e2e tests on existing cluster
222+
$(GINKGO) $(E2E_FLAGS) -trace -vv $(FOCUS) test/catalogd-e2e
223+
224+
catalogd-e2e: KIND_CLUSTER_NAME := catalogd-e2e
225+
catalogd-e2e: ISSUER_KIND := Issuer
226+
catalogd-e2e: ISSUER_NAME := selfsigned-issuer
227+
catalogd-e2e: CATALOGD_KUSTOMIZE_BUILD_DIR := catalogd/config/overlays/e2e
228+
catalogd-e2e: run catalogd-image-registry test-catalogd-e2e ## kind-clean Run e2e test suite on local kind cluster
229+
230+
## image-registry target has to come after run-latest-release,
231+
## because the image-registry depends on the olm-ca issuer.
232+
.PHONY: test-catalogd-upgrade-e2e
233+
test-catalogd-upgrade-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog
234+
test-catalogd-upgrade-e2e: export TEST_CLUSTER_CATALOG_IMAGE := docker-registry.catalogd-e2e.svc:5000/test-catalog:e2e
235+
test-catalogd-upgrade-e2e: ISSUER_KIND=ClusterIssuer
236+
test-catalogd-upgrade-e2e: ISSUER_NAME=olmv1-ca
237+
test-catalogd-upgrade-e2e: kind-cluster docker-build kind-load run-latest-release catalogd-image-registry catalogd-pre-upgrade-setup kind-deploy catalogd-post-upgrade-checks kind-clean ## Run upgrade e2e tests on a local kind cluster
238+
239+
.PHONY: catalogd-post-upgrade-checks
240+
catalogd-post-upgrade-checks:
241+
$(GINKGO) $(E2E_FLAGS) -trace -vv $(FOCUS) test/catalogd-upgrade-e2e
242+
243+
catalogd-pre-upgrade-setup:
244+
./test/tools/imageregistry/pre-upgrade-setup.sh ${TEST_CLUSTER_CATALOG_IMAGE} ${TEST_CLUSTER_CATALOG_NAME}
245+
246+
catalogd-image-registry: ## Setup in-cluster image registry
247+
./test/tools/imageregistry/registry.sh $(ISSUER_KIND) $(ISSUER_NAME)
248+
212249
.PHONY: extension-developer-e2e
213250
extension-developer-e2e: KUSTOMIZE_BUILD_DIR := config/overlays/cert-manager
214251
extension-developer-e2e: KIND_CLUSTER_NAME := operator-controller-ext-dev-e2e #EXHELP Run extension-developer e2e on local kind cluster
@@ -246,10 +283,9 @@ kind-load: $(KIND) #EXHELP Loads the currently constructed images into the KIND
246283
kind-deploy: export MANIFEST := ./operator-controller.yaml
247284
kind-deploy: export DEFAULT_CATALOG := ./catalogd/config/base/default/clustercatalogs/default-catalogs.yaml
248285
kind-deploy: manifests $(KUSTOMIZE)
249-
($(KUSTOMIZE) build $(KUSTOMIZE_BUILD_DIR) && echo "---" && $(KUSTOMIZE) build catalogd/config/overlays/cert-manager | sed "s/cert-git-version/cert-$(VERSION)/g") > $(MANIFEST)
286+
($(KUSTOMIZE) build $(KUSTOMIZE_BUILD_DIR) && echo "---" && $(KUSTOMIZE) build $(CATALOGD_KUSTOMIZE_BUILD_DIR) | sed "s/cert-git-version/cert-$(VERSION)/g") > $(MANIFEST)
250287
envsubst '$$DEFAULT_CATALOG,$$CERT_MGR_VERSION,$$INSTALL_DEFAULT_CATALOGS,$$MANIFEST' < scripts/install.tpl.sh | bash -s
251288

252-
253289
.PHONY: kind-cluster
254290
kind-cluster: $(KIND) #EXHELP Standup a kind cluster.
255291
-$(KIND) delete cluster --name $(KIND_CLUSTER_NAME)
@@ -302,7 +338,12 @@ go-build-linux: export GOARCH=amd64
302338
go-build-linux: $(BINARIES)
303339

304340
.PHONY: run
305-
run: docker-build kind-cluster kind-load kind-deploy #HELP Build the operator-controller then deploy it into a new kind cluster.
341+
run: docker-build kind-cluster kind-load kind-deploy wait #HELP Build the operator-controller then deploy it into a new kind cluster.
342+
343+
CATALOGD_NAMESPACE := olmv1-system
344+
wait:
345+
kubectl wait --for=condition=Available --namespace=$(CATALOGD_NAMESPACE) deployment/catalogd-controller-manager --timeout=60s
346+
kubectl wait --for=condition=Ready --namespace=$(CATALOGD_NAMESPACE) certificate/catalogd-service-cert # Avoid upgrade test flakes when reissuing cert
306347

307348
.PHONY: docker-build
308349
docker-build: build-linux #EXHELP Build docker image for operator-controller and catalog with GOOS=linux and local GOARCH.

‎catalogd/Makefile‎

Lines changed: 5 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,6 @@ else
2323
$(warning Could not find docker or podman in path! This may result in targets requiring a container runtime failing!)
2424
endif
2525

26-
# For standard development and release flows, we use the config/overlays/cert-manager overlay.
27-
KUSTOMIZE_OVERLAY := config/overlays/cert-manager
28-
2926
# bingo manages consistent tooling versions for things like kind, kustomize, etc.
3027
include ./../.bingo/Variables.mk
3128

@@ -38,14 +35,6 @@ ifeq ($(origin KIND_CLUSTER_NAME), undefined)
3835
KIND_CLUSTER_NAME := catalogd
3936
endif
4037

41-
# E2E configuration
42-
TESTDATA_DIR := testdata
43-
44-
CATALOGD_NAMESPACE := olmv1-system
45-
KIND_CLUSTER_IMAGE := kindest/node:v1.30.0@sha256:047357ac0cfea04663786a612ba1eaba9702bef25227a794b52890dd8bcd692e
46-
47-
GINKGO := go run github.com/onsi/ginkgo/v2/ginkgo
48-
4938
##@ General
5039

5140
# The help target prints out all targets with their descriptions organized
@@ -74,42 +63,6 @@ generate: $(CONTROLLER_GEN) ## Generate code and manifests.
7463
$(CONTROLLER_GEN) object:headerFile="../hack/boilerplate.go.txt" paths="./..."
7564
$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/base/crd/bases output:rbac:artifacts:config=config/base/rbac output:webhook:artifacts:config=config/base/manager/webhook/
7665

77-
FOCUS := $(if $(TEST),-v -focus "$(TEST)")
78-
ifeq ($(origin E2E_FLAGS), undefined)
79-
E2E_FLAGS :=
80-
endif
81-
test-e2e: ## Run the e2e tests on existing cluster
82-
$(GINKGO) $(E2E_FLAGS) -trace -vv $(FOCUS) test/e2e
83-
84-
e2e: KIND_CLUSTER_NAME := catalogd-e2e
85-
e2e: ISSUER_KIND := Issuer
86-
e2e: ISSUER_NAME := selfsigned-issuer
87-
e2e: KUSTOMIZE_OVERLAY := config/overlays/e2e
88-
e2e: run image-registry test-e2e kind-cluster-cleanup ## Run e2e test suite on local kind cluster
89-
90-
image-registry: ## Setup in-cluster image registry
91-
./test/tools/imageregistry/registry.sh $(ISSUER_KIND) $(ISSUER_NAME)
92-
93-
## image-registry target has to come after run-latest-release,
94-
## because the image-registry depends on the olm-ca issuer.
95-
.PHONY: test-upgrade-e2e
96-
test-upgrade-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog
97-
test-upgrade-e2e: export TEST_CLUSTER_CATALOG_IMAGE := docker-registry.catalogd-e2e.svc:5000/test-catalog:e2e
98-
test-upgrade-e2e: ISSUER_KIND=ClusterIssuer
99-
test-upgrade-e2e: ISSUER_NAME=olmv1-ca
100-
test-upgrade-e2e: kind-cluster cert-manager build-container kind-load run-latest-release image-registry pre-upgrade-setup only-deploy-manifest wait post-upgrade-checks kind-cluster-cleanup ## Run upgrade e2e tests on a local kind cluster
101-
102-
pre-upgrade-setup:
103-
./test/tools/imageregistry/pre-upgrade-setup.sh ${TEST_CLUSTER_CATALOG_IMAGE} ${TEST_CLUSTER_CATALOG_NAME}
104-
105-
.PHONY: run-latest-release
106-
run-latest-release:
107-
cd ..; curl -L -s https://github.com/operator-framework/operator-controller/releases/latest/download/install.sh | bash -s
108-
109-
.PHONY: post-upgrade-checks
110-
post-upgrade-checks:
111-
$(GINKGO) $(E2E_FLAGS) -trace -vv $(FOCUS) test/upgrade
112-
11366
##@ Build
11467

11568
BINARIES=catalogd
@@ -121,7 +74,7 @@ VERSION := $(shell git describe --tags --always --dirty)
12174
endif
12275
export VERSION
12376

124-
export VERSION_PKG := $(shell go list -mod=mod ./internal/version)
77+
export VERSION_PKG := $(shell go list -m)/internal/version
12578

12679
export GIT_COMMIT := $(shell git rev-parse HEAD)
12780
export GIT_VERSION := $(shell git describe --tags --always --dirty)
@@ -157,62 +110,17 @@ $(LINUX_BINARIES): BUILDBIN = bin/linux
157110
$(LINUX_BINARIES):
158111
GOOS=linux $(BUILDCMD)
159112

160-
161-
.PHONY: run
162-
run: generate kind-cluster install ## Create a kind cluster and install a local build of catalogd
163-
164113
.PHONY: build-container
165114
build-container: build-linux ## Build docker image for catalogd.
166115
$(CONTAINER_RUNTIME) build -f Dockerfile -t $(IMAGE) ./bin/linux
167116

168-
##@ Deploy
169-
170-
.PHONY: kind-cluster
171-
kind-cluster: $(KIND) kind-cluster-cleanup ## Standup a kind cluster
172-
$(KIND) create cluster --name $(KIND_CLUSTER_NAME) --image $(KIND_CLUSTER_IMAGE)
173-
$(KIND) export kubeconfig --name $(KIND_CLUSTER_NAME)
174-
175-
.PHONY: kind-cluster-cleanup
176-
kind-cluster-cleanup: $(KIND) ## Delete the kind cluster
177-
$(KIND) delete cluster --name $(KIND_CLUSTER_NAME)
178-
179117
.PHONY: kind-load
180-
kind-load: check-cluster $(KIND) ## Load the built images onto the local cluster
118+
kind-load: $(KIND) ## Load the built images onto the local cluster
181119
$(CONTAINER_RUNTIME) save $(IMAGE) | $(KIND) load image-archive /dev/stdin --name $(KIND_CLUSTER_NAME)
182120

183-
.PHONY: install
184-
install: check-cluster build-container kind-load deploy wait ## Install local catalogd to an existing cluster
185-
186-
.PHONY: deploy
187-
deploy: export MANIFEST="./catalogd.yaml"
188-
deploy: export DEFAULT_CATALOGS="./config/base/default/clustercatalogs/default-catalogs.yaml"
189-
deploy: $(KUSTOMIZE) ## Deploy Catalogd to the K8s cluster specified in ~/.kube/config with cert-manager and default clustercatalogs
190-
cd config/base/manager && $(KUSTOMIZE) edit set image controller=$(IMAGE) && cd ../../..
191-
$(KUSTOMIZE) build $(KUSTOMIZE_OVERLAY) | sed "s/cert-git-version/cert-$(GIT_VERSION)/g" > catalogd.yaml
192-
envsubst '$$CERT_MGR_VERSION,$$MANIFEST,$$DEFAULT_CATALOGS' < scripts/install.tpl.sh | bash -s
193-
194-
.PHONY: only-deploy-manifest
195-
only-deploy-manifest: $(KUSTOMIZE) ## Deploy just the Catalogd manifest--used in e2e testing where cert-manager is installed in a separate step
196-
cd config/base/manager && $(KUSTOMIZE) edit set image controller=$(IMAGE)
197-
$(KUSTOMIZE) build $(KUSTOMIZE_OVERLAY) | kubectl apply -f -
198-
199-
wait:
200-
kubectl wait --for=condition=Available --namespace=$(CATALOGD_NAMESPACE) deployment/catalogd-controller-manager --timeout=60s
201-
kubectl wait --for=condition=Ready --namespace=$(CATALOGD_NAMESPACE) certificate/catalogd-service-cert # Avoid upgrade test flakes when reissuing cert
202-
203-
204-
.PHONY: cert-manager
205-
cert-manager:
206-
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/${CERT_MGR_VERSION}/cert-manager.yaml
207-
kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager-webhook --timeout=60s
121+
##@ Deploy
208122

209-
.PHONY: demo-update
123+
# The demo script requires to install asciinema with: brew install asciinema to run on mac os envs.
124+
.PHONY: demo-update #HELP build demo
210125
demo-update:
211126
hack/scripts/generate-asciidemo.sh
212-
213-
.PHONY: check-cluster
214-
check-cluster:
215-
@kubectl config current-context >/dev/null 2>&1 || ( \
216-
echo "Error: Could not get current Kubernetes context. Maybe use 'run' or 'e2e' targets first?"; \
217-
exit 1; \
218-
)

‎catalogd/hack/scripts/generate-asciidemo.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ while getopts 'h' flag; do
3636
done
3737
set -u
3838

39-
WKDIR=$(mktemp -td generate-asciidemo.XXXXX)
39+
WKDIR=$(mktemp -d -t generate-asciidemo.XXXXX)
4040
if [ ! -d ${WKDIR} ]
4141
then
4242
echo "unable to create temporary workspace"

catalogd/internal/storage/http_precoditions_check.go renamed to catalogd/internal/storage/http_preconditions_check.go

File renamed without changes.

‎catalogd/internal/storage/localdir_test.go‎

Lines changed: 54 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package storage
22

33
import (
4+
"bytes"
45
"context"
56
"errors"
67
"fmt"
@@ -15,7 +16,10 @@ import (
1516
"testing"
1617
"testing/fstest"
1718

19+
"github.com/stretchr/testify/assert"
1820
"github.com/stretchr/testify/require"
21+
22+
"github.com/operator-framework/operator-registry/alpha/declcfg"
1923
)
2024

2125
const urlPrefix = "/catalogs/"
@@ -170,14 +174,10 @@ func TestLocalDirStoraget(t *testing.T) {
170174

171175
func TestLocalDirServerHandler(t *testing.T) {
172176
store := &LocalDirV1{RootDir: t.TempDir(), RootURL: &url.URL{Path: urlPrefix}}
173-
testFS := fstest.MapFS{
174-
"meta.json": &fstest.MapFile{
175-
Data: []byte(`{"foo":"bar"}`),
176-
},
177-
}
178-
if store.Store(context.Background(), "test-catalog", testFS) != nil {
177+
if store.Store(context.Background(), "test-catalog", createTestFS(t)) != nil {
179178
t.Fatal("failed to store test catalog and start server")
180179
}
180+
181181
testServer := httptest.NewServer(store.StorageServerHandler())
182182
defer testServer.Close()
183183

@@ -242,10 +242,12 @@ func TestLocalDirServerHandler(t *testing.T) {
242242
URLPath: "/catalogs/non-existent-catalog/api/v1/all",
243243
},
244244
{
245-
name: "Server returns 200 when path '/catalogs/<catalog>/api/v1/all' is queried, when catalog exists",
245+
name: "Server returns 200 with json-lines payload when path '/catalogs/<catalog>/api/v1/all' is queried, when catalog exists",
246246
expectedStatusCode: http.StatusOK,
247-
expectedContent: `{"foo":"bar"}`,
248-
URLPath: "/catalogs/test-catalog/api/v1/all",
247+
expectedContent: `{"image":"quaydock.io/namespace/bundle:0.0.3","name":"bundle.v0.0.1","package":"webhook_operator_test","properties":[{"type":"olm.bundle.object","value":{"data":"dW5pbXBvcnRhbnQK"}},{"type":"some.other","value":{"data":"arbitrary-info"}}],"relatedImages":[{"image":"testimage:latest","name":"test"}],"schema":"olm.bundle"}` + "\n" +
248+
`{"defaultChannel":"preview_test","name":"webhook_operator_test","schema":"olm.package"}` + "\n" +
249+
`{"entries":[{"name":"bundle.v0.0.1"}],"name":"preview_test","package":"webhook_operator_test","schema":"olm.channel"}`,
250+
URLPath: "/catalogs/test-catalog/api/v1/all",
249251
},
250252
} {
251253
t.Run(tc.name, func(t *testing.T) {
@@ -256,11 +258,14 @@ func TestLocalDirServerHandler(t *testing.T) {
256258
require.NoError(t, err)
257259

258260
require.Equal(t, tc.expectedStatusCode, resp.StatusCode)
261+
if resp.StatusCode == http.StatusOK {
262+
assert.Equal(t, "application/jsonl", resp.Header.Get("Content-Type"))
263+
}
259264

260-
var actualContent []byte
261-
actualContent, err = io.ReadAll(resp.Body)
265+
actualContent, err := io.ReadAll(resp.Body)
262266
require.NoError(t, err)
263-
require.Equal(t, tc.expectedContent, strings.TrimSpace(string(actualContent)))
267+
268+
require.Equal(t, strings.TrimSpace(tc.expectedContent), strings.TrimSpace(string(actualContent)))
264269
require.NoError(t, resp.Body.Close())
265270
})
266271
}
@@ -329,7 +334,7 @@ func TestMetasEndpoint(t *testing.T) {
329334
expectedContent: "",
330335
},
331336
{
332-
name: "valid query with packageName that returns multiple blobs",
337+
name: "valid query with packageName that returns multiple blobs in json-lines format",
333338
queryParams: "?package=webhook_operator_test",
334339
expectedStatusCode: http.StatusOK,
335340
expectedContent: `{"image":"quaydock.io/namespace/bundle:0.0.3","name":"bundle.v0.0.1","package":"webhook_operator_test","properties":[{"type":"olm.bundle.object","value":{"data":"dW5pbXBvcnRhbnQK"}},{"type":"some.other","value":{"data":"arbitrary-info"}}],"relatedImages":[{"image":"testimage:latest","name":"test"}],"schema":"olm.bundle"}
@@ -377,6 +382,9 @@ func TestMetasEndpoint(t *testing.T) {
377382
defer resp.Body.Close()
378383

379384
require.Equal(t, tc.expectedStatusCode, resp.StatusCode)
385+
if resp.StatusCode == http.StatusOK {
386+
assert.Equal(t, "application/jsonl", resp.Header.Get("Content-Type"))
387+
}
380388

381389
actualContent, err := io.ReadAll(resp.Body)
382390
require.NoError(t, err)
@@ -591,8 +599,38 @@ entries:
591599
testBundle := fmt.Sprintf(testBundleTemplate, testBundleImage, testBundleName, testPackageName, testBundleRelatedImageName, testBundleRelatedImageImage, testBundleObjectData)
592600
testChannel := fmt.Sprintf(testChannelTemplate, testPackageName, testChannelName, testBundleName)
593601
return &fstest.MapFS{
594-
"bundle.yaml": {Data: []byte(testBundle), Mode: os.ModePerm},
595-
"package.yaml": {Data: []byte(testPackage), Mode: os.ModePerm},
596-
"channel.yaml": {Data: []byte(testChannel), Mode: os.ModePerm},
602+
"test-catalog.yaml": {Data: []byte(
603+
generateJSONLinesOrFail(t, []byte(testBundle)) +
604+
generateJSONLinesOrFail(t, []byte(testPackage)) +
605+
generateJSONLinesOrFail(t, []byte(testChannel))),
606+
Mode: os.ModePerm},
597607
}
598608
}
609+
610+
// generateJSONLinesOrFail takes a byte slice of concatenated JSON objects and returns a JSONlines-formatted string
611+
// or raises a test failure in case of encountering any internal errors
612+
func generateJSONLinesOrFail(t *testing.T, in []byte) string {
613+
var out strings.Builder
614+
reader := bytes.NewReader(in)
615+
616+
err := declcfg.WalkMetasReader(reader, func(meta *declcfg.Meta, err error) error {
617+
if err != nil {
618+
return err
619+
}
620+
621+
if meta != nil && meta.Blob != nil {
622+
if meta.Blob[len(meta.Blob)-1] != '\n' {
623+
return fmt.Errorf("blob does not end with newline")
624+
}
625+
}
626+
627+
_, err = out.Write(meta.Blob)
628+
if err != nil {
629+
return err
630+
}
631+
return nil
632+
})
633+
require.NoError(t, err)
634+
635+
return out.String()
636+
}

0 commit comments

Comments
 (0)