Release 2022-05-18 - (expected chart version 4.12.0)

.envrc

@@ -1,5 +1,30 @@
-env=$(nix-build --no-out-link "$PWD/direnv.nix")
-PATH_add "${env}/bin"
+# Build a folder ./.env that contains all the tools
+# To speed up the nix evaluation, we only rebuild our environment when `./nix`
+# changes. We do this by adding its contents to the nix store and using its
+# store path as a cache key. We only use this to build the dev environment. This
+# will fail building in case anything inside the devEnv is reaching outside
+# the nix/ subfolder
+store_path=$(nix-store --add ./nix/)
+layout_dir=$(direnv_layout_dir)
+[[ -d "$layout_dir" ]] || mkdir -p "$layout_dir"
+if [[ ! -d ./.env || ! -f "$layout_dir/nix-rebuild" || "$store_path" != $(< "$layout_dir/nix-rebuild" ) ]]; then
+
+  # this builds "devEnv" defined in ./nix/default.nix and outputs to ./.env
+  nix-build "$store_path" -A devEnv --out-link ./.env
+
+  echo "$store_path" > "$layout_dir/nix-rebuild"
+fi
+
+PATH_add "./.env/bin"
+
+# source .profile from `$env`. This sets NIX_PATH to pkgs defined in
+# ./nix/default.nix. Tis is useful for nix tooling that runs inside this direnv,
+# e.g. "nix-shell -p foo" will get "foo" for pkgs defined in ./nix/default.nix
+ [[ -f "./.env/.profile" ]] && source_env "./.env/.profile"
 
 # allow local .envrc overrides
 [[ -f .envrc.local ]] && source_env .envrc.local
+
+# Locale
+export LC_ALL=en_US.UTF-8
+export LANG=en_US.UTF-8

.github/pull_request_template.md

@@ -3,7 +3,7 @@
  - [ ] The **PR Title** explains the impact of the change.
  - [ ] The **PR description** provides context as to why the change should occur and what the code contributes to that effect. This could also be a link to a JIRA ticket or a Github issue, if there is one.
  - [ ] If this PR changes development workflow or dependencies, they have been A) automated and B) documented under docs/developer/. All efforts have been taken to minimize development setup breakage or slowdown for co-workers.
- - [ ] If HTTP endpoint paths have been added or renamed, the **endpoint / config-flag checklist** (see Wire-employee only backend [wiki page](https://github.com/zinfra/backend-wiki/wiki/Checklists)) has been followed.
+ - [ ] If HTTP endpoint paths have been added or renamed, or feature configs have changed, the **endpoint / config-flag checklist** (see Wire-employee only backend [wiki page](https://github.com/zinfra/backend-wiki/wiki/Checklists)) has been followed.
  - [ ] If a cassandra schema migration has been added, I ran **`make git-add-cassandra-schema`** to update the cassandra schema documentation.
  - [ ] **changelog.d** contains the following bits of information ([details](https://github.com/wireapp/wire-server/blob/develop/docs/developer/changelog.md)):
    - [ ] A file with the changelog entry in one or more suitable sub-sections. The sub-sections are marked by directories inside `changelog.d`.

.github/workflows/ci.yml

@@ -25,8 +25,6 @@ jobs:
          signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
      - name: Build the wire-server-direnv
-       run: nix-build --no-out-link direnv.nix
+       run: nix-build --no-out-link ./nix -A devEnv
      - name: Install the wire-server-direnv
-       run: nix-env -f direnv.nix -i
-     - name: Ensure everything is formatted
-       run: make formatc
+       run: nix-env -if ./nix -A devEnv

.gitignore

@@ -110,3 +110,10 @@ cabal.project.local
 # Nix output symlinks
 result
 result-*
+
+# Created by .envrc
+.direnv
+.env
+
+# emacs misc
+.dir-locals.el

CHANGELOG.md

@@ -1,3 +1,203 @@
+# [2022-05-18] (Chart Release 4.12.0)
+
+## Release notes
+
+
+* If using [cert-manager](https://github.com/cert-manager/cert-manager), you need to have least version 1.0.0 (1.8.0 works at the time of writing) installed. Older cert-manager 0.15.X will no longer work. (#2401)
+
+* Upgrade team-settings version to 4.9.0-v0.29.7-0-142a76f (#2180)
+
+
+## API changes
+
+
+* Start version 2 of the public API. Main changes:
+
+   - Asset endpoints have lost their `v3` and `v4` suffixes. So for example
+     `/assets/v3` has been replaced by `/assets`.
+   - `GET /conversations/:conv/assets/:id` and `GET
+     /conversations/:conv/otr/assets/:id` have been removed.
+   - `GET /assets/:key/v3` has been removed. Use the qualified endpoint `GET
+     /assets/:domain/:key` instead.
+   - `DELETE /assets/:key/v3` has been removed. Use the qualified endpoint
+     `DELETE /assets/:domain/:key` instead.
+   - `GET /connections` has been removed. Use `POST /list-connections` instead.
+   - `POST /connections` has been removed. Use `POST /connections/:domain/:user` instead.
+   - `PUT /connections/:domain/:user` has been removed: use `POST` instead.
+   - `GET /conversations` has been removed. Use `POST /conversations/list-ids`
+     followed by `POST /conversations/list` instead.
+   - `POST /conversations/list/v2` has been replaced by `POST
+     /conversations/list`.
+   - `POST /conversations/:domain/:conv/members/v2` has lost its `v2` suffix, so
+     it is now `POST /conversations/:domain/:conv/members`.
+   - `GET /users`, `GET /users/by-handle` and `GET /users/handles` have been
+     removed. Use `POST /search/contacts` instead.
+   - `GET /users/:id` has been removed. Use the qualified endpoint `GET
+     /users/:domain/:id` instead.
+   - `GET /users/:id/clients` has been removed. Use the qualified endpoint `GET
+     /users/:domain/:id/clients` instead.
+   - `GET /users/:id/clients/:client` has been removed. Use the qualified
+     endpoint `GET /users/:domain/:id/clients/:client` instead.
+
+  Swagger documentation for the previous version of the API can be accessed at
+  `/v1/api/swagger-ui`. (#2297)
+
+* A new field `development` has been added to the object returned by `GET
+  /api-version`. Versions listed there are considered in flux, meaning that the
+  corresponding API contracts can change arbitrarily over time. Clients are free
+  to use development versions, as long as they are also listed in `supported`,
+  and failures due to incompatibilities are acceptable (e.g. in testing
+  environments). Backends are the authoritative source on whether a development
+  version can be used at all. If a development version should not be used, the
+  backend will not list it among the supported versions at all. (#2297)
+
+
+## Features
+
+
+* charts: Various new values can now be configured and some got changed
+
+  Allow new configurations in the brig chart:
+  * `config.emailSMS.user.invitationUrl`
+  * `config.emailSMS.team.tInvitationUrl`
+  * `config.emailSMS.team.tActivationUrl`
+  * `config.emailSMS.team.tCreatorWelcomeUrl`
+  * `config.emailSMS.team.tMemberWelcomeUrl`
+  * `config.setProviderSearchFilter`
+  * `config.setWhitelist`
+  * `config.setFeatureFlags`
+  * `config.setCustomerExtensions`
+
+  If any values in config.emailSMS.team are specified, all must be specified.
+
+  Allow new configurations in the gundeck chart:
+  * `config.perNativePushConcurrency`
+  * `config.maxConcurrentNativePushes.soft`
+  * `config.maxConcurrentNativePushes.hard`
+
+  Other changes:
+  * Default `maxTeamSize` changed to 10000 from 500. (#2347)
+
+* charts/nginx-ingress-services: Allow more fine-grained control over what services are installed. Upgrade Certificate/Issuer resources to 'cert-manager.io/v1' (#2401)
+
+* MLS implementation progress:
+
+   - remote key package claim is now supported (#2353)
+
+* charts/{brig,cargohold,galley,gundeck}: Allow not configuring AWS credentials and allow using a special service account.
+  This way, when operating wire in AWS cloud either instance profiles or IAM role attached to a service account can be used to communicate with AWS. (#2347)
+
+* Implement TURN service discovery using SRV records (#2389)
+
+
+## Bug fixes and other updates
+
+
+* When `config.enablePayment` and `FEATURE_ENABLE_PAYMENT` (`envVars`) were set,
+  the team-settings feature flag `FEATURE_ENABLE_PAYMENT` was rendered two times.
+  The new behavior is to give the `envVars` entry priority. I.e. when it's set,
+  it's used instead of the `config.enablePayment` value. (#2332)
+
+* Modify the nginz access control configuration to prevent clients connecting
+  to listeners with PROXY protocol enabled (such as the websocket listener) from
+  accessing a private metrics endpoint. (#2307)
+
+* Verification email is sent when external id is updated via SCIM (#2374)
+
+
+## Documentation
+
+
+* Move old /docs to /docs/legacy (leaving references). (#2328)
+
+* Fixup for #2321 (#2323)
+
+* Add pagination docs to `POST /list-connections` (#2369)
+
+* Documentation for the 2nd factor password challenge feature (#2329)
+
+* Documentation on how to enforce desktop application only for web app (#2334)
+
+* Documentation on how to enforce constant bit rate for all calls (#2336)
+
+* Documentation on how to disable media plugins for the web app (#2337)
+
+* Documentation on how to extra entropy in the web app (#2338)
+
+* Documentation on how to set the instance connection parameters and proxy settings (#2340)
+
+* Merged SAML/SCIM docs with its main documentation (#2356)
+
+
+## Internal changes
+
+
+* View and change team feature permissions apply to all features now (#2402)
+
+* Add sed to direnv (#2319)
+
+* Add python3 to nix development environment. It's needed by hack/bin/serve-charts.sh . (#2333)
+
+* Add a target to the Makefile to run ShellCheck. I.e. to run a linter on shell scripts. This will be used in the CI. For now, all scripts with linter issues are excluded from this check. (#2361)
+
+* Drop snappy support from bonanza (#2350)
+
+* Use cabal in buildah-based builds (#2341)
+
+* Fix flakyness of path traversal test (#2387)
+
+* Github Actions: disable mac builds (#2355)
+
+* Apply `versionMiddleware` last. This makes sure that every other middleware sees
+  the rewritten (unversioned) path. In particular, the prometheus middleware will
+  now only see paths it knows about, which prevents it from reporting "N/A" as the
+  path. (#2316)
+
+* Upgrade version of libzauth dependencies, notably sodiumoxide bindings to libsodium, and fix resulting errors and warnings. (#2327)
+
+* libzauth: Update sha256 for source in nix expression (#2354)
+
+* Log IO exceptions in Galley and Brig (#2385)
+
+* Generalise and move the Logger effect (#2306)
+
+* Fix a comment in a Makefile target (#2330)
+
+* Fix flaky MLS conversation creation test (#2386)
+
+* Fix flaky key package test (#2384)
+
+* Fix locale variables in Nix and .envrc (#2393)
+
+* Team Member API has been migrated to Servant (#2309)
+
+* Integration test for edge case: change external id before account registration (#2396)
+
+* Allow specifying 'redisAdditionalWrite' for a secondary redis to which gundeck will write in the context of a redis migration without downtime. (#2304)
+
+* Start TURN discovery only when the app starts and not when the Env is created (#2376)
+
+* Avoid using IN queries for fetching multiple conversations (#2397)
+
+* Remove oromolu GH action (has been moved to concourse https://github.com/zinfra/cailleach/pull/1033) (#2320)
+
+* Remove unused data type AllowedUserSearch (#2373)
+
+* docs: add latex to docs and publish pdf if exists (#2321)
+
+
+## Federation changes
+
+
+* We now fetch version information from other backends and negotiate a version to use. (#2297)
+
+* Fix assertion in testWelcomeNoKey (#2372)
+
+* Support remote welcome messages (#2368)
+
+* Implement remote admin action: Update receipt mode (#2141)
+
+
 # [2022-05-04] (Chart Release 4.11.0)
 
 ## Release notes

Makefile

@@ -1,5 +1,4 @@
 SHELL                 := /usr/bin/env bash
-LANG                  := en_US.UTF-8
 DOCKER_USER           ?= quay.io/wire
 # kubernetes namespace for running integration tests
 NAMESPACE             ?= test-$(USER)
@@ -8,13 +7,13 @@ DOCKER_TAG            ?= $(USER)
 # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything)
 HELM_SEMVER           ?= 0.0.42
 # The list of helm charts needed on internal kubernetes testing environments
-CHARTS_INTEGRATION    := wire-server databases-ephemeral fake-aws nginx-ingress-controller nginx-ingress-services wire-server-metrics fluent-bit kibana sftd restund coturn
+CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster fake-aws nginx-ingress-controller nginx-ingress-services wire-server-metrics fluent-bit kibana sftd restund coturn
 # The list of helm charts to publish on S3
 # FUTUREWORK: after we "inline local subcharts",
 # (e.g. move charts/brig to charts/wire-server/brig)
 # this list could be generated from the folder names under ./charts/ like so:
 # CHARTS_RELEASE := $(shell find charts/ -maxdepth 1 -type d | xargs -n 1 basename | grep -v charts)
-CHARTS_RELEASE        := wire-server redis-ephemeral databases-ephemeral fake-aws fake-aws-s3 fake-aws-sqs aws-ingress  fluent-bit kibana backoffice calling-test demo-smtp elasticsearch-curator elasticsearch-external elasticsearch-ephemeral minio-external cassandra-external nginx-ingress-controller nginx-ingress-services reaper wire-server-metrics sftd restund coturn
+CHARTS_RELEASE        := wire-server redis-ephemeral redis-cluster databases-ephemeral fake-aws fake-aws-s3 fake-aws-sqs aws-ingress  fluent-bit kibana backoffice calling-test demo-smtp elasticsearch-curator elasticsearch-external elasticsearch-ephemeral minio-external cassandra-external nginx-ingress-controller nginx-ingress-services reaper wire-server-metrics sftd restund coturn
 BUILDAH_PUSH          ?= 0
 KIND_CLUSTER_NAME     := wire-server
 BUILDAH_KIND_LOAD     ?= 1
@@ -76,6 +75,12 @@ endif
 ci: c
 	./hack/bin/cabal-run-integration.sh $(package)
 
+# Use ghcid to watch a particular package.
+# pass target=package:name to specify which target is watched.
+.PHONY: ghcid
+ghcid:
+	ghcid --command "cabal repl $(target)"
+
 # reset db using cabal
 .PHONY: db-reset-package
 db-reset-package: c
@@ -128,6 +133,10 @@ add-license:
 	@echo ""
 	@echo "you might want to run 'make formatf' now to make sure ormolu is happy"
 
+.PHONY: shellcheck
+shellcheck:
+	./hack/bin/shellcheck.sh
+
 # Clean
 .PHONY: clean
 clean:
@@ -209,7 +218,7 @@ docker-builder:
 
 .PHONY: docker-intermediate
 docker-intermediate:
-	# `docker-intermediate` needs to be built whenever code changes - this essentially runs `stack clean && stack install` on the whole repo
+	# `docker-intermediate` needs to be built whenever code changes - this essentially runs `cabal clean && cabal build all` on the whole repo
 	docker build -t $(DOCKER_USER)/ubuntu20-intermediate:$(DOCKER_TAG) -f build/ubuntu/Dockerfile.intermediate --build-arg builder=$(DOCKER_USER)/ubuntu20-builder:develop --build-arg deps=$(DOCKER_USER)/ubuntu20-deps:develop .;
 	docker tag $(DOCKER_USER)/ubuntu20-intermediate:$(DOCKER_TAG) $(DOCKER_USER)/ubuntu20-intermediate:latest;
 	if test -n "$$DOCKER_PUSH"; then docker login -u $(DOCKER_USERNAME) -p $(DOCKER_PASSWORD); docker push $(DOCKER_USER)/ubuntu20-intermediate:$(DOCKER_TAG); docker push $(DOCKER_USER)/ubuntu20-intermediate:latest; fi;
@@ -253,8 +262,8 @@ git-add-cassandra-schema: db-reset git-add-cassandra-schema-impl
 .PHONY: git-add-cassandra-schema-impl
 git-add-cassandra-schema-impl:
 	$(eval CASSANDRA_CONTAINER := $(shell docker ps | grep '/cassandra:' | perl -ne '/^(\S+)\s/ && print $$1'))
-	( echo '-- automatically generated with `make git-add-cassandra-schema`' ; docker exec -i $(CASSANDRA_CONTAINER) /usr/bin/cqlsh -e "DESCRIBE schema;" ) > ./docs/reference/cassandra-schema.cql
-	git add ./docs/reference/cassandra-schema.cql
+	( echo '-- automatically generated with `make git-add-cassandra-schema`' ; docker exec -i $(CASSANDRA_CONTAINER) /usr/bin/cqlsh -e "DESCRIBE schema;" ) > ./cassandra-schema.cql
+	git add ./cassandra-schema.cql
 
 .PHONY: cqlsh
 cqlsh:
@@ -434,7 +443,7 @@ echo-release-charts:
 
 .PHONY: buildah-docker
 buildah-docker: buildah-docker-nginz
-	./hack/bin/buildah-compile.sh
+	./hack/bin/buildah-compile.sh all
 	BUILDAH_PUSH=${BUILDAH_PUSH} KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME} BUILDAH_KIND_LOAD=${BUILDAH_KIND_LOAD}  ./hack/bin/buildah-make-images.sh
 
 .PHONY: buildah-docker-nginz

README.md

@@ -91,7 +91,7 @@ This requires a range of dependencies that depend on your platform/OS, such as:
 - Haskell & Rust compiler and package managers
 - Some package dependencies (libsodium, openssl, protobuf, icu, geoip, snappy, [cryptobox-c](https://github.com/wireapp/cryptobox-c), ...) that depend on your platform/OS
 
-See [docs/developer/dependencies.md](docs/developer/dependencies.md) for details.
+See [docs/developer/dependencies.md](docs/legacy/developer/dependencies.md) for details.
 
 Once all dependencies are set up, the following should succeed:
 

cabal.project

@@ -79,11 +79,6 @@ source-repository-package
     location: https://github.com/kim/hs-collectd
     tag: 885da222be2375f78c7be36127620ed772b677c9
 
-source-repository-package
-    type: git
-    location: https://github.com/kim/snappy-framing
-    tag: d99f702c0086729efd6848dea8a01e5266c3a61c
-
 source-repository-package
     type: git
     location: https://github.com/vincenthz/hs-certificate
@@ -149,11 +144,6 @@ source-repository-package
     location: https://github.com/wireapp/saml2-web-sso
     tag: 4227e38be5c0810012dc472fc6931f6087fbce68
 
-source-repository-package
-    type: git
-    location: https://github.com/wireapp/snappy
-    tag: b0e5c08af48911caecffa4fa6a3e74872018b258
-
 source-repository-package
     type: git
     location: https://gitlab.com/axeman/swagger