Release 2022-12-09 - (expected chart version 4.29.0)

CHANGELOG.md

@@ -1,3 +1,14 @@
+# [2022-12-09] (Chart Release )
+
+## Bug fixes and other updates
+
+* Prevention of storing unnecessary data in the database if adding a bot to a conversation fails. (#2870)
+
+## Internal changes
+
+* bump nginx-module-vts from v0.1.15 to v0.2.1 (#2827)
+* Build nginz and nginz_disco docker images using nix (#2796)
+
 # [2022-11-03] (Chart Release 4.26.0)
 
 ## Release notes

charts/nginz/templates/conf/_nginx.conf.tpl

@@ -37,6 +37,8 @@ http {
   types_hash_max_size 2048;
   map_hash_bucket_size 128;
 
+  variables_hash_bucket_size 256;
+
   server_names_hash_bucket_size 64;
   server_name_in_redirect off;
 

hack/bin/upload-image.sh

@@ -17,8 +17,8 @@ readonly DOCKER_TAG=${DOCKER_TAG:?"Please set the DOCKER_TAG env variable"}
 readonly usage="USAGE: $0 <image_attr>"
 readonly IMAGE_ATTR=${1:?$usage}
 
-SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-ROOT_DIR=$(cd -- "$SCRIPT_DIR/../../" &> /dev/null && pwd)
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+ROOT_DIR=$(cd -- "$SCRIPT_DIR/../../" &>/dev/null && pwd)
 readonly SCRIPT_DIR ROOT_DIR
 
 credsArgs=""
@@ -27,6 +27,39 @@ if [[ "${DOCKER_USER+x}" != "" ]]; then
     credsArgs="--dest-creds=$DOCKER_USER:$DOCKER_PASSWORD"
 fi
 
+# Retry a command with exponential backoff
+# quay.io sometimes rate-limits us, so try again.
+# Also, skopeo's retry logic doesn't properly work, look here if you want to see very badly written go code:
+# https://github.com/containers/skopeo/blob/869d496f185cc086f22d6bbb79bb57ac3a415617/vendor/github.com/containers/common/pkg/retry/retry.go#L52-L113
+function retry {
+    local maxAttempts=$1
+    local secondsDelay=1
+    local attemptCount=1
+    local output=
+    shift 1
+
+    while [ $attemptCount -le "$maxAttempts" ]; do
+        output=$("$@")
+        local status=$?
+
+        if [ $status -eq 0 ]; then
+            break
+        fi
+
+        if [ $attemptCount -lt "$maxAttempts" ]; then
+            echo "Command [$*] failed after attempt $attemptCount of $maxAttempts. Retrying in $secondsDelay second(s)." >&2
+            sleep $secondsDelay
+        elif [ $attemptCount -eq "$maxAttempts" ]; then
+            echo "Command [$*] failed after $attemptCount attempt(s)" >&2
+            return $status
+        fi
+        attemptCount=$((attemptCount + 1))
+        secondsDelay=$((secondsDelay * 2))
+    done
+
+    echo "$output"
+}
+
 tmp_link_store=$(mktemp -d)
 # Using dockerTools.streamLayeredImage outputs an executable which prints the
 # image tar on stdout when executed. This is done so we don't store large images
@@ -38,8 +71,8 @@ tmp_link_store=$(mktemp -d)
 image_stream_file="$tmp_link_store/image_stream"
 nix -v --show-trace -L build -f "$ROOT_DIR/nix" "$IMAGE_ATTR" -o "$image_stream_file"
 image_file="$tmp_link_store/image"
-"$image_stream_file" > "$image_file"
+"$image_stream_file" >"$image_file"
 repo=$(skopeo list-tags "docker-archive://$image_file" | jq -r '.Tags[0] | split(":") | .[0]')
-printf "*** Uploading $image_file to %s:%s" "$repo" "$DOCKER_TAG"
+printf "*** Uploading $image_file to %s:%s\n" "$repo" "$DOCKER_TAG"
 # shellcheck disable=SC2086
-skopeo --insecure-policy copy --retry-times 5 $credsArgs "docker-archive://$image_file" "docker://$repo:$DOCKER_TAG"
+retry 5 skopeo --insecure-policy copy --retry-times 5 $credsArgs "docker-archive://$image_file" "docker://$repo:$DOCKER_TAG"

hack/bin/upload-images.sh

@@ -17,8 +17,8 @@ readonly usage="USAGE: $0 <images_attr>"
 # nix attribute under wireServer from "$ROOT_DIR/nix" containing all the images
 readonly IMAGES_ATTR=${1:?$usage}
 
-SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-ROOT_DIR=$(cd -- "$SCRIPT_DIR/../../" &> /dev/null && pwd)
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+ROOT_DIR=$(cd -- "$SCRIPT_DIR/../../" &>/dev/null && pwd)
 readonly SCRIPT_DIR ROOT_DIR
 
 tmp_link_store=$(mktemp -d)
@@ -28,8 +28,12 @@ nix -v --show-trace -L build -f "$ROOT_DIR/nix" wireServer.imagesList -o "$image
 # Build everything first so we can benefit the most from having many cores.
 nix -v --show-trace -L build -f "$ROOT_DIR/nix" "wireServer.$IMAGES_ATTR" --no-link
 
-while IFS="" read -r image_name || [ -n "$image_name" ]
-do
+while IFS="" read -r image_name || [ -n "$image_name" ]; do
     printf '*** Uploading image %s\n' "$image_name"
     "$SCRIPT_DIR/upload-image.sh" "wireServer.$IMAGES_ATTR.$image_name"
-done < "$image_list_file"
+done <"$image_list_file"
+
+for image_name in nginz nginz-disco; do
+    printf '*** Uploading image %s\n' "$image_name"
+    "$SCRIPT_DIR/upload-image.sh" "$image_name"
+done

nix/default.nix

@@ -11,16 +11,18 @@ let
   };
 
   profileEnv = pkgs.writeTextFile {
-      name = "profile-env";
-      destination = "/.profile";
-      # This gets sourced by direnv. Set NIX_PATH, so `nix-shell` uses the same nixpkgs as here.
-      text = ''
-        export NIX_PATH=nixpkgs=${toString pkgs.path}
-        export LOCALE_ARCHIVE=${pkgs.glibcLocales}/lib/locale/locale-archive
-        '';
-    };
+    name = "profile-env";
+    destination = "/.profile";
+    # This gets sourced by direnv. Set NIX_PATH, so `nix-shell` uses the same nixpkgs as here.
+    text = ''
+      export NIX_PATH=nixpkgs=${toString pkgs.path}
+      export LOCALE_ARCHIVE=${pkgs.glibcLocales}/lib/locale/locale-archive
+    '';
+  };
 
   wireServer = import ./wire-server.nix pkgs;
+  nginz = pkgs.callPackage ./nginz.nix { };
+  nginz-disco = pkgs.callPackage ./nginz-disco.nix { };
 
   # packages necessary to build wire-server docs
   docsPkgs = [
@@ -65,4 +67,5 @@ let
     };
   mls-test-cli = pkgs.mls-test-cli;
   rusty-jwt-tools = pkgs.rusty-jwt-tools;
-in {inherit pkgs profileEnv wireServer docs docsEnv mls-test-cli;}
+in
+{ inherit pkgs profileEnv wireServer docs docsEnv mls-test-cli nginz nginz-disco; }

nix/nginz-disco.nix

@@ -0,0 +1,42 @@
+{ stdenv
+, dockerTools
+, gnugrep
+, coreutils
+, which
+, dumb-init
+, bashInteractive
+, lib
+, makeWrapper
+, writers
+, dig
+, gawk
+, diffutils
+}:
+let
+  nginz-disco = stdenv.mkDerivation {
+    name = "nginz-disco";
+    src = (writers.writeBash "nginz_disco.sh" ../tools/nginz_disco/nginz_disco.sh);
+    phases = "installPhase";
+    nativeBuildInputs = [ makeWrapper ];
+    installPhase = ''
+      mkdir -p $out/bin
+      cp $src $out/bin/nginz_disco.sh
+      wrapProgram $out/bin/nginz_disco.sh \
+        --prefix PATH : "${lib.makeBinPath [ gnugrep gawk dig diffutils ]}"
+    '';
+  };
+
+  nginz-disco-image = dockerTools.streamLayeredImage {
+    name = "quay.io/wire/nginz_disco";
+    maxLayers = 10;
+    contents = [
+      bashInteractive
+      coreutils
+      which
+    ];
+    config = {
+      Entrypoint = [ "${dumb-init}/bin/dumb-init" "--" "${nginz-disco}/bin/nginz_disco.sh" ];
+    };
+  };
+in
+nginz-disco-image

nix/nginz.nix

@@ -0,0 +1,80 @@
+{ stdenv
+, symlinkJoin
+, dockerTools
+, writeTextDir
+, runCommand
+, gnugrep
+, coreutils
+, which
+, inotify-tools
+, dumb-init
+, cacert
+, bashInteractive
+, lib
+, makeWrapper
+, writers
+, nginz
+}:
+let
+
+  nginzWithReloader = stdenv.mkDerivation {
+    name = "reload-script";
+    src = (writers.writeBash "nginz_reload.sh" ../services/nginz/nginz_reload.sh);
+    phases = "installPhase";
+    nativeBuildInputs = [ makeWrapper ];
+    installPhase = ''
+      mkdir -p $out/bin
+      cp $src $out/bin/nginz_reload.sh
+      wrapProgram $out/bin/nginz_reload.sh \
+        --prefix PATH : "${lib.makeBinPath [ inotify-tools nginz ]}"
+    '';
+  };
+
+  # copied from nixpkgs fakeNss, but using nginx as username
+  nginxFakeNss = symlinkJoin {
+    name = "fake-nss";
+    paths = [
+      (writeTextDir "etc/passwd" ''
+        root:x:0:0:root user:/var/empty:/bin/sh
+        nginx:x:101:101:nginx:/var/empty:/bin/sh
+        nobody:x:65534:65534:nobody:/var/empty:/bin/sh
+      '')
+      (writeTextDir "etc/group" ''
+        root:x:0:
+        nginx:x:101:
+        nobody:x:65534:
+      '')
+      (writeTextDir "etc/nsswitch.conf" ''
+        hosts: files dns
+      '')
+      (runCommand "var-empty" { } ''
+        mkdir -p $out/var/empty
+      '')
+      # it seems nginx still tries to log, and doesn't create
+      # these directories automatically
+      (runCommand "nginx-misc" { } ''
+        mkdir -p $out/var/log/nginx
+        mkdir -p $out/var/cache/nginx
+      '')
+    ];
+  };
+
+  nginzImage = dockerTools.streamLayeredImage {
+    name = "quay.io/wire/nginz";
+    maxLayers = 10;
+    contents = [
+      cacert
+      bashInteractive
+      gnugrep
+      which
+      coreutils
+      nginxFakeNss
+      nginz # so preStop lifecycle hook in cannon can nginx -c … quit
+    ];
+    config = {
+      Entrypoint = [ "${dumb-init}/bin/dumb-init" "--" "${nginzWithReloader}/bin/nginz_reload.sh" "-g" "daemon off;" "-c" "/etc/wire/nginz/conf/nginx.conf" ];
+      Env = [ "SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" ];
+    };
+  };
+in
+nginzImage

nix/overlay.nix

@@ -30,10 +30,11 @@ let
       src =
         if stdenv.isDarwin
         then
-          fetchurl {
-            url = darwinAmd64Url;
-            sha256 = darwinAmd64Sha256;
-          }
+          fetchurl
+            {
+              url = darwinAmd64Url;
+              sha256 = darwinAmd64Sha256;
+            }
         else
           fetchurl {
             url = linuxAmd64Url;

services/brig/src/Brig/Provider/API.hs

@@ -898,6 +898,14 @@ addBot zuid zcon cid add = do
   let sid = addBotService add
   -- Get the conversation and check preconditions
   cnv <- lift (liftSem $ GalleyProvider.getConv zuid cid) >>= maybeConvNotFound
+  -- Check that the user is a conversation admin and therefore is allowed to add a bot to this conversation.
+  -- Note that this precondition is also checked in the internal galley API,
+  -- but by having this check here we prevent any (useless) data to be written to the database
+  -- as well as the unnecessary creation of the bot via the external service API call.
+  -- However, in case we refine the roles model in the future, this check might not be granular enough.
+  -- In that case we should rather do an internal call to galley to check for the correct permissions.
+  -- Also see `removeBot` for a similar check.
+  guardConvAdmin cnv
   let mems = cnvMembers cnv
   unless (cnvType cnv == RegularConv) $
     throwStd invalidConv
@@ -974,6 +982,12 @@ removeBot :: Members '[GalleyProvider] r => UserId -> ConnId -> ConvId -> BotId
 removeBot zusr zcon cid bid = do
   -- Get the conversation and check preconditions
   cnv <- lift (liftSem $ GalleyProvider.getConv zusr cid) >>= maybeConvNotFound
+  -- Check that the user is a conversation admin and therefore is allowed to remove a bot from the conversation.
+  -- Note that this precondition is also checked in the internal galley API.
+  -- However, in case we refine the roles model in the future, this check might not be granular enough.
+  -- In that case we should rather do an internal call to galley to check for the correct permissions.
+  -- Also see `addBot` for a similar check.
+  guardConvAdmin cnv
   let mems = cnvMembers cnv
   unless (cnvType cnv == RegularConv) $
     throwStd invalidConv
@@ -985,6 +999,11 @@ removeBot zusr zcon cid bid = do
     Just _ -> do
       lift $ Public.RemoveBotResponse <$$> wrapHttpClient (deleteBot zusr (Just zcon) bid cid)
 
+guardConvAdmin :: Conversation -> ExceptT Error (AppT r) ()
+guardConvAdmin conv = do
+  let selfMember = cmSelf . cnvMembers $ conv
+  unless (memConvRoleName selfMember == roleNameWireAdmin) $ throwStd accessDenied
+
 --------------------------------------------------------------------------------
 -- Bot API