Merge branch 'develop' of github.com:matrix-org/sytest into anoa/dendrite_whitelist_annotation

* 'develop' of github.com:matrix-org/sytest: (42 commits)
  Fix OFFLINE support for the sytest-synapse docker image (#858)
  Add support for running worker mode against redis (#857)
  route GET account_data requests to client_reader (#855)
  Test that users receive updates for their own devices (#854)
  Add tests for SSO + user interactive authentication. (#845)
  Modify login tests to be spec compliant
  Make message history pagination test less flaky (#851)
  Small comment fixes for consistency (#808)
  Change room_fixture to run matrix_create_room_synced by default (#848)
  Update dendrite docs with new whitelist/blacklist info (#849)
  Add test for querying remote public rooms list with local server_name (#847)
  Add more logging to "Guest user can set display names" (#846)
  Add test for updating room aliases on upgraded room join (#844)
  Add tests to ensure that the operation cannot be modified during interactive auth. (#830)
  fix the CAS login test (#829)
  Update NOT_FOUND error to BAD_ALIAS error to be spec compliant. (#837)
  Missing bits
  Use r0 instead of 'unstable' for joined_members|rooms
  Add test to optionally keep other sessions when modifying a user's password. (#832)
  Remove tests that call query_auth endpoint. (#827)
  ...

Author: anoadragon453

.gitignore

@@ -11,4 +11,5 @@
 /var
 \#*
 .vscode
-.coverage*
+.coverage*
+.idea

docker/base.Dockerfile

@@ -60,6 +60,3 @@ ENV PGHOST=/var/run/postgresql
 ENV PGDATA=$PGHOST/data
 ENV PGUSER=postgres
 RUN for ver in `ls /usr/lib/postgresql | head -n 1`; do su -c '/usr/lib/postgresql/'$ver'/bin/initdb -E "UTF-8" --lc-collate="C" --lc-ctype="C" --username=postgres' postgres; done
-
-# Turn off all the fsync stuff for postgres
-RUN for ver in /etc/postgresql/*/main; do mkdir -p $ver/conf.d/; echo -e "fsync=off\nfull_page_writes=off" >> $ver/conf.d/fsync.conf; done

docs/dendrite-setup.md

@@ -41,14 +41,18 @@ SyTest will expect Dendrite to be at `../dendrite` relative to Sytest's root dir
 Simply run the following to execute tests:
 
 ```
-./run-tests.pl -I Dendrite::Monolith -W ../dendrite/testfile
+./run-tests.pl -I Dendrite::Monolith -W ../dendrite/sytest-whitelist -B ../dendrite/sytest-blacklist
 ```
 
 ## Useful flags
 
 * `-W` applies a test whitelist file, one of which is currently kept up to date
   with what sytests Dendrite passes
-  [here](https://github.com/matrix-org/dendrite/blob/master/testfile)
+  [here](https://github.com/matrix-org/dendrite/blob/master/sytest-whitelist)
+
+* `-B` applies a test blacklist file, one of which is currently kept up to date
+  with what sytests are currently flaky (fail *sometimes*) with Dendrite
+  [here](https://github.com/matrix-org/dendrite/blob/master/sytest-blacklist)
 
 * `-d` lets you set the path to Dendrite's `bin/` directory, in case it's
   somewhere other than `../dendrite/bin`

lib/SyTest/HTTPClient.pm

@@ -83,6 +83,8 @@ sub do_request
    my $self = shift;
    my %params = @_;
 
+   croak "must give a method" unless $params{method};
+
    my $uri = $self->full_uri_for( %params );
 
    # Also set verify_mode = 0 to not complain about self-signed SSL certs

lib/SyTest/Homeserver/Dendrite.pm

@@ -136,7 +136,10 @@ sub _get_config
       },
 
       database => {
-         map { $_ => $db_uri } qw(
+         # POSTGRES not set or is 0, use sqlite, which has separate .db files for each server
+         map { ( ! defined $ENV{'POSTGRES'} || $ENV{'POSTGRES'} == '0') ?
+         ($_ => "file:$self->{hs_dir}/" . $_ . ".db") :
+         ($_ => $db_uri) } qw(
             account device media_api sync_api room_server server_key
             federation_sender public_rooms_api naffka appservice
          ),

lib/SyTest/Homeserver/Synapse.pm

@@ -33,6 +33,7 @@ sub _init
 
    $self->{paths} = {};
    $self->{dendron} = '';
+   $self->{redis_host} = '';
 
    $self->SUPER::_init( $args );
 
@@ -282,6 +283,11 @@ sub start
         limit_usage_by_mau => "true",
         max_mau_value => 50000000,
 
+        redis => {
+           enabled => $self->{redis_host} ne '',
+           host    => $self->{redis_host},
+        },
+
         map {
            defined $self->{$_} ? ( $_ => $self->{$_} ) : ()
         } qw(
@@ -653,6 +659,7 @@ sub _init
    $self->SUPER::_init( @_ );
 
    $self->{dendron} = delete $args->{dendron_binary};
+   $self->{redis_host} = delete $args->{redis_host};
 
    if( my $level = delete $args->{torture_replication} ) {
       # torture the replication protocol a bit, to replicate bugs.
@@ -703,6 +710,7 @@ sub wrap_synapse_command
          "worker_log_config"       => $self->configure_logger("pusher"),
          "worker_replication_host" => "$bind_host",
          "worker_replication_port" => $self->{ports}{synapse_replication_tcp},
+         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type      => "http",
@@ -728,6 +736,7 @@ sub wrap_synapse_command
          "worker_log_config"       => $self->configure_logger("appservice"),
          "worker_replication_host" => "$bind_host",
          "worker_replication_port" => $self->{ports}{synapse_replication_tcp},
+         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type => "manhole",
@@ -753,6 +762,7 @@ sub wrap_synapse_command
          "worker_log_config"       => $self->configure_logger("federation_sender"),
          "worker_replication_host" => "$bind_host",
          "worker_replication_port" => $self->{ports}{synapse_replication_tcp},
+         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type => "manhole",
@@ -778,6 +788,7 @@ sub wrap_synapse_command
          "worker_log_config"       => $self->configure_logger("synchrotron"),
          "worker_replication_host" => "$bind_host",
          "worker_replication_port" => $self->{ports}{synapse_replication_tcp},
+         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type      => "http",
@@ -845,6 +856,7 @@ sub wrap_synapse_command
          "worker_log_config"       => $self->configure_logger("media_repository"),
          "worker_replication_host" => "$bind_host",
          "worker_replication_port" => $self->{ports}{synapse_replication_tcp},
+         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type      => "http",
@@ -912,6 +924,7 @@ sub wrap_synapse_command
          "worker_log_config"       => $self->configure_logger("user_dir"),
          "worker_replication_host" => "$bind_host",
          "worker_replication_port" => $self->{ports}{synapse_replication_tcp},
+         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type      => "http",
@@ -990,7 +1003,7 @@ sub wrap_synapse_command
             },
             {
                type => "manhole",
-               port => $self->{ports}{frontend_proxy},
+               port => $self->{ports}{frontend_proxy_manhole},
                bind_address => $bind_host,
             },
             {
@@ -1183,6 +1196,7 @@ sub generate_haproxy_map
 ^/_matrix/federation/v1/exchange_third_party_invite/  federation_reader
 ^/_matrix/federation/v1/send/                         federation_reader
 ^/_matrix/federation/v1/get_groups_publicised         federation_reader
+^/_matrix/federation/v1/user/devices/                 federation_reader
 ^/_matrix/key/v2/query                                federation_reader
 
 ^/_matrix/client/(api/v1|r0|unstable)/publicRooms$                client_reader
@@ -1197,9 +1211,12 @@ sub generate_haproxy_map
 ^/_matrix/client/versions$                                        client_reader
 ^/_matrix/client/(api/v1|r0|unstable)/voip/turnServer$            client_reader
 ^/_matrix/client/(r0|unstable)/register$                          client_reader
+^/_matrix/client/(r0|unstable)/auth/.*/fallback/web$              client_reader
 ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/messages$          client_reader
 ^/_matrix/client/(api/v1|r0|unstable)/get_groups_publicised$      client_reader
 ^/_matrix/client/(api/v1|r0|unstable)/joined_groups$              client_reader
+^/_matrix/client/(api/v1|r0|unstable)/publicised_groups$          client_reader
+^/_matrix/client/(api/v1|r0|unstable)/publicised_groups/          client_reader
 
 ^/_matrix/client/(api/v1|r0|unstable)/keys/upload  frontend_proxy
 
@@ -1216,10 +1233,13 @@ EOCONFIG
 sub generate_haproxy_get_map
 {
     return <<'EOCONFIG';
-^/_matrix/federation/v1/groups/                 federation_reader
-
-^/_matrix/client/(api/v1|r0|unstable)/groups/   client_reader
+# pushrules should be here, but the tests seem to be racy.
+# ^/_matrix/client/(api/v1|r0|unstable)/pushrules/            client_reader
+^/_matrix/client/(api/v1|r0|unstable)/groups/               client_reader
+^/_matrix/client/r0/user/[^/]*/account_data/                client_reader
+^/_matrix/client/r0/user/[^/]*/rooms/[^/]*/account_data/    client_reader
 
+^/_matrix/federation/v1/groups/                             federation_reader
 EOCONFIG
 }
 

lib/SyTest/HomeserverFactory/Synapse.pm

@@ -101,15 +101,17 @@ sub _init
    $self->{impl} = "SyTest::Homeserver::Synapse::ViaDendron";
    $self->{args}{dendron_binary} = "";
    $self->{args}{torture_replication} = 0;
+   $self->{args}{redis_host} = "";
 }
 
 sub get_options
 {
    my $self = shift;
 
    return (
-      'dendron-binary=s' => \$self->{args}{dendron_binary},
+      'dendron-binary=s'       => \$self->{args}{dendron_binary},
       'torture-replication:50' => \$self->{args}{torture_replication},
+      'redis-host=s'           => \$self->{args}{redis_host},
       $self->SUPER::get_options(),
    );
 }
@@ -125,6 +127,8 @@ sub print_usage
        --dendron-binary PATH    - path to the 'dendron' binary
 
        --torture-replication[=LEVEL] - enable torturing of the replication protocol
+
+       --redis-host HOST 	- if set then use redis for replication
 EOF
 }
 

scripts/synapse_sytest.sh

@@ -16,15 +16,26 @@ cd "$(dirname $0)/.."
 mkdir /work
 
 # PostgreSQL setup
-if [ -n "$MULTI_POSTGRES" ]; then
-    # In this mode we want to run synapse against multiple split out databases.
-
+if [ -n "$MULTI_POSTGRES" ] || [ -n "$POSTGRES" ]; then
     # We increase the max connections as we have more databases.
-    echo -e "max_connections=1000" >> /var/run/postgresql/data/postgresql.conf
+    sed -i -r "s/^max_connections.*$/max_connections = 500/" /var/run/postgresql/data/postgresql.conf
+
+    echo -e "fsync = off" >> /var/run/postgresql/data/postgresql.conf
+    echo -e "full_page_writes = off" >> /var/run/postgresql/data/postgresql.conf
 
     # Start the database
     su -c 'eatmydata /usr/lib/postgresql/*/bin/pg_ctl -w -D $PGDATA start' postgres
 
+    su -c psql postgres <<< "show config_file"
+    su -c psql postgres <<< "show max_connections"
+    su -c psql postgres <<< "show full_page_writes"
+    su -c psql postgres <<< "show fsync"
+fi
+
+# Now create the databases
+if [ -n "$MULTI_POSTGRES" ]; then
+    # In this mode we want to run synapse against multiple split out databases.
+
     # Make the test databases for the two Synapse servers that will be spun up
     su -c psql postgres <<EOF
 CREATE DATABASE pg1_main;
@@ -85,13 +96,11 @@ state_db:
 EOF
 
 elif [ -n "$POSTGRES" ]; then
+    # Env vars used by prep_sytest_for_postgres script.
     export PGUSER=postgres
     export POSTGRES_DB_1=pg1
     export POSTGRES_DB_2=pg2
 
-    # Start the database
-    su -c 'eatmydata /usr/lib/postgresql/*/bin/pg_ctl -w -D $PGDATA start' postgres
-
     # Write out the configuration for a PostgreSQL using Synapse
     ./scripts/prep_sytest_for_postgres.sh
 
@@ -105,14 +114,15 @@ if [ -n "$OFFLINE" ]; then
     # if we're in offline mode, just put synapse into the virtualenv, and
     # hope that the deps are up-to-date.
     #
-    # (`pip install -e` likes to reinstall setuptools even if it's already installed,
-    # so we just run setup.py explicitly.)
-    #
-    (cd /src && /venv/bin/python setup.py -q develop)
+    # --no-use-pep517 works around what appears to be a pip issue
+    # (https://github.com/pypa/pip/issues/5402 possibly) where pip wants
+    # to reinstall any requirements for the build system, even if they are
+    # already installed.
+    /venv/bin/pip install --no-index --no-use-pep517 /src
 else
     # We've already created the virtualenv, but lets double check we have all
     # deps.
-    /venv/bin/pip install -q --upgrade --no-cache-dir /src
+    /venv/bin/pip install -q --upgrade --no-cache-dir /src[redis]
     /venv/bin/pip install -q --upgrade --no-cache-dir \
         lxml psycopg2 coverage codecov tap.py coverage_enable_subprocess
 

tests/10apidoc/00prepare.pl

@@ -13,6 +13,8 @@ sub do_request_json_for
    my ( $user, %args ) = @_;
    is_User( $user ) or croak "Expected a User";
 
+   croak "must give a method" unless $args{method};
+
    my $user_id = $user->user_id;
    ( my $uri = delete $args{uri} ) =~ s/:user_id/$user_id/g;
 

tests/10apidoc/02login.pl

@@ -37,6 +37,7 @@
       my ( $http ) = @_;
 
       $http->do_request_json(
+         method => "GET",
          uri => "/r0/login",
       )->then( sub {
          my ( $body ) = @_;
@@ -81,7 +82,10 @@
 
          content => {
             type     => "m.login.password",
-            user     => $user_id,
+            identifier => {
+               type => "m.id.user",
+               user => $user_id,
+            },
             password => $password,
          },
       )->then( sub {
@@ -113,7 +117,10 @@
 
          content => {
             type     => "m.login.password",
-            user     => $user_id,
+            identifier => {
+               type => "m.id.user",
+               user => $user_id,
+            },
             password => $password,
             device_id => $device_id,
          },
@@ -145,7 +152,10 @@
 
          content => {
             type     => "m.login.password",
-            user     => $user_localpart,
+            identifier => {
+               type => "m.id.user",
+               user => $user_localpart,
+            },
             password => $password,
          },
       )->then( sub {
@@ -173,7 +183,10 @@
 
          content => {
             type     => "m.login.password",
-            user     => "i-ought-not-to-exist",
+            identifier => {
+               type => "m.id.user",
+               user => "i-ought-not-to-exist",
+            },
             password => "XXX",
          },
       )->main::expect_http_403;
@@ -192,7 +205,10 @@
 
          content => {
             type     => "m.login.password",
-            user     => $user_id,
+            identifier => {
+               type => "m.id.user",
+               user => $user_id,
+            },
             password => "${password}wrong",
          },
       )->main::expect_http_403->then( sub {
@@ -224,7 +240,10 @@ sub matrix_login_again_with_user
       uri     => "/r0/login",
       content  => {
          type     => "m.login.password",
-         user     => $user->user_id,
+         identifier => {
+            type => "m.id.user",
+            user => $user->user_id,
+         },
          password => $user->password,
          %args,
       },