Add a sytest blacklist file (#5611)

.buildkite/pipeline.yml

@@ -166,7 +166,26 @@ steps:
         - exit_status: 2
           limit: 2
 
-  - label: "SyTest - :python: 3 / :postgres: 9.6 / Monolith"
+
+  - label: "SyTest - :python: 3.5 / SQLite / Monolith"
+    agents:
+      queue: "medium"
+    command:
+      - "bash .buildkite/merge_base_branch.sh"
+      - "bash /synapse_sytest.sh"
+    plugins:
+      - docker#v3.0.1:
+          image: "matrixdotorg/sytest-synapse:py35"
+          propagate-environment: true
+          always-pull: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - label: "SyTest - :python: 3.5 / :postgres: 9.6 / Monolith"
     agents:
       queue: "xlarge"
     env:
@@ -179,17 +198,12 @@ steps:
           image: "matrixdotorg/sytest-synapse:dinsic-py3"
           propagate-environment: true
           always-pull: true
-          workdir: "/src"
-          entrypoint: "/bin/sh"
-          init: false
-          shell: ["-x", "-c"]
-          mount-buildkite-agent: false
-          volumes: ["./logs:/logs"]
-      - artifacts#v1.2.0:
-          upload: [ "logs/**/*.log", "logs/**/*.log.*", "logs/results.tap" ]
-      - matrix-org/annotate:
-          path: "logs/annotate.md"
-          style: "error"
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
 
   - label: "SyTest - :python: 3 / :postgres: 9.6 / Workers"
     agents:
@@ -205,15 +219,10 @@ steps:
           image: "matrixdotorg/sytest-synapse:dinsic-py3"
           propagate-environment: true
           always-pull: true
-          workdir: "/src"
-          entrypoint: "/bin/sh"
-          init: false
-          shell: ["-x", "-c"]
-          mount-buildkite-agent: false
-          volumes: ["./logs:/logs"]
-      - artifacts#v1.2.0:
-          upload: [ "logs/**/*.log", "logs/**/*.log.*", "logs/results.tap" ]
-      - matrix-org/annotate:
-          path: "logs/annotate.md"
-          style: "error"
-
+    soft_fail: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2

MANIFEST.in

@@ -8,6 +8,7 @@ include demo/README
 include demo/demo.tls.dh
 include demo/*.py
 include demo/*.sh
+include sytest-blacklist
 
 recursive-include synapse/storage/schema *.sql
 recursive-include synapse/storage/schema *.sql.postgres

changelog.d/5611.misc

@@ -0,0 +1 @@
+Add a blacklist file to the repo to blacklist certain sytests from failing CI.

sytest-blacklist

@@ -1,3 +1,39 @@
+# This file serves as a blacklist for SyTest tests that we expect will fail in
+# Synapse.
+#
+# Each line of this file is scanned by sytest during a run and if the line
+# exactly matches the name of a test, it will be marked as "expected fail",
+# meaning the test will still run, but failure will not mark the entire test
+# suite as failing.
+#
+# Test names are encouraged to have a bug accompanied with them, serving as an
+# explanation for why the test has been excluded.
+
+# Blacklisted due to https://github.com/matrix-org/synapse/issues/1679
+Remote room members also see posted message events
+
+# Blacklisted due to https://github.com/matrix-org/synapse/issues/2065
+Guest users can accept invites to private rooms over federation
+
+# Blacklisted due to https://github.com/vector-im/riot-web/issues/7211
+The only membership state included in a gapped incremental sync is for senders in the timeline
+
+# Blacklisted due to https://github.com/matrix-org/synapse/issues/1658
+Newly created users see their own presence in /initialSync (SYT-34)
+
+# Blacklisted due to https://github.com/matrix-org/synapse/issues/1396
+Should reject keys claiming to belong to a different user
+
+# Blacklisted due to https://github.com/matrix-org/synapse/issues/2306
+Users appear/disappear from directory when join_rules are changed
+Users appear/disappear from directory when history_visibility are changed
+
+# Blacklisted due to https://github.com/matrix-org/synapse/issues/1531
+Enabling an unknown default rule fails with 404
+
+# Blacklisted due to https://github.com/matrix-org/synapse/issues/1663
+New federated private chats get full presence information (SYN-115)
+
 # flaky test
 If remote user leaves room we no longer receive device updates