Merge branch 'aggregate_results' into 'master'

introduce result aggregation

See merge request sre18/break-it!18

.gitlab-ci.yml

@@ -6,14 +6,16 @@ variables:
 
 stages:
   - test
+  - aggregation
 
 style:
   stage: test
   script:
     - wget https://sre18.pages.rechenknecht.net/misc/pylintrc -O .pylintrc
     - pip3 install -r requirements.txt
+    - pip3 install -r requirements-dev.txt
     - pip3 install pylint
-    - pylint run.py tests util
+    - pylint run.py tests util tools
 
 groupA:
   stage: test
@@ -21,8 +23,12 @@ groupA:
     - docker:dind
   script:
     - dockerfiles/A/build.sh
-    - python3 run.py testing.marschke.me 9001 sre18groupa/smtp-server-group-a 6666
-  allow_failure: true
+    - mkdir -p results
+    - python3 run.py testing.marschke.me 9001 sre18groupa/smtp-server-group-a 6666 2>&1 | tee results/group-a
+  artifacts:
+    paths:
+      - results/group-a
+    expire_in: 1 week
 
 groupB:
   stage: test
@@ -30,8 +36,12 @@ groupB:
     - docker:dind
   script:
     - dockerfiles/B/build.sh
-    - python3 run.py testing.marschke.me 9001 sregroupb/smtp-server 8008
-  allow_failure: true
+    - mkdir -p results
+    - python3 run.py testing.marschke.me 9001 sregroupb/smtp-server 8008 2>&1 | tee results/group-b
+  artifacts:
+    paths:
+      - results/group-b
+    expire_in: 1 week
 
 groupC:
   stage: test
@@ -39,8 +49,12 @@ groupC:
     - docker:dind
   script:
     - dockerfiles/C/build.sh
-    - python3 run.py testing.marschke.me 9001 sregroupc/smtp-server 5555
-  allow_failure: true
+    - mkdir -p results
+    - python3 run.py testing.marschke.me 9001 sregroupc/smtp-server 5555 2>&1 | tee results/group-c
+  artifacts:
+    paths:
+      - results/group-c
+    expire_in: 1 week
 
 groupD:
   stage: test
@@ -48,8 +62,12 @@ groupD:
     - docker:dind
   script:
     - dockerfiles/D/build.sh
-    - python3 run.py testing.marschke.me 9001 sregroupd/sre-smtp-group-d 25
-  allow_failure: true
+    - mkdir -p results
+    - python3 run.py testing.marschke.me 9001 sregroupd/sre-smtp-group-d 25 2>&1 | tee results/group-d
+  artifacts:
+    paths:
+      - results/group-d
+    expire_in: 1 week
 
 groupE:
   stage: test
@@ -57,5 +75,28 @@ groupE:
     - docker:dind
   script:
     - dockerfiles/E/build.sh
-    - python3 run.py testing.marschke.me 9001 julianweise/simplesmtp 4431 --ssl --wait-up 2
-  allow_failure: true
+    - mkdir -p results
+    - python3 run.py testing.marschke.me 9001 julianweise/simplesmtp 4431 --ssl --wait-up 2 2>&1 | tee results/group-e
+  artifacts:
+    paths:
+      - results/group-e
+    expire_in: 1 week
+
+postfix:
+  stage: test
+  services:
+    - docker:dind
+  script:
+    - docker pull tozd/postfix
+    - mkdir -p results
+    - python3 run.py testing.marschke.me 9001 tozd/postfix 25 --wait-up 5 2>&1 | tee results/postfix
+  artifacts:
+    paths:
+      - results/postfix
+    expire_in: 1 week
+
+results:
+  stage: aggregation
+  script:
+    - pip3 install -r requirements-dev.txt
+    - python3 tools/result_aggregation.py

requirements-dev.txt

+tabulate

run.py

 import argparse
-import sys
 import unittest
 
 from tests import TestDrops, TestFuzzing, TestMail, TestSession
@@ -37,4 +36,4 @@ def main():
 
 
 if __name__ == '__main__':
-    sys.exit(main())
+    main()

tests/TestFuzzing.py

@@ -26,15 +26,13 @@ class TestFuzzing(BaseTest):
 
         while True:
             try:
-                self.client.recv(1)
+                if self.client.recv(1) == b'':
+                    return
             except socket.timeout:
                 break
 
         self.assertResponse([x for x in range(500, 510)], 'FAIL\r\n')
 
-    def test_unrecognized_command(self):
-        self.assertResponse(500, 'FAIL\r\n')
-
     def test_small_input(self):
         fuzzy_input = 'A' * 10**1
         self.assertResponse([x for x in range(500, 510)], '{}\r\n'.format(fuzzy_input))

tests/TestMail.py

-import subprocess
 from util import BaseTest
 
 
@@ -88,19 +87,19 @@ class TestMail(BaseTest):
     def test_mail_save(self):
         self.assertResponse(250, b'HELO localDomain\r\n')
         self.assertResponse(250, b'MAIL FROM:<user0@test.de>\r\n')
-        self.assertResponse(250, b'RCPT TO:<user1@localhost>\r\n')
-        self.assertResponse(250, b'RCPT TO:<user2@localhost>\r\n')
+        self.assertResponse(250, b'RCPT TO:<user1@server.marschke.me>\r\n')
+        self.assertResponse(250, b'RCPT TO:<user2@server.marschke.me>\r\n')
         self.assertResponse(354, b'DATA\r\n')
         self.client.send(b'SOME_RANDOM_TEXT\r\n')
         self.client.send(b'SOME_RANDOM_TEXT\r\n')
         self.assertResponse(250, b'.\r\n')
 
-        subprocess.check_output(['docker',
-                                 'exec',
-                                 '{}'.format(self.test_server.docker_id.decode('utf-8')),
-                                 'grep',
-                                 '-r',
-                                 'SOME_RANDOM_TEXT'])
+        #subprocess.check_output(['docker',
+        #                         'exec',
+        #                         '{}'.format(self.test_server.docker_id.decode('utf-8')),
+        #                         'grep',
+        #                         '-r',
+        #                        'SOME_RANDOM_TEXT'])
 
         self.assertResponse(221, 'QUIT\r\n')
 
@@ -109,7 +108,7 @@ class TestMail(BaseTest):
         self.assertResponse(250, 'MAIL FROM:<leonard@marschke.me>\r\n')
         self.assertResponse(250, 'RCPT TO:<firstAddress@{}>\r\n'.format(self.test_server.domain))
 
-        for i in range(10**3):
+        for i in range(800):
             self.assertResponse(250, 'RCPT TO:<address{}@{}>\r\n'.format(i, self.test_server.domain))
 
         self.assertResponse(354, 'DATA\r\n' + \

tests/TestSession.py

@@ -8,18 +8,18 @@ class TestSession(BaseTest):
     def test_ehlo(self):
         self.assertResponse(250, 'EHLO marschke.me\r\n')
 
-        self.assertResponse(500, 'FAIL\r\n')
+        self.assertResponse([500, 502], 'FAIL\r\n')
 
-    def test_post_invalid_parameter_stability_0(self):
-        self.assertResponse(250, b'EHLO localhost\r\n')
-        self.assertResponse(500, b'MAIL wrong input blabla\r\n')
+    def test_invalid_parameter_stability_0(self):
+        self.assertResponse(250, b'HELO localhost\r\n')
+        self.assertResponse(501, b'MAIL wrong input blabla\r\n')
         self.assertResponse(250, b'MAIL FROM:<correct.mail@test.de>\r\n')
 
-    def test_post_invalid_parameter_stability_1(self):
-        self.assertResponse(250, b'EHLO localhost\r\n')
+    def test_invalid_parameter_stability_1(self):
+        self.assertResponse(250, b'HELO localhost\r\n')
         self.assertResponse(250, b'MAIL FROM:<correct.mail@test.de>\r\n')
-        self.assertResponse(500, b'RCPT wrong input blabla\r\n')
-        self.assertResponse(250, b'RCPT TO:<correct@localhost>\r\n')
+        self.assertResponse(501, b'RCPT wrong input blabla\r\n')
+        self.assertResponse(250, b'RCPT TO:<correct@server.marschke.me>\r\n')
 
     def test_help(self):
         self.assertResponse(250, b'HELP\r\n')
@@ -40,34 +40,35 @@ class TestSession(BaseTest):
         self.assertResponse(501, b'EHLO\r\n')
 
     def test_mail_no_ehlo(self):
-        self.assertResponse(503, b'MAIL FROM:<correct.mail@test.de>\r\n')
+        self.assertResponse([x for x in range(500, 510)], b'MAIL FROM:<correct.mail@test.de>\r\n')
 
     def test_mail_double_send(self):
-        self.assertResponse(250, b'EHLO localDomain\r\n')
+        self.assertResponse(250, b'HELO localDomain\r\n')
         self.assertResponse(250, b'MAIL FROM:<correct.mail@test.de>\r\n')
         self.assertResponse(503, b'MAIL FROM:<correct2.mail@test.de>\r\n')
 
     def test_invalid_state_transition_03(self):
-        self.assertResponse(250, b'EHLO localhost\r\n')
+        self.assertResponse(250, b'HELO localhost\r\n')
         self.assertResponse(250, b'MAIL FROM:<leonard@marschke.me>\r\n')
-        self.assertResponse(503, b'DATA\r\n')
+        self.assertResponse([503, 554], b'DATA\r\n')
         self.assertResponse(250, b'RSET\r\n')
 
     def test_invalid_state_transition_04(self):
         self.assertResponse(250, b'RSET\r\n')
 
     def test_reset_operation(self):
-        self.assertResponse(250, b'EHLO localhost\r\n')
+        self.assertResponse(250, b'HELO localhost\r\n')
         self.assertResponse(250, b'MAIL FROM:<leonard@marschke.me>\r\n')
-        self.assertResponse(250, b'RCPT TO:<firstAddress@localhost>\r\n')
+        self.assertResponse(250, b'RCPT TO:<firstAddress@server.marschke.me>\r\n')
         self.assertResponse(250, b'RSET\r\n')
         self.assertResponse(503, b'DATA\r\n')
 
     def test_reset_operation_parameter(self):
-        self.assertResponse(250, b'EHLO localhost\r\n')
+        self.assertResponse(250, b'HELO localhost\r\n')
         self.assertResponse(250, b'MAIL FROM:<leonard@marschke.me>\r\n')
-        self.assertResponse(250, b'RCPT TO:<firstAddress@localhost>\r\n')
-        self.assertResponse(250, b'RSET MAIL FROM:<leonard@marschke.me>\r\n')
+        self.assertResponse(250, b'RCPT TO:<firstAddress@server.marschke.me>\r\n')
+        self.assertResponse(501, b'RSET MAIL FROM:<leonard@marschke.me>\r\n')
+        self.assertResponse(250, b'RSET\r\n')
         self.assertResponse(503, b'DATA\r\n')
 
     def test_noop_operation(self):
@@ -81,17 +82,25 @@ class TestSession(BaseTest):
         self.assertResponse(252, b'VRFY leonard@marschke.me\r\n')
 
     def test_multiple_token_wrong(self):
-        self.assertResponse(500, b'VERFY BLA BLUB')
+        self.assertResponse([500, 502], b'VERFY BLA BLUB\r\n')
 
     def test_token_wrong(self):
         self.assertResponse(500, b'VRFZ\r\n')
         self.assertResponse(250, b'NOOP\r\n')
 
+    def test_quit_operation(self):
+        self.assertResponse(250, b'HELO localDomain\r\n')
+
+        self.assertResponse(221, b'QUIT\r\n')
+
+        with self.assertRaises(AssertionError):
+            self.assertResponse([x for x in range(500, 510)], b'NONEXISTANT\r\n')
+
     def test_parse_helo_correct(self):
         self.assertResponse(250, b'HELO marschke.me\r\n')
 
     def test_invalid_state_transition_01(self):
-        self.assertResponse(503, b'RCPT TO:<firstAddress@localhost>\r\n')
+        self.assertResponse(503, b'RCPT TO:<firstAddress@server.marschke.me>\r\n')
 
     def test_invalid_state_transition_02(self):
         self.assertResponse(503, b'DATA\r\n')

tools/result_aggregation.py

+#!/usr/bin/env python3
+# pylint: disable=invalid-name
+
+import re
+from os import path
+
+from tabulate import tabulate
+
+groups = ['group-a', 'group-b', 'group-c', 'group-d', 'group-e', 'postfix']
+
+test_results = {}
+
+for group in groups:
+    group_result = {}
+
+    with open(path.join('results', group)) as file:
+        content = file.read()
+
+    for match in re.findall('(.*) \\((.*)\\) \\.\\.\\. (.*)', content):
+        group_result[match[0] + ' (' + match[1] + ')'] = match[2]
+
+    test_results[group] = group_result
+
+rows = []
+
+for test in test_results[groups[0]]:
+    row = [test]
+
+    for group in groups:
+        row.append(test_results[group][test])
+
+    rows.append(row)
+
+header = [
+    'Test',
+]
+
+for group in groups:
+    header.append(group)
+
+print(tabulate(rows, header, tablefmt="grid"))

util/BaseTest.py

@@ -45,15 +45,18 @@ class BaseTest(unittest.TestCase):
 
             return form_message
 
-        if isinstance(expected_code, int):
-            expected_code = [expected_code]
-
         if message:
             if isinstance(message, str):
                 self.client.send(message.encode('ASCII'))
             else:
                 self.client.send(message)
 
+        if not expected_code:
+            return
+
+        if isinstance(expected_code, int):
+            expected_code = [expected_code]
+
         response = ''
         while not response.endswith('\r\n'):
             try:

util/TestServer.py

@@ -10,7 +10,7 @@ class TestServer:
 
     def __init__(self, hostname, port, container_template, docker_port, ssl_on=False, wait_up=0.5):
         self.hostname = hostname
-        self.domain = 'localhost'
+        self.domain = 'server.marschke.me'
         self.port = port
         self.container_template = container_template
         self.docker_port = docker_port
@@ -30,6 +30,8 @@ class TestServer:
                                                   '--rm',
                                                   '-p',
                                                   '{}:{}'.format(self.port, self.docker_port),
+                                                  '-e',
+                                                  'HOST_NAME={}'.format(self.domain),
                                                   self.container_template])[:-1]
         time.sleep(self.wait_up)