introduce result aggregation

.gitlab-ci.yml

@@ -6,6 +6,7 @@ variables:
 
 stages:
   - test
+  - aggregation
 
 style:
   stage: test
@@ -21,8 +22,12 @@ groupA:
     - docker:dind
   script:
     - dockerfiles/A/build.sh
-    - python3 run.py testing.marschke.me 9001 sre18groupa/smtp-server-group-a 6666
-  allow_failure: true
+    - mkdir -p results
+    - python3 run.py testing.marschke.me 9001 sre18groupa/smtp-server-group-a 6666 | tee results/group-a
+  artifacts:
+    paths:
+      - results/group-a
+    expire_in: 1 week
 
 groupB:
   stage: test
@@ -30,7 +35,12 @@ groupB:
     - docker:dind
   script:
     - dockerfiles/B/build.sh
-    - python3 run.py testing.marschke.me 9001 sregroupb/smtp-server 8008
+    - mkdir -p results
+    - python3 run.py testing.marschke.me 9001 sregroupb/smtp-server 8008 | tee results/group-b
+  artifacts:
+    paths:
+      - results/group-b
+    expire_in: 1 week
   allow_failure: true
 
 groupC:
@@ -39,7 +49,12 @@ groupC:
     - docker:dind
   script:
     - dockerfiles/C/build.sh
-    - python3 run.py testing.marschke.me 9001 sregroupc/smtp-server 5555
+    - mkdir -p results
+    - python3 run.py testing.marschke.me 9001 sregroupc/smtp-server 5555 | tee results/group-c
+  artifacts:
+    paths:
+      - results/group-c
+    expire_in: 1 week
   allow_failure: true
 
 groupD:
@@ -48,7 +63,12 @@ groupD:
     - docker:dind
   script:
     - dockerfiles/D/build.sh
-    - python3 run.py testing.marschke.me 9001 sregroupd/sre-smtp-group-d 25
+    - mkdir -p results
+    - python3 run.py testing.marschke.me 9001 sregroupd/sre-smtp-group-d 25 | tee results/group-d
+  artifacts:
+    paths:
+      - results/group-d
+    expire_in: 1 week
   allow_failure: true
 
 groupE:
@@ -57,5 +77,16 @@ groupE:
     - docker:dind
   script:
     - dockerfiles/E/build.sh
-    - python3 run.py testing.marschke.me 9001 julianweise/simplesmtp 4431 --ssl --wait-up 2
+    - mkdir -p results
+    - python3 run.py testing.marschke.me 9001 julianweise/simplesmtp 4431 --ssl --wait-up 2 | tee results/group-e
+  artifacts:
+    paths:
+      - results/group-e
+    expire_in: 1 week
   allow_failure: true
+
+results:
+  stage: aggregation
+  script:
+    - pip3 install tabulate
+    - python3 tools/result_aggregation.py

tools/result_aggregation.py

+#!/usr/bin/env python3
+import re
+from os import path
+
+from tabulate import tabulate
+
+groups = ['a', 'b', 'c', 'd', 'e']
+
+test_results = {}
+
+for group in groups:
+    group_result = {}
+
+    with open(path.join('results', 'group-{}'.format(group))) as file:
+        content = file.read()
+
+    for match in re.findall('(.*) \((.*)\) \.\.\. (.*)', content):
+        group_result[match[0] + ' (' + match[1] + ')'] = match[2]
+
+    test_results[group] = group_result
+
+rows = []
+
+for test in test_results['a']:
+    rows.append([
+        test,
+        test_results['a'][test],
+        test_results['b'][test],
+        test_results['c'][test],
+        test_results['d'][test],
+        test_results['e'][test],
+    ])
+
+header = [
+    'Test',
+    'Group a',
+    'Group b',
+    'Group c',
+    'Group d',
+    'Group e',
+]
+
+print(tabulate(rows, header, tablefmt="grid"))