...
 
Commits (5)
......@@ -6,14 +6,16 @@ variables:
stages:
- test
- aggregation
style:
stage: test
script:
- wget https://sre18.pages.rechenknecht.net/misc/pylintrc -O .pylintrc
- pip3 install -r requirements.txt
- pip3 install -r requirements-dev.txt
- pip3 install pylint
- pylint run.py tests util
- pylint run.py tests util tools
groupA:
stage: test
......@@ -21,8 +23,12 @@ groupA:
- docker:dind
script:
- dockerfiles/A/build.sh
- python3 run.py testing.marschke.me 9001 sre18groupa/smtp-server-group-a 6666
allow_failure: true
- mkdir -p results
- python3 run.py testing.marschke.me 9001 sre18groupa/smtp-server-group-a 6666 2>&1 | tee results/group-a
artifacts:
paths:
- results/group-a
expire_in: 1 week
groupB:
stage: test
......@@ -30,8 +36,12 @@ groupB:
- docker:dind
script:
- dockerfiles/B/build.sh
- python3 run.py testing.marschke.me 9001 sregroupb/smtp-server 8008
allow_failure: true
- mkdir -p results
- python3 run.py testing.marschke.me 9001 sregroupb/smtp-server 8008 2>&1 | tee results/group-b
artifacts:
paths:
- results/group-b
expire_in: 1 week
groupC:
stage: test
......@@ -39,8 +49,12 @@ groupC:
- docker:dind
script:
- dockerfiles/C/build.sh
- python3 run.py testing.marschke.me 9001 sregroupc/smtp-server 5555
allow_failure: true
- mkdir -p results
- python3 run.py testing.marschke.me 9001 sregroupc/smtp-server 5555 2>&1 | tee results/group-c
artifacts:
paths:
- results/group-c
expire_in: 1 week
groupD:
stage: test
......@@ -48,8 +62,12 @@ groupD:
- docker:dind
script:
- dockerfiles/D/build.sh
- python3 run.py testing.marschke.me 9001 sregroupd/sre-smtp-group-d 25
allow_failure: true
- mkdir -p results
- python3 run.py testing.marschke.me 9001 sregroupd/sre-smtp-group-d 25 2>&1 | tee results/group-d
artifacts:
paths:
- results/group-d
expire_in: 1 week
groupE:
stage: test
......@@ -57,5 +75,28 @@ groupE:
- docker:dind
script:
- dockerfiles/E/build.sh
- python3 run.py testing.marschke.me 9001 julianweise/simplesmtp 4431 --ssl --wait-up 2
allow_failure: true
- mkdir -p results
- python3 run.py testing.marschke.me 9001 julianweise/simplesmtp 4431 --ssl --wait-up 2 2>&1 | tee results/group-e
artifacts:
paths:
- results/group-e
expire_in: 1 week
postfix:
stage: test
services:
- docker:dind
script:
- docker pull tozd/postfix
- mkdir -p results
- python3 run.py testing.marschke.me 9001 tozd/postfix 25 --wait-up 5 2>&1 | tee results/postfix
artifacts:
paths:
- results/postfix
expire_in: 1 week
results:
stage: aggregation
script:
- pip3 install -r requirements-dev.txt
- python3 tools/result_aggregation.py
import argparse
import sys
import unittest
from tests import TestDrops, TestFuzzing, TestMail, TestSession
......@@ -37,4 +36,4 @@ def main():
if __name__ == '__main__':
sys.exit(main())
main()
......@@ -26,15 +26,13 @@ class TestFuzzing(BaseTest):
while True:
try:
self.client.recv(1)
if self.client.recv(1) == b'':
return
except socket.timeout:
break
self.assertResponse([x for x in range(500, 510)], 'FAIL\r\n')
def test_unrecognized_command(self):
self.assertResponse(500, 'FAIL\r\n')
def test_small_input(self):
fuzzy_input = 'A' * 10**1
self.assertResponse([x for x in range(500, 510)], '{}\r\n'.format(fuzzy_input))
......
import subprocess
from util import BaseTest
......@@ -88,19 +87,19 @@ class TestMail(BaseTest):
def test_mail_save(self):
self.assertResponse(250, b'HELO localDomain\r\n')
self.assertResponse(250, b'MAIL FROM:<user0@test.de>\r\n')
self.assertResponse(250, b'RCPT TO:<user1@localhost>\r\n')
self.assertResponse(250, b'RCPT TO:<user2@localhost>\r\n')
self.assertResponse(250, b'RCPT TO:<user1@server.marschke.me>\r\n')
self.assertResponse(250, b'RCPT TO:<user2@server.marschke.me>\r\n')
self.assertResponse(354, b'DATA\r\n')
self.client.send(b'SOME_RANDOM_TEXT\r\n')
self.client.send(b'SOME_RANDOM_TEXT\r\n')
self.assertResponse(250, b'.\r\n')
subprocess.check_output(['docker',
'exec',
'{}'.format(self.test_server.docker_id.decode('utf-8')),
'grep',
'-r',
'SOME_RANDOM_TEXT'])
#subprocess.check_output(['docker',
# 'exec',
# '{}'.format(self.test_server.docker_id.decode('utf-8')),
# 'grep',
# '-r',
# 'SOME_RANDOM_TEXT'])
self.assertResponse(221, 'QUIT\r\n')
......@@ -109,7 +108,7 @@ class TestMail(BaseTest):
self.assertResponse(250, 'MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(250, 'RCPT TO:<firstAddress@{}>\r\n'.format(self.test_server.domain))
for i in range(10**3):
for i in range(800):
self.assertResponse(250, 'RCPT TO:<address{}@{}>\r\n'.format(i, self.test_server.domain))
self.assertResponse(354, 'DATA\r\n' + \
......
......@@ -8,18 +8,18 @@ class TestSession(BaseTest):
def test_ehlo(self):
self.assertResponse(250, 'EHLO marschke.me\r\n')
self.assertResponse(500, 'FAIL\r\n')
self.assertResponse([500, 502], 'FAIL\r\n')
def test_post_invalid_parameter_stability_0(self):
self.assertResponse(250, b'EHLO localhost\r\n')
self.assertResponse(500, b'MAIL wrong input blabla\r\n')
def test_invalid_parameter_stability_0(self):
self.assertResponse(250, b'HELO localhost\r\n')
self.assertResponse(501, b'MAIL wrong input blabla\r\n')
self.assertResponse(250, b'MAIL FROM:<correct.mail@test.de>\r\n')
def test_post_invalid_parameter_stability_1(self):
self.assertResponse(250, b'EHLO localhost\r\n')
def test_invalid_parameter_stability_1(self):
self.assertResponse(250, b'HELO localhost\r\n')
self.assertResponse(250, b'MAIL FROM:<correct.mail@test.de>\r\n')
self.assertResponse(500, b'RCPT wrong input blabla\r\n')
self.assertResponse(250, b'RCPT TO:<correct@localhost>\r\n')
self.assertResponse(501, b'RCPT wrong input blabla\r\n')
self.assertResponse(250, b'RCPT TO:<correct@server.marschke.me>\r\n')
def test_help(self):
self.assertResponse(250, b'HELP\r\n')
......@@ -40,34 +40,35 @@ class TestSession(BaseTest):
self.assertResponse(501, b'EHLO\r\n')
def test_mail_no_ehlo(self):
self.assertResponse(503, b'MAIL FROM:<correct.mail@test.de>\r\n')
self.assertResponse([x for x in range(500, 510)], b'MAIL FROM:<correct.mail@test.de>\r\n')
def test_mail_double_send(self):
self.assertResponse(250, b'EHLO localDomain\r\n')
self.assertResponse(250, b'HELO localDomain\r\n')
self.assertResponse(250, b'MAIL FROM:<correct.mail@test.de>\r\n')
self.assertResponse(503, b'MAIL FROM:<correct2.mail@test.de>\r\n')
def test_invalid_state_transition_03(self):
self.assertResponse(250, b'EHLO localhost\r\n')
self.assertResponse(250, b'HELO localhost\r\n')
self.assertResponse(250, b'MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(503, b'DATA\r\n')
self.assertResponse([503, 554], b'DATA\r\n')
self.assertResponse(250, b'RSET\r\n')
def test_invalid_state_transition_04(self):
self.assertResponse(250, b'RSET\r\n')
def test_reset_operation(self):
self.assertResponse(250, b'EHLO localhost\r\n')
self.assertResponse(250, b'HELO localhost\r\n')
self.assertResponse(250, b'MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(250, b'RCPT TO:<firstAddress@localhost>\r\n')
self.assertResponse(250, b'RCPT TO:<firstAddress@server.marschke.me>\r\n')
self.assertResponse(250, b'RSET\r\n')
self.assertResponse(503, b'DATA\r\n')
def test_reset_operation_parameter(self):
self.assertResponse(250, b'EHLO localhost\r\n')
self.assertResponse(250, b'HELO localhost\r\n')
self.assertResponse(250, b'MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(250, b'RCPT TO:<firstAddress@localhost>\r\n')
self.assertResponse(250, b'RSET MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(250, b'RCPT TO:<firstAddress@server.marschke.me>\r\n')
self.assertResponse(501, b'RSET MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(250, b'RSET\r\n')
self.assertResponse(503, b'DATA\r\n')
def test_noop_operation(self):
......@@ -81,17 +82,25 @@ class TestSession(BaseTest):
self.assertResponse(252, b'VRFY leonard@marschke.me\r\n')
def test_multiple_token_wrong(self):
self.assertResponse(500, b'VERFY BLA BLUB')
self.assertResponse([500, 502], b'VERFY BLA BLUB\r\n')
def test_token_wrong(self):
self.assertResponse(500, b'VRFZ\r\n')
self.assertResponse(250, b'NOOP\r\n')
def test_quit_operation(self):
self.assertResponse(250, b'HELO localDomain\r\n')
self.assertResponse(221, b'QUIT\r\n')
with self.assertRaises(AssertionError):
self.assertResponse([x for x in range(500, 510)], b'NONEXISTANT\r\n')
def test_parse_helo_correct(self):
self.assertResponse(250, b'HELO marschke.me\r\n')
def test_invalid_state_transition_01(self):
self.assertResponse(503, b'RCPT TO:<firstAddress@localhost>\r\n')
self.assertResponse(503, b'RCPT TO:<firstAddress@server.marschke.me>\r\n')
def test_invalid_state_transition_02(self):
self.assertResponse(503, b'DATA\r\n')
#!/usr/bin/env python3
# pylint: disable=invalid-name
import re
from os import path
from tabulate import tabulate
groups = ['group-a', 'group-b', 'group-c', 'group-d', 'group-e', 'postfix']
test_results = {}
for group in groups:
group_result = {}
with open(path.join('results', group)) as file:
content = file.read()
for match in re.findall('(.*) \\((.*)\\) \\.\\.\\. (.*)', content):
group_result[match[0] + ' (' + match[1] + ')'] = match[2]
test_results[group] = group_result
rows = []
for test in test_results[groups[0]]:
row = [test]
for group in groups:
row.append(test_results[group][test])
rows.append(row)
header = [
'Test',
]
for group in groups:
header.append(group)
print(tabulate(rows, header, tablefmt="grid"))
......@@ -45,15 +45,18 @@ class BaseTest(unittest.TestCase):
return form_message
if isinstance(expected_code, int):
expected_code = [expected_code]
if message:
if isinstance(message, str):
self.client.send(message.encode('ASCII'))
else:
self.client.send(message)
if not expected_code:
return
if isinstance(expected_code, int):
expected_code = [expected_code]
response = ''
while not response.endswith('\r\n'):
try:
......
......@@ -10,7 +10,7 @@ class TestServer:
def __init__(self, hostname, port, container_template, docker_port, ssl_on=False, wait_up=0.5):
self.hostname = hostname
self.domain = 'localhost'
self.domain = 'server.marschke.me'
self.port = port
self.container_template = container_template
self.docker_port = docker_port
......@@ -30,6 +30,8 @@ class TestServer:
'--rm',
'-p',
'{}:{}'.format(self.port, self.docker_port),
'-e',
'HOST_NAME={}'.format(self.domain),
self.container_template])[:-1]
time.sleep(self.wait_up)
......