...
 
Commits (15)
......@@ -6,6 +6,7 @@ variables:
stages:
- test
- aggregation
style:
stage: test
......@@ -21,8 +22,12 @@ groupA:
- docker:dind
script:
- dockerfiles/A/build.sh
- python3 run.py testing.marschke.me 9001 sre18groupa/smtp-server-group-a 6666
allow_failure: true
- mkdir -p results
- python3 run.py testing.marschke.me 9001 sre18groupa/smtp-server-group-a 6666 2>&1 | tee results/group-a
artifacts:
paths:
- results/group-a
expire_in: 1 week
groupB:
stage: test
......@@ -30,8 +35,12 @@ groupB:
- docker:dind
script:
- dockerfiles/B/build.sh
- python3 run.py testing.marschke.me 9001 sregroupb/smtp-server 8008
allow_failure: true
- mkdir -p results
- python3 run.py testing.marschke.me 9001 sregroupb/smtp-server 8008 2>&1 | tee results/group-b
artifacts:
paths:
- results/group-b
expire_in: 1 week
groupC:
stage: test
......@@ -39,8 +48,12 @@ groupC:
- docker:dind
script:
- dockerfiles/C/build.sh
- python3 run.py testing.marschke.me 9001 sregroupc/smtp-server 5555
allow_failure: true
- mkdir -p results
- python3 run.py testing.marschke.me 9001 sregroupc/smtp-server 5555 2>&1 | tee results/group-c
artifacts:
paths:
- results/group-c
expire_in: 1 week
groupD:
stage: test
......@@ -48,8 +61,12 @@ groupD:
- docker:dind
script:
- dockerfiles/D/build.sh
- python3 run.py testing.marschke.me 9001 sregroupd/sre-smtp-group-d 25
allow_failure: true
- mkdir -p results
- python3 run.py testing.marschke.me 9001 sregroupd/sre-smtp-group-d 25 2>&1 | tee results/group-d
artifacts:
paths:
- results/group-d
expire_in: 1 week
groupE:
stage: test
......@@ -57,5 +74,15 @@ groupE:
- docker:dind
script:
- dockerfiles/E/build.sh
- python3 run.py testing.marschke.me 9001 julianweise/simplesmtp 4431 --ssl --wait-up 2
allow_failure: true
- mkdir -p results
- python3 run.py testing.marschke.me 9001 julianweise/simplesmtp 4431 --ssl --wait-up 2 2>&1 | tee results/group-e
artifacts:
paths:
- results/group-e
expire_in: 1 week
results:
stage: aggregation
script:
- pip3 install tabulate
- python3 tools/result_aggregation.py
......@@ -4,8 +4,10 @@ at least the smtp servers of the other groups..
## Run Tests
`python3 run.py testing.marschke.me 9001 smtp-server-group-a 6666
```bash
python3 run.py testing.marschke.me 9001 smtp-server-group-a 6666
python3 run.py testing.marschke.me 9001 smtp-server-group-b 8008
python3 run.py testing.marschke.me 9001 smtp-server-group-c 5555
python3 run.py testing.marschke.me 9001 smtp-server-group-d 25
python3 run.py testing.marschke.me 9001 julianweise/simplesmtp 4431 --ssl --wait-up 2`
python3 run.py testing.marschke.me 9001 julianweise/simplesmtp 4431 --ssl --wait-up 2
```
import argparse
import sys
import unittest
from tests import TestDrops, TestFuzzing, TestMail, TestSession
......@@ -37,4 +36,4 @@ def main():
if __name__ == '__main__':
sys.exit(main())
main()
......@@ -35,5 +35,17 @@ class TestFuzzing(BaseTest):
def test_unrecognized_command(self):
self.assertResponse(500, 'FAIL\r\n')
def test_small_input(self):
fuzzy_input = 'A' * 10**1
self.assertResponse([x for x in range(500, 510)], '{}\r\n'.format(fuzzy_input))
def test_big_input(self):
fuzzy_input = 'A' * 10**3
self.assertResponse([x for x in range(500, 510)], '{}\r\n'.format(fuzzy_input))
def test_huge_input(self):
fuzzy_input = 'A' * 10**5
self.assertResponse([x for x in range(500, 510)], '{}\r\n'.format(fuzzy_input))
def test_sentry_issue_49(self):
self.assertResponse(500, b'\xff\xf4\xff\xfd\x06' + "\r\n".encode('ASCII'))
self.assertResponse([x for x in range(500, 510)], b'\xff\xf4\xff\xfd\x06' + "\r\n".encode('ASCII'))
......@@ -103,3 +103,74 @@ class TestMail(BaseTest):
'SOME_RANDOM_TEXT'])
self.assertResponse(221, 'QUIT\r\n')
def test_mail_many_recipients(self):
self.assertResponse(250, 'HELO marschke.me\r\n')
self.assertResponse(250, 'MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(250, 'RCPT TO:<firstAddress@{}>\r\n'.format(self.test_server.domain))
for i in range(10**3):
self.assertResponse(250, 'RCPT TO:<address{}@{}>\r\n'.format(i, self.test_server.domain))
self.assertResponse(354, 'DATA\r\n' + \
'Date: Thu, 10 May 2018 10:33:29 -0100\r\n' + \
'FROM: Leonard Marschke<leonard@marschke.me>\r\n' + \
'Subject: Testmail\r\n' + \
'To: firstaddress@{}\r\n'.format(self.test_server.domain) + \
'\r\n' + \
'Hi all, this is\r\n' + \
'a second line.\r\n' + \
'And a third.\r\n' + \
'Single point incoming.\r\n' + \
'..\r\n' + \
'BR\r\n')
self.assertResponse(250, '.\r\n')
self.assertResponse(221, 'QUIT\r\n')
def test_mail_many_mails(self):
for _ in range(10**3):
self.assertResponse(250, 'HELO marschke.me\r\n')
self.assertResponse(250, 'MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(250, 'RCPT TO:<firstAddress@{}>\r\n'.format(self.test_server.domain))
self.assertResponse(250, 'RCPT TO:<secondAddress@{}>\r\n'.format(self.test_server.domain))
self.assertResponse(354, 'DATA\r\n' + \
'Date: Thu, 10 May 2018 10:33:29 -0100\r\n' + \
'FROM: Leonard Marschke<leonard@marschke.me>\r\n' + \
'Subject: Testmail\r\n' + \
'To: firstaddress@{}\r\n'.format(self.test_server.domain) + \
'\r\n' + \
'Hi all, this is\r\n' + \
'a second line.\r\n' + \
'And a third.\r\n' + \
'Single point incoming.\r\n' + \
'..\r\n' + \
'BR\r\n')
self.assertResponse(250, '.\r\n')
self.assertResponse(221, 'QUIT\r\n')
def test_mail_huge_mails(self):
self.assertResponse(250, 'HELO marschke.me\r\n')
self.assertResponse(250, 'MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(250, 'RCPT TO:<firstAddress@{}>\r\n'.format(self.test_server.domain))
self.assertResponse(250, 'RCPT TO:<secondAddress@{}>\r\n'.format(self.test_server.domain))
huge_lines = ''
for i in range(10**3):
huge_line = 'Line {} {}\r\n'.format(str(i), 'A' * 10**2)
huge_lines += huge_line
self.assertResponse(354, 'DATA\r\n' + \
'Date: Thu, 10 May 2018 10:33:29 -0100\r\n' + \
'FROM: Leonard Marschke<leonard@marschke.me>\r\n' + \
'Subject: Testmail\r\n' + \
'To: firstaddress@{}\r\n'.format(self.test_server.domain) + \
'\r\n' + \
'{}\r\n'.format(huge_lines) + \
'..\r\n' + \
'BR\r\n')
self.assertResponse(250, '.\r\n')
self.assertResponse(221, 'QUIT\r\n')
......@@ -10,13 +10,13 @@ class TestSession(BaseTest):
self.assertResponse(500, 'FAIL\r\n')
def test_post_invalid_parameter_stability_0(self):
self.assertResponse(250, b'EHLO localhost\r\n')
def test_invalid_parameter_stability_0(self):
self.assertResponse(250, b'HELO localhost\r\n')
self.assertResponse(500, b'MAIL wrong input blabla\r\n')
self.assertResponse(250, b'MAIL FROM:<correct.mail@test.de>\r\n')
def test_post_invalid_parameter_stability_1(self):
self.assertResponse(250, b'EHLO localhost\r\n')
def test_invalid_parameter_stability_1(self):
self.assertResponse(250, b'HELO localhost\r\n')
self.assertResponse(250, b'MAIL FROM:<correct.mail@test.de>\r\n')
self.assertResponse(500, b'RCPT wrong input blabla\r\n')
self.assertResponse(250, b'RCPT TO:<correct@localhost>\r\n')
......@@ -43,12 +43,12 @@ class TestSession(BaseTest):
self.assertResponse(503, b'MAIL FROM:<correct.mail@test.de>\r\n')
def test_mail_double_send(self):
self.assertResponse(250, b'EHLO localDomain\r\n')
self.assertResponse(250, b'HELO localDomain\r\n')
self.assertResponse(250, b'MAIL FROM:<correct.mail@test.de>\r\n')
self.assertResponse(503, b'MAIL FROM:<correct2.mail@test.de>\r\n')
def test_invalid_state_transition_03(self):
self.assertResponse(250, b'EHLO localhost\r\n')
self.assertResponse(250, b'HELO localhost\r\n')
self.assertResponse(250, b'MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(503, b'DATA\r\n')
self.assertResponse(250, b'RSET\r\n')
......@@ -57,14 +57,14 @@ class TestSession(BaseTest):
self.assertResponse(250, b'RSET\r\n')
def test_reset_operation(self):
self.assertResponse(250, b'EHLO localhost\r\n')
self.assertResponse(250, b'HELO localhost\r\n')
self.assertResponse(250, b'MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(250, b'RCPT TO:<firstAddress@localhost>\r\n')
self.assertResponse(250, b'RSET\r\n')
self.assertResponse(503, b'DATA\r\n')
def test_reset_operation_parameter(self):
self.assertResponse(250, b'EHLO localhost\r\n')
self.assertResponse(250, b'HELO localhost\r\n')
self.assertResponse(250, b'MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(250, b'RCPT TO:<firstAddress@localhost>\r\n')
self.assertResponse(250, b'RSET MAIL FROM:<leonard@marschke.me>\r\n')
......@@ -88,7 +88,13 @@ class TestSession(BaseTest):
self.assertResponse(250, b'NOOP\r\n')
def test_quit_operation(self):
pass
self.assertResponse(250, b'HELO localDomain\r\n')
self.assertResponse(221, b'QUIT\r\n')
with self.assertRaises(AssertionError):
self.assertResponse([x for x in range(500, 510)], b'NONEXISTANT\r\n')
def test_parse_helo_correct(self):
self.assertResponse(250, b'HELO marschke.me\r\n')
......
#!/usr/bin/env python3
import re
from os import path
from tabulate import tabulate
groups = ['a', 'b', 'c', 'd', 'e']
test_results = {}
for group in groups:
group_result = {}
with open(path.join('results', 'group-{}'.format(group))) as file:
content = file.read()
for match in re.findall('(.*) \((.*)\) \.\.\. (.*)', content):
group_result[match[0] + ' (' + match[1] + ')'] = match[2]
test_results[group] = group_result
rows = []
for test in test_results['a']:
rows.append([
test,
test_results['a'][test],
test_results['b'][test],
test_results['c'][test],
test_results['d'][test],
test_results['e'][test],
])
header = [
'Test',
'Group a',
'Group b',
'Group c',
'Group d',
'Group e',
]
print(tabulate(rows, header, tablefmt="grid"))
......@@ -35,32 +35,38 @@ class BaseTest(unittest.TestCase):
raise self.failureException("{} not in {}: {}".format(value, expected_list, msg))
def assertResponse(self, expected_code, message=None): # pylint: disable=invalid-name
def format_message(message):
if not message or isinstance(message, bytes):
return message
if message[-2:] == '\r\n':
return message[:-2]
return message
def format_message(form_message):
if not form_message or isinstance(form_message, bytes):
return form_message
if form_message[-2:] == '\r\n':
form_message = form_message[:-2]
if len(form_message) > 100:
form_message = '{} ..[{}x].. {}'.format(form_message[:50], len(form_message)-100, form_message[-50:])
if isinstance(expected_code, int):
expected_code = [expected_code]
return form_message
if message:
if isinstance(message, str):
message = message.encode('ASCII')
self.client.send(message)
message = format_message(message)
self.client.send(message.encode('ASCII'))
else:
self.client.send(message)
if not expected_code:
return
if isinstance(expected_code, int):
expected_code = [expected_code]
response = ''
while not response.endswith('\r\n'):
try:
byte = self.client.recv(1)
except socket.timeout:
raise self.failureException("Timeout with message <{}>".format(message))
raise self.failureException("Timeout with message <{}>".format(format_message(message)))
if byte == b'':
raise self.failureException("No code raised, but expected {}, message <{}>".format(
expected_code, message))
expected_code, format_message(message)))
response += byte.decode('ASCII')
self.assertValueListEqual(int(response[:3]), expected_code, msg="Message <{}>".format(message))
self.assertValueListEqual(int(response[:3]), expected_code, msg="Message <{}>".format(format_message(message)))
if response[3] == '-':
self.assertResponse(expected_code)
......@@ -6,7 +6,7 @@ import time
class TestServer:
CONNECTION_TIMEOUT = 5
CONNECTION_TIMEOUT = 20
def __init__(self, hostname, port, container_template, docker_port, ssl_on=False, wait_up=0.5):
self.hostname = hostname
......@@ -37,7 +37,7 @@ class TestServer:
if not self.docker_id:
raise AssertionError('No docker ID set!')
subprocess.run(['docker', 'kill', self.docker_id], stdout=subprocess.PIPE)
subprocess.run(['docker', 'kill', self.docker_id], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
self.docker_id = None
def cleanup(self):
......