Commit feb8fd7b authored by Leonard Marschke's avatar Leonard Marschke

introduce postfix as reference

parent 1cea3c85
Pipeline #6696 failed with stages
in 9 minutes and 37 seconds
......@@ -13,8 +13,9 @@ style:
script:
- wget https://sre18.pages.rechenknecht.net/misc/pylintrc -O .pylintrc
- pip3 install -r requirements.txt
- pip3 install -r requirements-dev.txt
- pip3 install pylint
- pylint run.py tests util
- pylint run.py tests util tools
groupA:
stage: test
......@@ -81,8 +82,21 @@ groupE:
- results/group-e
expire_in: 1 week
postfix:
stage: test
services:
- docker:dind
script:
- docker pull tozd/postfix
- mkdir -p results
- python3 run.py testing.marschke.me 9001 tozd/postfix 25 --wait-up 5 2>&1 | tee results/postfix
artifacts:
paths:
- results/postfix
expire_in: 1 week
results:
stage: aggregation
script:
- pip3 install tabulate
- pip3 install -r requirements-dev.txt
- python3 tools/result_aggregation.py
......@@ -26,15 +26,13 @@ class TestFuzzing(BaseTest):
while True:
try:
self.client.recv(1)
if self.client.recv(1) == b'':
return
except socket.timeout:
break
self.assertResponse([x for x in range(500, 510)], 'FAIL\r\n')
def test_unrecognized_command(self):
self.assertResponse(500, 'FAIL\r\n')
def test_small_input(self):
fuzzy_input = 'A' * 10**1
self.assertResponse([x for x in range(500, 510)], '{}\r\n'.format(fuzzy_input))
......
import subprocess
from util import BaseTest
......@@ -88,19 +87,19 @@ class TestMail(BaseTest):
def test_mail_save(self):
self.assertResponse(250, b'HELO localDomain\r\n')
self.assertResponse(250, b'MAIL FROM:<user0@test.de>\r\n')
self.assertResponse(250, b'RCPT TO:<user1@localhost>\r\n')
self.assertResponse(250, b'RCPT TO:<user2@localhost>\r\n')
self.assertResponse(250, b'RCPT TO:<user1@server.marschke.me>\r\n')
self.assertResponse(250, b'RCPT TO:<user2@server.marschke.me>\r\n')
self.assertResponse(354, b'DATA\r\n')
self.client.send(b'SOME_RANDOM_TEXT\r\n')
self.client.send(b'SOME_RANDOM_TEXT\r\n')
self.assertResponse(250, b'.\r\n')
subprocess.check_output(['docker',
'exec',
'{}'.format(self.test_server.docker_id.decode('utf-8')),
'grep',
'-r',
'SOME_RANDOM_TEXT'])
#subprocess.check_output(['docker',
# 'exec',
# '{}'.format(self.test_server.docker_id.decode('utf-8')),
# 'grep',
# '-r',
# 'SOME_RANDOM_TEXT'])
self.assertResponse(221, 'QUIT\r\n')
......@@ -109,7 +108,7 @@ class TestMail(BaseTest):
self.assertResponse(250, 'MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(250, 'RCPT TO:<firstAddress@{}>\r\n'.format(self.test_server.domain))
for i in range(10**3):
for i in range(800):
self.assertResponse(250, 'RCPT TO:<address{}@{}>\r\n'.format(i, self.test_server.domain))
self.assertResponse(354, 'DATA\r\n' + \
......
......@@ -8,18 +8,18 @@ class TestSession(BaseTest):
def test_ehlo(self):
self.assertResponse(250, 'EHLO marschke.me\r\n')
self.assertResponse(500, 'FAIL\r\n')
self.assertResponse([500, 502], 'FAIL\r\n')
def test_invalid_parameter_stability_0(self):
self.assertResponse(250, b'HELO localhost\r\n')
self.assertResponse(500, b'MAIL wrong input blabla\r\n')
self.assertResponse(501, b'MAIL wrong input blabla\r\n')
self.assertResponse(250, b'MAIL FROM:<correct.mail@test.de>\r\n')
def test_invalid_parameter_stability_1(self):
self.assertResponse(250, b'HELO localhost\r\n')
self.assertResponse(250, b'MAIL FROM:<correct.mail@test.de>\r\n')
self.assertResponse(500, b'RCPT wrong input blabla\r\n')
self.assertResponse(250, b'RCPT TO:<correct@localhost>\r\n')
self.assertResponse(501, b'RCPT wrong input blabla\r\n')
self.assertResponse(250, b'RCPT TO:<correct@server.marschke.me>\r\n')
def test_help(self):
self.assertResponse(250, b'HELP\r\n')
......@@ -40,7 +40,7 @@ class TestSession(BaseTest):
self.assertResponse(501, b'EHLO\r\n')
def test_mail_no_ehlo(self):
self.assertResponse(503, b'MAIL FROM:<correct.mail@test.de>\r\n')
self.assertResponse([x for x in range(500, 510)], b'MAIL FROM:<correct.mail@test.de>\r\n')
def test_mail_double_send(self):
self.assertResponse(250, b'HELO localDomain\r\n')
......@@ -50,7 +50,7 @@ class TestSession(BaseTest):
def test_invalid_state_transition_03(self):
self.assertResponse(250, b'HELO localhost\r\n')
self.assertResponse(250, b'MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(503, b'DATA\r\n')
self.assertResponse([503, 554], b'DATA\r\n')
self.assertResponse(250, b'RSET\r\n')
def test_invalid_state_transition_04(self):
......@@ -59,15 +59,16 @@ class TestSession(BaseTest):
def test_reset_operation(self):
self.assertResponse(250, b'HELO localhost\r\n')
self.assertResponse(250, b'MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(250, b'RCPT TO:<firstAddress@localhost>\r\n')
self.assertResponse(250, b'RCPT TO:<firstAddress@server.marschke.me>\r\n')
self.assertResponse(250, b'RSET\r\n')
self.assertResponse(503, b'DATA\r\n')
def test_reset_operation_parameter(self):
self.assertResponse(250, b'HELO localhost\r\n')
self.assertResponse(250, b'MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(250, b'RCPT TO:<firstAddress@localhost>\r\n')
self.assertResponse(250, b'RSET MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(250, b'RCPT TO:<firstAddress@server.marschke.me>\r\n')
self.assertResponse(501, b'RSET MAIL FROM:<leonard@marschke.me>\r\n')
self.assertResponse(250, b'RSET\r\n')
self.assertResponse(503, b'DATA\r\n')
def test_noop_operation(self):
......@@ -81,7 +82,7 @@ class TestSession(BaseTest):
self.assertResponse(252, b'VRFY leonard@marschke.me\r\n')
def test_multiple_token_wrong(self):
self.assertResponse(500, b'VERFY BLA BLUB')
self.assertResponse([500, 502], b'VERFY BLA BLUB\r\n')
def test_token_wrong(self):
self.assertResponse(500, b'VRFZ\r\n')
......@@ -99,7 +100,7 @@ class TestSession(BaseTest):
self.assertResponse(250, b'HELO marschke.me\r\n')
def test_invalid_state_transition_01(self):
self.assertResponse(503, b'RCPT TO:<firstAddress@localhost>\r\n')
self.assertResponse(503, b'RCPT TO:<firstAddress@server.marschke.me>\r\n')
def test_invalid_state_transition_02(self):
self.assertResponse(503, b'DATA\r\n')
#!/usr/bin/env python3
# pylint: disable=invalid-name
import re
from os import path
from tabulate import tabulate
groups = ['a', 'b', 'c', 'd', 'e']
groups = ['group-a', 'group-b', 'group-c', 'group-d', 'group-e', 'postfix']
test_results = {}
......@@ -14,7 +16,7 @@ for group in groups:
with open(path.join('results', 'group-{}'.format(group))) as file:
content = file.read()
for match in re.findall('(.*) \((.*)\) \.\.\. (.*)', content):
for match in re.findall('(.*) \\((.*)\\) \\.\\.\\. (.*)', content):
group_result[match[0] + ' (' + match[1] + ')'] = match[2]
test_results[group] = group_result
......@@ -22,22 +24,18 @@ for group in groups:
rows = []
for test in test_results['a']:
rows.append([
test,
test_results['a'][test],
test_results['b'][test],
test_results['c'][test],
test_results['d'][test],
test_results['e'][test],
])
row = [test]
for group in groups:
row.append(test_results[group][test])
rows.append(row)
header = [
'Test',
'Group a',
'Group b',
'Group c',
'Group d',
'Group e',
]
for group in groups:
header.append(group)
print(tabulate(rows, header, tablefmt="grid"))
......@@ -10,7 +10,7 @@ class TestServer:
def __init__(self, hostname, port, container_template, docker_port, ssl_on=False, wait_up=0.5):
self.hostname = hostname
self.domain = 'localhost'
self.domain = 'server.marschke.me'
self.port = port
self.container_template = container_template
self.docker_port = docker_port
......@@ -30,6 +30,8 @@ class TestServer:
'--rm',
'-p',
'{}:{}'.format(self.port, self.docker_port),
'-e',
'HOST_NAME={}'.format(self.domain),
self.container_template])[:-1]
time.sleep(self.wait_up)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment