Merge branch 'aggregate_results' into 'master' (003c3f55) · Commits · sre18 / break-it

.gitlab-ci.yml

+52 −11

Original line number	Diff line number	Diff line
		@@ -6,14 +6,16 @@ variables:

		stages:
		- test
		- aggregation

		style:
		stage: test
		script:
		- wget https://sre18.pages.rechenknecht.net/misc/pylintrc -O .pylintrc
		- pip3 install -r requirements.txt
		- pip3 install -r requirements-dev.txt
		- pip3 install pylint
		- pylint run.py tests util
		- pylint run.py tests util tools

		groupA:
		stage: test
		@@ -21,8 +23,12 @@ groupA:
		- docker:dind
		script:
		- dockerfiles/A/build.sh
		- python3 run.py testing.marschke.me 9001 sre18groupa/smtp-server-group-a 6666
		allow_failure: true
		- mkdir -p results
		- python3 run.py testing.marschke.me 9001 sre18groupa/smtp-server-group-a 6666 2>&1 \| tee results/group-a
		artifacts:
		paths:
		- results/group-a
		expire_in: 1 week

		groupB:
		stage: test
		@@ -30,8 +36,12 @@ groupB:
		- docker:dind
		script:
		- dockerfiles/B/build.sh
		- python3 run.py testing.marschke.me 9001 sregroupb/smtp-server 8008
		allow_failure: true
		- mkdir -p results
		- python3 run.py testing.marschke.me 9001 sregroupb/smtp-server 8008 2>&1 \| tee results/group-b
		artifacts:
		paths:
		- results/group-b
		expire_in: 1 week

		groupC:
		stage: test
		@@ -39,8 +49,12 @@ groupC:
		- docker:dind
		script:
		- dockerfiles/C/build.sh
		- python3 run.py testing.marschke.me 9001 sregroupc/smtp-server 5555
		allow_failure: true
		- mkdir -p results
		- python3 run.py testing.marschke.me 9001 sregroupc/smtp-server 5555 2>&1 \| tee results/group-c
		artifacts:
		paths:
		- results/group-c
		expire_in: 1 week

		groupD:
		stage: test
		@@ -48,8 +62,12 @@ groupD:
		- docker:dind
		script:
		- dockerfiles/D/build.sh
		- python3 run.py testing.marschke.me 9001 sregroupd/sre-smtp-group-d 25
		allow_failure: true
		- mkdir -p results
		- python3 run.py testing.marschke.me 9001 sregroupd/sre-smtp-group-d 25 2>&1 \| tee results/group-d
		artifacts:
		paths:
		- results/group-d
		expire_in: 1 week

		groupE:
		stage: test
		@@ -57,5 +75,28 @@ groupE:
		- docker:dind
		script:
		- dockerfiles/E/build.sh
		- python3 run.py testing.marschke.me 9001 julianweise/simplesmtp 4431 --ssl --wait-up 2
		allow_failure: true
		- mkdir -p results
		- python3 run.py testing.marschke.me 9001 julianweise/simplesmtp 4431 --ssl --wait-up 2 2>&1 \| tee results/group-e
		artifacts:
		paths:
		- results/group-e
		expire_in: 1 week

		postfix:
		stage: test
		services:
		- docker:dind
		script:
		- docker pull tozd/postfix
		- mkdir -p results
		- python3 run.py testing.marschke.me 9001 tozd/postfix 25 --wait-up 5 2>&1 \| tee results/postfix
		artifacts:
		paths:
		- results/postfix
		expire_in: 1 week

		results:
		stage: aggregation
		script:
		- pip3 install -r requirements-dev.txt
		- python3 tools/result_aggregation.py

requirements-dev.txt

0 → 100644

+1 −0

Original line number	Diff line number	Diff line
		tabulate

run.py

+1 −2

Original line number	Diff line number	Diff line
		import argparse
		import sys
		import unittest

		from tests import TestDrops, TestFuzzing, TestMail, TestSession
		@@ -37,4 +36,4 @@ def main():


		if __name__ == '__main__':
		sys.exit(main())
		main()

tests/TestFuzzing.py

+2 −4

Original line number	Diff line number	Diff line
		@@ -26,15 +26,13 @@ class TestFuzzing(BaseTest):

		while True:
		try:
		self.client.recv(1)
		if self.client.recv(1) == b'':
		return
		except socket.timeout:
		break

		self.assertResponse([x for x in range(500, 510)], 'FAIL\r\n')

		def test_unrecognized_command(self):
		self.assertResponse(500, 'FAIL\r\n')

		def test_small_input(self):
		fuzzy_input = 'A' * 10**1
		self.assertResponse([x for x in range(500, 510)], '{}\r\n'.format(fuzzy_input))

tests/TestMail.py

+9 −10

Original line number	Diff line number	Diff line
		import subprocess
		from util import BaseTest


		@@ -88,19 +87,19 @@ class TestMail(BaseTest):
		def test_mail_save(self):
		self.assertResponse(250, b'HELO localDomain\r\n')
		self.assertResponse(250, b'MAIL FROM:<user0@test.de>\r\n')
		self.assertResponse(250, b'RCPT TO:<user1@localhost>\r\n')
		self.assertResponse(250, b'RCPT TO:<user2@localhost>\r\n')
		self.assertResponse(250, b'RCPT TO:<user1@server.marschke.me>\r\n')
		self.assertResponse(250, b'RCPT TO:<user2@server.marschke.me>\r\n')
		self.assertResponse(354, b'DATA\r\n')
		self.client.send(b'SOME_RANDOM_TEXT\r\n')
		self.client.send(b'SOME_RANDOM_TEXT\r\n')
		self.assertResponse(250, b'.\r\n')

		subprocess.check_output(['docker',
		'exec',
		'{}'.format(self.test_server.docker_id.decode('utf-8')),
		'grep',
		'-r',
		'SOME_RANDOM_TEXT'])
		#subprocess.check_output(['docker',
		# 'exec',
		# '{}'.format(self.test_server.docker_id.decode('utf-8')),
		# 'grep',
		# '-r',
		# 'SOME_RANDOM_TEXT'])

		self.assertResponse(221, 'QUIT\r\n')

		@@ -109,7 +108,7 @@ class TestMail(BaseTest):
		self.assertResponse(250, 'MAIL FROM:<leonard@marschke.me>\r\n')
		self.assertResponse(250, 'RCPT TO:<firstAddress@{}>\r\n'.format(self.test_server.domain))

		for i in range(10**3):
		for i in range(800):
		self.assertResponse(250, 'RCPT TO:<address{}@{}>\r\n'.format(i, self.test_server.domain))

		self.assertResponse(354, 'DATA\r\n' + \