-
Notifications
You must be signed in to change notification settings - Fork 0
/
tests.py
executable file
·172 lines (136 loc) · 6.63 KB
/
tests.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
#!/usr/bin/python
from subprocess import Popen, PIPE, STDOUT
import os.path
import sys
#-------------- test cases --------------#
basic_test_data = [
#config error checking
("No Such Plugboard Config File Test", "", ["plugboards/nulll.pb"], "", "FAIL"),
#minimal config checking
("Minimal Config Test", "", ["plugboards/null.pb"], "", "PASS"),
#basic IO checking
("Basic Rotor I/O Test", "A",["rotors/I.rot","plugboards/I.pb"], "N", "PASS"),
("Invalid Input Test", "a", ["rotors/I.rot","plugboards/I.pb"], "N", "FAIL"),
#first char map checking
("First Char Mapping Test 1", "A", ["rotors/I.rot","plugboards/null.pb"], "N", "PASS"),
("First Char Mapping Test 2", "B", ["rotors/II.rot","plugboards/I.pb"], "M", "PASS"),
("First Char Mapping Test 3", "C", ["rotors/VII.rot","plugboards/I.pb"], "D", "PASS"),
("First Char Mapping Test 4", "D", ["rotors/III.rot","rotors/VII.rot","plugboards/II.pb"], "X", "PASS"),
]
advanced_test_data = [
#full reflector functionality checking
("Full Reflector Test", "ABCDEFGHIJKLMNOPQRSTUVWXYZ", ["plugboards/null.pb"] ,"NOPQRSTUVWXYZABCDEFGHIJKLM", "PASS"),
#full plugboard functionality checking
("Full Plugbaord Test 1", "ABCDEFGHIJKLMNOPQRSTUVWXYZ", ["plugboards/I.pb"], "NOPQRSTUMWXYIABCDEFGHZJKLV", "PASS"),
("Full Plugbaord Test 2", "ABCDEFGHIJKLMNOPQRSTUVWXYZ", ["plugboards/II.pb"], "NOPQISTUEWMYKABCDVFGHRJZLX", "PASS"),
#full basic rotor functionality checking
("Full Basic Rotor Test 1", "AAAAAAAAAAAAAAAAAAAAAAAAAA", ["rotors/I.rot", "plugboards/null.pb"], "NNNNNNNNNNNNNNNNNNNNNNNNNN", "PASS"),
("Full Basic Rotor Test 2", "AAAAAAAAAAAAAAAAAAAAAAAAAA", ["rotors/II.rot", "plugboards/null.pb"], "PLPLPLPLPLPLPLPLPLPLPLPLPL", "PASS"),
#full complex rotor functionality checking
("Full Rotor Test 1", "AAAAAAAAAAAAAAAAAAAAAAAAAA", ["rotors/III.rot", "plugboards/null.pb"], "LLFORJEVOKWPPKRFJMGQVJMQUR", "PASS"),
("Full Rotor Test 2", "DDDDDDDDDDDDDDDDDDDDDDDDDD", ["rotors/IV.rot", "plugboards/null.pb"], "KYRVLIQWULYGVOAGPVAQGLIASM", "PASS"),
("Full Rotor Test 3", "KLKLKLKLKLKLKLKLKLKLKLKLKL", ["rotors/V.rot", "plugboards/null.pb"], "PKJSRUTBANMJIWVSRNMJISRVUQ", "PASS"),
#full rotor cascade checking
("Rotor Cascade Test 1", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
["rotors/I.rot", "rotors/II.rot", "plugboards/null.pb"],
"LLLLLLLLLLLLLLLLLLLLLLLLLLPPPPPPPPPPPPPPPPPPPPPPPPPP", "PASS"),
("Rotor Cascade Test 2", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
["rotors/I.rot", "rotors/II.rot", "plugboards/IV.pb"],
"ZZZZZZZZZZZZZZZZZZZZZZZZZZXXXXXXXXXXXXXXXXXXXXXXXXXX", "PASS"),
("Rotor Cascade Test 3", "ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZ",
["rotors/II.rot", "rotors/II.rot", "plugboards/null.pb"],
"NSPURWTYVAXCZEBGDIFKHMJOLQROTQVSXUZWBYDAFCHEJGLINKPM", "PASS"),
("Rotor Cascade Test 4", "ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZ",
["rotors/II.rot", "rotors/II.rot", "plugboards/III.pb"],
"NQLWRUTCKVIYUEBGFXDKMJHRPSROPSVQIWGZBCQAFYMEHGNXLVTJ", "PASS"),
("Rotor Cascade Test 5", "ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZ",
["rotors/VI.rot", "rotors/IV.rot", "plugboards/II.pb"],
"MGDSBKVCZTGFAGSJKPFDIZGIGMFHLWODQOXBJBWIPZHQMCZUDJSN", "PASS")
]
unzip_call = Popen(["gunzip", "-c", "moby.txt.gz"], stdout=PIPE)
moby = unzip_call.communicate()[0]
robust_test_data = [
#robustness testing (will only pass if the implementation is "spot on")
("Robustness Test 1", moby, ["rotors/III.rot", "rotors/VII.rot", "plugboards/II.pb"],
"0315f65c4d79ad398a2c59ab638b81ba8fe34480bb8cce49cbe3908065c12527 -", "PASS")
]
#----------- helper functions -----------#
def check(name):
if not os.path.exists(name) :
print "Cannot find ", name, ", automated test aborted."
sys.exit(-1)
def run_tests(test_data, check_sha):
num_tests = len(test_data)
passed = 0
for test_case in test_data:
test_name = test_case[0]
message = test_case[1]
settings = test_case[2]
expected = test_case[3]
test_type = test_case[4]
enigma_call = Popen(["./enigma"] + settings, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
output = enigma_call.communicate(message)
status = enigma_call.returncode
actual = output[0].strip('\n')
inverse_call = Popen(["./enigma"] + settings, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
output = inverse_call.communicate(actual)
inverse = output[0].strip('\n')
inverse_error = (inverse != message)
if check_sha:
message = "moby.txt.gz"
sha_call = Popen(["sha256sum"], stdin=PIPE, stdout=PIPE)
actual = sha_call.communicate(actual)[0].strip('\n')
if test_type == "PASS" and ( actual != expected or status != 0 or (inverse_error and not check_sha) ):
print "\nFAILED: ", test_name
print "(Your enigma program should pass this test case)"
print " config:",
for arg in ["./enigma"] + settings:
print arg,
print ""
print " input: ", message
print " expected: ", expected
print " actual: ", actual
if not check_sha:
print " self inverse:", not inverse_error
print " exit code:", status
elif test_type == "FAIL" and actual == expected and status == 0:
print "\nFAILED: ", test_name
print "(Your enigma program should produce an error in this test case)"
print " config:",
for arg in ["./enigma"] + settings:
print arg,
print ""
print " input: ", message
print " not expecting:", expected
print " actual: ", actual
print " exit code: ", status
else:
passed += 1
return (passed,num_tests)
#----------- Main Test Script -----------#
print "---- Automated testing for the C++ Enigma exercise ----\n"
#compile the student's submission
print "> Running make:"
make_cmd = Popen(["make"], stdout=PIPE, stderr=STDOUT)
output = make_cmd.communicate()
status = make_cmd.returncode
print output[0]
if status != 0:
print "Failed to compile, automated test aborted."
sys.exit(-1)
#check binary was created and config files exist
check("enigma")
check("rotors")
check("plugboards")
#run the tests
print "> Running basic tests:"
basic_results = run_tests(basic_test_data, False)
print "> Running advanced tests:"
adv_results = run_tests(advanced_test_data, False)
print "> Running robustness tests:\n"
robust_results = run_tests(robust_test_data, True)
#print test summary
print "Automated Test Summary:"
print " Basic Tests: ", basic_results[0], "/", basic_results[1]
print " Advanced Tests: ", adv_results[0], "/", adv_results[1]
print " Robustness Tests:", robust_results[0], "/", robust_results[1]