code_exec_server/test_reqs.py at main · cassanof/code_exec_server · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
"""
Testing program to send a bunch of reqs
"""
import time
import code_exec_reqs

CODE_PASS = """
assert True
"""


CODE_FAIL = """
assert False, "This should fail"
"""

print("###### Testing simple pass/fail cases ######")

pass_req = code_exec_reqs.exec_test("http://127.0.0.1:8000", CODE_PASS, "")
print(pass_req)

fail_req = code_exec_reqs.exec_test("http://127.0.0.1:8000", CODE_FAIL, "")
print(fail_req)

print("##### Testing large code payload ######")

CODE_PASS_LARGE = (CODE_PASS * 100_000) + "\nprint('done pass large')"
pass_req = code_exec_reqs.exec_test("http://127.0.0.1:8000", CODE_PASS_LARGE, "")
print(pass_req)

CODE_FAIL_LARGE = "\nprint('doing fail large')\n" + (CODE_FAIL * 100_000)
fail_req = code_exec_reqs.exec_test("http://127.0.0.1:8000", CODE_FAIL_LARGE, "")
print(fail_req)

print("##### Testing stdin capture ######")
CODE_STDIN = """
inp = input()
assert inp == "Hello, World!"
print("stdin works")
"""
stdin_req = code_exec_reqs.exec_test(
    "http://127.0.0.1:8000", CODE_STDIN, "", stdin="Hello, World!")
print(stdin_req)

print("##### Testing string escape cases ######")
CODE_ESCAPE = """
print("Hello,'' World!")
print('a\\nb\\nc\\nd\\nef')
print("abc'def")
"""
escape_req = code_exec_reqs.exec_test(
    "http://127.0.0.1:8000", CODE_ESCAPE, "")
print(escape_req)

print("##### Testing memory overflow case ######")
CODE_OVERFLOW = """
import resource # limit memory to 4GB
resource.setrlimit(resource.RLIMIT_AS, (4 * 1024**3, 4 * 1024**3))
a = []
while True:
    a.append([42]*100000)
"""
codeoverflow_req = code_exec_reqs.exec_test(
        "http://127.0.0.1:8000", CODE_OVERFLOW, "")
print(codeoverflow_req)

print("###### Testing child trying to kill parent process ######")
CODE_KILL = """
import os
print(os.kill(os.getppid(), 9))
"""
kill_req = code_exec_reqs.exec_test(
    "http://127.0.0.1:8000", CODE_KILL, "")
print(kill_req)

print("###### Testing multiple pass/fail cases with Python ######")

pass_req = code_exec_reqs.exec_test_multipl_e(
    "http://127.0.0.1:8000", CODE_PASS, "", "python"
)
print(pass_req)

fail_req = code_exec_reqs.exec_test_multipl_e(
    "http://127.0.0.1:8000", CODE_FAIL,  "", "python"
)
print(fail_req)

print("###### Testing batched pass/fail cases with Python ######")

# batched
codes = [
    CODE_PASS,
    CODE_FAIL,
    CODE_PASS,
    CODE_FAIL,
    CODE_PASS,
]
tests = ["" for _ in range(len(codes))]
batched_req = code_exec_reqs.exec_test_batched(
    "http://127.0.0.1:8000", codes, tests)
print(batched_req)
assert len(batched_req) == len(codes)


print("###### Testing multiple pass/fail cases with TypeScript. also capture stdout/stderr ######")

CODE_TS_PASS = """
console.log("Hello, World!");
"""

# needs to exit 1 to fail
CODE_TS_FAIL = """
console.error("This should fail");
process.exit(1);
"""

pass_req = code_exec_reqs.exec_test_multipl_e(
    "http://127.0.0.1:8000", CODE_TS_PASS, "", "ts"
)
print(pass_req)

fail_req = code_exec_reqs.exec_test_multipl_e(
    "http://127.0.0.1:8000", CODE_TS_FAIL, "", "ts"
)

print(fail_req)

print("###### Testing multiple pass/fail cases with JavaScript. also capture stdout/stderr ######")

pass_req = code_exec_reqs.exec_test_multipl_e(
    "http://127.0.0.1:8000", CODE_TS_PASS, "", "javascript"
)
print(pass_req)

fail_req = code_exec_reqs.exec_test_multipl_e(
    "http://127.0.0.1:8000", CODE_TS_FAIL, "", "javascript"
)

print(fail_req)

print("###### Testing batched pass/fail cases with TypeScript ######")

# batched
codes = [
    CODE_TS_PASS,
    CODE_TS_FAIL,
    CODE_TS_PASS,
    CODE_TS_FAIL,
    CODE_TS_PASS,
]

tests = ["" for _ in range(len(codes))]
batched_req = code_exec_reqs.exec_test_batched(
    "http://127.0.0.1:8000", codes, tests, "ts")
print(batched_req)


print("###### Testing timeout case with Python ######")
# timeout on python
CODE_TIMEOUT = """
while True:
    pass
"""
time_now = time.time()
timeout_req = code_exec_reqs.exec_test(
    "http://127.0.0.1:8000", CODE_TIMEOUT, "", timeout=5)
time_after = time.time()
print("Time elapsed (seconds):", time_after - time_now)
print(timeout_req)