summer 2020 version of server_bench, take 1
This commit is contained in:
Godmar Back
parent
3baf28ccae
commit
fac1440934
465
tests/server_bench.py
Executable file
465
tests/server_bench.py
Executable file
@ -0,0 +1,465 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
#
|
||||
# Benchmark an HTTP server with wrk
|
||||
#
|
||||
# @author gback; Spring 2016, Spring 2018, Summer 2002
|
||||
#
|
||||
|
||||
import getopt, sys, os, subprocess, signal, re, json, resource, time, socket, atexit
|
||||
from collections import namedtuple
|
||||
|
||||
from http.client import HTTPConnection, OK
|
||||
|
||||
runconfig = namedtuple(
|
||||
"runconfig",
|
||||
[
|
||||
"name", # name of configuration
|
||||
"nthreads", # number of threads
|
||||
"nconnections", # number of conn per thread
|
||||
"timeout", # number of conn per thread
|
||||
"window", # number of conn per thread
|
||||
"overlap", # number of conn per thread
|
||||
"description", # description
|
||||
"path", # target path to be retrieved
|
||||
"duration",
|
||||
],
|
||||
) # duration (with unit as string, i.e. "1s")
|
||||
|
||||
VERSION = "1.1"
|
||||
server_exe = "./server"
|
||||
server_root = "_serverroot_"
|
||||
wrk_exe = "/home/courses/cs3214/bin/wrk"
|
||||
nthreads = 64
|
||||
|
||||
# tests will be run in this order
|
||||
tests = [
|
||||
runconfig(
|
||||
name="login40",
|
||||
timeout="1s",
|
||||
nthreads=40,
|
||||
nconnections=40,
|
||||
duration="10s",
|
||||
path="/api/login",
|
||||
description="""
|
||||
Can your server handle 40 parallel connections request /api/login?
|
||||
""",
|
||||
window=500,
|
||||
overlap=250,
|
||||
),
|
||||
runconfig(
|
||||
name="login500",
|
||||
timeout="5s",
|
||||
nthreads=nthreads,
|
||||
nconnections=500,
|
||||
duration="10s",
|
||||
path="/api/login",
|
||||
description="""
|
||||
Using 500 connections, each of which is repeatedly requesting /api/login (~2bytes in
|
||||
HTTP body). We believe this should be enough to make the server CPU bound.
|
||||
""",
|
||||
window=1000,
|
||||
overlap=500,
|
||||
),
|
||||
runconfig(
|
||||
name="login10k",
|
||||
timeout="5s",
|
||||
nthreads=nthreads,
|
||||
nconnections=10000,
|
||||
duration="30s",
|
||||
path="/api/login",
|
||||
description="""
|
||||
Handling 10k simultaneous connections has been a target of scalability since 1999:
|
||||
http://www.kegel.com/c10k.html
|
||||
Can your server handle it?
|
||||
""",
|
||||
window=1000,
|
||||
overlap=500,
|
||||
),
|
||||
runconfig(
|
||||
name="wwwcsvt100",
|
||||
timeout="5s",
|
||||
nthreads=nthreads,
|
||||
nconnections=100,
|
||||
duration="20s",
|
||||
path="/www.cs.vt.edu-20200417.html",
|
||||
description="""
|
||||
The home page of the CS Department, as of 4/17/2020, is about 66KB large (not counting embedded objects).
|
||||
If 100 clients accessed it simultaneously, how much throughput could they expect?
|
||||
""",
|
||||
window=1000,
|
||||
overlap=500,
|
||||
),
|
||||
runconfig(
|
||||
name="doom100",
|
||||
timeout="10s",
|
||||
nthreads=40,
|
||||
nconnections=40,
|
||||
duration="20s",
|
||||
path="/large",
|
||||
description="""
|
||||
According to https://mobiforge.com/research-analysis/the-web-is-doom the combined size of all
|
||||
objects that make an average web page was 2,250kBytes as of April 2016. If these were transferred
|
||||
all in a single objects, how much throughput would you get?
|
||||
This should max out the 10Gbps Ethernet links, even with only 40 connections.
|
||||
""",
|
||||
window=500,
|
||||
overlap=250,
|
||||
),
|
||||
]
|
||||
testsbyname = dict((c.name, c) for c in tests)
|
||||
teststorun = map(lambda t: t.name, tests)
|
||||
|
||||
|
||||
def listtests():
|
||||
for test in tests:
|
||||
print(
|
||||
"""
|
||||
Test: %s
|
||||
Connections: %d
|
||||
Duration: %s
|
||||
Path: %s
|
||||
Description: %s
|
||||
"""
|
||||
% (test.name, test.nconnections, test.duration, test.path, test.description)
|
||||
)
|
||||
|
||||
|
||||
script_dir = "/".join(os.path.realpath(__file__).split("/")[:-1])
|
||||
if script_dir == "":
|
||||
script_dir = "."
|
||||
|
||||
script_dir = os.path.realpath(script_dir)
|
||||
|
||||
|
||||
def usage():
|
||||
print(
|
||||
"""
|
||||
Usage: %s [-hv] [-l] [-s server] [-R serverroot] [-t test1,test2,...] [url]
|
||||
|
||||
-h display this help
|
||||
-v run verbose
|
||||
-s path to server executable, default %s
|
||||
-R server_root path to server root, default %s
|
||||
-t test run just the tests specified
|
||||
-l list available tests with their descriptions
|
||||
-i activate ink tracing tool
|
||||
url URL where your server can be reached, i.e.
|
||||
http://hickory.rlogin:12306/
|
||||
|
||||
This script must be started on two different rlogin nodes.
|
||||
On the first node, run it without a URL to start the server.
|
||||
|
||||
Then run it on a second node with the URL printed out by the
|
||||
first run.
|
||||
|
||||
"""
|
||||
% (sys.argv[0], server_exe, server_root)
|
||||
)
|
||||
|
||||
|
||||
try:
|
||||
opts, args = getopt.getopt(sys.argv[1:], "ihvs:R:t:l", ["help", "verbose"])
|
||||
except getopt.GetoptError as err:
|
||||
print(str(err))
|
||||
usage()
|
||||
sys.exit(2)
|
||||
|
||||
verbose = False
|
||||
hostname = socket.gethostname()
|
||||
useInk = False
|
||||
|
||||
for opt, arg in opts:
|
||||
if opt == "-h":
|
||||
usage()
|
||||
sys.exit(0)
|
||||
if opt == "-v":
|
||||
verbose = True
|
||||
elif opt == "-i":
|
||||
useInk = True
|
||||
elif opt == "-s":
|
||||
server_exe = arg
|
||||
elif opt == "-R":
|
||||
server_root = arg
|
||||
elif opt == "-l":
|
||||
listtests()
|
||||
sys.exit(0)
|
||||
elif opt == "-t":
|
||||
teststorun = arg.split(",")
|
||||
else:
|
||||
assert False, "unhandled option"
|
||||
|
||||
|
||||
def raise_fd_limit():
|
||||
print("I will now try to raise the file descriptor limit")
|
||||
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
|
||||
resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
|
||||
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
|
||||
print("Your server process can open %d file descriptors simultaneously." % soft)
|
||||
|
||||
|
||||
def raise_thread_limit():
|
||||
print("I will now try to raise the max number of threads you can spawn")
|
||||
soft, hard = resource.getrlimit(resource.RLIMIT_NPROC)
|
||||
resource.setrlimit(resource.RLIMIT_NPROC, (hard, hard))
|
||||
soft, hard = resource.getrlimit(resource.RLIMIT_NPROC)
|
||||
print("Your server process can spawn %d threads simultaneously." % soft)
|
||||
|
||||
|
||||
#
|
||||
# Start the server.
|
||||
#
|
||||
def start_server(root_dir):
|
||||
print("I will now prepare your server for benchmarking.")
|
||||
if not os.access(server_exe, os.X_OK):
|
||||
print("Did not find server executable: %s" % (server_exe))
|
||||
sys.exit(-1)
|
||||
|
||||
# prepare files to be served
|
||||
print("I will use the directory %s to store 2 files" % (root_dir))
|
||||
if not os.access(root_dir, os.W_OK):
|
||||
os.mkdir(root_dir)
|
||||
|
||||
def make_synthetic_content(sz):
|
||||
return "0123456789ABCDEF" * int(sz / 16)
|
||||
|
||||
def write_file(name, content):
|
||||
with open("%s/%s" % (root_dir, name), "wb") as sfile:
|
||||
sfile.write(content.encode("utf-8"))
|
||||
sfile.close()
|
||||
|
||||
sfilecontent = make_synthetic_content(1024)
|
||||
write_file("small", sfilecontent)
|
||||
lfilecontent = make_synthetic_content(2250 * 1024)
|
||||
write_file("large", lfilecontent)
|
||||
wwwcscont = open("%s/res/www.cs.vt.edu-20200417.html" % script_dir).read()
|
||||
write_file("www.cs.vt.edu-20200417.html", wwwcscont)
|
||||
|
||||
port = (os.getpid() % 10000) + 20000
|
||||
|
||||
cmd = [server_exe, "-p", str(port), "-R", root_dir, "-s"]
|
||||
|
||||
raise_fd_limit()
|
||||
raise_thread_limit()
|
||||
|
||||
server = subprocess.Popen(cmd, stdout=open(os.devnull, "w"), stderr=sys.stderr)
|
||||
|
||||
def clean_up_testing():
|
||||
try:
|
||||
os.kill(server.pid, signal.SIGKILL)
|
||||
except:
|
||||
pass
|
||||
|
||||
atexit.register(clean_up_testing)
|
||||
|
||||
print("I will now test that your server works.")
|
||||
|
||||
def test_server():
|
||||
http_conn = HTTPConnection(hostname, port)
|
||||
http_conn.connect()
|
||||
for url, expected in zip(
|
||||
["/small", "/large", "/www.cs.vt.edu-20200417.html", "/api/login"],
|
||||
[sfilecontent, lfilecontent, wwwcscont, "{}"],
|
||||
):
|
||||
http_conn.request("GET", url)
|
||||
server_response = http_conn.getresponse()
|
||||
sfile = server_response.read().decode("utf-8")
|
||||
if server_response.status != OK:
|
||||
print(
|
||||
"Server returned %s for %s, expected %d."
|
||||
% (server_response.status, url, OK)
|
||||
)
|
||||
sys.exit(-1)
|
||||
|
||||
if (
|
||||
isinstance(expected, int)
|
||||
and len(sfile) >= expected
|
||||
or isinstance(expected, (str, bytes))
|
||||
and sfile == expected
|
||||
):
|
||||
print("Retrieved %s ok." % (url))
|
||||
else:
|
||||
print("Did not find expected content at %s." % (url))
|
||||
sys.exit(-1)
|
||||
|
||||
http_conn.close()
|
||||
|
||||
for tries in range(10):
|
||||
try:
|
||||
time.sleep(1)
|
||||
test_server()
|
||||
break
|
||||
except Exception as e:
|
||||
print(f'starting server failed: {e}')
|
||||
pass
|
||||
|
||||
if tries == 9:
|
||||
print("Your server did not start, giving up after 10 tries")
|
||||
sys.exit(0)
|
||||
|
||||
this_script = os.path.realpath(sys.argv[0])
|
||||
print(
|
||||
f"""
|
||||
Congratulations, you are now ready to run the benchmark!
|
||||
Now, find another unloaded rlogin machine and run:
|
||||
|
||||
{this_script} http://{hostname}:{port}/
|
||||
|
||||
To use the ink tool, instead run
|
||||
|
||||
{this_script} -i http://{hostname}:{port}/
|
||||
|
||||
When you are done, don't forget to hit ^C here.
|
||||
|
||||
Your server's stdout is going to /dev/null.
|
||||
Your server's stderr is going to the driver's stderr.
|
||||
"""
|
||||
)
|
||||
sys.stdout.flush()
|
||||
server.wait()
|
||||
|
||||
|
||||
def start_wrk(url, test):
|
||||
|
||||
cmd = [ wrk_exe ]
|
||||
cmd += [] if useInk else ['--no-trace']
|
||||
cmd += [
|
||||
"-c", str(test.nconnections),
|
||||
"-t", str(test.nthreads),
|
||||
"-d", test.duration,
|
||||
"-r", test.name + ".tar",
|
||||
"-x", test.timeout,
|
||||
"-w", str(test.window),
|
||||
"-o", str(test.overlap),
|
||||
"-s", script_dir + "/cs3214bench.lua",
|
||||
url + test.path,
|
||||
]
|
||||
if verbose:
|
||||
print("I will now run", " ".join(cmd))
|
||||
|
||||
resfile = "ssresults.json"
|
||||
luajson = "%s/JSON.lua" % script_dir
|
||||
assert os.access(luajson, os.R_OK)
|
||||
server = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=sys.stdout,
|
||||
stderr=sys.stderr,
|
||||
env=dict(os.environ, JSON_OUTPUT_FILE=resfile, JSON_LUA=luajson),
|
||||
)
|
||||
server.wait()
|
||||
with open(resfile) as jfile:
|
||||
r = json.load(jfile)
|
||||
os.unlink(resfile)
|
||||
return r
|
||||
|
||||
|
||||
if len(args) == 0:
|
||||
start_server(server_root)
|
||||
else:
|
||||
url = args[0]
|
||||
# strip ending / since the path args contain them
|
||||
while url.endswith("/"):
|
||||
url = url[:-1]
|
||||
|
||||
if hostname in url:
|
||||
print("Please do not start the client on the same machine as the server.")
|
||||
sys.exit(-1)
|
||||
|
||||
raise_fd_limit()
|
||||
raise_thread_limit()
|
||||
results = dict(version=VERSION)
|
||||
ran = []
|
||||
for testname in teststorun:
|
||||
ran.append(testname)
|
||||
if testname not in testsbyname:
|
||||
print("Test: %s not found, skipping" % testname)
|
||||
continue
|
||||
|
||||
test = testsbyname[testname]
|
||||
print("Now running test: %s\n" % (testname))
|
||||
try:
|
||||
results[testname] = start_wrk(url, test)
|
||||
except Exception as e:
|
||||
print("An exception occurred %s, skipping this test" % (str(e)))
|
||||
|
||||
ofilename = "pserv.results.%d.json" % (os.getpid())
|
||||
print("Writing results to %s" % ofilename)
|
||||
with open(ofilename, "w") as ofile:
|
||||
json.dump(results, ofile)
|
||||
|
||||
print(
|
||||
"""
|
||||
Submit your results to the scoreboard with ~cs3214/bin/sspostresults.py %s
|
||||
"""
|
||||
% ofilename
|
||||
)
|
||||
|
||||
with open(ofilename, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
score = 0
|
||||
# the following rubric encode the performance expectations for this semester
|
||||
# there are 20 pts currently, 4 pts per benchmark.
|
||||
#
|
||||
# If errors (p) is set, deduct p pts if there are one or more errors
|
||||
#
|
||||
# If served (a, b) is set, deduct b pts unless a fraction of a clients was served
|
||||
#
|
||||
rubric = {
|
||||
# 200 or more yields 4 pts, 100 or more yields 2 points
|
||||
"login40": {"rps": [(320, 4), (160, 2)], "errors": -1},
|
||||
"login500": {"rps": [(800, 4), (500, 2)], "errors": -1},
|
||||
"login10k": {"rps": [(650, 4), (450, 2)], "served": (0.80, 2)},
|
||||
# these max out the 10GBps link, so these are MByte/s
|
||||
"wwwcsvt100": {"mbps": [(900, 4), (800, 2)]},
|
||||
"doom100": {"mbps": [(900, 4), (800, 2)]},
|
||||
}
|
||||
extra = ""
|
||||
for test in ran:
|
||||
points = rubric[test]
|
||||
category = 0
|
||||
if "rps" in points:
|
||||
rps = 1e3 * (
|
||||
data[test]["summary"]["requests"] / data[test]["summary"]["duration"]
|
||||
)
|
||||
for requiredmin, value in points["rps"]:
|
||||
if rps > requiredmin:
|
||||
category += value
|
||||
break
|
||||
if test == "login10k" and rps > 900:
|
||||
extra = "+10 points extra credit for login10k! If your error count isn't > 5,000..."
|
||||
if "mbps" in points:
|
||||
mbps = (
|
||||
1e6
|
||||
* data[test]["summary"]["bytes"]
|
||||
/ data[test]["summary"]["duration"]
|
||||
/ 1024
|
||||
/ 1024
|
||||
)
|
||||
for requiredmin, value in points["mbps"]:
|
||||
if mbps > requiredmin:
|
||||
category += value
|
||||
break
|
||||
if "served" in points:
|
||||
(threshold, deduction) = points["served"]
|
||||
percent = (
|
||||
data[test]["summary"]["served"] / data[test]["summary"]["connections"]
|
||||
)
|
||||
if percent < threshold:
|
||||
category = max(category - deduction, 0)
|
||||
if "errors" in rubric[test]:
|
||||
errors = sum(data[test]["summary"]["errors"].values())
|
||||
if errors > 0:
|
||||
category = max(category - rubric[test], 0)
|
||||
print("%s: %d/4" % (test, category))
|
||||
score += category
|
||||
|
||||
print("Your server got a performance score of %d/20" % score)
|
||||
if extra != "":
|
||||
print(extra)
|
||||
|
||||
print(
|
||||
"""
|
||||
Submit your individual <test_name>.tar reports with ~cs3214/bin/p4api.sh\nThis will return a link to visualize your server's performance.
|
||||
"""
|
||||
)
|
||||
Loading…
x
Reference in New Issue
Block a user