diff --git a/tests/server_bench.py b/tests/server_bench.py new file mode 100755 index 0000000..e792f2c --- /dev/null +++ b/tests/server_bench.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python3 + +# +# Benchmark an HTTP server with wrk +# +# @author gback; Spring 2016, Spring 2018, Summer 2002 +# + +import getopt, sys, os, subprocess, signal, re, json, resource, time, socket, atexit +from collections import namedtuple + +from http.client import HTTPConnection, OK + +runconfig = namedtuple( + "runconfig", + [ + "name", # name of configuration + "nthreads", # number of threads + "nconnections", # number of conn per thread + "timeout", # number of conn per thread + "window", # number of conn per thread + "overlap", # number of conn per thread + "description", # description + "path", # target path to be retrieved + "duration", + ], +) # duration (with unit as string, i.e. "1s") + +VERSION = "1.1" +server_exe = "./server" +server_root = "_serverroot_" +wrk_exe = "/home/courses/cs3214/bin/wrk" +nthreads = 64 + +# tests will be run in this order +tests = [ + runconfig( + name="login40", + timeout="1s", + nthreads=40, + nconnections=40, + duration="10s", + path="/api/login", + description=""" + Can your server handle 40 parallel connections request /api/login? + """, + window=500, + overlap=250, + ), + runconfig( + name="login500", + timeout="5s", + nthreads=nthreads, + nconnections=500, + duration="10s", + path="/api/login", + description=""" + Using 500 connections, each of which is repeatedly requesting /api/login (~2bytes in + HTTP body). We believe this should be enough to make the server CPU bound. + """, + window=1000, + overlap=500, + ), + runconfig( + name="login10k", + timeout="5s", + nthreads=nthreads, + nconnections=10000, + duration="30s", + path="/api/login", + description=""" + Handling 10k simultaneous connections has been a target of scalability since 1999: + http://www.kegel.com/c10k.html + Can your server handle it? + """, + window=1000, + overlap=500, + ), + runconfig( + name="wwwcsvt100", + timeout="5s", + nthreads=nthreads, + nconnections=100, + duration="20s", + path="/www.cs.vt.edu-20200417.html", + description=""" + The home page of the CS Department, as of 4/17/2020, is about 66KB large (not counting embedded objects). + If 100 clients accessed it simultaneously, how much throughput could they expect? + """, + window=1000, + overlap=500, + ), + runconfig( + name="doom100", + timeout="10s", + nthreads=40, + nconnections=40, + duration="20s", + path="/large", + description=""" + According to https://mobiforge.com/research-analysis/the-web-is-doom the combined size of all + objects that make an average web page was 2,250kBytes as of April 2016. If these were transferred + all in a single objects, how much throughput would you get? + This should max out the 10Gbps Ethernet links, even with only 40 connections. + """, + window=500, + overlap=250, + ), +] +testsbyname = dict((c.name, c) for c in tests) +teststorun = map(lambda t: t.name, tests) + + +def listtests(): + for test in tests: + print( + """ +Test: %s +Connections: %d +Duration: %s +Path: %s +Description: %s +""" + % (test.name, test.nconnections, test.duration, test.path, test.description) + ) + + +script_dir = "/".join(os.path.realpath(__file__).split("/")[:-1]) +if script_dir == "": + script_dir = "." + +script_dir = os.path.realpath(script_dir) + + +def usage(): + print( + """ +Usage: %s [-hv] [-l] [-s server] [-R serverroot] [-t test1,test2,...] [url] + + -h display this help + -v run verbose + -s path to server executable, default %s + -R server_root path to server root, default %s + -t test run just the tests specified + -l list available tests with their descriptions + -i activate ink tracing tool + url URL where your server can be reached, i.e. + http://hickory.rlogin:12306/ + +This script must be started on two different rlogin nodes. +On the first node, run it without a URL to start the server. + +Then run it on a second node with the URL printed out by the +first run. + + """ + % (sys.argv[0], server_exe, server_root) + ) + + +try: + opts, args = getopt.getopt(sys.argv[1:], "ihvs:R:t:l", ["help", "verbose"]) +except getopt.GetoptError as err: + print(str(err)) + usage() + sys.exit(2) + +verbose = False +hostname = socket.gethostname() +useInk = False + +for opt, arg in opts: + if opt == "-h": + usage() + sys.exit(0) + if opt == "-v": + verbose = True + elif opt == "-i": + useInk = True + elif opt == "-s": + server_exe = arg + elif opt == "-R": + server_root = arg + elif opt == "-l": + listtests() + sys.exit(0) + elif opt == "-t": + teststorun = arg.split(",") + else: + assert False, "unhandled option" + + +def raise_fd_limit(): + print("I will now try to raise the file descriptor limit") + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard)) + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + print("Your server process can open %d file descriptors simultaneously." % soft) + + +def raise_thread_limit(): + print("I will now try to raise the max number of threads you can spawn") + soft, hard = resource.getrlimit(resource.RLIMIT_NPROC) + resource.setrlimit(resource.RLIMIT_NPROC, (hard, hard)) + soft, hard = resource.getrlimit(resource.RLIMIT_NPROC) + print("Your server process can spawn %d threads simultaneously." % soft) + + +# +# Start the server. +# +def start_server(root_dir): + print("I will now prepare your server for benchmarking.") + if not os.access(server_exe, os.X_OK): + print("Did not find server executable: %s" % (server_exe)) + sys.exit(-1) + + # prepare files to be served + print("I will use the directory %s to store 2 files" % (root_dir)) + if not os.access(root_dir, os.W_OK): + os.mkdir(root_dir) + + def make_synthetic_content(sz): + return "0123456789ABCDEF" * int(sz / 16) + + def write_file(name, content): + with open("%s/%s" % (root_dir, name), "wb") as sfile: + sfile.write(content.encode("utf-8")) + sfile.close() + + sfilecontent = make_synthetic_content(1024) + write_file("small", sfilecontent) + lfilecontent = make_synthetic_content(2250 * 1024) + write_file("large", lfilecontent) + wwwcscont = open("%s/res/www.cs.vt.edu-20200417.html" % script_dir).read() + write_file("www.cs.vt.edu-20200417.html", wwwcscont) + + port = (os.getpid() % 10000) + 20000 + + cmd = [server_exe, "-p", str(port), "-R", root_dir, "-s"] + + raise_fd_limit() + raise_thread_limit() + + server = subprocess.Popen(cmd, stdout=open(os.devnull, "w"), stderr=sys.stderr) + + def clean_up_testing(): + try: + os.kill(server.pid, signal.SIGKILL) + except: + pass + + atexit.register(clean_up_testing) + + print("I will now test that your server works.") + + def test_server(): + http_conn = HTTPConnection(hostname, port) + http_conn.connect() + for url, expected in zip( + ["/small", "/large", "/www.cs.vt.edu-20200417.html", "/api/login"], + [sfilecontent, lfilecontent, wwwcscont, "{}"], + ): + http_conn.request("GET", url) + server_response = http_conn.getresponse() + sfile = server_response.read().decode("utf-8") + if server_response.status != OK: + print( + "Server returned %s for %s, expected %d." + % (server_response.status, url, OK) + ) + sys.exit(-1) + + if ( + isinstance(expected, int) + and len(sfile) >= expected + or isinstance(expected, (str, bytes)) + and sfile == expected + ): + print("Retrieved %s ok." % (url)) + else: + print("Did not find expected content at %s." % (url)) + sys.exit(-1) + + http_conn.close() + + for tries in range(10): + try: + time.sleep(1) + test_server() + break + except Exception as e: + print(f'starting server failed: {e}') + pass + + if tries == 9: + print("Your server did not start, giving up after 10 tries") + sys.exit(0) + + this_script = os.path.realpath(sys.argv[0]) + print( + f""" +Congratulations, you are now ready to run the benchmark! +Now, find another unloaded rlogin machine and run: + +{this_script} http://{hostname}:{port}/ + +To use the ink tool, instead run + +{this_script} -i http://{hostname}:{port}/ + +When you are done, don't forget to hit ^C here. + +Your server's stdout is going to /dev/null. +Your server's stderr is going to the driver's stderr. +""" + ) + sys.stdout.flush() + server.wait() + + +def start_wrk(url, test): + + cmd = [ wrk_exe ] + cmd += [] if useInk else ['--no-trace'] + cmd += [ + "-c", str(test.nconnections), + "-t", str(test.nthreads), + "-d", test.duration, + "-r", test.name + ".tar", + "-x", test.timeout, + "-w", str(test.window), + "-o", str(test.overlap), + "-s", script_dir + "/cs3214bench.lua", + url + test.path, + ] + if verbose: + print("I will now run", " ".join(cmd)) + + resfile = "ssresults.json" + luajson = "%s/JSON.lua" % script_dir + assert os.access(luajson, os.R_OK) + server = subprocess.Popen( + cmd, + stdout=sys.stdout, + stderr=sys.stderr, + env=dict(os.environ, JSON_OUTPUT_FILE=resfile, JSON_LUA=luajson), + ) + server.wait() + with open(resfile) as jfile: + r = json.load(jfile) + os.unlink(resfile) + return r + + +if len(args) == 0: + start_server(server_root) +else: + url = args[0] + # strip ending / since the path args contain them + while url.endswith("/"): + url = url[:-1] + + if hostname in url: + print("Please do not start the client on the same machine as the server.") + sys.exit(-1) + + raise_fd_limit() + raise_thread_limit() + results = dict(version=VERSION) + ran = [] + for testname in teststorun: + ran.append(testname) + if testname not in testsbyname: + print("Test: %s not found, skipping" % testname) + continue + + test = testsbyname[testname] + print("Now running test: %s\n" % (testname)) + try: + results[testname] = start_wrk(url, test) + except Exception as e: + print("An exception occurred %s, skipping this test" % (str(e))) + + ofilename = "pserv.results.%d.json" % (os.getpid()) + print("Writing results to %s" % ofilename) + with open(ofilename, "w") as ofile: + json.dump(results, ofile) + + print( + """ + Submit your results to the scoreboard with ~cs3214/bin/sspostresults.py %s + """ + % ofilename + ) + + with open(ofilename, "r") as f: + data = json.load(f) + + score = 0 + # the following rubric encode the performance expectations for this semester + # there are 20 pts currently, 4 pts per benchmark. + # + # If errors (p) is set, deduct p pts if there are one or more errors + # + # If served (a, b) is set, deduct b pts unless a fraction of a clients was served + # + rubric = { + # 200 or more yields 4 pts, 100 or more yields 2 points + "login40": {"rps": [(320, 4), (160, 2)], "errors": -1}, + "login500": {"rps": [(800, 4), (500, 2)], "errors": -1}, + "login10k": {"rps": [(650, 4), (450, 2)], "served": (0.80, 2)}, + # these max out the 10GBps link, so these are MByte/s + "wwwcsvt100": {"mbps": [(900, 4), (800, 2)]}, + "doom100": {"mbps": [(900, 4), (800, 2)]}, + } + extra = "" + for test in ran: + points = rubric[test] + category = 0 + if "rps" in points: + rps = 1e3 * ( + data[test]["summary"]["requests"] / data[test]["summary"]["duration"] + ) + for requiredmin, value in points["rps"]: + if rps > requiredmin: + category += value + break + if test == "login10k" and rps > 900: + extra = "+10 points extra credit for login10k! If your error count isn't > 5,000..." + if "mbps" in points: + mbps = ( + 1e6 + * data[test]["summary"]["bytes"] + / data[test]["summary"]["duration"] + / 1024 + / 1024 + ) + for requiredmin, value in points["mbps"]: + if mbps > requiredmin: + category += value + break + if "served" in points: + (threshold, deduction) = points["served"] + percent = ( + data[test]["summary"]["served"] / data[test]["summary"]["connections"] + ) + if percent < threshold: + category = max(category - deduction, 0) + if "errors" in rubric[test]: + errors = sum(data[test]["summary"]["errors"].values()) + if errors > 0: + category = max(category - rubric[test], 0) + print("%s: %d/4" % (test, category)) + score += category + + print("Your server got a performance score of %d/20" % score) + if extra != "": + print(extra) + + print( + """ + Submit your individual .tar reports with ~cs3214/bin/p4api.sh\nThis will return a link to visualize your server's performance. + """ + )