summer 2020 version of server_bench, take 1

tests/server_bench.py

@@ -0,0 +1,465 @@
+#!/usr/bin/env python3
+
+#
+# Benchmark an HTTP server with wrk
+#
+# @author gback; Spring 2016, Spring 2018, Summer 2002
+#
+
+import getopt, sys, os, subprocess, signal, re, json, resource, time, socket, atexit
+from collections import namedtuple
+
+from http.client import HTTPConnection, OK
+
+runconfig = namedtuple(
+    "runconfig",
+    [
+        "name",  # name of configuration
+        "nthreads",  # number of threads
+        "nconnections",  # number of conn per thread
+        "timeout",  # number of conn per thread
+        "window",  # number of conn per thread
+        "overlap",  # number of conn per thread
+        "description",  # description
+        "path",  # target path to be retrieved
+        "duration",
+    ],
+)  # duration (with unit as string, i.e. "1s")
+
+VERSION = "1.1"
+server_exe = "./server"
+server_root = "_serverroot_"
+wrk_exe = "/home/courses/cs3214/bin/wrk"
+nthreads = 64
+
+# tests will be run in this order
+tests = [
+    runconfig(
+        name="login40",
+        timeout="1s",
+        nthreads=40,
+        nconnections=40,
+        duration="10s",
+        path="/api/login",
+        description="""
+    Can your server handle 40 parallel connections request /api/login?
+    """,
+        window=500,
+        overlap=250,
+    ),
+    runconfig(
+        name="login500",
+        timeout="5s",
+        nthreads=nthreads,
+        nconnections=500,
+        duration="10s",
+        path="/api/login",
+        description="""
+    Using 500 connections, each of which is repeatedly requesting /api/login (~2bytes in
+    HTTP body).  We believe this should be enough to make the server CPU bound.
+    """,
+        window=1000,
+        overlap=500,
+    ),
+    runconfig(
+        name="login10k",
+        timeout="5s",
+        nthreads=nthreads,
+        nconnections=10000,
+        duration="30s",
+        path="/api/login",
+        description="""
+    Handling 10k simultaneous connections has been a target of scalability since 1999:
+    http://www.kegel.com/c10k.html
+    Can your server handle it?
+    """,
+        window=1000,
+        overlap=500,
+    ),
+    runconfig(
+        name="wwwcsvt100",
+        timeout="5s",
+        nthreads=nthreads,
+        nconnections=100,
+        duration="20s",
+        path="/www.cs.vt.edu-20200417.html",
+        description="""
+    The home page of the CS Department, as of 4/17/2020, is about 66KB large (not counting embedded objects).
+    If 100 clients accessed it simultaneously, how much throughput could they expect?
+    """,
+        window=1000,
+        overlap=500,
+    ),
+    runconfig(
+        name="doom100",
+        timeout="10s",
+        nthreads=40,
+        nconnections=40,
+        duration="20s",
+        path="/large",
+        description="""
+    According to https://mobiforge.com/research-analysis/the-web-is-doom the combined size of all
+    objects that make an average web page was 2,250kBytes as of April 2016. If these were transferred
+    all in a single objects, how much throughput would you get?
+    This should max out the 10Gbps Ethernet links, even with only 40 connections.
+    """,
+        window=500,
+        overlap=250,
+    ),
+]
+testsbyname = dict((c.name, c) for c in tests)
+teststorun = map(lambda t: t.name, tests)
+
+
+def listtests():
+    for test in tests:
+        print(
+            """
+Test:           %s
+Connections:    %d
+Duration:       %s
+Path:           %s
+Description:    %s
+"""
+            % (test.name, test.nconnections, test.duration, test.path, test.description)
+        )
+
+
+script_dir = "/".join(os.path.realpath(__file__).split("/")[:-1])
+if script_dir == "":
+    script_dir = "."
+
+script_dir = os.path.realpath(script_dir)
+
+
+def usage():
+    print(
+        """
+Usage: %s [-hv] [-l] [-s server] [-R serverroot] [-t test1,test2,...] [url]
+
+   -h                     display this help
+   -v                     run verbose
+   -s                     path to server executable, default %s
+   -R server_root         path to server root, default %s
+   -t test                run just the tests specified
+   -l                     list available tests with their descriptions
+   -i                     activate ink tracing tool
+   url                    URL where your server can be reached, i.e.
+                            http://hickory.rlogin:12306/
+
+This script must be started on two different rlogin nodes.
+On the first node, run it without a URL to start the server.
+
+Then run it on a second node with the URL printed out by the
+first run.
+
+    """
+        % (sys.argv[0], server_exe, server_root)
+    )
+
+
+try:
+    opts, args = getopt.getopt(sys.argv[1:], "ihvs:R:t:l", ["help", "verbose"])
+except getopt.GetoptError as err:
+    print(str(err))
+    usage()
+    sys.exit(2)
+
+verbose = False
+hostname = socket.gethostname()
+useInk = False
+
+for opt, arg in opts:
+    if opt == "-h":
+        usage()
+        sys.exit(0)
+    if opt == "-v":
+        verbose = True
+    elif opt == "-i":
+        useInk = True
+    elif opt == "-s":
+        server_exe = arg
+    elif opt == "-R":
+        server_root = arg
+    elif opt == "-l":
+        listtests()
+        sys.exit(0)
+    elif opt == "-t":
+        teststorun = arg.split(",")
+    else:
+        assert False, "unhandled option"
+
+
+def raise_fd_limit():
+    print("I will now try to raise the file descriptor limit")
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    print("Your server process can open %d file descriptors simultaneously." % soft)
+
+
+def raise_thread_limit():
+    print("I will now try to raise the max number of threads you can spawn")
+    soft, hard = resource.getrlimit(resource.RLIMIT_NPROC)
+    resource.setrlimit(resource.RLIMIT_NPROC, (hard, hard))
+    soft, hard = resource.getrlimit(resource.RLIMIT_NPROC)
+    print("Your server process can spawn %d threads simultaneously." % soft)
+
+
+#
+# Start the server.
+#
+def start_server(root_dir):
+    print("I will now prepare your server for benchmarking.")
+    if not os.access(server_exe, os.X_OK):
+        print("Did not find server executable: %s" % (server_exe))
+        sys.exit(-1)
+
+    # prepare files to be served
+    print("I will use the directory %s to store 2 files" % (root_dir))
+    if not os.access(root_dir, os.W_OK):
+        os.mkdir(root_dir)
+
+    def make_synthetic_content(sz):
+        return "0123456789ABCDEF" * int(sz / 16)
+
+    def write_file(name, content):
+        with open("%s/%s" % (root_dir, name), "wb") as sfile:
+            sfile.write(content.encode("utf-8"))
+            sfile.close()
+
+    sfilecontent = make_synthetic_content(1024)
+    write_file("small", sfilecontent)
+    lfilecontent = make_synthetic_content(2250 * 1024)
+    write_file("large", lfilecontent)
+    wwwcscont = open("%s/res/www.cs.vt.edu-20200417.html" % script_dir).read()
+    write_file("www.cs.vt.edu-20200417.html", wwwcscont)
+
+    port = (os.getpid() % 10000) + 20000
+
+    cmd = [server_exe, "-p", str(port), "-R", root_dir, "-s"]
+
+    raise_fd_limit()
+    raise_thread_limit()
+
+    server = subprocess.Popen(cmd, stdout=open(os.devnull, "w"), stderr=sys.stderr)
+
+    def clean_up_testing():
+        try:
+            os.kill(server.pid, signal.SIGKILL)
+        except:
+            pass
+
+    atexit.register(clean_up_testing)
+
+    print("I will now test that your server works.")
+
+    def test_server():
+        http_conn = HTTPConnection(hostname, port)
+        http_conn.connect()
+        for url, expected in zip(
+            ["/small", "/large", "/www.cs.vt.edu-20200417.html", "/api/login"],
+            [sfilecontent, lfilecontent, wwwcscont, "{}"],
+        ):
+            http_conn.request("GET", url)
+            server_response = http_conn.getresponse()
+            sfile = server_response.read().decode("utf-8")
+            if server_response.status != OK:
+                print(
+                    "Server returned %s for %s, expected %d."
+                    % (server_response.status, url, OK)
+                )
+                sys.exit(-1)
+
+            if (
+                isinstance(expected, int)
+                and len(sfile) >= expected
+                or isinstance(expected, (str, bytes))
+                and sfile == expected
+            ):
+                print("Retrieved %s ok." % (url))
+            else:
+                print("Did not find expected content at %s." % (url))
+                sys.exit(-1)
+
+        http_conn.close()
+
+    for tries in range(10):
+        try:
+            time.sleep(1)
+            test_server()
+            break
+        except Exception as e:
+            print(f'starting server failed: {e}')
+            pass
+
+    if tries == 9:
+        print("Your server did not start, giving up after 10 tries")
+        sys.exit(0)
+
+    this_script = os.path.realpath(sys.argv[0])
+    print(
+        f"""
+Congratulations, you are now ready to run the benchmark!
+Now, find another unloaded rlogin machine and run:
+
+{this_script} http://{hostname}:{port}/
+
+To use the ink tool, instead run 
+
+{this_script} -i http://{hostname}:{port}/
+
+When you are done, don't forget to hit ^C here.
+
+Your server's stdout is going to /dev/null.
+Your server's stderr is going to the driver's stderr.
+"""
+    )
+    sys.stdout.flush()
+    server.wait()
+
+
+def start_wrk(url, test):
+
+    cmd = [ wrk_exe ]
+    cmd += [] if useInk else ['--no-trace']
+    cmd += [
+        "-c", str(test.nconnections),
+        "-t", str(test.nthreads),
+        "-d", test.duration,
+        "-r", test.name + ".tar",
+        "-x", test.timeout,
+        "-w", str(test.window),
+        "-o", str(test.overlap),
+        "-s", script_dir + "/cs3214bench.lua",
+        url + test.path,
+    ]
+    if verbose:
+        print("I will now run", " ".join(cmd))
+
+    resfile = "ssresults.json"
+    luajson = "%s/JSON.lua" % script_dir
+    assert os.access(luajson, os.R_OK)
+    server = subprocess.Popen(
+        cmd,
+        stdout=sys.stdout,
+        stderr=sys.stderr,
+        env=dict(os.environ, JSON_OUTPUT_FILE=resfile, JSON_LUA=luajson),
+    )
+    server.wait()
+    with open(resfile) as jfile:
+        r = json.load(jfile)
+        os.unlink(resfile)
+        return r
+
+
+if len(args) == 0:
+    start_server(server_root)
+else:
+    url = args[0]
+    # strip ending / since the path args contain them
+    while url.endswith("/"):
+        url = url[:-1]
+
+    if hostname in url:
+        print("Please do not start the client on the same machine as the server.")
+        sys.exit(-1)
+
+    raise_fd_limit()
+    raise_thread_limit()
+    results = dict(version=VERSION)
+    ran = []
+    for testname in teststorun:
+        ran.append(testname)
+        if testname not in testsbyname:
+            print("Test: %s not found, skipping" % testname)
+            continue
+
+        test = testsbyname[testname]
+        print("Now running test: %s\n" % (testname))
+        try:
+            results[testname] = start_wrk(url, test)
+        except Exception as e:
+            print("An exception occurred %s, skipping this test" % (str(e)))
+
+    ofilename = "pserv.results.%d.json" % (os.getpid())
+    print("Writing results to %s" % ofilename)
+    with open(ofilename, "w") as ofile:
+        json.dump(results, ofile)
+
+    print(
+        """
+    Submit your results to the scoreboard with ~cs3214/bin/sspostresults.py %s
+        """
+        % ofilename
+    )
+
+    with open(ofilename, "r") as f:
+        data = json.load(f)
+
+    score = 0
+    # the following rubric encode the performance expectations for this semester
+    # there are 20 pts currently, 4 pts per benchmark.
+    #
+    # If errors (p) is set, deduct p pts if there are one or more errors
+    #
+    # If served (a, b) is set, deduct b pts unless a fraction of a clients was served
+    #
+    rubric = {
+        # 200 or more yields 4 pts, 100 or more yields 2 points
+        "login40": {"rps": [(320, 4), (160, 2)], "errors": -1},
+        "login500": {"rps": [(800, 4), (500, 2)], "errors": -1},
+        "login10k": {"rps": [(650, 4), (450, 2)], "served": (0.80, 2)},
+        # these max out the 10GBps link, so these are MByte/s
+        "wwwcsvt100": {"mbps": [(900, 4), (800, 2)]},
+        "doom100": {"mbps": [(900, 4), (800, 2)]},
+    }
+    extra = ""
+    for test in ran:
+        points = rubric[test]
+        category = 0
+        if "rps" in points:
+            rps = 1e3 * (
+                data[test]["summary"]["requests"] / data[test]["summary"]["duration"]
+            )
+            for requiredmin, value in points["rps"]:
+                if rps > requiredmin:
+                    category += value
+                    break
+            if test == "login10k" and rps > 900:
+                extra = "+10 points extra credit for login10k! If your error count isn't > 5,000..."
+        if "mbps" in points:
+            mbps = (
+                1e6
+                * data[test]["summary"]["bytes"]
+                / data[test]["summary"]["duration"]
+                / 1024
+                / 1024
+            )
+            for requiredmin, value in points["mbps"]:
+                if mbps > requiredmin:
+                    category += value
+                    break
+        if "served" in points:
+            (threshold, deduction) = points["served"]
+            percent = (
+                data[test]["summary"]["served"] / data[test]["summary"]["connections"]
+            )
+            if percent < threshold:
+                category = max(category - deduction, 0)
+        if "errors" in rubric[test]:
+            errors = sum(data[test]["summary"]["errors"].values())
+            if errors > 0:
+                category = max(category - rubric[test], 0)
+        print("%s: %d/4" % (test, category))
+        score += category
+
+    print("Your server got a performance score of %d/20" % score)
+    if extra != "":
+        print(extra)
+
+    print(
+        """
+    Submit your individual <test_name>.tar reports with ~cs3214/bin/p4api.sh\nThis will return a link to visualize your server's performance.
+        """
+    )