Add benchmarks for clap solvers.

benchmarks/bench_clap.py

@@ -0,0 +1,85 @@
+import argparse
+
+import numpy as np
+import pyperf
+
+
+def get_solvers():
+    from laptools.clap import costs as costs_old
+    from laptools.clap_new import costs as costs_new
+
+    return {
+        "clap_old": costs_old,
+        "clap_new": costs_new,
+    }
+
+
+def time_func(n_inner_loops, solver, shape):
+    cost_matrix = np.random.random(shape)
+    t0 = pyperf.perf_counter()
+    for i in range(n_inner_loops):
+        solver(cost_matrix)
+    return pyperf.perf_counter() - t0
+
+
+def get_bench_name(size, solver_name):
+    return "{}x{}-{}".format(size[0], size[1], solver_name)
+
+
+def parse_args(benchopts):
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--min-row-size-pow",
+        type=int,
+        metavar="POW",
+        default=1,
+        help="Smallest number of rows is 2^POW.",
+    )
+    parser.add_argument(
+        "--min-col-size-pow",
+        type=int,
+        metavar="POW",
+        default=1,
+        help="Smallest number of cols is 2^POW.",
+    )
+    parser.add_argument(
+        "--max-row-size-pow",
+        type=int,
+        metavar="POW",
+        default=2,
+        help="Largest number of rows is 2^POW.",
+    )
+    parser.add_argument(
+        "--max-col-size-pow",
+        type=int,
+        metavar="POW",
+        default=2,
+        help="Largest number of cols is 2^POW.",
+    )
+    return parser.parse_args(benchopts)
+
+
+def add_cmdline_args(cmd, args):
+    cmd.append("--")
+    cmd.extend(args.benchopts)
+
+
+def main():
+    runner = pyperf.Runner(add_cmdline_args=add_cmdline_args)
+    runner.argparser.add_argument("benchopts", nargs="*")
+    args = parse_args(runner.parse_args().benchopts)
+
+    solvers = get_solvers()
+    sizes = [
+        (n_rows, n_cols)
+        for n_rows in 2 ** np.arange(args.min_row_size_pow, args.max_row_size_pow + 1)
+        for n_cols in 2 ** np.arange(args.min_col_size_pow, args.max_col_size_pow + 1)
+    ]
+    for size in sizes:
+        for solver_name, solver_func in solvers.items():
+            bench_name = get_bench_name(size, solver_name)
+            runner.bench_time_func(bench_name, time_func, solver_func, size)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/plot_clap.py

@@ -0,0 +1,78 @@
+import argparse
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pyperf
+
+
+def get_solver_from_bench(bench):
+    """Extract the solver from a benchmark.
+
+    Assumes each benchmark's name is of the format:
+
+        "n_rows n_cols solver_name"
+    """
+    _, solver = bench.get_name().split("-")
+    return solver
+
+
+def get_size_from_bench(bench):
+    """Extract the problem size from a benchmark.
+
+    Assumes each benchmark's name is of the format:
+
+        "n_rows n_cols solver_name"
+    """
+    size, _ = bench.get_name().split("-")
+    n_rows, n_cols = size.split("x")
+    return (int(n_rows), int(n_cols))
+
+
+def get_solver_to_benches(suite):
+    """Get a map from each solver to a list of its benchmarks."""
+    solver_to_benches = {}
+    for bench in suite:
+        solver = get_solver_from_bench(bench)
+        if solver not in solver_to_benches:
+            solver_to_benches[solver] = []
+        solver_to_benches[solver].append(bench)
+    return solver_to_benches
+
+
+def get_data_from_benches(benches):
+    """Extract the problem size and runtimes from each benchmark."""
+    sizes = [get_size_from_bench(bench) for bench in benches]
+    times = [bench.get_values() for bench in benches]
+    return np.array(sizes), np.array(times)
+
+
+# TODO: can't do a loglog plot
+def plot_suite(suite):
+    """Plot the performance of each solver."""
+    solver_to_benches = get_solver_to_benches(suite)
+    for solver, benches in solver_to_benches.items():
+        sizes, times = get_data_from_benches(benches)
+        medians = np.median(times, axis=1)
+        sizes_str = ["{}x{}".format(n_rows, n_cols) for n_rows, n_cols in sizes]
+        plt.plot(sizes_str, medians, label=solver)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("suitefile")
+    parser.add_argument("outputfile")
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    suite = pyperf.BenchmarkSuite.load(args.suitefile)
+    plot_suite(suite)
+    plt.legend()
+    plt.xlabel("Problem size")
+    plt.ylabel("Time to solve")
+    plt.savefig(args.outputfile)
+
+
+if __name__ == "__main__":
+    main()

src/laptools/__init__.py

@@ -13,4 +13,4 @@
 
 from . import clap, clap_new, lap
 
-__all__ = ["clap", "lap"]
+__all__ = ["clap", "clap_new", "lap"]

src/laptools/clap_new.py

@@ -77,7 +77,7 @@ def costs(cost_matrix):
     # Since there are at least as many columns as rows, row_idxs should
     # be identical to np.arange(n_rows). We depend on this.
     row_idxs = np.arange(n_rows)
-    row4col, col4row, u, v = lap.solve(cost_matrix)
+    row4col, col4row, u, v = lap._solve(cost_matrix)
 
     # Column vector of costs of each assignment in the lsap solution.
     lsap_costs = cost_matrix[row_idxs, col4row]

tox.ini

@@ -69,6 +69,6 @@ extras = perf
 commands =
     # https://pyperf.readthedocs.io/en/latest/runner.html#runner-cli
     # tox -e perf -- --values=10 --processes=100 # Run 100 trials for each method on each of 10 different matrices
-    # tox -e perf -- -- --min-size-pow=3 --max-size-pow=5 # Use matrices ranging in size from 2^3 to 2^5
-    python bench.py -o {envtmpdir}/bench.json --values=1 --processes=1 {posargs}
-    python plot.py {envtmpdir}/bench.json bench.pdf
+    # tox -e perf -- -- --min-row-size-pow=3 --max-row-size-pow=5 --min-col-size-pow=3 --max-col-size-pow=5 # Use matrices ranging in size from 2^3 to 2^5
+    python bench_clap.py -o {envtmpdir}/bench.json --values=1 --processes=1 {posargs}
+    python plot_clap.py {envtmpdir}/bench.json bench_clap.pdf