diff --git a/benchmarks/bench_clap.pdf b/benchmarks/bench_clap.pdf new file mode 100644 index 0000000..82a0fa2 Binary files /dev/null and b/benchmarks/bench_clap.pdf differ diff --git a/benchmarks/bench_clap.py b/benchmarks/bench_clap.py new file mode 100644 index 0000000..69d1f03 --- /dev/null +++ b/benchmarks/bench_clap.py @@ -0,0 +1,85 @@ +import argparse + +import numpy as np +import pyperf + + +def get_solvers(): + from laptools.clap import costs as costs_old + from laptools.clap_new import costs as costs_new + + return { + "clap_old": costs_old, + "clap_new": costs_new, + } + + +def time_func(n_inner_loops, solver, shape): + cost_matrix = np.random.random(shape) + t0 = pyperf.perf_counter() + for i in range(n_inner_loops): + solver(cost_matrix) + return pyperf.perf_counter() - t0 + + +def get_bench_name(size, solver_name): + return "{}x{}-{}".format(size[0], size[1], solver_name) + + +def parse_args(benchopts): + parser = argparse.ArgumentParser() + parser.add_argument( + "--min-row-size-pow", + type=int, + metavar="POW", + default=1, + help="Smallest number of rows is 2^POW.", + ) + parser.add_argument( + "--min-col-size-pow", + type=int, + metavar="POW", + default=1, + help="Smallest number of cols is 2^POW.", + ) + parser.add_argument( + "--max-row-size-pow", + type=int, + metavar="POW", + default=2, + help="Largest number of rows is 2^POW.", + ) + parser.add_argument( + "--max-col-size-pow", + type=int, + metavar="POW", + default=2, + help="Largest number of cols is 2^POW.", + ) + return parser.parse_args(benchopts) + + +def add_cmdline_args(cmd, args): + cmd.append("--") + cmd.extend(args.benchopts) + + +def main(): + runner = pyperf.Runner(add_cmdline_args=add_cmdline_args) + runner.argparser.add_argument("benchopts", nargs="*") + args = parse_args(runner.parse_args().benchopts) + + solvers = get_solvers() + sizes = [ + (n_rows, n_cols) + for n_rows in 2 ** np.arange(args.min_row_size_pow, args.max_row_size_pow + 1) + for n_cols in 2 ** np.arange(args.min_col_size_pow, args.max_col_size_pow + 1) + ] + for size in sizes: + for solver_name, solver_func in solvers.items(): + bench_name = get_bench_name(size, solver_name) + runner.bench_time_func(bench_name, time_func, solver_func, size) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/plot_clap.py b/benchmarks/plot_clap.py new file mode 100644 index 0000000..1b889be --- /dev/null +++ b/benchmarks/plot_clap.py @@ -0,0 +1,78 @@ +import argparse + +import matplotlib.pyplot as plt +import numpy as np +import pyperf + + +def get_solver_from_bench(bench): + """Extract the solver from a benchmark. + + Assumes each benchmark's name is of the format: + + "n_rows n_cols solver_name" + """ + _, solver = bench.get_name().split("-") + return solver + + +def get_size_from_bench(bench): + """Extract the problem size from a benchmark. + + Assumes each benchmark's name is of the format: + + "n_rows n_cols solver_name" + """ + size, _ = bench.get_name().split("-") + n_rows, n_cols = size.split("x") + return (int(n_rows), int(n_cols)) + + +def get_solver_to_benches(suite): + """Get a map from each solver to a list of its benchmarks.""" + solver_to_benches = {} + for bench in suite: + solver = get_solver_from_bench(bench) + if solver not in solver_to_benches: + solver_to_benches[solver] = [] + solver_to_benches[solver].append(bench) + return solver_to_benches + + +def get_data_from_benches(benches): + """Extract the problem size and runtimes from each benchmark.""" + sizes = [get_size_from_bench(bench) for bench in benches] + times = [bench.get_values() for bench in benches] + return np.array(sizes), np.array(times) + + +# TODO: can't do a loglog plot +def plot_suite(suite): + """Plot the performance of each solver.""" + solver_to_benches = get_solver_to_benches(suite) + for solver, benches in solver_to_benches.items(): + sizes, times = get_data_from_benches(benches) + medians = np.median(times, axis=1) + sizes_str = ["{}x{}".format(n_rows, n_cols) for n_rows, n_cols in sizes] + plt.plot(sizes_str, medians, label=solver) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("suitefile") + parser.add_argument("outputfile") + return parser.parse_args() + + +def main(): + args = parse_args() + suite = pyperf.BenchmarkSuite.load(args.suitefile) + plot_suite(suite) + plt.legend() + plt.xlabel("Problem size") + plt.ylabel("Time to solve") + plt.savefig(args.outputfile) + + +if __name__ == "__main__": + main() diff --git a/src/laptools/__init__.py b/src/laptools/__init__.py index acfedcb..67eeefd 100644 --- a/src/laptools/__init__.py +++ b/src/laptools/__init__.py @@ -13,4 +13,4 @@ from . import clap, clap_new, lap -__all__ = ["clap", "lap"] +__all__ = ["clap", "clap_new", "lap"] diff --git a/src/laptools/clap_new.py b/src/laptools/clap_new.py index 40ed234..9d9da2b 100644 --- a/src/laptools/clap_new.py +++ b/src/laptools/clap_new.py @@ -77,7 +77,7 @@ def costs(cost_matrix): # Since there are at least as many columns as rows, row_idxs should # be identical to np.arange(n_rows). We depend on this. row_idxs = np.arange(n_rows) - row4col, col4row, u, v = lap.solve(cost_matrix) + row4col, col4row, u, v = lap._solve(cost_matrix) # Column vector of costs of each assignment in the lsap solution. lsap_costs = cost_matrix[row_idxs, col4row] diff --git a/tox.ini b/tox.ini index 8d00cbf..6ec041d 100644 --- a/tox.ini +++ b/tox.ini @@ -69,6 +69,6 @@ extras = perf commands = # https://pyperf.readthedocs.io/en/latest/runner.html#runner-cli # tox -e perf -- --values=10 --processes=100 # Run 100 trials for each method on each of 10 different matrices - # tox -e perf -- -- --min-size-pow=3 --max-size-pow=5 # Use matrices ranging in size from 2^3 to 2^5 - python bench.py -o {envtmpdir}/bench.json --values=1 --processes=1 {posargs} - python plot.py {envtmpdir}/bench.json bench.pdf + # tox -e perf -- -- --min-row-size-pow=3 --max-row-size-pow=5 --min-col-size-pow=3 --max-col-size-pow=5 # Use matrices ranging in size from 2^3 to 2^5 + python bench_clap.py -o {envtmpdir}/bench.json --values=1 --processes=1 {posargs} + python plot_clap.py {envtmpdir}/bench.json bench_clap.pdf