project_6/cccl_upstream/benchmarks/scripts/sol.py

#!/usr/bin/env python3

import argparse
import os
import re

import cccl
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns


def is_finite(x):
    if isinstance(x, float):
        return x != np.inf and x != -np.inf
    return True


def filter_by_problem_size(df):
    min_elements_pow2 = 28
    if "Elements{io}[pow2]" in df.columns:
        df["Elements{io}[pow2]"] = df["Elements{io}[pow2]"].astype(int)
        df = df[df["Elements{io}[pow2]"] >= min_elements_pow2]
    return df


def filter_by_offset_type(df):
    if "OffsetT{ct}" in df.columns:
        filtered = df[
            (df["OffsetT{ct}"] == "I32") | (df["OffsetT{ct}"] == "U32")
        ]  # only use 32-bit offset types
        if not filtered.empty:  # some benchmarks only use a 64-bit offset type
            df = filtered
    return df


def filter_by_type(df):
    if "T{ct}" in df:
        # df = df[df['T{ct}'].str.contains('64')]
        df = df[~df["T{ct}"].str.contains("C")]
    elif "KeyT{ct}" in df:
        # df = df[df['KeyT{ct}'].str.contains('64')]
        df = df[~df["KeyT{ct}"].str.contains("C")]
    return df


def alg_dfs(files, alg_regex):
    pattern = re.compile(alg_regex)
    result = {}
    for file in files:
        storage = cccl.bench.SQLiteStorage(file)
        for algname in storage.algnames():
            if pattern.match(algname):
                for subbench in storage.subbenches(algname):
                    df = storage.alg_to_df(algname, subbench)
                    df = df.map(lambda x: x if is_finite(x) else np.nan)
                    df = df.dropna(subset=["center"], how="all")
                    df = filter_by_type(
                        filter_by_offset_type(filter_by_problem_size(df))
                    )
                    df = df.filter(items=["ctk", "cccl", "gpu", "variant", "bw"])
                    fused_algname = algname.replace("bench.", "") + "." + subbench
                    if df.empty:
                        print(
                            f"WARNING: Skipped {fused_algname} because no data is present"
                        )
                        print(df)
                        continue
                    if df["bw"].dropna().empty:
                        print(
                            f"WARNING: Skipped {fused_algname} because it does not report bandwidth"
                        )
                        continue
                    df["variant"] = df["variant"].astype(str)
                    df["bw"] = df["bw"] * 100
                    if fused_algname in result:
                        result[fused_algname] = pd.concat([result[fused_algname], df])
                    else:
                        result[fused_algname] = df
                    print(fused_algname)
    return result


def alg_bws(dfs, verbose):
    medians = None
    for algname in dfs:
        df = dfs[algname]
        df["alg"] = algname
        if df is None:
            medians = df
        else:
            medians = pd.concat([medians, df])
    # print more information if it's not unique across all runs or when requested (verbose)
    medians["hue"] = ""
    if verbose or medians["cccl"].unique().size > 1:
        medians["hue"] = medians["hue"] + "CCCL " + medians["cccl"].astype(str) + " "
    gpuname = (
        medians["gpu"]
        if verbose
        else medians["gpu"].astype(str).map(lambda x: x[: x.find("(") - 1])
    )
    medians["hue"] = medians["hue"] + gpuname + " "
    if medians["variant"].unique().size > 1:
        variant = (
            medians["variant"]
            .astype(str)
            .map(lambda x: (" " + x if x != "base" else ""))
        )
        medians["hue"] = medians["hue"] + variant + " "
    if verbose or medians["ctk"].unique().size > 1:
        medians["hue"] = medians["hue"] + "CTK " + medians["ctk"].astype(str)
    return medians.drop(columns=["ctk", "cccl", "gpu", "variant"])


def file_exists(value):
    if not os.path.isfile(value):
        raise argparse.ArgumentTypeError(f"The file '{value}' does not exist.")
    return value


def plot_sol(medians, box):
    if box:
        ax = sns.boxenplot(data=medians, x="alg", y="bw", hue="hue")
    else:
        ax = sns.barplot(
            data=medians,
            x="alg",
            y="bw",
            hue="hue",
            errorbar=lambda x: (x.min(), x.max()),
        )
        ax.bar_label(ax.containers[0], fmt="%.1f")
        for container in ax.containers[1:]:
            labels = [
                f"{c:.1f}\n({(c / f) * 100:.0f}%)"
                for f, c in zip(ax.containers[0].datavalues, container.datavalues)
            ]
            ax.bar_label(container, labels=labels)

    ax.legend(title=None)
    ax.set_xlabel("Algorithm")
    ax.set_ylabel("Bandwidth (%SOL)")
    ax.set_xticklabels(
        ax.get_xticklabels(), rotation=30, rotation_mode="anchor", ha="right"
    )
    ax.set_ylim([0, 100])
    plt.show()


def print_speedup(medians):
    m = medians.groupby(["alg", "hue"], sort=False).mean()
    m["speedup"] = m["bw"] / m.groupby(["alg"])["bw"].transform("first")
    print("# Speedups:")
    print()
    print(m.drop(columns="bw").sort_values(by="speedup", ascending=False).to_markdown())


def parse_args():
    parser = argparse.ArgumentParser(description="Analyze benchmark results.")
    parser.add_argument(
        "files", type=file_exists, nargs="+", help="At least one file is required."
    )
    parser.add_argument("--box", action="store_true", help="Plot box instead of bar.")
    parser.add_argument("-v", action="store_true", help="Verbose legend.")
    parser.add_argument(
        "-R", type=str, default=".*", help="Regex for benchmarks selection."
    )
    return parser.parse_args()


def sol():
    args = parse_args()
    dfs = alg_dfs(args.files, args.R)
    if not dfs:
        print("ERROR: No benchmark data to process (all benchmarks were skipped).")
        return
    medians = alg_bws(dfs, args.v)
    print_speedup(medians)
    plot_sol(medians, args.box)


if __name__ == "__main__":
    sol()