113 lines
3.4 KiB
Plaintext
113 lines
3.4 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "64cea749",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
"from haven import haven_jupyter as hj\n",
|
|
"from haven import haven_results as hr\n",
|
|
"from haven import haven_utils as hu\n",
|
|
"\n",
|
|
"# path to where the experiments got saved\n",
|
|
"savedir_base = '/home/toolkit/trl/results'\n",
|
|
"exp_list = None\n",
|
|
"\n",
|
|
"# get experiments from the exp config\n",
|
|
"exp_config_fname = None\n",
|
|
"if exp_config_fname:\n",
|
|
" config = hu.load_py(exp_config_fname)\n",
|
|
" exp_list = []\n",
|
|
" for exp_group in [\n",
|
|
" \"example\"\n",
|
|
" ]:\n",
|
|
" exp_list += config.EXP_GROUPS[exp_group]\n",
|
|
"\n",
|
|
"# filter exps\n",
|
|
"\n",
|
|
"filterby_list = None\n",
|
|
"# filterby_list =[{'dataset':'mnist'}]\n",
|
|
"\n",
|
|
"# get experiments\n",
|
|
"rm = hr.ResultManager(exp_list=exp_list,\n",
|
|
" savedir_base=savedir_base,\n",
|
|
" filterby_list=filterby_list,\n",
|
|
" verbose=0,\n",
|
|
" exp_groups=None,\n",
|
|
" job_scheduler='slurm'\n",
|
|
" )\n",
|
|
"\n",
|
|
"# specify display parameters\n",
|
|
"\n",
|
|
"# groupby_list = ['dataset']\n",
|
|
"# title_list = ['dataset']\n",
|
|
"# legend_list = ['model']\n",
|
|
"y_metrics = ['train_loss']\n",
|
|
"x_metric = 'epoch'\n",
|
|
"\n",
|
|
"# launch dashboard\n",
|
|
"hj.get_dashboard(rm, vars(), wide_display=False, enable_datatables=False)\n",
|
|
" "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "94dd14d0",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
"# get table \n",
|
|
"rm.get_score_df().head()\n",
|
|
"\n",
|
|
"# get latex \n",
|
|
"# print(rm.get_latex_table(legend=['dataset'], metrics=['train_loss'], decimals=1, caption=\"Results\", label='tab:results'))\n",
|
|
"\n",
|
|
"# get custom plots\n",
|
|
"fig = rm.get_plot_all(\n",
|
|
" # order='metrics_by_groups',\n",
|
|
" # avg_across='runs',\n",
|
|
" y_metric_list=y_metrics, \n",
|
|
" x_metric=x_metric,\n",
|
|
" # legend_fontsize=18,\n",
|
|
" # x_fontsize=20,\n",
|
|
" # y_fontsize=20,\n",
|
|
" # xtick_fontsize=20,\n",
|
|
" # ytick_fontsize=20,\n",
|
|
" # title_fontsize=24,\n",
|
|
" # legend_list=['model], \n",
|
|
" # title_list = ['dataset'], \n",
|
|
" # title_format='Dataset:{}',\n",
|
|
" # log_metric_list = ['train_loss'], \n",
|
|
" # groupby_list = ['dataset'],\n",
|
|
" # map_ylabel_list=[{'train_loss':'Train loss'}],\n",
|
|
" # map_xlabel_list=[{'epoch':'Epoch'}],\n",
|
|
" # figsize=(15,5),\n",
|
|
" # plot_confidence=False,\n",
|
|
" # savedir_plots='%s' % (name)\n",
|
|
")\n",
|
|
" "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6068885d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
" !pip install --upgrade git+https://github.com/haven-ai/haven-ai\n",
|
|
" "
|
|
]
|
|
}
|
|
],
|
|
"metadata": {},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|