Skip to content

Instantly share code, notes, and snippets.

@acmiyaguchi

acmiyaguchi/README.md

Last active March 10, 2023 03:04
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save acmiyaguchi/36e5df4f0fd989878acbd8defdc6e34b to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save acmiyaguchi/36e5df4f0fd989878acbd8defdc6e34b to your computer and use it in GitHub Desktop.
Download ZIP
GKNet evaluation reproduction
Raw
README.md
Display the source blob
Display the rendered blob
Raw
test_pretrained_results.ipynb
{
"cells": [
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"# pretrained model results\n",
"\n",
"2023-03-09 "
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>task</th>\n",
" <th>success</th>\n",
" <th>total</th>\n",
" <th>wall_time</th>\n",
" <th>exp_id</th>\n",
" <th>dataset</th>\n",
" <th>accuracy</th>\n",
" <th>fps</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>dbmctdet</td>\n",
" <td>9108</td>\n",
" <td>9354</td>\n",
" <td>113.399522</td>\n",
" <td>model_alexnet_ajd</td>\n",
" <td>jac_coco_36</td>\n",
" <td>0.973701</td>\n",
" <td>82.487120</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>dbmctdet</td>\n",
" <td>9203</td>\n",
" <td>9354</td>\n",
" <td>124.360462</td>\n",
" <td>model_dla34_ajd</td>\n",
" <td>jac_coco_36</td>\n",
" <td>0.983857</td>\n",
" <td>75.216832</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>dbmctdet</td>\n",
" <td>9162</td>\n",
" <td>9354</td>\n",
" <td>124.072763</td>\n",
" <td>model_resnet18_ajd</td>\n",
" <td>jac_coco_36</td>\n",
" <td>0.979474</td>\n",
" <td>75.391244</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>dbmctdet</td>\n",
" <td>9189</td>\n",
" <td>9354</td>\n",
" <td>122.886355</td>\n",
" <td>model_resnet50_ajd</td>\n",
" <td>jac_coco_36</td>\n",
" <td>0.982360</td>\n",
" <td>76.119110</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>dbmctdet</td>\n",
" <td>9201</td>\n",
" <td>9354</td>\n",
" <td>114.280679</td>\n",
" <td>model_vgg16_ajd</td>\n",
" <td>jac_coco_36</td>\n",
" <td>0.983643</td>\n",
" <td>81.851106</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>dbmctdet_cornell</td>\n",
" <td>20930</td>\n",
" <td>22110</td>\n",
" <td>78.452531</td>\n",
" <td>model_alexnet_cornell</td>\n",
" <td>cornell</td>\n",
" <td>0.946630</td>\n",
" <td>281.826472</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>dbmctdet_cornell</td>\n",
" <td>21399</td>\n",
" <td>22110</td>\n",
" <td>141.670923</td>\n",
" <td>model_dla34_cornell</td>\n",
" <td>cornell</td>\n",
" <td>0.967843</td>\n",
" <td>156.065899</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>dbmctdet_cornell</td>\n",
" <td>21162</td>\n",
" <td>22110</td>\n",
" <td>77.845522</td>\n",
" <td>model_resnet18_cornell</td>\n",
" <td>cornell</td>\n",
" <td>0.957123</td>\n",
" <td>284.024046</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>dbmctdet_cornell</td>\n",
" <td>21260</td>\n",
" <td>22110</td>\n",
" <td>78.786531</td>\n",
" <td>model_resnet50_cornell</td>\n",
" <td>cornell</td>\n",
" <td>0.961556</td>\n",
" <td>280.631723</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>dbmctdet_cornell</td>\n",
" <td>21319</td>\n",
" <td>22110</td>\n",
" <td>78.759574</td>\n",
" <td>model_vgg16_cornell</td>\n",
" <td>cornell</td>\n",
" <td>0.964224</td>\n",
" <td>280.727776</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" task success total wall_time exp_id \\\n",
"0 dbmctdet 9108 9354 113.399522 model_alexnet_ajd \n",
"1 dbmctdet 9203 9354 124.360462 model_dla34_ajd \n",
"2 dbmctdet 9162 9354 124.072763 model_resnet18_ajd \n",
"3 dbmctdet 9189 9354 122.886355 model_resnet50_ajd \n",
"4 dbmctdet 9201 9354 114.280679 model_vgg16_ajd \n",
"5 dbmctdet_cornell 20930 22110 78.452531 model_alexnet_cornell \n",
"6 dbmctdet_cornell 21399 22110 141.670923 model_dla34_cornell \n",
"7 dbmctdet_cornell 21162 22110 77.845522 model_resnet18_cornell \n",
"8 dbmctdet_cornell 21260 22110 78.786531 model_resnet50_cornell \n",
"9 dbmctdet_cornell 21319 22110 78.759574 model_vgg16_cornell \n",
"\n",
" dataset accuracy fps \n",
"0 jac_coco_36 0.973701 82.487120 \n",
"1 jac_coco_36 0.983857 75.216832 \n",
"2 jac_coco_36 0.979474 75.391244 \n",
"3 jac_coco_36 0.982360 76.119110 \n",
"4 jac_coco_36 0.983643 81.851106 \n",
"5 cornell 0.946630 281.826472 \n",
"6 cornell 0.967843 156.065899 \n",
"7 cornell 0.957123 284.024046 \n",
"8 cornell 0.961556 280.631723 \n",
"9 cornell 0.964224 280.727776 "
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from pathlib import Path\n",
"import json\n",
"import pandas as pd\n",
"\n",
"def parse_log_line(line):\n",
" data = line.split(\":\")[1:]\n",
" return json.loads(\":\".join(data))\n",
"\n",
"paths = Path(\"../exp\").glob(\"**/log.txt\")\n",
"results = []\n",
"for path in paths:\n",
" text = path.read_text()\n",
" # first line is options, second line is timing, third line is results\n",
" lines = text.strip().split(\"\\n\")\n",
" if len(lines) < 3:\n",
" print(f\"skipping {path}\")\n",
" continue\n",
" # each line is a timestamp: json object\n",
" opts = parse_log_line(lines[0])\n",
" result = parse_log_line(lines[2])\n",
" result[\"task\"] = opts[\"task\"]\n",
" result[\"exp_id\"] = opts[\"exp_id\"]\n",
" result[\"dataset\"] = opts[\"dataset\"]\n",
" results.append(result)\n",
"\n",
"df = pd.DataFrame(results)\n",
"df[\"accuracy\"] = df.success / df.total\n",
"df[\"fps\"] = df.total / df.wall_time\n",
"df"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"| exp_id | dataset | accuracy | fps |\n",
"|:-----------------------|:------------|-----------:|---------:|\n",
"| model_alexnet_ajd | jac_coco_36 | 0.973701 | 82.4871 |\n",
"| model_dla34_ajd | jac_coco_36 | 0.983857 | 75.2168 |\n",
"| model_resnet18_ajd | jac_coco_36 | 0.979474 | 75.3912 |\n",
"| model_resnet50_ajd | jac_coco_36 | 0.98236 | 76.1191 |\n",
"| model_vgg16_ajd | jac_coco_36 | 0.983643 | 81.8511 |\n",
"| model_alexnet_cornell | cornell | 0.94663 | 281.826 |\n",
"| model_dla34_cornell | cornell | 0.967843 | 156.066 |\n",
"| model_resnet18_cornell | cornell | 0.957123 | 284.024 |\n",
"| model_resnet50_cornell | cornell | 0.961556 | 280.632 |\n",
"| model_vgg16_cornell | cornell | 0.964224 | 280.728 |\n"
]
}
],
"source": [
"# get the results as a markdown table\n",
"res = df[[\"exp_id\", \"dataset\", \"accuracy\", \"fps\"]]\n",
"print(res.to_markdown(index=False))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.5"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "5fba4eb1ddfb0b5f2c0f3e2fd25cac3f968691247a9de4806ec9bb18fee8fadd"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment