{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a45459fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from collections import defaultdict\n",
    "import re\n",
    "import pandas as pd\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81fe734f",
   "metadata": {},
   "source": [
    "# 1) Parse results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "592ad0a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "result_folder = 'log/classification'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1d8b58b",
   "metadata": {},
   "outputs": [],
   "source": [
    "experiments = os.listdir(result_folder)\n",
    "experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f18cf365",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_last_line(filename, keyword):\n",
    "    pattern = re.compile(keyword)\n",
    "    \n",
    "    foundline = None\n",
    "    for line in open(filename, 'r'):\n",
    "        for match in re.finditer(pattern, line):\n",
    "            foundline = line\n",
    "    return foundline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac4a19c9",
   "metadata": {},
   "source": [
    "## Unpack experiment names:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c168ffaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = defaultdict(lambda: defaultdict(list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ff14707",
   "metadata": {},
   "outputs": [],
   "source": [
    "for exp in experiments:\n",
    "    stats[exp]['random_rotations'] = True if 'ROT' in exp else False\n",
    "    stats[exp]['scaling_interval'] = exp.split('_S')[1].split('_T')[0]\n",
    "    stats[exp]['translation_distance'] = exp.split('_T')[1].split('PCA')[0]\n",
    "    stats[exp]['pca_align'] = exp.split('PCA_')[1].split('_')[0]\n",
    "    stats[exp]['center_align'] = exp.split('TL_')[1].split('_')[0]\n",
    "    stats[exp]['unscale'] = exp.split('UC_')[1].split('_')[0]    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "058c28e3",
   "metadata": {},
   "source": [
    "## Collect stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "535c7a04",
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames = os.listdir('log')\n",
    "filenames = [f for f in filenames if f.endswith('.log')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47b3fd33",
   "metadata": {},
   "outputs": [],
   "source": [
    "for exp in experiments:\n",
    "    #filename = os.path.join(result_folder, exp, os.listdir(os.path.join(result_folder, \n",
    "    #                                                             exp))[0], 'logs', 'pointnet_cls.txt')\n",
    "    \n",
    "    for file in filenames: # iterate over all logs:\n",
    "        filename = os.path.join('log', file)\n",
    "        idx_line = find_last_line(filename, 'Saving at log/classification')\n",
    "        idx = idx_line.split('classification/')[1].split('/')[0]\n",
    "        if idx == exp:\n",
    "            best_acc_line = find_last_line(filename, 'Best Instance Accuracy: ')\n",
    "            stats[exp]['Best Instance Accuracy'] += [float(best_acc_line.split('Best Instance Accuracy: ')[1].split(',')[0])]\n",
    "            stats[exp]['Best Class Accuracy'] += [float(best_acc_line.split('Class Accuracy: ')[1].strip())]\n",
    "\n",
    "            final_acc_line = find_last_line(filename, 'Test Instance Accuracy: .*, Class Accuracy: ')\n",
    "            \n",
    "            \n",
    "            stats[exp]['Final Instance Accuracy'] += [float(final_acc_line.split('Test Instance Accuracy: ')[1].split(',')[0])]\n",
    "            stats[exp]['Final Class Accuracy'] +=  [float(final_acc_line.split('Class Accuracy: ')[1].strip())]\n",
    "\n",
    "            # rotations\n",
    "            for invariance in ['rotation', 'translation', 'scaling']:\n",
    "                for metric in ['Instance', 'Class']:\n",
    "                    acc_line = find_last_line(filename, f'{invariance} Worst-case Test {metric}')\n",
    "                    stats[exp][f'Worst-case {invariance} {metric} Accuracy'] += [float(acc_line.split(\n",
    "                        f'Worst-case Test {metric} Accuracy: ')[1].split(' ')[0])]\n",
    "                    stats[exp][f'Average-case {invariance} {metric} Accuracy'] += [float(acc_line.split(\n",
    "                        f'Average-case Test {metric} Accuracy: ')[1].strip())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6851b950",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c4aaee6",
   "metadata": {},
   "outputs": [],
   "source": [
    "average_stats = defaultdict(lambda: defaultdict(list))\n",
    "for exp in experiments:\n",
    "    for metric in stats[exp].keys():\n",
    "        if not 'Instance' in metric and not 'Best' in metric:\n",
    "            if isinstance(stats[exp][metric], list):\n",
    "                mean = np.mean(stats[exp][metric]) \n",
    "                std  = np.std(stats[exp][metric])\n",
    "                average_stats[exp][metric] = f'{mean:2.2%} (+- {std:2.2%})'\n",
    "            else:\n",
    "                average_stats[exp][metric] = stats[exp][metric]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64ad9ca2",
   "metadata": {},
   "outputs": [],
   "source": [
    "average_stats"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de9f4844",
   "metadata": {},
   "source": [
    "# Write CSV summar:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7422f60",
   "metadata": {},
   "outputs": [],
   "source": [
    "summary_file_name = 'results_summary_recomputed.csv'\n",
    "\n",
    "df = pd.DataFrame.from_dict(data=average_stats, orient='index')\n",
    "df.to_csv(summary_file_name, header=True, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b47115f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
