Faster Multi-Object Segmentation using Parallel Quadratic Pseudo-Boolean Optimization, ICCV 2021 Paper
Authors: Patrick M. Jensen (patmjen@dtu.dk) and Niels Jeppesen (niejep@dtu.dk)
This notebook is used to analyze the benchmark results for the comparison between P-QPBO and a few non-QPBO solves (see paper for more details).
Note: The code for the non-QPBO methods used to generate the CSV files summarised here is not included in this paper's subplementary material. However, the code is available at https://github.com/patmjen/maxflow_algorithms.
import numpy as np
import pandas as pd
from glob import glob
import matplotlib.pyplot as plt
qpbo_data = pd.read_csv('../benchmark/nuclei_benchmarks/qpbo/parallel_qpbo_nuclei_benchmark_results_20201011-172857.csv')
most_data = pd.concat([pd.read_csv(f) for f in glob('../benchmark/nuclei_benchmarks/all/*.csv')])
most_data['Name'] = most_data['file_name'].apply(lambda n: n.split('/')[-1][:-3])
most_data['SolveTime'] = most_data['solve_time']
most_data['ShortName'] = most_data['algorithm']
most_data['NucleiCount'] = most_data['num_blocks']
most_data['CpuCount'] = most_data['num_threads']
peibfs_data = pd.concat([pd.read_csv(f) for f in glob('../benchmark/nuclei_benchmarks/peibfs/*.csv')])
peibfs_data['Name'] = peibfs_data['file_name'].apply(lambda n: n.split('/')[-1][:-3])
peibfs_data['SolveTime'] = peibfs_data['solve_time']
peibfs_data['ShortName'] = peibfs_data['algorithm']
peibfs_data['NucleiCount'] = peibfs_data['num_blocks']
peibfs_data['CpuCount'] = peibfs_data['num_threads']
# Make a nice dataframe which only has the columns we need
keep_columns = ['Name', 'ShortName', 'NucleiCount', 'SolveTime', 'CpuCount']
all_data = pd.concat([most_data[keep_columns], qpbo_data[keep_columns], peibfs_data[keep_columns]])
# Only keep entries for datasets where all algorithms have run at least once
keep_names = list(set(most_data['Name']).intersection(set(qpbo_data['Name'])).intersection(set(peibfs_data['Name'])))
all_data = all_data[all_data['Name'].isin(keep_names)]
print(f'Number of datasets: {len(keep_names)}/670')
# Make sure all algorithms are present
data = all_data.groupby(['Name', 'ShortName', 'CpuCount']).min().reset_index()
data['ShortName'].unique()
# Compute speed-up w.r.t K-QPBO
data2 = data.set_index('Name')
data2['SpeedUp'] = 1 / data2['SolveTime'].divide(data2[data2['ShortName'] == 'QPBO']['SolveTime'])
print('Mean speed-up')
print(data2.groupby(['ShortName', 'CpuCount']).mean()['SpeedUp'])
print('Speed-up std.')
print(data2.groupby(['ShortName', 'CpuCount']).std()['SpeedUp'])
benchmark_paths = glob('../benchmark/nerve_benchmarks/parallel_qpbo_benchmark_results_*.csv')
benchmark_paths
df_qpbo = pd.concat([pd.read_csv(p) for p in benchmark_paths])
df_qpbo = df_qpbo[df_qpbo['CpuCount'] <= 32]
df_qpbo_group = df_qpbo.groupby(['NodeCount', 'EdgeCount', 'Class', 'SystemCpu', 'CpuCount'])
df_qpbo_group[['SolveTime']].describe()
qpbo_mins = df_qpbo_group['SolveTime'].min().reset_index()
qpbo_mins_n1 = qpbo_mins[qpbo_mins['NodeCount'] == 363748800].copy()
qpbo_mins_n1['SpeedUp'] = qpbo_mins_n1[qpbo_mins_n1['Class'] == 'QPBOInt']['SolveTime'].iloc[0] / qpbo_mins_n1['SolveTime']
qpbo_mins_n1[['Class', 'CpuCount', 'SolveTime', 'SpeedUp']]
We read the results from the CSV files.
benchmark_paths = glob('../benchmark/nerve_benchmarks/bench*.csv')
benchmark_paths
df_maxflow = pd.concat([pd.read_csv(p) for p in benchmark_paths])
df_maxflow = df_maxflow[df_maxflow['num_threads'] <= 32]
df_maxflow_group = df_maxflow.groupby(['num_nodes', 'num_term_arcs', 'bench_name', 'num_threads'])
df_maxflow_group[['solve_time']].describe()
maxflow_mins = df_maxflow_group['solve_time'].min().reset_index()
maxflow_mins_n1 = maxflow_mins[maxflow_mins['num_nodes'] == 363748800].copy()
maxflow_mins_n1['SpeedUp'] = qpbo_mins_n1[qpbo_mins_n1['Class'] == 'QPBOInt']['SolveTime'].iloc[0] / maxflow_mins_n1['solve_time']
maxflow_mins_n1[['bench_name', 'num_threads', 'solve_time', 'SpeedUp']]
maxflow_mins_n1.groupby(['bench_name'])['SpeedUp'].max()
print('EIBFS-QPBO estimate:', maxflow_mins_n1.groupby(['bench_name'])['SpeedUp'].max()['ICCV2021 EIBFS N1'] * 2)