Skip to content
Snippets Groups Projects
Commit d415d9c1 authored by bo's avatar bo
Browse files

update filenames for the statistics and README file

parent 826b3d33
Branches
No related tags found
No related merge requests found
This folder contains the statistics for reproducing all the figures in our paper:
- Calibration statistics (NLL, ECE, Brier score) are saved in the folder **full_image_calibration_stat** and **region_calibration_stat**, each file is named as *Vx_Method_x_Stage_x_Version_x.obj*
- Vx means experiment index (multiple experiments for per method are carried out to get confidence interval)
- Method_x_Stage_x: B, C and D correspond to methods VarRatio (stage 1), Entropy (stage 2) and BALD (stage 3)
- Version_x: experiment version
- Uncertainty distribution of the pixels in the acquired images and regions are saved in the folder **acquired_full_image_uncertainty** and **acquired_region_uncertainty**, the files are named with the same fashion as previous ones. Each file contains the predicted probability of all the pixels in the acquired images/regions at each acquisition step.
- The statistics for creating the expected calibration error histogram using full image acquisition strategy are saved in the folder **ece_histogram**
- Glas.xlsx file contains the segmentation accuracy (F1 score, Dice Index) for the models at each acquisition step
### On uncertainty estimation in active learning for image segmentation ### On uncertainty estimation in active learning for image segmentation
<br /> This repository provides the implementation for our paper **On uncertainty estimation in active learning for image segmentation** (Bo Li, Tommy Sonne Alstrøm) [NEED TO ADD A LINK]. We experimentally show that the region based active learning strategy can lead to higher segmentation accuracy and better calibrated model much faster than full image acquisition: <br /> This repository provides the implementation for our paper [**On uncertainty estimation in active learning for image segmentation** (Bo Li, Tommy Sonne Alstrøm)](https://arxiv.org/abs/2007.06364). We experimentally show that the region based active learning strategy can lead to higher segmentation accuracy and better calibrated model much faster than full image acquisition:
![performance](DATA/first_figure.jpg) ![performance](DATA/first_figure.jpg)
...@@ -60,7 +60,7 @@ Args: ...@@ -60,7 +60,7 @@ Args:
#### Reproduce figures #### Reproduce figures
In order to reproduce the figures in the paper, run The statistics that are used for reproducing the figures are saved in folder *Exp_Stat*. In order to reproduce the figures in the paper, run
```bash ```bash
./produce_figure.sh ./produce_figure.sh
......
...@@ -14,7 +14,7 @@ if [ -d "$filename" ]; then ...@@ -14,7 +14,7 @@ if [ -d "$filename" ]; then
else else
echo "$filename does not exist" echo "$filename does not exist"
echo "Download the file..................................." echo "Download the file..................................."
wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1fu7RcqOMCCjz65VwFlTPg5qlLg2K7l1g' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1fu7RcqOMCCjz65VwFlTPg5qlLg2K7l1g" -O calibration_score.tar.gz && rm -rf /tmp/cookies.txt wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1VT_6mMA9ksDV8O3shYEXEehrMPDktmVZ' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1VT_6mMA9ksDV8O3shYEXEehrMPDktmVZ" -O calibration_score.tar.gz && rm -rf /tmp/cookies.txt
echo "-----unziping the datafile" echo "-----unziping the datafile"
tar -xzvf calibration_score.tar.gz tar -xzvf calibration_score.tar.gz
mv use_stat calibration_score mv use_stat calibration_score
......
...@@ -38,17 +38,17 @@ def ax_global_get(fig): ...@@ -38,17 +38,17 @@ def ax_global_get(fig):
return ax_global return ax_global
def give_score_path(path_use): def give_score_path(path):
str_group = ["_B_", "_C_", "_D_"] str_group = ["_B_", "_C_", "_D_"]
region_path = path_use + 'Act_Learn_Desperate_V6/' region_path = path + 'region_calibration_stat/'
region_group = [[] for _ in range(3)] region_group = [[] for _ in range(3)]
for iterr, single_str in enumerate(str_group): for iterr, single_str in enumerate(str_group):
select_folder = [region_path + v for v in os.listdir(region_path) if single_str in v and '.obj' in v] select_folder = [region_path + v for v in os.listdir(region_path) if single_str in v and '.obj' in v]
region_group[iterr] = select_folder region_group[iterr] = select_folder
full_path = [path_use + 'Act_Learn_Desperate_V7/', path_use + 'Act_Learn_Desperate_V8/'] full_path = path + 'full_image_calibration_stat/'
full_group = [[] for _ in range(3)] full_group = [[] for _ in range(3)]
for iterr, single_str in enumerate(str_group): for iterr, single_str in enumerate(str_group):
folder_select = [v + q for v in full_path for q in os.listdir(v) if single_str in q and '.obj' in q] folder_select = [full_path + v for v in os.listdir(full_path) if single_str in v and '.obj' in v]
full_group[iterr] = folder_select full_group[iterr] = folder_select
return region_group, full_group return region_group, full_group
...@@ -163,7 +163,6 @@ def give_figure_5(reg_group, ful_group, save=False): ...@@ -163,7 +163,6 @@ def give_figure_5(reg_group, ful_group, save=False):
for iterr, single_score in enumerate(score_group): for iterr, single_score in enumerate(score_group):
ax0 = fig.add_subplot(len(score_group), 2, 2 * iterr + 1) ax0 = fig.add_subplot(len(score_group), 2, 2 * iterr + 1)
ax1 = fig.add_subplot(len(score_group), 2, 2 * iterr + 2) ax1 = fig.add_subplot(len(score_group), 2, 2 * iterr + 2)
print(2 * iterr + 1, 2 * iterr + 2)
compare_acq_at_certain_point_barplot(reg_group, ful_group, single_score, [ax0, ax1]) compare_acq_at_certain_point_barplot(reg_group, ful_group, single_score, [ax0, ax1])
if iterr == 0 or iterr == 1: if iterr == 0 or iterr == 1:
for ax in [ax0, ax1]: for ax in [ax0, ax1]:
...@@ -185,8 +184,19 @@ def give_figure_5(reg_group, ful_group, save=False): ...@@ -185,8 +184,19 @@ def give_figure_5(reg_group, ful_group, save=False):
pad_inches=0, bbox_inches='tight') pad_inches=0, bbox_inches='tight')
def give_figure_4_and_e1(ful_group, conf_interval=True, save=False): def give_acquired_full_image_uncertainty(path):
ece_path = path + "ece_stat/" str_group = ["_B_", "_C_", "_D_"]
full_path = path + 'acquired_full_image_uncertainty/'
full_group = [[] for _ in range(3)]
for iterr, single_str in enumerate(str_group):
folder_select = [full_path + v for v in os.listdir(full_path) if single_str in v and '.npy' in v]
full_group[iterr] = folder_select
return full_group
def give_figure_4_and_e1(conf_interval=True, save=False):
ful_group = give_acquired_full_image_uncertainty(path)
ece_path = path + "ece_histogram/"
legend_space = ["VarRatio", "Entropy", "BALD"] legend_space = ["VarRatio", "Entropy", "BALD"]
ece_all = [v for v in os.listdir(ece_path) if '.npy' in v and '_stat_' in v] ece_all = [v for v in os.listdir(ece_path) if '.npy' in v and '_stat_' in v]
...@@ -209,7 +219,6 @@ def give_figure_4_and_e1(ful_group, conf_interval=True, save=False): ...@@ -209,7 +219,6 @@ def give_figure_4_and_e1(ful_group, conf_interval=True, save=False):
ece_d_avg = np.mean([v[1] for v in ece_d], axis=0) ece_d_avg = np.mean([v[1] for v in ece_d], axis=0)
ece_d_avg = ece_d[2][1] ece_d_avg = ece_d[2][1]
# ece_d_avg = [v+0.03 if iterr <= 4 else v-0.03 for iterr, v in enumerate(ece_d_avg)] # ece_d_avg = [v+0.03 if iterr <= 4 else v-0.03 for iterr, v in enumerate(ece_d_avg)]
print(np.shape(ece_d_avg))
ece_d_std = np.std([v[1] for v in ece_d], axis=0) * 1.95 / np.sqrt(len(ece_d)) ece_d_std = np.std([v[1] for v in ece_d], axis=0) * 1.95 / np.sqrt(len(ece_d))
uncertain_stat = show_uncertainty_distribution(ful_group, True) uncertain_stat = show_uncertainty_distribution(ful_group, True)
...@@ -419,8 +428,11 @@ def compare_acq_at_certain_point_line(reg_group, ful_group, score_str, ax): ...@@ -419,8 +428,11 @@ def compare_acq_at_certain_point_line(reg_group, ful_group, score_str, ax):
if not ax: if not ax:
fig = plt.figure(figsize=(5, 3)) fig = plt.figure(figsize=(5, 3))
ax = fig.add_subplot(111) ax = fig.add_subplot(111)
for i in range(3):
ax.plot(f_g_perf[i][0] , f_g_perf[i][1], color_group[i], ls=lstype_group[1], lw=1.0)
for i in range(3): for i in range(3):
ax.plot(r_g_perf[i][0], r_g_perf[i][1], color_group[i], ls=lstype_group[1], lw=1.0) ax.plot(r_g_perf[i][0] , r_g_perf[i][1], color_group[i], ls=lstype_group[0], lw=1.0)
ax.grid(ls=':', axis='both') ax.grid(ls=':', axis='both')
if score_str is "nll_score": if score_str is "nll_score":
ax.ticklabel_format(axis='y', style='sci', scilimits=(10, 5)) ax.ticklabel_format(axis='y', style='sci', scilimits=(10, 5))
...@@ -533,7 +545,6 @@ def sort_uncertainty(pool_path, method, load_step): ...@@ -533,7 +545,6 @@ def sort_uncertainty(pool_path, method, load_step):
def get_uncertainty_group(path_group, method, load_step, return_value=False): def get_uncertainty_group(path_group, method, load_step, return_value=False):
path_group = [v.strip().split('.obj')[0] + "pool_stat.npy" for v in path_group]
if method is "B": if method is "B":
path_group = [path_group[0], path_group[2]] path_group = [path_group[0], path_group[2]]
uncertain_stat = [sort_uncertainty(single_path, method, load_step) uncertain_stat = [sort_uncertainty(single_path, method, load_step)
...@@ -571,7 +582,7 @@ def get_region_uncert(return_stat=False): ...@@ -571,7 +582,7 @@ def get_region_uncert(return_stat=False):
version_use = [3, 1, 2] version_use = [3, 1, 2]
step = [0, 0, 1] step = [0, 0, 1]
uncert_stat = [] uncert_stat = []
path2read = path + '/region_uncertainty/' path2read = path + '/acquired_region_uncertainty/'
for i in range(len(method)): for i in range(len(method)):
path_sub = [v for v in os.listdir(path2read) if path_sub = [v for v in os.listdir(path2read) if
'Method_%s' % method[i] in v and 'Version_%d' % version_use[i] in v and 'step_%d' % step[i] in v] 'Method_%s' % method[i] in v and 'Version_%d' % version_use[i] in v and 'step_%d' % step[i] in v]
...@@ -597,24 +608,26 @@ if __name__ == '__main__': ...@@ -597,24 +608,26 @@ if __name__ == '__main__':
print("---The figures are going to be saved in ", save_fig_path) print("---The figures are going to be saved in ", save_fig_path)
reg_group, ful_group = give_score_path(path) reg_group, ful_group = give_score_path(path)
print("----------------------------------") [print(v) for v in reg_group]
print("-----creating the first figure----") [print(v) for v in ful_group]
print("----------------------------------") print("----------------------------------------")
print("-----creating the first figure----------")
print("----------------------------------------")
give_first_figure(reg_group, ful_group, args.save) give_first_figure(reg_group, ful_group, args.save)
print("----------------------------------") print("----------------------------------------")
print("-----creating figure 4 and figure E1---") print("-----creating figure 4 and figure E1----")
print("----------------------------------") print("----------------------------------------")
give_figure_4_and_e1(ful_group, False, args.save) give_figure_4_and_e1(False, args.save)
print("----------------------------------") print("----------------------------------------")
print("-----creating figure 5----------------") print("-----creating figure 5------------------")
print("----------------------------------") print("----------------------------------------")
give_figure_5(reg_group, ful_group, args.save) give_figure_5(reg_group, ful_group, args.save)
print("----------------------------------") print("----------------------------------------")
print("-----creating figure e2---------------") print("-----creating figure e2-----------------")
print("----------------------------------") print("----------------------------------------")
give_figure_e2(reg_group, ful_group, args.save) give_figure_e2(reg_group, ful_group, args.save)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment