diff --git a/Exp_Stat/README.md b/Exp_Stat/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..959212c7184de27282418fb522d946c33244adb5
--- /dev/null
+++ b/Exp_Stat/README.md
@@ -0,0 +1,10 @@
+This folder contains the statistics for reproducing all the figures in our paper:
+
+- Calibration statistics (NLL, ECE, Brier score) are saved in the folder **full_image_calibration_stat** and **region_calibration_stat**, each file is named as *Vx_Method_x_Stage_x_Version_x.obj*
+ - Vx means experiment index (multiple experiments for per method are carried out to get confidence interval)
+ - Method_x_Stage_x: B, C and D correspond to methods VarRatio (stage 1), Entropy (stage 2) and BALD (stage 3)
+ - Version_x: experiment version
+- Uncertainty distribution of the pixels in the acquired images and regions are saved in the folder **acquired_full_image_uncertainty** and **acquired_region_uncertainty**, the files are named with the same fashion as previous ones. Each file contains the predicted probability of all the pixels in the acquired images/regions at each acquisition step.
+- The statistics for creating the expected calibration error histogram using full image acquisition strategy are saved in the folder **ece_histogram**
+- Glas.xlsx file contains the segmentation accuracy (F1 score, Dice Index) for the models at each acquisition step
+
diff --git a/README.md b/README.md
index 6e3461ad0d7ff5264c0ba9b4405f4e85980e0bac..883f5907eef7003c0ae3b0c8a875c165baf413a7 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
### On uncertainty estimation in active learning for image segmentation
-<br /> This repository provides the implementation for our paper **On uncertainty estimation in active learning for image segmentation** (Bo Li, Tommy Sonne Alstrøm) [NEED TO ADD A LINK]. We experimentally show that the region based active learning strategy can lead to higher segmentation accuracy and better calibrated model much faster than full image acquisition:
+<br /> This repository provides the implementation for our paper [**On uncertainty estimation in active learning for image segmentation** (Bo Li, Tommy Sonne Alstrøm)](https://arxiv.org/abs/2007.06364). We experimentally show that the region based active learning strategy can lead to higher segmentation accuracy and better calibrated model much faster than full image acquisition:
@@ -60,7 +60,7 @@ Args:
#### Reproduce figures
-In order to reproduce the figures in the paper, run
+The statistics that are used for reproducing the figures are saved in folder *Exp_Stat*. In order to reproduce the figures in the paper, run
```bash
./produce_figure.sh
diff --git a/produce_figure.sh b/produce_figure.sh
index bcb1a31ff108b783c1e2642b0876de4064ddfbea..7aa9bf99b91e72020b2507eeb70c513e7d7716b8 100644
--- a/produce_figure.sh
+++ b/produce_figure.sh
@@ -14,7 +14,7 @@ if [ -d "$filename" ]; then
else
echo "$filename does not exist"
echo "Download the file..................................."
- wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1fu7RcqOMCCjz65VwFlTPg5qlLg2K7l1g' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1fu7RcqOMCCjz65VwFlTPg5qlLg2K7l1g" -O calibration_score.tar.gz && rm -rf /tmp/cookies.txt
+ wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1VT_6mMA9ksDV8O3shYEXEehrMPDktmVZ' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1VT_6mMA9ksDV8O3shYEXEehrMPDktmVZ" -O calibration_score.tar.gz && rm -rf /tmp/cookies.txt
echo "-----unziping the datafile"
tar -xzvf calibration_score.tar.gz
mv use_stat calibration_score
diff --git a/visualize_calibration_score.py b/visualize_calibration_score.py
index 086148794cc61aeb99def65ff668eaf5359d9ddc..56658a3e2de887a626b0fb836fb90477cf428d01 100644
--- a/visualize_calibration_score.py
+++ b/visualize_calibration_score.py
@@ -38,17 +38,17 @@ def ax_global_get(fig):
return ax_global
-def give_score_path(path_use):
+def give_score_path(path):
str_group = ["_B_", "_C_", "_D_"]
- region_path = path_use + 'Act_Learn_Desperate_V6/'
+ region_path = path + 'region_calibration_stat/'
region_group = [[] for _ in range(3)]
for iterr, single_str in enumerate(str_group):
select_folder = [region_path + v for v in os.listdir(region_path) if single_str in v and '.obj' in v]
region_group[iterr] = select_folder
- full_path = [path_use + 'Act_Learn_Desperate_V7/', path_use + 'Act_Learn_Desperate_V8/']
+ full_path = path + 'full_image_calibration_stat/'
full_group = [[] for _ in range(3)]
for iterr, single_str in enumerate(str_group):
- folder_select = [v + q for v in full_path for q in os.listdir(v) if single_str in q and '.obj' in q]
+ folder_select = [full_path + v for v in os.listdir(full_path) if single_str in v and '.obj' in v]
full_group[iterr] = folder_select
return region_group, full_group
@@ -163,7 +163,6 @@ def give_figure_5(reg_group, ful_group, save=False):
for iterr, single_score in enumerate(score_group):
ax0 = fig.add_subplot(len(score_group), 2, 2 * iterr + 1)
ax1 = fig.add_subplot(len(score_group), 2, 2 * iterr + 2)
- print(2 * iterr + 1, 2 * iterr + 2)
compare_acq_at_certain_point_barplot(reg_group, ful_group, single_score, [ax0, ax1])
if iterr == 0 or iterr == 1:
for ax in [ax0, ax1]:
@@ -183,10 +182,21 @@ def give_figure_5(reg_group, ful_group, save=False):
if save is True:
plt.savefig(save_fig_path + 'overall_calibration.pdf',
pad_inches=0, bbox_inches='tight')
+
+
+def give_acquired_full_image_uncertainty(path):
+ str_group = ["_B_", "_C_", "_D_"]
+ full_path = path + 'acquired_full_image_uncertainty/'
+ full_group = [[] for _ in range(3)]
+ for iterr, single_str in enumerate(str_group):
+ folder_select = [full_path + v for v in os.listdir(full_path) if single_str in v and '.npy' in v]
+ full_group[iterr] = folder_select
+ return full_group
-def give_figure_4_and_e1(ful_group, conf_interval=True, save=False):
- ece_path = path + "ece_stat/"
+def give_figure_4_and_e1(conf_interval=True, save=False):
+ ful_group = give_acquired_full_image_uncertainty(path)
+ ece_path = path + "ece_histogram/"
legend_space = ["VarRatio", "Entropy", "BALD"]
ece_all = [v for v in os.listdir(ece_path) if '.npy' in v and '_stat_' in v]
@@ -209,7 +219,6 @@ def give_figure_4_and_e1(ful_group, conf_interval=True, save=False):
ece_d_avg = np.mean([v[1] for v in ece_d], axis=0)
ece_d_avg = ece_d[2][1]
# ece_d_avg = [v+0.03 if iterr <= 4 else v-0.03 for iterr, v in enumerate(ece_d_avg)]
- print(np.shape(ece_d_avg))
ece_d_std = np.std([v[1] for v in ece_d], axis=0) * 1.95 / np.sqrt(len(ece_d))
uncertain_stat = show_uncertainty_distribution(ful_group, True)
@@ -419,8 +428,11 @@ def compare_acq_at_certain_point_line(reg_group, ful_group, score_str, ax):
if not ax:
fig = plt.figure(figsize=(5, 3))
ax = fig.add_subplot(111)
+
+ for i in range(3):
+ ax.plot(f_g_perf[i][0] , f_g_perf[i][1], color_group[i], ls=lstype_group[1], lw=1.0)
for i in range(3):
- ax.plot(r_g_perf[i][0], r_g_perf[i][1], color_group[i], ls=lstype_group[1], lw=1.0)
+ ax.plot(r_g_perf[i][0] , r_g_perf[i][1], color_group[i], ls=lstype_group[0], lw=1.0)
ax.grid(ls=':', axis='both')
if score_str is "nll_score":
ax.ticklabel_format(axis='y', style='sci', scilimits=(10, 5))
@@ -533,7 +545,6 @@ def sort_uncertainty(pool_path, method, load_step):
def get_uncertainty_group(path_group, method, load_step, return_value=False):
- path_group = [v.strip().split('.obj')[0] + "pool_stat.npy" for v in path_group]
if method is "B":
path_group = [path_group[0], path_group[2]]
uncertain_stat = [sort_uncertainty(single_path, method, load_step)
@@ -571,7 +582,7 @@ def get_region_uncert(return_stat=False):
version_use = [3, 1, 2]
step = [0, 0, 1]
uncert_stat = []
- path2read = path + '/region_uncertainty/'
+ path2read = path + '/acquired_region_uncertainty/'
for i in range(len(method)):
path_sub = [v for v in os.listdir(path2read) if
'Method_%s' % method[i] in v and 'Version_%d' % version_use[i] in v and 'step_%d' % step[i] in v]
@@ -597,24 +608,26 @@ if __name__ == '__main__':
print("---The figures are going to be saved in ", save_fig_path)
reg_group, ful_group = give_score_path(path)
- print("----------------------------------")
- print("-----creating the first figure----")
- print("----------------------------------")
+ [print(v) for v in reg_group]
+ [print(v) for v in ful_group]
+ print("----------------------------------------")
+ print("-----creating the first figure----------")
+ print("----------------------------------------")
give_first_figure(reg_group, ful_group, args.save)
- print("----------------------------------")
- print("-----creating figure 4 and figure E1---")
- print("----------------------------------")
+ print("----------------------------------------")
+ print("-----creating figure 4 and figure E1----")
+ print("----------------------------------------")
- give_figure_4_and_e1(ful_group, False, args.save)
- print("----------------------------------")
- print("-----creating figure 5----------------")
- print("----------------------------------")
+ give_figure_4_and_e1(False, args.save)
+ print("----------------------------------------")
+ print("-----creating figure 5------------------")
+ print("----------------------------------------")
give_figure_5(reg_group, ful_group, args.save)
- print("----------------------------------")
- print("-----creating figure e2---------------")
- print("----------------------------------")
+ print("----------------------------------------")
+ print("-----creating figure e2-----------------")
+ print("----------------------------------------")
give_figure_e2(reg_group, ful_group, args.save)