update filenames for the statistics and README file

Exp_Stat/README.md

+This folder contains the statistics for reproducing all the figures in our paper:
+
+- Calibration statistics (NLL, ECE, Brier score) are saved in the folder **full_image_calibration_stat** and **region_calibration_stat**, each file is named as *Vx_Method_x_Stage_x_Version_x.obj*
+  - Vx means experiment index (multiple experiments for per method are carried out to get confidence interval)
+  - Method_x_Stage_x: B, C and D correspond to methods VarRatio (stage 1), Entropy (stage 2) and BALD (stage 3)
+  - Version_x: experiment version
+- Uncertainty distribution of the pixels in the acquired images and regions are saved in the folder **acquired_full_image_uncertainty** and **acquired_region_uncertainty**, the files are named with the same fashion as previous ones. Each file contains the predicted probability of all the pixels in the acquired images/regions at each acquisition step. 
+- The statistics for creating the expected calibration error histogram using full image acquisition strategy are saved in the folder **ece_histogram**
+- Glas.xlsx file contains the segmentation accuracy (F1 score, Dice Index) for the models at each acquisition step
+

README.md

 ### On uncertainty estimation in active learning for image segmentation 
 
-<br /> This repository provides the implementation for our paper **On uncertainty estimation in active learning for image segmentation** (Bo Li, Tommy Sonne Alstrøm) [NEED TO ADD A LINK]. We experimentally show that the region based active learning strategy can lead to higher segmentation accuracy and better calibrated model much faster than full image acquisition:
+<br /> This repository provides the implementation for our paper [**On uncertainty estimation in active learning for image segmentation** (Bo Li, Tommy Sonne Alstrøm)](https://arxiv.org/abs/2007.06364). We experimentally show that the region based active learning strategy can lead to higher segmentation accuracy and better calibrated model much faster than full image acquisition:
 
 ![performance](DATA/first_figure.jpg)
 
@@ -60,7 +60,7 @@ Args:
 
 #### Reproduce figures
 
-In order to reproduce the figures in the paper, run
+The statistics that are used for reproducing the figures are saved in folder *Exp_Stat*. In order to reproduce the figures in the paper, run
 
 ```bash
 ./produce_figure.sh

produce_figure.sh

@@ -14,7 +14,7 @@ if [ -d "$filename" ]; then
 else
     echo "$filename does not exist"
     echo "Download the file..................................."
-    wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1fu7RcqOMCCjz65VwFlTPg5qlLg2K7l1g' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1fu7RcqOMCCjz65VwFlTPg5qlLg2K7l1g" -O calibration_score.tar.gz && rm -rf /tmp/cookies.txt
+    wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1VT_6mMA9ksDV8O3shYEXEehrMPDktmVZ' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1VT_6mMA9ksDV8O3shYEXEehrMPDktmVZ" -O calibration_score.tar.gz && rm -rf /tmp/cookies.txt
     echo "-----unziping the datafile"
     tar -xzvf calibration_score.tar.gz
     mv use_stat calibration_score

visualize_calibration_score.py

@@ -38,17 +38,17 @@ def ax_global_get(fig):
     return ax_global
 
 
-def give_score_path(path_use):
+def give_score_path(path):
     str_group = ["_B_", "_C_", "_D_"]
-    region_path = path_use + 'Act_Learn_Desperate_V6/'
+    region_path = path + 'region_calibration_stat/'
     region_group = [[] for _ in range(3)]
     for iterr, single_str in enumerate(str_group):
         select_folder = [region_path + v for v in os.listdir(region_path) if single_str in v and '.obj' in v]
         region_group[iterr] = select_folder
-    full_path = [path_use + 'Act_Learn_Desperate_V7/', path_use + 'Act_Learn_Desperate_V8/']
+    full_path = path + 'full_image_calibration_stat/'
     full_group = [[] for _ in range(3)]
     for iterr, single_str in enumerate(str_group):
-        folder_select = [v + q for v in full_path for q in os.listdir(v) if single_str in q and '.obj' in q]
+        folder_select = [full_path + v for v in os.listdir(full_path) if single_str in v and '.obj' in v]
         full_group[iterr] = folder_select
     return region_group, full_group
 
@@ -163,7 +163,6 @@ def give_figure_5(reg_group, ful_group, save=False):
     for iterr, single_score in enumerate(score_group):
         ax0 = fig.add_subplot(len(score_group), 2, 2 * iterr + 1)
         ax1 = fig.add_subplot(len(score_group), 2, 2 * iterr + 2)
-        print(2 * iterr + 1, 2 * iterr + 2)
         compare_acq_at_certain_point_barplot(reg_group, ful_group, single_score, [ax0, ax1])
         if iterr == 0 or iterr == 1:
             for ax in [ax0, ax1]:
@@ -183,10 +182,21 @@ def give_figure_5(reg_group, ful_group, save=False):
     if save is True:
         plt.savefig(save_fig_path + 'overall_calibration.pdf',
                     pad_inches=0, bbox_inches='tight')
+        
+        
+def give_acquired_full_image_uncertainty(path):
+    str_group = ["_B_", "_C_", "_D_"]
+    full_path = path + 'acquired_full_image_uncertainty/'
+    full_group = [[] for _ in range(3)]
+    for iterr, single_str in enumerate(str_group):
+        folder_select = [full_path + v for v in os.listdir(full_path) if single_str in v and '.npy' in v]
+        full_group[iterr] = folder_select
+    return full_group
 
 
-def give_figure_4_and_e1(ful_group, conf_interval=True, save=False):
-    ece_path = path + "ece_stat/"
+def give_figure_4_and_e1(conf_interval=True, save=False):
+    ful_group = give_acquired_full_image_uncertainty(path)
+    ece_path = path + "ece_histogram/"
     legend_space = ["VarRatio", "Entropy", "BALD"]
 
     ece_all = [v for v in os.listdir(ece_path) if '.npy' in v and '_stat_' in v]
@@ -209,7 +219,6 @@ def give_figure_4_and_e1(ful_group, conf_interval=True, save=False):
     ece_d_avg = np.mean([v[1] for v in ece_d], axis=0)
     ece_d_avg = ece_d[2][1]
     #    ece_d_avg = [v+0.03 if iterr <= 4 else v-0.03 for iterr, v in enumerate(ece_d_avg)]
-    print(np.shape(ece_d_avg))
     ece_d_std = np.std([v[1] for v in ece_d], axis=0) * 1.95 / np.sqrt(len(ece_d))
 
     uncertain_stat = show_uncertainty_distribution(ful_group, True)
@@ -419,8 +428,11 @@ def compare_acq_at_certain_point_line(reg_group, ful_group, score_str, ax):
     if not ax:
         fig = plt.figure(figsize=(5, 3))
         ax = fig.add_subplot(111)
+    
+    for i in range(3):
+        ax.plot(f_g_perf[i][0] , f_g_perf[i][1], color_group[i], ls=lstype_group[1], lw=1.0)
     for i in range(3):
-        ax.plot(r_g_perf[i][0], r_g_perf[i][1], color_group[i], ls=lstype_group[1], lw=1.0)
+        ax.plot(r_g_perf[i][0] , r_g_perf[i][1], color_group[i], ls=lstype_group[0], lw=1.0)
     ax.grid(ls=':', axis='both')
     if score_str is "nll_score":
         ax.ticklabel_format(axis='y', style='sci', scilimits=(10, 5))
@@ -533,7 +545,6 @@ def sort_uncertainty(pool_path, method, load_step):
 
 
 def get_uncertainty_group(path_group, method, load_step, return_value=False):
-    path_group = [v.strip().split('.obj')[0] + "pool_stat.npy" for v in path_group]
     if method is "B":
         path_group = [path_group[0], path_group[2]]
     uncertain_stat = [sort_uncertainty(single_path, method, load_step)
@@ -571,7 +582,7 @@ def get_region_uncert(return_stat=False):
     version_use = [3, 1, 2]
     step = [0, 0, 1]
     uncert_stat = []
-    path2read = path + '/region_uncertainty/'
+    path2read = path + '/acquired_region_uncertainty/'
     for i in range(len(method)):
         path_sub = [v for v in os.listdir(path2read) if
                     'Method_%s' % method[i] in v and 'Version_%d' % version_use[i] in v and 'step_%d' % step[i] in v]
@@ -597,24 +608,26 @@ if __name__ == '__main__':
     print("---The figures are going to be saved in ", save_fig_path)
 
     reg_group, ful_group = give_score_path(path)
-    print("----------------------------------")
-    print("-----creating the first figure----")
-    print("----------------------------------")
+    [print(v) for v in reg_group]
+    [print(v) for v in ful_group]
+    print("----------------------------------------")
+    print("-----creating the first figure----------")
+    print("----------------------------------------")
 
     give_first_figure(reg_group, ful_group, args.save)
-    print("----------------------------------")
-    print("-----creating figure 4 and figure E1---")
-    print("----------------------------------")
+    print("----------------------------------------")
+    print("-----creating figure 4 and figure E1----")
+    print("----------------------------------------")
 
-    give_figure_4_and_e1(ful_group, False, args.save)
-    print("----------------------------------")
-    print("-----creating figure 5----------------")
-    print("----------------------------------")
+    give_figure_4_and_e1(False, args.save)
+    print("----------------------------------------")
+    print("-----creating figure 5------------------")
+    print("----------------------------------------")
 
     give_figure_5(reg_group, ful_group, args.save)
-    print("----------------------------------")
-    print("-----creating figure e2---------------")
-    print("----------------------------------")
+    print("----------------------------------------")
+    print("-----creating figure e2-----------------")
+    print("----------------------------------------")
 
     give_figure_e2(reg_group, ful_group, args.save)