From 826b3d33c67409ab7d81e378ebe955aaa4923c6d Mon Sep 17 00:00:00 2001
From: bo li <s161488@student.dtu.dk>
Date: Fri, 3 Jul 2020 16:24:25 +0200
Subject: [PATCH] update bash script
---
README.md | 16 ++++++++-----
data_utils/glanddata.py | 11 ++++-----
data_utils/prepare_data.py | 2 +-
produce_figure.sh | 21 ++++++++++++----
requirement.sh | 49 ++++++++++++++++++++++++++++++--------
5 files changed, 71 insertions(+), 28 deletions(-)
diff --git a/README.md b/README.md
index 238c72b..6e3461a 100644
--- a/README.md
+++ b/README.md
@@ -10,22 +10,26 @@
```bash
git clone https://lab.compute.dtu.dk/s161488/region_active_learning.git
- cd Region_Active_Learning
+ cd region_active_learning
+ chmod +x requirement.sh
+ chmod +x produce_figure.sh
```
-2. Prepare the Dataset and pertained resnet-50 ckpt
+2. Create a virtual env with the required packages
```bash
- ./requirement.sh
+ conda env create -f active_learning.yaml
+ source activate act
```
-3. Create a virtual env with the required packages
+
+2. Prepare the Dataset and pertained resnet-50 ckpt
```bash
- conda env create --act active_learning.yaml
- source activate act
+ ./requirement.sh
```
+
#### Evaluate the model
In order to evaluate the model at each acquisition step, run
diff --git a/data_utils/glanddata.py b/data_utils/glanddata.py
index e9ff99c..14fb1f1 100644
--- a/data_utils/glanddata.py
+++ b/data_utils/glanddata.py
@@ -17,7 +17,7 @@ from skimage.morphology import dilation, disk
from scipy import ndimage
import matplotlib.pyplot as plt
-path_mom = "/home/blia/Region_Active_Learning/DATA/" # the directory for saving data
+path_mom = "DATA/" # the directory for saving data
# path_mom = "/Users/bo/Documents/Exp_Data/"
path_use = path_mom + 'Data'
if not os.path.exists(path_use):
@@ -167,6 +167,7 @@ def transfer_data_to_dict():
print("Saving the data in path:", filename.split(".")[0])
else:
print("Oh, this is the first time of creating this file")
+ print("Creating the training data npy for GlaS")
np.save(filename.split(".")[0], data)
@@ -235,12 +236,10 @@ def transfer_data_to_dict_test():
print("Remove the existing data file", os.remove(filename))
print("Saving the data in path:", filename.split(".")[0])
else:
- print("The file is not existed")
+ print("Creating GlaS test data (benign)")
print("Saving the data in path:", filename.split(".")[0])
-
np.save(filename.split(".")[0], data)
-
data1 = defaultdict(list)
data1['image'] = image_mali
data1['label'] = label_mali
@@ -252,6 +251,6 @@ def transfer_data_to_dict_test():
print("Remove the existing data file", os.remove(filename))
print("Saving the data in path:", filename.split(".")[0])
else:
- print("The file is not existed")
-
+ print("Creating GlaS test data (malignant)")
+ print("Saving the data in path:", filename.split(".")[0])
np.save(filename.split(".")[0], data1)
diff --git a/data_utils/prepare_data.py b/data_utils/prepare_data.py
index 75282b8..8042309 100644
--- a/data_utils/prepare_data.py
+++ b/data_utils/prepare_data.py
@@ -10,7 +10,7 @@ import tensorflow as tf
import cv2
import matplotlib.pyplot as plt
-path_mom = "/home/blia/Region_Active_Learning/DATA/" # NOTE, NEED TO BE MANUALLY DEFINED
+path_mom = "DATA/" # NOTE, NEED TO BE MANUALLY DEFINED
def prepare_train_data(path, select_benign_train, select_mali_train):
diff --git a/produce_figure.sh b/produce_figure.sh
index 0cc5406..bcb1a31 100644
--- a/produce_figure.sh
+++ b/produce_figure.sh
@@ -1,17 +1,28 @@
#!/bin/bash
-cd Exp_Stat
+expdir=Exp_Stat
+if [ -d "$expdir" ]; then
+ echo "folder $expdir already exists"
+ echo "next step, download the statistics to reproduce figures"
+else
+ mkdir $expdir
+ echo "make folder $expdir for saving statistics"
+fi
+cd $expdir
filename=calibration_score
if [ -d "$filename" ]; then
- echo "$filename exists"
- echo "-----next step, reproducing the figures...................."
+ echo "YEAH! $filename exists"
else
echo "$filename does not exist"
- echo "Download the file-------------------"
- wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /t$
+ echo "Download the file..................................."
+ wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1fu7RcqOMCCjz65VwFlTPg5qlLg2K7l1g' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1fu7RcqOMCCjz65VwFlTPg5qlLg2K7l1g" -O calibration_score.tar.gz && rm -rf /tmp/cookies.txt
echo "-----unziping the datafile"
tar -xzvf calibration_score.tar.gz
mv use_stat calibration_score
rm calibration_score.tar.gz
fi
cd ..
+echo "Start to reproduce figures"
python3 visualize_calibration_score.py --save True --path Exp_Stat/calibration_score/
+echo "-----------------------------------"
+echo "YEAH, FINISH REPRODUCING FIGURES :)"
+echo "-----------------------------------"
\ No newline at end of file
diff --git a/requirement.sh b/requirement.sh
index 4931236..4ea20f7 100644
--- a/requirement.sh
+++ b/requirement.sh
@@ -1,17 +1,46 @@
#!/bin/bash
cd DATA
-wget https://warwick.ac.uk/fac/sci/dcs/research/tia/glascontest/download/warwick_qu_dataset_released_2016_07_08.zip
-unzip warwick_qu_dataset_released_2016_07_08.zip
-mv 'Warwick QU Dataset (Released 2016_07_08)' gland_data
-rm -rf warwick_qu_dataset_released_2016_07_08.zip
-rm -rf __MACOSX
+datafolder=gland_data
+if [ -d "$datafolder" ]; then
+ echo "YEAH, $datafolder exist"
+ echo "next download the resnet-v2-50 pretrained weight from tensorflow website..............."
+else
+ echo "download the GlaS dataset ...................................."
+ wget https://warwick.ac.uk/fac/sci/dcs/research/tia/glascontest/download/warwick_qu_dataset_released_2016_07_08.zip
+ unzip warwick_qu_dataset_released_2016_07_08.zip
+ mv 'Warwick QU Dataset (Released 2016_07_08)' gland_data
+ rm -rf warwick_qu_dataset_released_2016_07_08.zip
+ rm -rf __MACOSX
+fi
cd ..
+pwd
+echo "download the resnet-v2-50 pretrained weight............."
+pretrainfolder=pretrain_model
+if [ -d "$pretrainfolder" ]; then
+ echo "YEAH, folder $pretrainfolder exists"
+else
+ echo "create the folder to save resnet-v2-50 pretrained weight"
+ mkdir $pretrainfolder
+fi
+resnetname=resnet_v2_50.ckpt
cd pretrain_model
-wget http://download.tensorflow.org/models/resnet_v2_50_2017_04_14.tar.gz
-tar -xvf resnet_v2_50_2017_04_14.tar.gz
-rm resnet_v2_50_2017_04_14.tar.gz
-rm train.graph
-rm eval.graph
+if [ -f "$resnetname" ]; then
+ echo "YEAH, $resnetname exists"
+ echo "next prepare the dataset ...................."
+else
+ echo "download the resnet-v2-50 pretrained weight............."
+ wget http://download.tensorflow.org/models/resnet_v2_50_2017_04_14.tar.gz
+ tar -xvf resnet_v2_50_2017_04_14.tar.gz
+ rm resnet_v2_50_2017_04_14.tar.gz
+ rm train.graph
+ rm eval.graph
+fi
cd ..
+echo "print current directory"
+pwd
+echo "prepare the dataset"
python3 -c 'import data_utils.glanddata as gd;gd.transfer_data_to_dict()'
python3 -c 'import data_utils.glanddata as gd;gd.transfer_data_to_dict_test()'
+echo "-------------------------------"
+echo "YEAH, FINISH PREPARING THE DATA"
+echo "-------------------------------"
--
GitLab