A
A PPENDIX - S OURCE CODE
# draw_figures . py
# Draws f i g u r e s from the p r e d i c t i o n s . import numpy as np
import skimage , os from glob import glob
import matplotlib . pyplot as p l t
from skimage . u t i l . montage import montage2d
def npy_to_figure ( i ) :
path_img = ’ r e s u l t s / p r e di c t /img_%d . npy ’ % i path_pred = ’ r e s u l t s / p r e dict / pred_%d . npy ’ % i path_truth = ’ r e s u l t s / pr e di c t / truth_%d . npy ’ % i path_out = ’ r e s u l t s / f i g u r e s/%d . png ’ % i
spine_image = np . load ( path_img )
spine_image = np . squeeze ( spine_image , a x i s =3) spine_pred = np . load ( path_pred )
spine_pred = np . squeeze ( spine_pred , a x i s =3) spine_mask = np . load ( path_truth )
spine_mask = np . squeeze ( spine_mask , a x i s =3)
stacked = np . dstack ( ( spine_image , spine_pred , spine_mask ) ) stacked = np . transpose ( stacked , ( 2 , 0 , 1) )
p l t . imsave ( path_out , montage2d ( stacked , grid_shape =( 3 , 32) ) , ,→ cmap= ’ inferno ’ )
i f __name__ == "__main__" :
imgs_path = ’ r e s u l t s / p r e dic t / ’
all_images=glob ( os . path . j o i n ( imgs_path , ’ img_ * . npy ’ ) ) for i in range(len( all_images ) ) :
npy_to_figure ( i )
# draw_hist_from_csv . py
# Draws histograms from the spine metrics csv . import pandas as pd
import matplotlib . pyplot as p l t def s p l i t _ a t _ i ( x , j ) :
FN_t , FN_v = s p l i t _ a t _ i (FN, v a l i d a t i o n _ s p l i t _ i )
accuracy = (cm[ 3 ] + cm[ 0 ] ) / (cm[ 0 ] + cm[ 1 ] + cm[ 2 ] + cm[ 3 ] )
test_name = " gt_ " + test_name
# save spine r e s u l t s on csv
,→ f i l t e r _ l i s t ]
# evaluate_spines . py
df = pd . DataFrame . from_records ( spine_csv , index= s e l f . ,→ v a l i d a t i o n _ c a s e s )
df . to_csv ( o u t _ f i l e )
def pixel_confusion_matrix ( y_true , y_pred ) :
# truth_0 = np . count_nonzero (~ y_true )
# truth_1 = np . count_nonzero ( y_true ) pred_0 = np . count_nonzero (~ y_pred ) pred_1 = np . count_nonzero ( y_pred )
# a s s e r t truth_0 + truth_1 == pred_0 + pred_1 == 64*64*32 background = (~ y_true ) * (~ y_pred )
i n t e r s e c t i o n = y_true * y_pred
common_0 = np . count_nonzero ( background ) common_1 = np . count_nonzero ( i n t e r s e c t i o n )
pass spine_image = np . load ( path_img )
spine_pred = np . load ( path_pred )
spine_pred = ( spine_pred > 0 . 5 ) * 1.0 spine_pred = spine_pred . astype ( ’ bool ’ ) spine_mask = np . load ( path_truth )
spine_mask = spine_mask . astype ( ’ bool ’ ) return spine_image , spine_pred , spine_mask spine_image = np . load ( path_img )
spine_pred1 = np . load ( path_pred1 )
spine_pred1 = spine_pred1 . astype ( ’ bool ’ ) spine_pred2 = np . load ( path_pred2 )
spine_pred2 = ( spine_pred2 > 0 . 5 ) * 1.0 spine_pred2 = spine_pred2 . astype ( ’ bool ’ ) spine_mask = np . load ( path_truth )
spine_mask = spine_mask . astype ( ’ bool ’ )
return spine_image , spine_pred1 , spine_pred2 , spine_mask def e v a l u a t e _ t r a i n ( ) :
imgs_path = ’ r e s u l t s / p r e dic t / ’
all_images=glob ( os . path . j o i n ( imgs_path , ’ img_ * . npy ’ ) ) absolute_cm_all_spines = [ 0 , 0 , 0 , 0]
training_cm = [ 0 , 0 , 0 , 0]
v a l i d a t i o n _ s p l i t _ i = 3056 vm = ValidationMetrics ( )
for i in range( 0 , len( all_images ) ) :
_ , prediction , truth = load_image_pred_mask ( i ) cm = pixel_confusion_matrix ( truth , prediction )
# print_confusion_matrix ( normalize_confusion_matrix (cm) ) vm. add_cm(cm)
absolute_cm_all_spines = add_to_cm ( absolute_cm_all_spines ,
,→ cm)
i f i == v a l i d a t i o n _ s p l i t _ i − 1 :
training_cm = [ x for x in absolute_cm_all_spines ] print( ’ Confusion matrix with only t r a i n i n g spines : ’ ) print_confusion_matrix ( ( absolute_cm_all_spines ) ) print( ’ Confusion matrix with a l l spines at once : ’ )
print_confusion_matrix ( ( absolute_cm_all_spines ) ) print( ’ Vali da tio n : ’ )
validation_cm = [ absolute_cm_all_spines [ i ] − training_cm [ i ] ,→ for i in range( 4 ) ]
_ , pred1 , pred2 , tr uth = load_image_pred12_mask ( i ) cma = pixel_confusion_matrix ( truth , pred1 )
cmb = pixel_confusion_matrix ( truth , pred2 ) cmc = pixel_confusion_matrix ( pred1 , pred2 )
# print_confusion_matrix ( normalize_confusion_matrix (cm) ) vma. add_cm(cma)
vmb. add_cm(cmb) vmc . add_cm(cmc)
cm_p1_gt = add_to_cm ( cm_p1_gt , cma) cm_p2_gt = add_to_cm ( cm_p2_gt , cmb) cm_p2_p1 = add_to_cm ( cm_p2_p1 , cmc)
print( ’ Pred1 compared to GT ’ ) print_confusion_matrix ( cm_p1_gt ) print( ’ Pred2 compared to GT ’ ) print_confusion_matrix ( cm_p2_gt ) print( ’ Pred2 compared to Pred1 ’ ) print_confusion_matrix ( cm_p2_p1 )
vma. save ( " . / spine_scores_test_p1_gt . csv " ) vmb. save ( " . / spine_scores_test_p2_gt . csv " ) vmc . save ( " . / spine_scores_test_p2_p1 . csv " ) i f __name__ == "__main__" :
e v a l u a t e _ t r a i n ( ) e v a l u a t e _ t e s t ( )
# example_dendrite . py
# Shows a sample d e n t r i t e import numpy as np
from skimage import img_as_float import matplotlib . pyplot as p l t t r y:
return image
i f __name__ == "__main__" : z = 35
img = read_image_to_array ( ’ data / data / i f 6 .1.1−1 enero / spine . n i i . ,→ gz ’ )
mask = read_image_to_array ( ’ data / data / i f 6 .1.1−1 enero / tru th . n i i ,→ . gz ’ )
mask = mask_150_255 (mask) . astype ( ’ bool ’ ) p l t . f i g u r e ( f i g s i z e =(12 , 18) )
stacked_mask = np . transpose (mask , ( 2 , 0 , 1) )
p l t . imsave ( ’ example_mask . png ’ , stacked_mask [ z ] , cmap= ’ gray ’ ) p l t . f i g u r e ( f i g s i z e =(12 , 18) )
stacked_img = np . transpose ( img , ( 2 , 0 , 1) )
p l t . imsave ( ’ example_img . png ’ , stacked_img [ z ] , cmap= ’ gray ’ )
# to rgb
stacked_img = np . stack ( ( stacked_img , ) *3 , −1) img_layer = stacked_img [ z ]
img_layer [ stacked_mask [ z ] , 0] = img_layer [ stacked_mask [ z ] , 0]
,→ * 0 . 5 + 127
img_layer [ stacked_mask [ z ] , 1] = img_layer [ stacked_mask [ z ] , 1]
,→ * 0 . 7
img_layer [ stacked_mask [ z ] , 2] = img_layer [ stacked_mask [ z ] , 2]
,→ * 0 . 7
p l t . imsave ( ’ example_mask_img . png ’ , img_layer )
# nidata . py
# Converts data from n i i format to npy format
# ready to be used by the unet import numpy as np
import skimage , os
from skimage . measure import l ab el , regionprops from skimage . u t i l . montage import montage2d from scipy import ndimage as ndi
from skimage import img_as_float import matplotlib . pyplot as p l t from glob import glob
def save_montage_img_mask ( path_out , image , mask) : p l t . f i g u r e ( f i g s i z e =(12 , 18) )
i f f i l t e r _ s p i n e s ! = True or ( r . area > 8 and spine_mask . all_images , all_masks , a l l _ p r e d = get_images (BASE_IMG_PATH)
# normalize images
all_images = [ ( x−np . amin ( x ) ) / (np . amax( x )−np . amin ( x ) ) for x in ,→ all_images ]
# TODO: b i n a r i z e al l _p re d ( doesnt make sense i f we r e s i z e i t
,→ anyway )
# Now the mask contains 150 f o r d e n d r i t e s and 255 f o r spines ,
# we only want to keep sp ine s . Same f o r pred all_masks = [ mask_150_255 ( x ) for x in all_masks ] a l l _ p r e d = [ mask_150_255 ( x ) for x in a l l _ p r e d ]
# so we combine images and masks into a new array of shape ,→ (1024 , 1024 , 112 , 2)
for i in range(len( spine_images_and_masks ) ) :
imgmasks = np . s p l i t ( spine_images_and_masks [ i ] , 2 , a x i s =−1)
,→ which r e t u r n s
for i in range(len( spine_images_and_preds ) ) :
imgmasks = np . s p l i t ( spine_images_and_preds [ i ] , 2 , a x i s =−1)
all_images , all_masks , _ = get_images (BASE_IMG_PATH)
# Now the mask contains 150 f o r d e n d r i t e s and 255 f o r spines ,
# we only want to keep sp ine s
all_masks = [ mask_150_255 ( x ) for x in all_masks ] return all_images , all_masks
def g e t _ t r a i n _ s p i n e s ( ) :
all_images , all_masks = get_train_images ( )
all_spine_images , all_spine_masks = get_spines ( all_images , ,→ all_masks )
return all_spine_images , all_spine_masks
# t a r g e t _ s i z e = ( 3 2 , 32 , 32)
# w i l l return ( 3 2 , 32 , 32 , 1) np . save ( img_path , all_spine_images )
img_path=os . path . j o i n ( ’ data ’ , ’ spines_train_mask_gray . npy ’ ) np . save ( img_path , all_spine_masks )
# T e s t
all_spine_images , all_spine_masks , all_spine_pred =
,→ g e t _ t e s t _ s p i n e s _ r e s i z e ( t a r g e t _ s i z e , use_pred_as_gt=True ) img_path=os . path . j o i n ( ’ data ’ , ’ spines_test_gt_image . npy ’ )
np . save ( img_path , all_spine_images )
img_path=os . path . j o i n ( ’ data ’ , ’ spines_test_gt_mask . npy ’ ) np . save ( img_path , all_spine_masks )
img_path=os . path . j o i n ( ’ data ’ , ’ spines_test_gt_pred . npy ’ ) np . save ( img_path , all_spine_pred )
def load_train_spines ( ) :
return imgs , masks , pred1
images = [ read_image_to_array ( x ) for x in all_images ] masks = [ read_image_to_array ( x ) for x in all_masks ] pred = [ read_image_to_array ( x ) for x in a l l _ p r e d ]
def get_spines ( all_images , all_masks ) : all_spine_images = [ ]
return a
p l t . subplot ( 3 , 1 , 3)
p l t . h i s t ( sizez , 100 , [ 0 , 1 0 0 ] ) # h i s t ( data , bins , range ) p l t . minorticks_on ( )
p l t . x l a b e l ( ’ z s i z e of the spine ’ ) p l t . y l a b e l ( ’number of spines ’ ) p l t . s a v e f i g ( ’ h i s t _ s p i n e s . svg ’ ) p l t . s a v e f i g ( ’ h i s t _ s p i n e s . png ’ ) i f __name__ == "__main__" :
s a v e _ s p i n e s _ t o _ f i l e s ( ( 6 4 , 64 , 32) )
# unet . py
from keras . l a y e r s import Conv3D, Conv3DTranspose , MaxPooling3D , ,→ BatchNormalization , A c t i v a t i o n
from keras . optimizers import *
from keras . c a l l b a c k s import ModelCheckpoint , TensorBoard , ,→ EarlyStopping , CSVLogger , ReduceLROnPlateau
import keras . backend as K
conv = Conv3D( features , ker ne l_size , padding= ’same ’ ,
,→ k e r n e l _ i n i t i a l i z e r = ’ he_normal ’ ) (input)
conv = Conv3D( features , ker ne l_size , padding= ’same ’ , ,→ k e r n e l _ i n i t i a l i z e r = ’ he_normal ’ ) ( conv )
i f batch_norm :
conv = BatchNormalization ( a x i s =4) ( conv ) conv = A c t i v a t i o n ( ’ r e l u ’ ) ( conv )
pool = MaxPooling3D ( pool_size=pool_size ) ( conv ) return conv , pool conv = Conv3D( features , ker ne l_size , padding= ’same ’ ,
,→ k e r n e l _ i n i t i a l i z e r = ’ he_normal ’ ) ( merge ) s e l f . img_layers = img_layers def load_data ( s e l f ) :
imgs , masks = load_train_spines ( )
imgs = img_as_float ( imgs ) . astype (np . f l o a t 3 2 ) model = Model ( inputs=input1 , outputs=conv10 )
# o r i g i n a l : l o s s = ’ binary_crossentropy ’ , l r =1e−4
# l o s s = d i c e _ c o e f _ l o s s
model .compile( optimizer=Adam( l r =1e−4) , l o s s = ’
,→ binary_crossentropy ’ , metrics =[ ’ accuracy ’ , mean_pred ,→ , ’ binary_crossentropy ’ , global_dice_coef ] )
print( model . summary ( ) )
imgs_train , imgs_mask_train , imgs_test = s e l f . load_data ( ) print( " loading data done" )
model = s e l f . get_unet ( )
,→ batch_size=batch_size , write_graph=True , write_grads ,→ =False , write_images=False , embeddings_freq =0 ,
,→ embeddings_layer_names=None, embeddings_metadata=
model . f i t ( imgs_train , imgs_mask_train , batch_size=
,→ batch_size , epochs=500 , verbose =1 , v a l i d a t i o n _ s p l i t ,→ = 0 . 2 , s h u f f l e =True , c a l l b a c k s = c a l l b a c k s )
def pr e dict ( s e l f ) :
print( " loading data " )
imgs_train , imgs_mask_train , imgs_test = s e l f . load_data ( ) print( " loading data done" )
model = s e l f . get_unet ( ) print( " got unet " )
model . load_weights ( ’ r e s u l t s / unet_t . hdf5 ’ )
imgs_pred = model . pr e di ct ( imgs_train , batch_size =2 , ,→ verbose =1)
imgs_train , imgs_mask_train , imgs_pred1 = s e l f . ,→ load_data_pred1 ( )
print( " loading data done" ) model = s e l f . get_unet ( )
def grayscale_to_binary ( img , threshold = 0 . 5 ) : im = ( img > threshold ) * 1.0
return im
# additional metric which shows the mean o f the p r e d i c t i o n
# u s e f u l because 0 . 0 means that a l l images are black , and 1 . 0 = ,→ white
def mean_pred ( y_true , y_pred ) :
m = K . mean( y_pred ) return m
# d i c e c o e f but applied to the e n t i r e batch at once ( instead of ,→ each image i n d i v i d u a l l y )
def global_dice_coef ( y_true , y_pred ) : smooth = 1 .
return −global_dice_coef ( y_true , y_pred ) i f __name__ == ’ __main__ ’ :
parser = argparse . ArgumentParser ( )
parser . add_argument ( ’−p ’ , action= ’ s t o r e _ t r u e ’ , help= ’ pre d ic t
B IBLIOGRAPHY
[1] V. Dumoulin and F. Visin, “A guide to convolution arithmetic for deep learning,”
ArXiv e-prints, mar 2016. (document),2.6,2.7,2.8,A
[2] P. Penzes, M. E. Cahill, K. A. Jones, J.-E. VanLeeuwen, and K. M. Woolfrey,
“Dendritic spine pathology in neuropsychiatric disorders,” Nat Neurosci, vol. 14, no. 3, pp. 285–293, Mar 2011, 21346746[pmid]. [Online]. Available:
http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3530413/1.1
[3] F. Janoos, K. Mosaliganti, X. Xu, R. Machiraju, K. Huang, and S. T. Wong,
“Robust 3d reconstruction and identification of dendritic spines from optical microscopy imaging,” Medical Image Analysis, vol. 13, no. 1, pp. 167 – 179, 2009, includes Special Section on Medical Image Analysis on the 2006 Workshop Microscopic Image Analysis with Applications in Biology. [Online]. Available:
http://www.sciencedirect.com/science/article/pii/S13618415080008321.1 [4] S. Basu, P. K. Saha, M. Roszkowska, M. Magnowska, E. Baczynska, N. Das,
D. Plewczynski, and J. Wlodarczyk, “Quantitative 3-d morphometric analysis of individual dendritic spines,”Scientific Reports, vol. 8, no. 1, p. 3545, 2018. [Online].
Available:https://doi.org/10.1038/s41598-018-21753-81.1 [5] T. Worbs and R. Förster, “4d-tracking with imaris.”1.1
[6] K. Simonyan and A. Zisserman, “Very deep convolutional networks for large-scale image recognition,” CoRR, vol. abs/1409.1556, 2014. [Online]. Available:
http://arxiv.org/abs/1409.15561.3
[7] J. Long, E. Shelhamer, and T. Darrell, “Fully convolutional networks for semantic segmentation,”CoRR, vol. abs/1411.4038, 2014. [Online]. Available:
http://arxiv.org/abs/1411.40381.3
[8] O. Ronneberger, P. Fischer, and T. Brox, “U-net: Convolutional networks for biomedical image segmentation,”CoRR, vol. abs/1505.04597, 2015. [Online].
Available:http://arxiv.org/abs/1505.045972.4
[9] S. Ioffe and C. Szegedy, “Batch normalization: Accelerating deep network training by reducing internal covariate shift,”CoRR, vol. abs/1502.03167, 2015. [Online].
Available:http://arxiv.org/abs/1502.031672.4.2
[10] N. Srivastava, G. Hinton, A. Krizhevsky, I. Sutskever, and R. Salakhutdinov,
“Dropout: A simple way to prevent neural networks from overfitting,” vol. 15, pp. 1929–1958, 06 2014.2.4.2
[11] K. He, X. Zhang, S. Ren, and J. Sun, “Delving deep into rectifiers: Surpassing human-level performance on imagenet classification,”CoRR, vol. abs/1502.01852, 2015. [Online]. Available:http://arxiv.org/abs/1502.018522.4.2
[12] F. Cholletet al., “Keras,”https://keras.io, 2015. 2.6 Sources of external figures
Figures not listed here are original work.
1.1https://commons.wikimedia.org/wiki/File:Dendrite_(PSF).png 1.2https://pixabay.com/p-1246693/
1.3https://en.wikipedia.org/wiki/File:Colored_neural_network.svg 2.10https://commons.wikimedia.org/wiki/File:Ramp_function.svg 2.11https://commons.wikimedia.org/wiki/File:Logistic-curve.svg 2.6,2.7,2.8: [1]