add Multi C training code and dependencies (DO NOT MERGE!)

histomicstk/segmentationschool/Codes/InitializeFolderStructure.py

@@ -1,7 +1,7 @@
-import os,girder_client
+import os
 import numpy as np
 from glob import glob
-from shutil import rmtree,copy#,move,copyfile
+from shutil import rmtree,move,copyfile,copy
 
 def purge_training_set(args):
     rmtree(args.base_dir + '/' + args.project + '/Permanent/')
@@ -41,39 +41,24 @@ def initFolder(args):
     dirs = {'imExt': '.jpeg'}
     dirs['basedir'] = args.base_dir
     dirs['maskExt'] = '.png'
-    dirs['modeldir'] = 'MODELS'
-    dirs['tempdirLR'] = 'TempLR'
-    dirs['tempdirHR'] = 'TempHR'
-    dirs['pretraindir'] = 'Deeplab_network'
-    dirs['training_data_dir'] = 'TRAINING_data'
-    dirs['validation_data_dir'] = 'HOLDOUT_data'
+    dirs['modeldir'] = '/MODELS/'
+    dirs['tempdirLR'] = '/TempLR/'
+    dirs['tempdirHR'] = '/TempHR/'
+    dirs['pretraindir'] = '/Deeplab_network/'
+    dirs['training_data_dir'] = '/TRAINING_data/'
+    dirs['validation_data_dir'] = '/HOLDOUT_data/'
     dirs['model_init'] = 'deeplab_resnet.ckpt'
     dirs['project']= '/' + args.project
-    dirs['data_dir_HR'] = args.base_dir + args.project + 'Permanent/HR'
-    dirs['data_dir_LR'] = args.base_dir + args.project + 'Permanent/LR'
+    dirs['data_dir_HR'] = args.base_dir + args.project + '/Permanent/HR/'
+    dirs['data_dir_LR'] = args.base_dir + args.project + '/Permanent/LR/'
     initializeFolderStructure(dirs,args)
     print('Please add xmls/svs files to the newest TRAINING_data folder.')
 
 
 def initializeFolderStructure(dirs,args):
+    make_folder(dirs['basedir'] +dirs['project'] + dirs['modeldir'] + str(0) + '/LR/')
+    make_folder(dirs['basedir'] +dirs['project']+ dirs['modeldir'] + str(0) + '/HR/')
 
-    folder_base = args.base_dir
-    base_directory_id = folder_base.split('/')[-2]
-    _ = os.system("printf '\nIn the base directory: {} {}\n'".format(base_directory_id,folder_base))
-
-    folder_project = args.project
-    project_directory_id = folder_project.split('/')[-2]
-    _ = os.system("printf '\nIn the base directory: {}{}\n'".format(project_directory_id,folder_base))
-
-    gc = girder_client.GirderClient(apiUrl=args.girderApiUrl)
-    gc.setToken(args.girderToken)
-
-    # modeldir = gc.createFolder(project_directory_id, dirs['modeldir'])
-
-
-    # make_folder(dirs['basedir'] +dirs['project'] + dirs['modeldir'] + str(0) + '/LR/')
-    # make_folder(dirs['basedir'] +dirs['project']+ dirs['modeldir'] + str(0) + '/HR/')
-
     if args.transfer==' ':
         pass
     else:
@@ -103,6 +88,37 @@ def initializeFolderStructure(dirs,args):
         for file in pretrain_filesHR:
             copy(file,dirs['basedir'] +dirs['project']+ dirs['modeldir'] + str(0) + '/HR/')
 
-    training_data_dir = gc.createFolder(project_directory_id, dirs['training_data_dir'])
 
-    print(training_data_dir, 'this is training data dir')
+    make_folder(dirs['basedir']+dirs['project'] + dirs['training_data_dir'] + str(0))
+
+    make_folder(dirs['basedir'] +dirs['project']+ dirs['tempdirLR'] + '/regions')
+    make_folder(dirs['basedir'] +dirs['project']+ dirs['tempdirLR'] + '/masks')
+
+    make_folder(dirs['basedir'] +dirs['project']+ dirs['tempdirLR'] + '/Augment' +'/regions')
+    make_folder(dirs['basedir'] +dirs['project']+ dirs['tempdirLR'] + '/Augment' +'/masks')
+
+    make_folder(dirs['basedir']+dirs['project'] + dirs['tempdirHR'] + '/regions')
+    make_folder(dirs['basedir'] +dirs['project']+ dirs['tempdirHR'] + '/masks')
+
+    make_folder(dirs['basedir']+dirs['project'] + dirs['tempdirHR'] + '/Augment' +'/regions')
+    make_folder(dirs['basedir']+dirs['project']+ dirs['tempdirHR'] + '/Augment' +'/masks')
+
+    make_folder(dirs['basedir'] +dirs['project']+ dirs['modeldir'])
+    make_folder(dirs['basedir'] +dirs['project']+ dirs['training_data_dir'])
+    make_folder(dirs['basedir'] +dirs['project']+ dirs['validation_data_dir'])
+
+
+    make_folder(dirs['basedir'] +dirs['project']+ '/Permanent' +'/LR/'+ 'regions/')
+    make_folder(dirs['basedir'] +dirs['project']+ '/Permanent' +'/LR/'+ 'masks/')
+    make_folder(dirs['basedir'] +dirs['project']+ '/Permanent' +'/HR/'+ 'regions/')
+    make_folder(dirs['basedir'] +dirs['project']+ '/Permanent' +'/HR/'+ 'masks/')
+
+    make_folder(dirs['basedir'] +dirs['project']+ dirs['training_data_dir'])
+
+    make_folder(dirs['basedir'] + '/Codes/Deeplab_network/datasetLR')
+    make_folder(dirs['basedir'] + '/Codes/Deeplab_network/datasetHR')
+
+
+def make_folder(directory):
+    if not os.path.exists(directory):
+        os.makedirs(directory) # make directory if it does not exit already # make new directory # Check if folder exists, if not make it

histomicstk/segmentationschool/Codes/IterativePredict.py

@@ -2,28 +2,28 @@
 import numpy as np
 import os
 import sys
-# import argparse
+import argparse
 import multiprocessing
 import lxml.etree as ET
 import warnings
 import time
 
-sys.path.append('..')
+sys.path.append(os.getcwd()+'/Codes')
 
 from glob import glob
 from subprocess import call
 from joblib import Parallel, delayed
 from skimage.io import imread, imsave
 from skimage.transform import resize
 from scipy.ndimage.measurements import label
-# from skimage.segmentation import clear_border
+from skimage.segmentation import clear_border
 from skimage.morphology import remove_small_objects
-# from skimage import color
+from skimage import color
 from shutil import rmtree
-from .IterativeTraining import get_num_classes
-from .get_choppable_regions import get_choppable_regions
-from .get_network_performance import get_perf
-# from matplotlib import pyplot as plt
+from IterativeTraining import get_num_classes
+from get_choppable_regions import get_choppable_regions
+from get_network_performance import get_perf
+from matplotlib import pyplot as plt
 """
 Pipeline code to find gloms from WSI