v1

.gitattributes

@@ -32,3 +32,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/*.* filter=lfs diff=lfs merge=lfs -text
+weights/*.* filter=lfs diff=lfs merge=lfs -text
+app_files/*.* filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,112 @@
+## Daniel Buscombe, Marda Science LLC 2023
+# This file contains many functions originally from Doodleverse https://github.com/Doodleverse programs
+
+import gradio as gr
+import numpy as np
+import sys, json, os
+sys.path.insert(1, 'app_files'+os.sep+'src')
+from sedinet_eval import *
+
+###===================================================
+def estimate_siso_simo_1image(vars, im, greyscale,
+                       dropout, weights_path): # numclass, name, mode, res_folder,
+                    #    batch_size, ):#, scale): #
+    """
+    This function uses a sedinet model for continuous prediction on 1 image
+    """
+    SM = make_sedinet_siso_simo(vars, greyscale, dropout)
+    SM.load_weights(weights_path)
+
+    # im = Image.open(image).convert('LA')
+    #im = im.resize((IM_HEIGHT, IM_HEIGHT))
+    im = Image.fromarray(im)
+    im = np.array(im)[:,:,0]
+    nx,ny = np.shape(im)
+    if (nx!=IM_HEIGHT) or (ny!=IM_HEIGHT):
+        im = im[int(nx/2)-int(IM_HEIGHT/2):int(nx/2)+int(IM_HEIGHT/2), int(ny/2)-int(IM_HEIGHT/2):int(ny/2)+int(IM_HEIGHT/2)]
+
+    if DO_STANDARDIZE==True:
+       im = do_standardize(im)
+    else:
+       im = np.array(im) / 255.0
+
+    result = SM.predict(np.expand_dims(np.expand_dims(im, axis=2), axis=0))
+    result = [float(r[0]) for r in result]
+
+    return result
+
+###===================================================
+def grainsize(input_img, dims=(1024, 1024)):
+
+    configfile = 'weights/config_usace_combined2021_2022_v12.json'
+    weights_path = 'weights/sandsnap_merged_1024_modelrevOct2022_v12_simo_batch10_im1024_9vars_mse_noaug.hdf5'
+
+    # load the user configs
+    with open(os.getcwd()+os.sep+configfile) as f:
+      config = json.load(f)
+
+    ###===================================================
+    dropout = config["dropout"]
+    greyscale = config['greyscale']
+
+    try:
+       greyscale = config['greyscale']
+    except:
+       greyscale = 'true'
+
+    #output variables
+    vars = [k for k in config.keys() if not np.any([k.startswith('base'), k.startswith('MAX_LR'),
+            k.startswith('MIN_LR'), k.startswith('DO_AUG'), k.startswith('SHALLOW'),
+            k.startswith('res_folder'), k.startswith('train_csvfile'), k.startswith('csvfile'),
+            k.startswith('test_csvfile'), k.startswith('name'), k.startswith('val_csvfile'),
+            k.startswith('greyscale'), k.startswith('aux_in'),
+            k.startswith('dropout'), k.startswith('N'),k.startswith('scale'),
+            k.startswith('numclass')])]
+    vars = sorted(vars)
+
+    #this relates to 'mimo' and 'miso' modes that are planned for the future but not currently implemented
+    auxin = [k for k in config.keys() if k.startswith('aux_in')]
+
+    if len(auxin) > 0:
+       auxin = config[auxin[0]]   ##at least for now, just one 'auxilliary' (numerical/categorical) input in addition to imagery
+       if len(vars) ==1:
+          mode = 'miso'
+       elif len(vars) >1:
+          mode = 'mimo'
+    else:
+       if len(vars) ==1:
+          mode = 'siso'
+       elif len(vars) >1:
+          mode = 'simo'
+
+    print("Mode: %s" % (mode))
+
+    result = estimate_siso_simo_1image(vars, input_img, greyscale,
+                       dropout, weights_path)
+    
+    result = np.array(result)
+    print(result)
+    
+    plt.plot(np.hstack((result[:3], result[4:])),[10,16,25,50,65,75,84,90],  'k-o')
+    plt.xlabel('Grain size (pixels)')
+    plt.ylabel('Percent finer')
+    plt.savefig("psd.png", dpi=300, bbox_inches="tight")
+
+    return 'mean grain size = %f pixels' % (result[4]), '90th percentile grain size = %f pixels' % (result[-1]), plt
+
+title = "SandSnap/SediNet Model Demo- Measure grain size from image of sand!" 
+description = "Allows upload of imagery and download of grain size statistics. Statistics are unscaled (i.e. in pixels)" 
+
+examples = [
+['examples/IMG_20210922_170908944_cropped.jpg'],
+['examples/20210208_172834_cropped.jpg'],
+['examples/20220101_165359_cropped.jpg']
+] 
+
+inp = gr.Image()
+out2 = gr.Plot(type='matplotlib')
+
+Segapp = gr.Interface(grainsize, inp, ["text", "text", out2], title = title, description = description, examples=examples)
+#, allow_flagging='manual', flagging_options=["bad", "ok", "good", "perfect"], flagging_dir="flagged")
+
+Segapp.launch(enable_queue=True)

app_files/src/defaults.py

@@ -0,0 +1,110 @@
+# Written by Dr Daniel Buscombe, Marda Science LLC
+# for the SandSnap Program
+#
+# MIT License
+#
+# Copyright (c) 2020-2021, Marda Science LLC
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+##> Release v1.4 (Aug 2021)
+
+## Contains values for defaults that you may change.
+## They are listed in order of likelihood that you might change them:
+
+# size of image in pixels. keep this consistent in training and application
+# suggestd: 512 -- 1024 (larger = larger GPU required)
+# integer
+IM_HEIGHT = 1024
+IM_WIDTH = IM_HEIGHT
+
+# number of images to feed the network per step in epoch #suggested: as many as you have gpu memory for, probably
+# integer
+
+# BATCH_SIZE =8
+# BATCH_SIZE =10
+BATCH_SIZE =12
+
+#use an ensemble of batch sizes like this
+#BATCH_SIZE = [7,12,14]
+
+# if True, use a smaller (shallower) network architecture
+##True or False ##False=larger network
+SHALLOW = False #True
+
+## if True, carry out data augmentation. 2 x number of images used in training
+##True or False
+DO_AUG = False #True
+
+# maximum learning rate ##1e-1 -- 1e-5
+MAX_LR = 1e-4
+# MAX_LR = 1e-5
+# MAX_LR = 5e-3
+# MAX_LR = 5e-4
+
+# max. number of training epics (20 -1000)
+# integer
+NUM_EPOCHS = 300
+
+## loss function for continuous models (2 choices)
+#CONT_LOSS = 'pinball'
+CONT_LOSS = 'mse'
+
+## loss function for categorical (disrete) models (2 choices)
+CAT_LOSS = 'focal'
+#CAT_LOSS = 'categorical_crossentropy'
+
+# optimizer (gradient descent solver) good alternative == 'rmsprop'
+OPT = 'adam'
+
+# base number of conv2d filters in categorical models
+# integer
+BASE_CAT = 30
+
+# base number of conv2d filters in continuous models
+# integer
+# BASE_CONT = 30
+BASE_CONT = 10
+
+# number of Dense units for continuous prediction
+# integer
+# CONT_DENSE_UNITS = 3072
+CONT_DENSE_UNITS = 2048
+# CONT_DENSE_UNITS = 1024
+
+# number of Dense units for categorical prediction
+# integer
+CAT_DENSE_UNITS = 128
+
+# set to False if you wish to use cpu (not recommended)
+##True or False
+USE_GPU = True
+
+## standardize imagery (recommended)
+DO_STANDARDIZE = True
+
+
+# STOP_PATIENCE = 10
+
+# FACTOR = 0.2
+
+# MIN_DELTA = 0.0001
+
+# MIN_LR = 1e-4

app_files/src/imports.py

@@ -0,0 +1,123 @@
+# Written by Dr Daniel Buscombe, Marda Science LLC
+# for the SandSnap Program
+#
+# MIT License
+#
+# Copyright (c) 2020-2021, Marda Science LLC
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+##> Release v1.4 (Aug 2021)
+
+###===================================================
+# import libraries
+import gc, os, sys, shutil
+
+###===================================================
+# import and set global variables from defaults.py
+from defaults import *
+
+global IM_HEIGHT, IM_WIDTH
+
+global NUM_EPOCHS, SHALLOW
+
+global VALID_BATCH_SIZE, BATCH_SIZE
+
+VALID_BATCH_SIZE = BATCH_SIZE
+
+global MAX_LR, OPT, USE_GPU, DO_AUG, DO_STANDARDIZE
+
+
+# global STOP_PATIENCE, FACTOR, MIN_DELTA, MIN_LR
+
+# global MIN_DELTA, FACTOR, STOP_PATIENCE
+##====================================================
+
+# import tensorflow.compat.v1 as tf1
+# config = tf1.ConfigProto()
+# config.gpu_options.allow_growth = True  # dynamically grow the memory used on the GPU
+# config.log_device_placement = True  # to log device placement (on which device the operation ran)
+# sess = tf1.Session(config=config)
+# tf1.keras.backend.set_session(sess)
+
+# PREDICT = False
+#
+# ##OS
+# if PREDICT == True:
+#    os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
+
+##TF/keras
+if USE_GPU == True:
+   ##use the first available GPU
+   os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+else:
+   ## to use the CPU (not recommended):
+   os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
+
+import numpy as np
+import tensorflow as tf
+
+# from tensorflow.keras import mixed_precision
+# mixed_precision.set_global_policy('mixed_float16')
+
+SEED=42
+np.random.seed(SEED)
+AUTO = tf.data.experimental.AUTOTUNE # used in tf.data.Dataset API
+
+tf.random.set_seed(SEED)
+
+print("Version: ", tf.__version__)
+print("Eager mode: ", tf.executing_eagerly())
+print('GPU name: ', tf.config.experimental.list_physical_devices('GPU'))
+print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
+
+from tensorflow.keras.layers import Input, Dense, MaxPool2D, GlobalMaxPool2D
+from tensorflow.keras.layers import Dropout, MaxPooling2D, GlobalAveragePooling2D
+from tensorflow.keras.models import Model, Sequential
+from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau, LearningRateScheduler
+from tensorflow.keras.layers import DepthwiseConv2D, Conv2D, SeparableConv2D
+from tensorflow.keras.layers import BatchNormalization, Activation, concatenate
+
+try:
+    from tensorflow.keras.utils import plot_model
+except:
+    pass
+
+import tensorflow.keras.backend as K
+from tensorflow.keras.utils import to_categorical
+import tensorflow_addons as tfa
+
+##SKLEARN
+from sklearn.preprocessing import RobustScaler #MinMaxScaler
+from sklearn.metrics import confusion_matrix, classification_report
+
+##OTHER
+from PIL import Image
+from glob import glob
+import matplotlib.pyplot as plt
+import pandas as pd
+import itertools
+import joblib
+import random
+from tempfile import TemporaryFile
+import tensorflow_addons as tfa
+import tqdm
+
+from skimage.transform import AffineTransform, warp #rotate,

app_files/src/sedinet_eval.py

@@ -0,0 +1,287 @@
+# Written by Dr Daniel Buscombe, Marda Science LLC
+# for the SandSnap Program
+#
+# MIT License
+#
+# Copyright (c) 2020-2021, Marda Science LLC
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+##> Release v1.4 (Aug 2021)
+
+###===================================================
+# import libraries
+from sedinet_models import *
+
+###===================================================
+def get_data_generator(df, indices, greyscale, tilesize,batch_size=16):
+    """
+    This function generates data for a batch of images and no metric, for  # "unseen" samples
+    """
+
+    for_training = False
+    images = []
+    while True:
+        for i in indices:
+            r = df.iloc[i]
+            file = r['files']
+
+            # if greyscale==True:
+            #    im = Image.open(file).convert('LA')
+            # else:
+            #    im = Image.open(file)
+            # im = im.resize((IM_HEIGHT, IM_HEIGHT))
+            # im = np.array(im) / 255.0
+
+            if greyscale==True:
+               im = Image.open(file).convert('LA')
+               #im = im.resize((IM_HEIGHT, IM_HEIGHT))
+               im = np.array(im)[:,:,0]
+               nx,ny = np.shape(im)
+               if (nx!=tilesize) or (ny!=tilesize):
+                   im = im[int(nx/2)-int(tilesize/2):int(nx/2)+int(tilesize/2), int(ny/2)-int(tilesize/2):int(ny/2)+int(tilesize/2)]
+
+            else:
+               im = Image.open(file)
+               #im = im.resize((IM_HEIGHT, IM_HEIGHT))
+               im = np.array(im)
+               nx,ny,nz = np.shape(im)
+               if (nx!=tilesize) or (ny!=tilesize):
+                   im = im[int(nx/2)-int(tilesize/2):int(nx/2)+int(tilesize/2), int(ny/2)-int(tilesize/2):int(ny/2)+int(tilesize/2)]
+
+            if greyscale==True:
+               images.append(np.expand_dims(im, axis=2)) #[:,:,0]
+            else:
+               images.append(im)
+
+            if len(images) >= batch_size:
+                yield np.array(images)
+                images = []
+        if not for_training:
+            break
+
+###===================================================
+def get_data_generator_1vars(df, indices, for_training, vars, greyscale,
+                                tilesize, batch_size=16):
+    """
+    This function generates data for a batch of images and 1 associated metric
+    """
+    images, p1s = [], []
+    while True:
+        for i in indices:
+            r = df.iloc[i]
+            file, p1 = r['files'], r[vars[0]]
+            #im = Image.open(file).convert('LA')
+            #im = im.resize((IM_HEIGHT, IM_HEIGHT))
+            #im = np.array(im) / 255.0
+            #im2 = np.rot90(im)
+
+            # if greyscale==True:
+            #    im = Image.open(file).convert('LA')
+            # else:
+            #    im = Image.open(file)
+            # im = im.resize((IM_HEIGHT, IM_HEIGHT))
+            # im = np.array(im) / 255.0
+
+            if greyscale==True:
+               im = Image.open(file).convert('LA')
+               #im = im.resize((IM_HEIGHT, IM_HEIGHT))
+               im = np.array(im)[:,:,0]
+               nx,ny = np.shape(im)
+               if (nx!=tilesize) or (ny!=tilesize):
+                   im = im[int(nx/2)-int(tilesize/2):int(nx/2)+int(tilesize/2), int(ny/2)-int(tilesize/2):int(ny/2)+int(tilesize/2)]
+
+            else:
+               im = Image.open(file)
+               #im = im.resize((IM_HEIGHT, IM_HEIGHT))
+               im = np.array(im)
+               nx,ny,nz = np.shape(im)
+               if (nx!=tilesize) or (ny!=tilesize):
+                   im = im[int(nx/2)-int(tilesize/2):int(nx/2)+int(tilesize/2), int(ny/2)-int(tilesize/2):int(ny/2)+int(tilesize/2)]
+
+
+            if greyscale==True:
+               images.append(np.expand_dims(im, axis=2))
+            else:
+               images.append(im)
+
+            p1s.append(p1)
+            if len(images) >= batch_size:
+                yield np.array(images), [np.array(p1s)]
+                images, p1s = [], []
+        if not for_training:
+            break
+
+###===================================================
+def estimate_categorical(vars, csvfile, res_folder, dropout,
+                         numclass, greyscale, name, mode):
+   """
+   This function uses a SediNet model for categorical prediction
+   """
+
+   ID_MAP = dict(zip(np.arange(numclass), [str(k) for k in range(numclass)]))
+
+   ##======================================
+   ## this randomly selects imagery for training and testing imagery sets
+   ## while also making sure that both training and tetsing sets have
+   ## at least 3 examples of each category
+   test_idx, test_df. _ = get_df(csvfile,fortrain=True)
+
+   # for 16GB RAM, used maximum of 200 samples to test on
+   # need to change batch gnerator into a better keras one
+
+   valid_gen = get_data_generator_1image(test_df, test_idx, True, ID_MAP,
+                vars[0],  len(train_idx), greyscale, False, IM_HEIGHT) #np.min((200, len(train_idx))),
+
+   if SHALLOW is True:
+      if DO_AUG is True:
+          weights_path = name+"_"+mode+"_batch"+str(BATCH_SIZE)+"_im"+str(IM_HEIGHT)+\
+                   "_"+str(IM_WIDTH)+"_shallow_"+vars[0]+"_"+CAT_LOSS+"_aug.hdf5"
+      else:
+          weights_path = name+"_"+mode+"_batch"+str(BATCH_SIZE)+"_im"+str(IM_HEIGHT)+\
+                   "_"+str(IM_WIDTH)+"_shallow_"+vars[0]+"_"+CAT_LOSS+"_noaug.hdf5"
+   else:
+      if DO_AUG is True:
+           weights_path = name+"_"+mode+"_batch"+str(BATCH_SIZE)+"_im"+str(IM_HEIGHT)+\
+                   "_"+str(IM_WIDTH)+"_"+vars[0]+"_"+CAT_LOSS+"_aug.hdf5"
+      else:
+           weights_path = name+"_"+mode+"_batch"+str(BATCH_SIZE)+"_im"+str(IM_HEIGHT)+\
+                   "_"+str(IM_WIDTH)+"_"+vars[0]+"_"+CAT_LOSS+"_noaug.hdf5"
+
+
+   if not os.path.exists(weights_path):
+       weights_path = res_folder + os.sep+ weights_path
+       print("Using %s" % (weights_path))
+
+   if numclass>0:
+      ID_MAP = dict(zip(np.arange(numclass), [str(k) for k in range(numclass)]))
+
+   SM = make_cat_sedinet(ID_MAP, dropout)
+
+   if type(BATCH_SIZE)==list:
+       predict_test_train_cat(test_df, None, test_idx, None, vars[0],
+                         SMs, [i for i in ID_MAP.keys()], weights_path, greyscale,
+                         name, DO_AUG, IM_HEIGHT)
+   else:
+       predict_test_train_cat(test_df, None, test_idx, None, vars[0],
+                         SM, [i for i in ID_MAP.keys()], weights_path, greyscale,
+                         name, DO_AUG, IM_HEIGHT)
+
+   K.clear_session()
+
+   ##===================================
+   ## move model files and plots to the results folder
+   tidy(name, res_folder)
+
+###===================================================
+def estimate_siso_simo(vars, csvfile, greyscale,
+                       dropout, numclass, name, mode, res_folder,#scale,
+                       batch_size, weights_path):
+   """
+   This function uses a sedinet model for continuous prediction
+   """
+
+   if not os.path.exists(weights_path):
+       weights_path = res_folder + os.sep+ weights_path
+       print("Using %s" % (weights_path))
+
+   ##======================================
+   ## this randomly selects imagery for training and testing imagery sets
+   ## while also making sure that both training and tetsing sets have
+   ## at least 3 examples of each category
+   #train_idx, train_df = get_df(train_csvfile)
+   train_idx, train_df,split = get_df(csvfile)
+
+   ##==============================================
+   ## create a sedinet model to estimate category
+   SM = make_sedinet_siso_simo(vars, greyscale, dropout)
+
+   # if scale==True:
+   #     CS = []
+   #     for var in vars:
+   #        cs = RobustScaler() #MinMaxScaler()
+   #        if split:
+   #            cs.fit_transform(
+   #              np.r_[train_df[0][var].values].reshape(-1,1)
+   #              )
+   #        else:
+   #            cs.fit_transform(
+   #              np.r_[train_df[var].values].reshape(-1,1)
+   #              )
+   #        CS.append(cs)
+   #        del cs
+   # else:
+   #     CS = []
+
+
+   do_aug = False
+   for_training = False
+   if type(train_df)==list:
+       print('Reading in all files and memory mapping in batches ... takes a while')
+       train_gen = []
+       for df,id in zip(train_df,train_idx):
+           train_gen.append(get_data_generator_Nvars_siso_simo(df, id, for_training,
+                   vars, len(id), greyscale,  do_aug, DO_STANDARDIZE, IM_HEIGHT))#CS,
+
+       x_train = []; vals = []; files = []
+       for gen in train_gen:
+           a, b = next(gen)
+           outfile = TemporaryFile()
+           files.append(outfile)
+           dt  = a.dtype; sh = a.shape
+           fp = np.memmap(outfile, dtype=dt, mode='w+', shape=sh)
+           fp[:] = a[:]
+           fp.flush()
+           del a
+           del fp
+           a = np.memmap(outfile, dtype=dt, mode='r', shape=sh)
+           x_train.append(a)
+           vals.append(b)
+
+   else:
+       train_gen = get_data_generator_Nvars_siso_simo(train_df, train_idx, for_training,
+                vars, len(train_idx), greyscale,do_aug, DO_STANDARDIZE, IM_HEIGHT)# CS,
+
+       x_train, vals = next(train_gen)
+
+   # test model
+   # if numclass==0:
+   x_test=None
+   test_vals = None
+   if type(BATCH_SIZE)==list:
+       predict_test_train_siso_simo(x_train, vals, x_test, test_vals, vars, #train_df, None, train_idx, None,
+                            SMs, weights_path, name, mode, greyscale, #CS,
+                            dropout,  DO_AUG, DO_STANDARDIZE,counter)#scale,
+   else:
+       if type(x_train)==list:
+           for counter, x in enumerate(x_train):
+               #print(counter)
+               predict_test_train_siso_simo(x, vals[counter], x_test, test_vals, vars,
+                                SM, weights_path, name, mode, greyscale, #CS,
+                                dropout, DO_AUG, DO_STANDARDIZE,counter)#scale,
+       else:
+           predict_test_train_siso_simo(x_train,vals, x_test, test_vals, vars,
+                            SM, weights_path, name, mode, greyscale,# CS,
+                            dropout,DO_AUG, DO_STANDARDIZE,counter)# scale
+       K.clear_session()
+
+   ##===================================
+   ## move model files and plots to the results folder
+   tidy(name, res_folder)

app_files/src/sedinet_infer.py

@@ -0,0 +1,544 @@
+# Written by Dr Daniel Buscombe, Marda Science LLC
+# for the SandSnap Program
+#
+# MIT License
+#
+# Copyright (c) 2020-2021, Marda Science LLC
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+##> Release v1.4 (Aug 2021)
+
+from sedinet_models import *
+
+###===================================================
+def run_training_siso_simo(vars, train_csvfile, test_csvfile, val_csvfile, name, res_folder,
+                           mode, greyscale, dropout, numclass): #scale
+   """
+   This function generates, trains and evaluates a sedinet model for
+   continuous prediction
+   """
+
+   if numclass>0:
+      ID_MAP = dict(zip(np.arange(numclass), [str(k) for k in range(numclass)]))
+
+   # ##======================================
+   # ## this randomly selects imagery for training and testing imagery sets
+   # ## while also making sure that both training and tetsing sets have
+   # ## at least 3 examples of each category
+   # train_idx, train_df, _ = get_df(train_csvfile,fortrain=True)
+   # test_idx, test_df, _ = get_df(test_csvfile,fortrain=True)
+
+   ##==============================================
+   ## create a sedinet model to estimate category
+   if numclass>0:
+      SM = make_cat_sedinet(ID_MAP, dropout)
+   else:
+      SM = make_sedinet_siso_simo(vars, greyscale, dropout)
+
+    # if scale==True:
+    #     CS = []
+    #     for var in vars:
+    #        cs = RobustScaler() ##alternative = MinMaxScaler()
+    #        cs.fit_transform(
+    #           np.r_[train_df[var].values, test_df[var].values].reshape(-1,1)
+    #           )
+    #        CS.append(cs)
+    #        del cs
+    # else:
+    #     CS = []
+
+   ##==============================================
+   ## train model
+   if numclass==0:
+      if type(BATCH_SIZE)==list:
+         SMs = []; weights_path = []
+         for batch_size, valid_batch_size in zip(BATCH_SIZE, VALID_BATCH_SIZE):
+            sm, wp,train_df, test_df, val_df, train_idx, test_idx, val_idx = train_sedinet_siso_simo(SM, name,
+                                                  train_csvfile, test_csvfile, val_csvfile, vars, mode, greyscale, #CS,
+                                                  dropout, batch_size, valid_batch_size,
+                                                  res_folder)#, scale)
+            SMs.append(sm)
+            weights_path.append(wp)
+            gc.collect()
+
+      else:
+         SM, weights_path,train_df, test_df, val_df, train_idx, test_idx, val_idx = train_sedinet_siso_simo(SM, name,
+                                                  train_csvfile, test_csvfile, val_csvfile, vars, mode, greyscale, #CS,
+                                                  dropout, BATCH_SIZE, VALID_BATCH_SIZE,
+                                                  res_folder)#, scale)
+   else:
+      if type(BATCH_SIZE)==list:
+         SMs = []; weights_path = []
+         for batch_size, valid_batch_size in zip(BATCH_SIZE, VALID_BATCH_SIZE):
+            sm, wp = train_sedinet_cat(SM, train_df, test_df, train_idx,
+                         test_idx, ID_MAP, vars, greyscale, name, mode,
+                         batch_size, valid_batch_size, res_folder)
+            SMs.append(sm)
+            weights_path.append(wp)
+            gc.collect()
+
+      else:
+          SM, weights_path = train_sedinet_cat(SM, train_df, test_df, train_idx,
+                         test_idx, ID_MAP, vars, greyscale, name, mode,
+                         BATCH_SIZE, VALID_BATCH_SIZE, res_folder)
+
+
+      classes = np.arange(len(ID_MAP))
+
+   K.clear_session()
+
+   ##==============================================
+   # test model
+   do_aug = False
+   for_training = False
+   if type(test_df)==list:
+       print('Reading in all train files and memory mapping in batches ... takes a while')
+       test_gen = []
+       for df,id in zip(test_df,test_idx):
+           test_gen.append(get_data_generator_Nvars_siso_simo(df, id, for_training,
+                   vars, len(id), greyscale,  do_aug, DO_STANDARDIZE, IM_HEIGHT)) #CS,
+
+       x_test = []; test_vals = []; files = []
+       for gen in test_gen:
+           a, b = next(gen)
+           outfile = TemporaryFile()
+           files.append(outfile)
+           dt  = a.dtype; sh = a.shape
+           fp = np.memmap(outfile, dtype=dt, mode='w+', shape=sh)
+           fp[:] = a[:]
+           fp.flush()
+           del a
+           del fp
+           a = np.memmap(outfile, dtype=dt, mode='r', shape=sh)
+           x_test.append(a)
+           test_vals.append(b)
+
+
+   else:
+    #    train_gen = get_data_generator_Nvars_siso_simo(train_df, train_idx, for_training,
+    #             vars, len(train_idx), greyscale,  do_aug, DO_STANDARDIZE, IM_HEIGHT)#CS,
+
+    #    x_train, train_vals = next(train_gen)
+
+       test_gen = get_data_generator_Nvars_siso_simo(test_df, test_idx, for_training,
+                vars, len(test_idx), greyscale, do_aug, DO_STANDARDIZE, IM_HEIGHT)
+       
+       x_test, test_vals = next(test_gen)
+
+#    if numclass==0:
+#     #   suffix = 'train'
+#       if type(BATCH_SIZE)==list:
+#           count_in = 0
+#           predict_train_siso_simo(x_train, train_vals, vars, #train_df, test_df, train_idx, test_idx, vars,  x_test, test_vals,
+#                                 SMs, weights_path, name, mode, greyscale,# CS,
+#                                 dropout, DO_AUG,DO_STANDARDIZE, count_in)#scale,
+#       else:
+#           if type(x_train)==list:
+#               for count_in, (a, b) in enumerate(zip(x_train, train_vals)): #x_test, test_vals
+#                   predict_train_siso_simo(a, b, vars, #train_df, test_df, train_idx, test_idx, vars, c, d,
+#                                         SM, weights_path, name, mode, greyscale,# CS,
+#                                         dropout, DO_AUG,DO_STANDARDIZE, count_in)#scale,
+#                   plot_all_save_all(weights_path, vars)
+
+#           else:
+#               count_in = 0; consolidate = False
+#               predict_train_siso_simo(x_train, train_vals, vars, #train_df, test_df, train_idx, test_idx, vars, x_test, test_vals,
+#                                     SM, weights_path, name, mode, greyscale,# CS,
+#                                     dropout,  DO_AUG,DO_STANDARDIZE, count_in)#scale,
+
+
+   if numclass==0:
+      if type(BATCH_SIZE)==list:
+          count_in = 0
+          predict_train_siso_simo(x_test, test_vals, vars, 
+                                SMs, weights_path, name, mode, greyscale,
+                                dropout, DO_AUG,DO_STANDARDIZE, count_in)
+      else:
+          if type(x_test)==list:
+              for count_in, (a, b) in enumerate(zip(x_test, test_vals)): 
+                  predict_train_siso_simo(a, b, vars, 
+                                        SM, weights_path, name, mode, greyscale,
+                                        dropout, DO_AUG,DO_STANDARDIZE, count_in)
+                  plot_all_save_all(weights_path, vars)
+
+          else:
+              count_in = 0; #consolidate = False
+              predict_train_siso_simo(x_test, test_vals, vars, 
+                                    SM, weights_path, name, mode, greyscale,
+                                    dropout,  DO_AUG,DO_STANDARDIZE, count_in)
+
+   else:
+      if type(BATCH_SIZE)==list:
+          predict_test_train_cat(train_df, test_df, train_idx, test_idx, vars[0],
+                             SMs, [i for i in ID_MAP.keys()], weights_path, greyscale,
+                             name, DO_AUG,DO_STANDARDIZE)
+      else:
+          predict_test_train_cat(train_df, test_df, train_idx, test_idx, vars[0],
+                             SM, [i for i in ID_MAP.keys()], weights_path, greyscale,
+                             name, DO_AUG,DO_STANDARDIZE)
+
+   K.clear_session()
+
+   #
+
+   ##===================================
+   ## move model files and plots to the results folder
+   tidy(name, res_folder)
+
+
+###==================================
+def train_sedinet_cat(SM, train_csvfile, test_csvfile, #train_df, test_df, train_idx, test_idx,
+                      ID_MAP, vars, greyscale, name, mode, batch_size, valid_batch_size,
+                      res_folder):
+    """
+    This function trains an implementation of SediNet
+    """
+    ##================================
+    ## create training and testing file generators, set the weights path,
+    ## plot the model, and create a callback list for model training
+    for_training=True
+    train_gen = get_data_generator_1image(train_df, train_idx, for_training, ID_MAP,
+                vars[0], batch_size, greyscale, DO_AUG, DO_STANDARDIZE, IM_HEIGHT) ##BATCH_SIZE
+    do_aug = False
+    valid_gen = get_data_generator_1image(test_df, test_idx, for_training, ID_MAP,
+                vars[0], valid_batch_size, greyscale, do_aug, DO_STANDARDIZE, IM_HEIGHT) ##VALID_BATCH_SIZE
+
+    if SHALLOW is True:
+       if DO_AUG is True:
+           weights_path = name+"_"+mode+"_batch"+str(batch_size)+"_im"+str(IM_HEIGHT)+\
+                   "_"+str(IM_WIDTH)+"_shallow_"+vars[0]+"_"+CAT_LOSS+"_aug.hdf5"
+       else:
+           weights_path = name+"_"+mode+"_batch"+str(batch_size)+"_im"+str(IM_HEIGHT)+\
+                   "_"+str(IM_WIDTH)+"_shallow_"+vars[0]+"_"+CAT_LOSS+"_noaug.hdf5"
+    else:
+       if DO_AUG is True:
+           weights_path = name+"_"+mode+"_batch"+str(batch_size)+"_im"+str(IM_HEIGHT)+\
+                   "_"+str(IM_WIDTH)+"_"+vars[0]+"_"+CAT_LOSS+"_aug.hdf5"
+       else:
+           weights_path = name+"_"+mode+"_batch"+str(batch_size)+"_im"+str(IM_HEIGHT)+\
+                   "_"+str(IM_WIDTH)+"_"+vars[0]+"_"+CAT_LOSS+"_noaug.hdf5"
+
+    if os.path.exists(weights_path):
+        SM.load_weights(weights_path)
+        print("==========================================")
+        print("Loading weights that already exist: %s" % (weights_path)  )
+        print("Skipping model training")
+
+    elif os.path.exists(res_folder+os.sep+weights_path):
+        weights_path = res_folder+os.sep+weights_path
+        SM.load_weights(weights_path)
+        print("==========================================")
+        print("Loading weights that already exist: %s" % (weights_path)  )
+        print("Skipping model training")
+
+    else:
+
+        try:
+           plot_model(SM, weights_path.replace('.hdf5', '_model.png'),
+                      show_shapes=True, show_layer_names=True)
+        except:
+           pass
+
+        callbacks_list = [
+             ModelCheckpoint(weights_path, monitor='val_loss', verbose=1,
+                             save_best_only=True, mode='min',
+                             save_weights_only = True)
+         ]
+
+        print("=========================================")
+        print("[INFORMATION] schematic of the model has been written out to: "+\
+              weights_path.replace('.hdf5', '_model.png'))
+        print("[INFORMATION] weights will be written out to: "+weights_path)
+
+        ##==============================================
+        ## set checkpoint file and parameters that control early stopping,
+        ## and reduction of learning rate if and when validation
+        ## scores plateau upon successive epochs
+        # reduceloss_plat = ReduceLROnPlateau(monitor='val_loss', factor=FACTOR,
+        #                   patience=STOP_PATIENCE, verbose=1, mode='auto', min_delta=MIN_DELTA,
+        #                   cooldown=STOP_PATIENCE, min_lr=MIN_LR)
+        #
+        earlystop = EarlyStopping(monitor="val_loss", mode="min", patience=10)
+
+        model_checkpoint = ModelCheckpoint(weights_path, monitor='val_loss',
+                           verbose=1, save_best_only=True, mode='min',
+                           save_weights_only = True)
+
+        ##==============================================
+        ## train the model
+
+        ## with non-adaptive exponentially decreasing learning rate
+        #exponential_decay_fn = exponential_decay(MAX_LR, NUM_EPOCHS)
+
+        #lr_scheduler = LearningRateScheduler(exponential_decay_fn)
+
+        callbacks_list = [model_checkpoint, earlystop] #lr_scheduler
+
+        ## train the model
+        history = SM.fit(train_gen,
+                        steps_per_epoch=len(train_idx)//batch_size, ##BATCH_SIZE
+                        epochs=NUM_EPOCHS,
+                        callbacks=callbacks_list,
+                        validation_data=valid_gen, #use_multiprocessing=True,
+                        validation_steps=len(test_idx)//valid_batch_size) #max_queue_size=10 ##VALID_BATCH_SIZE
+
+        ###===================================================
+        ## Plot the loss and accuracy as a function of epoch
+        plot_train_history_1var(history)
+        # plt.savefig(vars+'_'+str(IM_HEIGHT)+'_batch'+str(batch_size)+'_history.png', ##BATCH_SIZE
+        #             dpi=300, bbox_inches='tight')
+        plt.savefig(weights_path.replace('.hdf5','_history.png'),dpi=300, bbox_inches='tight')
+        plt.close('all')
+
+        # serialize model to JSON to use later to predict
+        model_json = SM.to_json()
+        with open(weights_path.replace('.hdf5','.json'), "w") as json_file:
+           json_file.write(model_json)
+
+    return SM, weights_path
+
+
+###===================================================
+def train_sedinet_siso_simo(SM, name, train_csvfile, test_csvfile, val_csvfile, #train_df, test_df, train_idx, test_idx,
+                            vars, mode, greyscale, dropout, batch_size, valid_batch_size,#CS,
+                            res_folder):#, scale):
+    """
+    This function trains an implementation of sedinet
+    """
+
+    ##==============================================
+    ## create training and testing file generators, set the weights path,
+    ## plot the model, and create a callback list for model training
+
+    # get a string saying how many variables, fr the output files
+    varstring = str(len(vars))+'vars' #''.join([str(k)+'_' for k in vars])
+
+    # mae the appropriate weights file
+    if SHALLOW is True:
+       if DO_AUG is True:
+          # if len(CS)>0:#scale is True:
+          #     weights_path = name+"_"+mode+"_batch"+str(batch_size)+"_im"+str(IM_HEIGHT)+\
+          #          "_"+str(IM_WIDTH)+"_shallow_"+varstring+"_"+CONT_LOSS+"_aug_scale.hdf5"
+          # else:
+          weights_path = name+"_"+mode+"_batch"+str(batch_size)+"_im"+str(IM_HEIGHT)+\
+               "_"+str(IM_WIDTH)+"_shallow_"+varstring+"_"+CONT_LOSS+"_aug.hdf5"
+       else:
+          # if len(CS)>0:#scale is True:
+          #     weights_path = name+"_"+mode+"_batch"+str(batch_size)+"_im"+str(IM_HEIGHT)+\
+          #          "_"+str(IM_WIDTH)+"_shallow_"+varstring+"_"+CONT_LOSS+"_noaug_scale.hdf5"
+          # else:
+          weights_path = name+"_"+mode+"_batch"+str(batch_size)+"_im"+str(IM_HEIGHT)+\
+               "_"+str(IM_WIDTH)+"_shallow_"+varstring+"_"+CONT_LOSS+"_noaug.hdf5"
+    else:
+       if DO_AUG is True:
+          # if len(CS)>0:#scale is True:
+          #     weights_path = name+"_"+mode+"_batch"+str(batch_size)+"_im"+str(IM_HEIGHT)+\
+          #          "_"+str(IM_WIDTH)+"_"+varstring+"_"+CONT_LOSS+"_aug_scale.hdf5"
+          # else:
+          weights_path = name+"_"+mode+"_batch"+str(batch_size)+"_im"+str(IM_HEIGHT)+\
+               "_"+str(IM_WIDTH)+"_"+varstring+"_"+CONT_LOSS+"_aug.hdf5"
+       else:
+          # if len(CS)>0:#scale is True:
+          #     weights_path = name+"_"+mode+"_batch"+str(batch_size)+"_im"+str(IM_HEIGHT)+\
+          #          "_"+str(IM_WIDTH)+"_"+varstring+"_"+CONT_LOSS+"_noaug_scale.hdf5"
+          # else:
+          weights_path = name+"_"+mode+"_batch"+str(batch_size)+"_im"+str(IM_HEIGHT)+\
+               "_"+varstring+"_"+CONT_LOSS+"_noaug.hdf5"
+
+
+    # if it already exists, skip training
+    if os.path.exists(weights_path):
+        SM.load_weights(weights_path)
+        print("==========================================")
+        print("Loading weights that already exist: %s" % (weights_path)  )
+        print("Skipping model training")
+
+        ##======================================
+        ## this randomly selects imagery for training and testing imagery sets
+        ## while also making sure that both training and tetsing sets have
+        ## at least 3 examples of each category
+        train_idx, train_df, _ = get_df(train_csvfile,fortrain=False)
+        test_idx, test_df, _ = get_df(test_csvfile,fortrain=False)
+        val_idx, test_df, _ = get_df(val_csvfile,fortrain=False)
+
+
+        for_training = False
+        train_gen = get_data_generator_Nvars_siso_simo(train_df, train_idx, for_training,
+                                                       vars, batch_size, greyscale,
+                                                       DO_AUG, DO_STANDARDIZE, IM_HEIGHT) # CS,
+        do_aug = False
+        valid_gen = get_data_generator_Nvars_siso_simo(val_df, val_idx, for_training,
+                                                       vars, valid_batch_size, greyscale,
+                                                      do_aug, DO_STANDARDIZE, IM_HEIGHT) ##only augment training # CS,
+
+        # do_aug = False
+        # test_gen = get_data_generator_Nvars_siso_simo(test_df, test_idx, for_training,
+        #                                                vars, valid_batch_size, greyscale,
+        #                                               do_aug, DO_STANDARDIZE, IM_HEIGHT) ##only augment training # CS,
+
+    # if it already exists in res_folder, skip training
+    elif os.path.exists(res_folder+os.sep+weights_path):
+        weights_path = res_folder+os.sep+weights_path
+        SM.load_weights(weights_path)
+        print("==========================================")
+        print("Loading weights that already exist: %s" % (weights_path)  )
+        print("Skipping model training")
+
+        ##======================================
+        ## this randomly selects imagery for training and testing imagery sets
+        ## while also making sure that both training and tetsing sets have
+        ## at least 3 examples of each category
+        train_idx, train_df, _ = get_df(train_csvfile,fortrain=False)
+        test_idx, test_df, _ = get_df(test_csvfile,fortrain=False)
+        val_idx, val_df, _ = get_df(val_csvfile,fortrain=False)
+
+        for_training = False
+        train_gen = get_data_generator_Nvars_siso_simo(train_df, train_idx, for_training,
+                                                       vars, batch_size, greyscale,
+                                                       DO_AUG, DO_STANDARDIZE, IM_HEIGHT) # CS,
+        do_aug = False
+        valid_gen = get_data_generator_Nvars_siso_simo(val_df, val_idx, for_training,
+                                                       vars, valid_batch_size, greyscale,
+                                                       do_aug, DO_STANDARDIZE, IM_HEIGHT) ##only augment training # CS,
+
+        # do_aug = False
+        # test_gen = get_data_generator_Nvars_siso_simo(test_df, test_idx, for_training,
+        #                                                vars, valid_batch_size, greyscale,
+        #                                                do_aug, DO_STANDARDIZE, IM_HEIGHT) ##only augment training # CS,
+
+    else: #train
+
+        ##======================================
+        ## this randomly selects imagery for training and testing imagery sets
+        ## while also making sure that both training and tetsing sets have
+        ## at least 3 examples of each category
+        train_idx, train_df, _ = get_df(train_csvfile,fortrain=True)
+        test_idx, test_df, _ = get_df(test_csvfile,fortrain=True)
+        val_idx, val_df, _ = get_df(val_csvfile,fortrain=True)
+
+        for_training = True
+        train_gen = get_data_generator_Nvars_siso_simo(train_df, train_idx, for_training,
+                                                       vars, batch_size, greyscale,
+                                                       DO_AUG, DO_STANDARDIZE, IM_HEIGHT) # CS,
+        # do_aug = False
+        # test_gen = get_data_generator_Nvars_siso_simo(test_df, test_idx, for_training,
+        #                                                vars, valid_batch_size, greyscale,
+        #                                                do_aug, DO_STANDARDIZE, IM_HEIGHT) ##only augment training # CS,
+
+        do_aug = False
+        valid_gen = get_data_generator_Nvars_siso_simo(val_df, val_idx, for_training,
+                                                       vars, valid_batch_size, greyscale,
+                                                       do_aug, DO_STANDARDIZE, IM_HEIGHT) ##only augment training # CS,
+
+        # if scaler=true (CS=[]), dump out scalers to pickle file
+        # if len(CS)==0:
+        #     pass
+        # else:
+            # joblib.dump(CS, weights_path.replace('.hdf5','_scaler.pkl'))
+            # print('Wrote scaler to pkl file')
+
+        try: # plot the model if pydot/graphviz installed
+            plot_model(SM, weights_path.replace('.hdf5', '_model.png'),
+                       show_shapes=True, show_layer_names=True)
+            print("model schematic written to: "+\
+                  weights_path.replace('.hdf5', '_model.png'))
+        except:
+            pass
+
+        print("==========================================")
+        print("weights will be written out to: "+weights_path)
+
+        ##==============================================
+        ## set checkpoint file and parameters that control early stopping,
+        ## and reduction of learning rate if and when validation scores plateau upon successive epochs
+        # reduceloss_plat = ReduceLROnPlateau(monitor='val_loss', factor=FACTOR,
+        #                                     patience=STOP_PATIENCE, verbose=1, mode='auto',
+        #                                     min_delta=MIN_DELTA, cooldown=5,
+        #                                     min_lr=MIN_LR)
+
+        earlystop = EarlyStopping(monitor="val_loss", mode="min",
+                                  patience=10)
+
+        # set model checkpoint. only save best weights, based on min validation loss
+        model_checkpoint = ModelCheckpoint(weights_path, monitor='val_loss', verbose=1,
+                                           save_best_only=True, mode='min',
+                                           save_weights_only = True)
+
+
+        #tqdm_callback = tfa.callbacks.TQDMProgressBar()
+        # callbacks_list = [model_checkpoint, reduceloss_plat, earlystop] #, tqdm_callback]
+
+        try: #write summary of the model to txt file
+            with open(weights_path.replace('.hdf5','') + '_report.txt','w') as fh:
+                # Pass the file handle in as a lambda function to make it callable
+                SM.summary(print_fn=lambda x: fh.write(x + '\n'))
+            fh.close()
+            print("model summary written to: "+ \
+                  weights_path.replace('.hdf5','') + '_report.txt')
+            with open(weights_path.replace('.hdf5','') + '_report.txt','r') as fh:
+                tmp = fh.readlines()
+            print("===============================================")
+            print("Total parameters: %s" %\
+                 (''.join(tmp).split('Total params:')[-1].split('\n')[0]))
+            fh.close()
+            print("===============================================")
+        except:
+            pass
+
+        ##==============================================
+        ## train the model
+
+        ## non-adaptive exponentially decreasing learning rate
+        # exponential_decay_fn = exponential_decay(MAX_LR, NUM_EPOCHS)
+
+        #lr_scheduler = LearningRateScheduler(exponential_decay_fn)
+
+        callbacks_list = [model_checkpoint, earlystop] #lr_scheduler
+
+        ## train the model
+        history = SM.fit(train_gen,
+                        steps_per_epoch=len(train_idx)//batch_size, ##BATCH_SIZE
+                        epochs=NUM_EPOCHS,
+                        callbacks=callbacks_list,
+                        validation_data=valid_gen, #use_multiprocessing=True,
+                        validation_steps=len(val_idx)//valid_batch_size) #max_queue_size=10 ##VALID_BATCH_SIZE
+
+
+        ###===================================================
+        ## Plot the loss and accuracy as a function of epoch
+        if len(vars)==1:
+           plot_train_history_1var_mae(history)
+        else:
+           plot_train_history_Nvar(history, vars, len(vars))
+
+        varstring = ''.join([str(k)+'_' for k in vars])
+        plt.savefig(weights_path.replace('.hdf5', '_history.png'), dpi=300,
+                    bbox_inches='tight')
+        plt.close('all')
+
+        # serialize model to JSON to use later to predict
+        model_json = SM.to_json()
+        with open(weights_path.replace('.hdf5','.json'), "w") as json_file:
+           json_file.write(model_json)
+
+    return SM, weights_path,train_df, test_df, val_df, train_idx, test_idx, val_idx
+
+#

app_files/src/sedinet_models.py

@@ -0,0 +1,144 @@
+# Written by Dr Daniel Buscombe, Marda Science LLC
+# for the SandSnap Program
+#
+# MIT License
+#
+# Copyright (c) 2020-2021, Marda Science LLC
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+##> Release v1.4 (Aug 2021)
+
+###===================================================
+# import libraries
+from sedinet_utils import *
+
+###===================================================
+def conv_block2(inp, filters=32, bn=True, pool=True, drop=True):
+   """
+   This function generates a SediNet convolutional block
+   """
+   # _ = Conv2D(filters=filters, kernel_size=3, activation='relu',
+   #            kernel_initializer='he_uniform')(inp)
+
+   #relu creating dead neurons?
+   _ = SeparableConv2D(filters=filters, kernel_size=3, activation='relu')(inp) #'relu' #kernel_initializer='he_uniform'
+   if bn:
+       _ = BatchNormalization()(_)
+   if pool:
+       _ = MaxPool2D()(_)
+   if drop:
+       _ = Dropout(0.2)(_)
+   return _
+
+###===================================================
+def make_cat_sedinet(ID_MAP, dropout):
+    """
+    This function creates an implementation of SediNet for estimating
+	sediment category
+    """
+
+    base = BASE_CAT ##30
+
+    input_layer = Input(shape=(IM_HEIGHT, IM_WIDTH, 3))
+    _ = conv_block2(input_layer, filters=base, bn=False, pool=False, drop=False) #x #
+    _ = conv_block2(_, filters=base*2, bn=False, pool=True,drop=False)
+    _ = conv_block2(_, filters=base*3, bn=False, pool=True,drop=False)
+    _ = conv_block2(_, filters=base*4, bn=False, pool=True,drop=False)
+
+    bottleneck = GlobalMaxPool2D()(_)
+    bottleneck = Dropout(dropout)(bottleneck)
+
+     # for class prediction
+    _ = Dense(units=CAT_DENSE_UNITS, activation='relu')(bottleneck)  ##128
+    output = Dense(units=len(ID_MAP), activation='softmax', name='output')(_)
+
+    model = Model(inputs=input_layer, outputs=[output])
+
+    OPT = tf.keras.optimizers.Adam(learning_rate=MAX_LR)
+
+    if CAT_LOSS == 'focal':
+       model.compile(optimizer=OPT,
+                  loss={'output': tfa.losses.SigmoidFocalCrossEntropy() },
+                  metrics={'output': 'accuracy'})
+    else:
+       model.compile(optimizer=OPT, #'adam',
+                  loss={'output': CAT_LOSS}, #'categorical_crossentropy'
+                  metrics={'output': 'accuracy'})
+
+
+    print("==========================================")
+    print('[INFORMATION] Model summary:')
+    model.summary()
+    return model
+
+
+###===================================================
+def make_sedinet_siso_simo(vars, greyscale, dropout):
+    """
+    This function creates an implementation of SediNet for estimating
+	sediment metric on a continuous scale
+    """
+
+    base = BASE_CONT ##30 ## suggested range = 20 -- 40
+    if greyscale==True:
+       input_layer = Input(shape=(IM_HEIGHT, IM_WIDTH, 1))
+    else:
+       input_layer = Input(shape=(IM_HEIGHT, IM_WIDTH, 3))
+
+    _ = conv_block2(input_layer, filters=base, bn=False, pool=False, drop=False) #x #
+    _ = conv_block2(_, filters=base*2, bn=False, pool=True,drop=False)
+    _ = conv_block2(_, filters=base*3, bn=False, pool=True,drop=False)
+    _ = conv_block2(_, filters=base*4, bn=False, pool=True,drop=False)
+    _ = conv_block2(_, filters=base*5, bn=False, pool=True,drop=False)
+
+    if not SHALLOW:
+       _ = conv_block2(_, filters=base*6, bn=False, pool=True,drop=False)
+       _ = conv_block2(_, filters=base*7, bn=False, pool=True,drop=False)
+       _ = conv_block2(_, filters=base*8, bn=False, pool=True,drop=False)
+       _ = conv_block2(_, filters=base*9, bn=False, pool=True,drop=False)
+
+    _ = BatchNormalization(axis=-1)(_)
+    bottleneck = GlobalMaxPool2D()(_)
+    bottleneck = Dropout(dropout)(bottleneck)
+
+    units = CONT_DENSE_UNITS ## suggested range 512 -- 1024
+    _ = Dense(units=units, activation='relu')(bottleneck) #'relu'
+
+    ##would it be better to predict the full vector directly instread of one by one?
+    outputs = []
+    for var in vars:
+       outputs.append(Dense(units=1, activation='linear', name=var+'_output')(_) ) #relu
+
+    if CONT_LOSS == 'pinball':
+       loss = dict(zip([k+"_output" for k in vars], [tfa.losses.PinballLoss(tau=.5) for k in vars]))
+    else: ## 'mse'
+       loss = dict(zip([k+"_output" for k in vars], ['mse' for k in vars])) #loss = tf.keras.losses.MeanSquaredError(reduction=tf.keras.losses.Reduction.NONE)  # Sum of squared error
+
+    metrics = dict(zip([k+"_output" for k in vars], ['mae' for k in vars]))
+
+    OPT = tf.keras.optimizers.Adam(learning_rate=MAX_LR)
+
+    model = Model(inputs=input_layer, outputs=outputs)
+    model.compile(optimizer=OPT,loss=loss, metrics=metrics)
+    #print("==========================================")
+    #print('[INFORMATION] Model summary:')
+    #model.summary()
+    return model

app_files/src/sedinet_utils.py

@@ -0,0 +1,2117 @@
+# Written by Dr Daniel Buscombe, Marda Science LLC
+# for the SandSnap Program
+#
+# MIT License
+#
+# Copyright (c) 2020-2021, Marda Science LLC
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+##> Release v1.4 (Aug 2021)
+
+from imports import *
+from matplotlib import MatplotlibDeprecationWarning
+
+import warnings
+warnings.filterwarnings(action="ignore",category=MatplotlibDeprecationWarning)
+
+###===================================================
+## FUNCTIONS FOR LEARNING RATE SCHEDULER
+
+def exponential_decay(lr0, s):
+    def exponential_decay_fn(epoch):
+        return lr0 * 0.1 **(epoch / s)
+    return exponential_decay_fn
+
+###===================================================
+## IMAGE AUGMENTATION FUNCTIONS (for DO_AUG=True)
+
+
+# def h_flip(image):
+#     return  np.fliplr(image)
+
+def v_flip(image):
+    return np.flipud(image)
+
+def warp_shift(image):
+    shift= random.randint(25,200)
+    transform = AffineTransform(translation=(0,shift))
+    warp_image = warp(image, transform, mode="wrap")
+    return warp_image
+
+def apply_aug(im):
+    return [im,v_flip(warp_shift(im))] #, clockwise_rotation(im), h_flip(im)]
+
+
+##========================================================
+def rescale(dat,
+    mn,
+    mx):
+    '''
+    rescales an input dat between mn and mx
+    '''
+    m = min(dat.flatten())
+    M = max(dat.flatten())
+    return (mx-mn)*(dat-m)/(M-m)+mn
+
+def do_standardize(img):
+    #standardization using adjusted standard deviation
+    N = np.shape(img)[0] * np.shape(img)[1]
+    s = np.maximum(np.std(img), 1.0/np.sqrt(N))
+    m = np.mean(img)
+    img = (img - m) / s
+    img = rescale(img, 0, 1)
+    del m, s, N
+
+    return img
+
+###===================================================
+### IMAGE BATCH GENERATOR FUNCTIONS
+
+def get_data_generator_Nvars_siso_simo(df, indices, for_training, vars,
+                                       batch_size, greyscale, do_aug,#CS,
+                                       standardize, tilesize):
+    """
+    This function generates data for a batch of images and N associated metrics
+    """
+
+    ##print(do_aug)
+
+    if len(vars)==1:
+       images, p1s = [], []
+    elif len(vars)==2:
+       images, p1s, p2s = [], [], []
+    elif len(vars)==3:
+       images, p1s, p2s, p3s = [], [], [], []
+    elif len(vars)==4:
+       images, p1s, p2s, p3s, p4s = [], [], [], [], []
+    elif len(vars)==5:
+       images, p1s, p2s, p3s, p4s, p5s = [], [], [], [], [], []
+    elif len(vars)==6:
+       images, p1s, p2s, p3s, p4s, p5s, p6s =\
+        [], [], [], [], [], [], []
+    elif len(vars)==7:
+       images, p1s, p2s, p3s, p4s, p5s, p6s, p7s =\
+        [], [], [], [], [], [], [], []
+    elif len(vars)==8:
+       images, p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s =\
+        [], [], [], [], [], [], [], [], []
+    elif len(vars)==9:
+       images, p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s, p9s =\
+        [], [], [], [], [], [], [], [], [], []
+
+    while True:
+        for i in indices:
+            r = df.iloc[i]
+            if len(vars)==1:
+               file, p1 = r['filenames'], r[vars[0]]
+            if len(vars)==2:
+               file, p1, p2 = r['filenames'], r[vars[0]], r[vars[1]]
+            if len(vars)==3:
+               file, p1, p2, p3 = \
+               r['filenames'], r[vars[0]], r[vars[1]], r[vars[2]]
+            if len(vars)==4:
+               file, p1, p2, p3, p4 = \
+               r['filenames'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]]
+            if len(vars)==5:
+               file, p1, p2, p3, p4, p5 = \
+               r['filenames'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]]
+            if len(vars)==6:
+               file, p1, p2, p3, p4, p5, p6 = \
+               r['filenames'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[vars[5]]
+            if len(vars)==7:
+               file, p1, p2, p3, p4, p5, p6, p7 = \
+               r['filenames'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[vars[5]], r[vars[6]]
+            if len(vars)==8:
+               file, p1, p2, p3, p4, p5, p6, p7, p8 = \
+               r['filenames'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[vars[5]], r[vars[6]], r[vars[7]]
+            elif len(vars)==9:
+               file, p1, p2, p3, p4, p5, p6, p7, p8, p9 = \
+               r['filenames'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[vars[5]], r[vars[6]], r[vars[7]], r[vars[8]]
+
+            if greyscale==True:
+               im = Image.open(file).convert('LA')
+               #im = im.resize((IM_HEIGHT, IM_HEIGHT))
+               im = np.array(im)[:,:,0]
+               nx,ny = np.shape(im)
+               if (nx!=tilesize) or (ny!=tilesize):
+                   im = im[int(nx/2)-int(tilesize/2):int(nx/2)+int(tilesize/2), int(ny/2)-int(tilesize/2):int(ny/2)+int(tilesize/2)]
+
+            else:
+               im = Image.open(file)
+               #im = im.resize((IM_HEIGHT, IM_HEIGHT))
+               im = np.array(im)
+               nx,ny,nz = np.shape(im)
+               if (nx!=tilesize) or (ny!=tilesize):
+                   im = im[int(nx/2)-int(tilesize/2):int(nx/2)+int(tilesize/2), int(ny/2)-int(tilesize/2):int(ny/2)+int(tilesize/2)]
+
+            if standardize==True:
+               im = do_standardize(im)
+            else:
+               im = np.array(im) / 255.0
+
+            #if np.ndim(im)==2:
+            #   im = np.dstack((im, im , im)) ##np.expand_dims(im[:,:,0], axis=2)
+
+            #im = im[:,:,:3]
+
+            if greyscale==True:
+               if do_aug==True:
+                   aug = apply_aug(im)
+                   images.append(aug)
+                   if len(vars)==1:
+                      p1s.append([p1 for k in range(2)])
+                   elif len(vars)==2:
+                      p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)])
+                   elif len(vars)==3:
+                      p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)])
+                      p3s.append([p3 for k in range(2)]);
+                   elif len(vars)==4:
+                      p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)])
+                      p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)])
+                   elif len(vars)==5:
+                      p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)])
+                      p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)])
+                      p5s.append([p5 for k in range(2)]);
+                   elif len(vars)==6:
+                      p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)])
+                      p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)])
+                      p5s.append([p5 for k in range(2)]); p6s.append([p6 for k in range(2)])
+                   elif len(vars)==7:
+                      p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)])
+                      p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)])
+                      p5s.append([p5 for k in range(2)]); p6s.append([p6 for k in range(2)])
+                      p7s.append([p7 for k in range(2)]);
+                   elif len(vars)==8:
+                      p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)])
+                      p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)])
+                      p5s.append([p5 for k in range(2)]); p6s.append([p6 for k in range(2)])
+                      p7s.append([p7 for k in range(2)]); p8s.append([p8 for k in range(2)])
+                   elif len(vars)==9:
+                      p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)])
+                      p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)])
+                      p5s.append([p5 for k in range(2)]); p6s.append([p6 for k in range(2)])
+                      p7s.append([p7 for k in range(2)]); p8s.append([p8 for k in range(2)])
+                      p9s.append([p9 for k in range(2)])
+
+               else:
+                   images.append(np.expand_dims(im, axis=2))
+                   if len(vars)==1:
+                      p1s.append(p1)
+                   elif len(vars)==2:
+                      p1s.append(p1); p2s.append(p2)
+                   elif len(vars)==3:
+                      p1s.append(p1); p2s.append(p2)
+                      p3s.append(p3);
+                   elif len(vars)==4:
+                      p1s.append(p1); p2s.append(p2)
+                      p3s.append(p3); p4s.append(p4)
+                   elif len(vars)==5:
+                      p1s.append(p1); p2s.append(p2)
+                      p3s.append(p3); p4s.append(p4)
+                      p5s.append(p5);
+                   elif len(vars)==6:
+                      p1s.append(p1); p2s.append(p2)
+                      p3s.append(p3); p4s.append(p4)
+                      p5s.append(p5); p6s.append(p6)
+                   elif len(vars)==7:
+                      p1s.append(p1); p2s.append(p2)
+                      p3s.append(p3); p4s.append(p4)
+                      p5s.append(p5); p6s.append(p6)
+                      p7s.append(p7);
+                   elif len(vars)==8:
+                      p1s.append(p1); p2s.append(p2)
+                      p3s.append(p3); p4s.append(p4)
+                      p5s.append(p5); p6s.append(p6)
+                      p7s.append(p7); p8s.append(p8)
+                   elif len(vars)==9:
+                      p1s.append(p1); p2s.append(p2)
+                      p3s.append(p3); p4s.append(p4)
+                      p5s.append(p5); p6s.append(p6)
+                      p7s.append(p7); p8s.append(p8)
+                      p9s.append(p9)
+
+            else:
+               if do_aug==True:
+                   aug = apply_aug(im)
+                   images.append(aug)
+                   if len(vars)==1:
+                      p1s.append([p1 for k in range(2)])
+                   elif len(vars)==2:
+                      p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)])
+                   elif len(vars)==3:
+                      p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)])
+                      p3s.append([p3 for k in range(2)]);
+                   elif len(vars)==4:
+                      p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)])
+                      p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)])
+                   elif len(vars)==5:
+                      p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)])
+                      p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)])
+                      p5s.append([p5 for k in range(2)]);
+                   elif len(vars)==6:
+                      p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)])
+                      p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)])
+                      p5s.append([p5 for k in range(2)]); p6s.append([p6 for k in range(2)])
+                   elif len(vars)==7:
+                      p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)])
+                      p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)])
+                      p5s.append([p5 for k in range(2)]); p6s.append([p6 for k in range(2)])
+                      p7s.append([p7 for k in range(2)]);
+                   elif len(vars)==8:
+                      p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)])
+                      p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)])
+                      p5s.append([p5 for k in range(2)]); p6s.append([p6 for k in range(2)])
+                      p7s.append([p7 for k in range(2)]); p8s.append([p8 for k in range(2)])
+                   elif len(vars)==9:
+                      p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)])
+                      p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)])
+                      p5s.append([p5 for k in range(2)]); p6s.append([p6 for k in range(2)])
+                      p7s.append([p7 for k in range(2)]); p8s.append([p8 for k in range(2)])
+                      p9s.append([p9 for k in range(2)])
+
+               else:
+                   images.append(im)
+                   if len(vars)==1:
+                      p1s.append(p1)
+                   elif len(vars)==2:
+                      p1s.append(p1); p2s.append(p2)
+                   elif len(vars)==3:
+                      p1s.append(p1); p2s.append(p2)
+                      p3s.append(p3);
+                   elif len(vars)==4:
+                      p1s.append(p1); p2s.append(p2)
+                      p3s.append(p3); p4s.append(p4)
+                   elif len(vars)==5:
+                      p1s.append(p1); p2s.append(p2)
+                      p3s.append(p3); p4s.append(p4)
+                      p5s.append(p5);
+                   elif len(vars)==6:
+                      p1s.append(p1); p2s.append(p2)
+                      p3s.append(p3); p4s.append(p4)
+                      p5s.append(p5); p6s.append(p6)
+                   elif len(vars)==7:
+                      p1s.append(p1); p2s.append(p2)
+                      p3s.append(p3); p4s.append(p4)
+                      p5s.append(p5); p6s.append(p6)
+                      p7s.append(p7);
+                   elif len(vars)==8:
+                      p1s.append(p1); p2s.append(p2)
+                      p3s.append(p3); p4s.append(p4)
+                      p5s.append(p5); p6s.append(p6)
+                      p7s.append(p7); p8s.append(p8)
+                   elif len(vars)==9:
+                      p1s.append(p1); p2s.append(p2)
+                      p3s.append(p3); p4s.append(p4)
+                      p5s.append(p5); p6s.append(p6)
+                      p7s.append(p7); p8s.append(p8)
+                      p9s.append(p9)
+
+            if len(images) >= batch_size:
+               if len(vars)==1:
+                  # if len(CS)==0:
+                  p1s = np.squeeze(np.array(p1s))
+                  # else:
+                  #     p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+                  if do_aug==True:
+                     if len(images) >= batch_size:
+                        if greyscale==False:
+                           images = np.array(np.vstack(images))
+                        else:
+                           images = np.expand_dims(np.array(np.vstack(images)), axis=-1)
+                           p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1)
+                        yield images,[p1s]
+                  else:
+                     if len(images) >= batch_size:
+                           #p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1)
+                           yield np.array(images),[np.array(p1s)]
+                  images, p1s = [], []
+
+               elif len(vars)==2:
+                  # if len(CS)==0:
+                  p1s = np.squeeze(np.array(p1s))
+                  p2s = np.squeeze(np.array(p2s))
+                  # else:
+                  #     p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+                  #     p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1)))
+                  if do_aug==True:
+                     if len(images) >= batch_size:
+                        if greyscale==False:
+                           images = np.array(np.vstack(images))
+                        else:
+                           images = np.expand_dims(np.array(np.vstack(images)), axis=-1)
+                           p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1)
+                           p2s = np.expand_dims(np.vstack(p2s).flatten(),axis=-1)
+                        yield images,[p1s, p2s]
+                  else:
+                     if len(images) >= batch_size:
+                        yield np.array(images),[np.array(p1s), np.array(p2s)]
+                  images, p1s, p2s = [], [], []
+
+               elif len(vars)==3:
+                  # if len(CS)==0:
+                  p1s = np.squeeze(np.array(p1s))
+                  p2s = np.squeeze(np.array(p2s))
+                  p3s = np.squeeze(np.array(p3s))
+                  # else:
+                  #     p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+                  #     p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1)))
+                  #     p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1)))
+                  if do_aug==True:
+                     if len(images) >= batch_size:
+                        if greyscale==False:
+                           images = np.array(np.vstack(images))
+                        else:
+                           images = np.expand_dims(np.array(np.vstack(images)), axis=-1)
+                           p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1)
+                           p2s = np.expand_dims(np.vstack(p2s).flatten(),axis=-1)
+                           p3s = np.expand_dims(np.vstack(p3s).flatten(),axis=-1)
+                        yield images,[p1s, p2s, p3s]
+                  else:
+                     if len(images) >= batch_size:
+                        yield np.array(images),[np.array(p1s), np.array(p2s), np.array(p3s)]
+                  images, p1s, p2s, p3s = [], [], [], []
+
+               elif len(vars)==4:
+                  # if len(CS)==0:
+                  p1s = np.squeeze(np.array(p1s))
+                  p2s = np.squeeze(np.array(p2s))
+                  p3s = np.squeeze(np.array(p3s))
+                  p4s = np.squeeze(np.array(p4s))
+                  # else:
+                  #     p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+                  #     p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1)))
+                  #     p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1)))
+                  #     p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1)))
+                  if do_aug==True:
+                     if len(images) >= batch_size:
+                        if greyscale==False:
+                           images = np.array(np.vstack(images))
+                        else:
+                           images = np.expand_dims(np.array(np.vstack(images)), axis=-1)
+                           p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1)
+                           p2s = np.expand_dims(np.vstack(p2s).flatten(),axis=-1)
+                           p3s = np.expand_dims(np.vstack(p3s).flatten(),axis=-1)
+                           p4s = np.expand_dims(np.vstack(p4s).flatten(),axis=-1)
+                        yield images,[p1s, p2s, p3s, p4s]
+                  else:
+                     if len(images) >= batch_size:
+                        yield np.array(images),[np.array(p1s), np.array(p2s), np.array(p3s),
+                              np.array(p4s)]
+                  images, p1s, p2s, p3s, p4s = [], [], [], [], []
+
+               elif len(vars)==5:
+                  # if len(CS)==0:
+                  p1s = np.squeeze(np.array(p1s))
+                  p2s = np.squeeze(np.array(p2s))
+                  p3s = np.squeeze(np.array(p3s))
+                  p4s = np.squeeze(np.array(p4s))
+                  p5s = np.squeeze(np.array(p5s))
+                  # else:
+                  #     p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+                  #     p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1)))
+                  #     p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1)))
+                  #     p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1)))
+                  #     p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1)))
+                  if do_aug==True:
+                     if len(images) >= batch_size:
+                        if greyscale==False:
+                           images = np.array(np.vstack(images))
+                        else:
+                           images = np.expand_dims(np.array(np.vstack(images)), axis=-1)
+                           p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1)
+                           p2s = np.expand_dims(np.vstack(p2s).flatten(),axis=-1)
+                           p3s = np.expand_dims(np.vstack(p3s).flatten(),axis=-1)
+                           p4s = np.expand_dims(np.vstack(p4s).flatten(),axis=-1)
+                           p5s = np.expand_dims(np.vstack(p5s).flatten(),axis=-1)
+                        yield images,[p1s, p2s, p3s, p4s, p5s]
+                  else:
+                     if len(images) >= batch_size:
+                        yield np.array(images),[np.array(p1s), np.array(p2s), np.array(p3s),
+                              np.array(p4s), np.array(p5s)]
+                  images, p1s, p2s, p3s, p4s, p5s = [], [], [], [], [], []
+
+               elif len(vars)==6:
+                  # if len(CS)==0:
+                  p1s = np.squeeze(np.array(p1s))
+                  p2s = np.squeeze(np.array(p2s))
+                  p3s = np.squeeze(np.array(p3s))
+                  p4s = np.squeeze(np.array(p4s))
+                  p5s = np.squeeze(np.array(p5s))
+                  p6s = np.squeeze(np.array(p6s))
+                  # else:
+                  #     p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+                  #     p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1)))
+                  #     p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1)))
+                  #     p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1)))
+                  #     p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1)))
+                  #     p6s = np.squeeze(CS[5].transform(np.array(p6s).reshape(-1, 1)))
+                  if do_aug==True:
+                     if len(images) >= batch_size:
+                        if greyscale==False:
+                           images = np.array(np.vstack(images))
+                        else:
+                           images = np.expand_dims(np.array(np.vstack(images)), axis=-1)
+                           p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1)
+                           p2s = np.expand_dims(np.vstack(p2s).flatten(),axis=-1)
+                           p3s = np.expand_dims(np.vstack(p3s).flatten(),axis=-1)
+                           p4s = np.expand_dims(np.vstack(p4s).flatten(),axis=-1)
+                           p5s = np.expand_dims(np.vstack(p5s).flatten(),axis=-1)
+                           p6s = np.expand_dims(np.vstack(p6s).flatten(),axis=-1)
+                        yield images,[p1s, p2s, p3s, p4s, p5s, p6s]
+                  else:
+                     if len(images) >= batch_size:
+                        yield np.array(images),[np.array(p1s), np.array(p2s), np.array(p3s),
+                              np.array(p4s), np.array(p5s), np.array(p6s)]
+                  images, p1s, p2s, p3s, p4s, p5s, p6s = \
+                  [], [], [], [], [], [], []
+
+               elif len(vars)==7:
+                  # if len(CS)==0:
+                  p1s = np.squeeze(np.array(p1s))
+                  p2s = np.squeeze(np.array(p2s))
+                  p3s = np.squeeze(np.array(p3s))
+                  p4s = np.squeeze(np.array(p4s))
+                  p5s = np.squeeze(np.array(p5s))
+                  p6s = np.squeeze(np.array(p6s))
+                  p7s = np.squeeze(np.array(p7s))
+                  # else:
+                  #     p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+                  #     p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1)))
+                  #     p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1)))
+                  #     p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1)))
+                  #     p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1)))
+                  #     p6s = np.squeeze(CS[5].transform(np.array(p6s).reshape(-1, 1)))
+                  #     p7s = np.squeeze(CS[6].transform(np.array(p7s).reshape(-1, 1)))
+                  if do_aug==True:
+                     if len(images) >= batch_size:
+                        if greyscale==False:
+                           images = np.array(np.vstack(images))
+                        else:
+                           images = np.expand_dims(np.array(np.vstack(images)), axis=-1)
+                           p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1)
+                           p2s = np.expand_dims(np.vstack(p2s).flatten(),axis=-1)
+                           p3s = np.expand_dims(np.vstack(p3s).flatten(),axis=-1)
+                           p4s = np.expand_dims(np.vstack(p4s).flatten(),axis=-1)
+                           p5s = np.expand_dims(np.vstack(p5s).flatten(),axis=-1)
+                           p6s = np.expand_dims(np.vstack(p6s).flatten(),axis=-1)
+                           p7s = np.expand_dims(np.vstack(p7s).flatten(),axis=-1)
+                        yield images,[p1s, p2s, p3s, p4s, p5s, p6s, p7s]
+                  else:
+                     if len(images) >= batch_size:
+                        yield np.array(images),[np.array(p1s), np.array(p2s), np.array(p3s),
+                              np.array(p4s), np.array(p5s), np.array(p6s),
+                              np.array(p7s)]
+                  images, p1s, p2s, p3s, p4s, p5s, p6s, p7s = \
+                  [], [], [], [], [], [], [], []
+
+               elif len(vars)==8:
+                  # if len(CS)==0:
+                  p1s = np.squeeze(np.array(p1s))
+                  p2s = np.squeeze(np.array(p2s))
+                  p3s = np.squeeze(np.array(p3s))
+                  p4s = np.squeeze(np.array(p4s))
+                  p5s = np.squeeze(np.array(p5s))
+                  p6s = np.squeeze(np.array(p6s))
+                  p7s = np.squeeze(np.array(p7s))
+                  p8s = np.squeeze(np.array(p8s))
+                  # else:
+                  #     p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+                  #     p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1)))
+                  #     p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1)))
+                  #     p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1)))
+                  #     p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1)))
+                  #     p6s = np.squeeze(CS[5].transform(np.array(p6s).reshape(-1, 1)))
+                  #     p7s = np.squeeze(CS[6].transform(np.array(p7s).reshape(-1, 1)))
+                  #     p8s = np.squeeze(CS[7].transform(np.array(p8s).reshape(-1, 1)))
+                  if do_aug==True:
+                     if len(images) >= batch_size:
+                        if greyscale==False:
+                           images = np.array(np.vstack(images))
+                        else:
+                           images = np.expand_dims(np.array(np.vstack(images)), axis=-1)
+                           p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1)
+                           p2s = np.expand_dims(np.vstack(p2s).flatten(),axis=-1)
+                           p3s = np.expand_dims(np.vstack(p3s).flatten(),axis=-1)
+                           p4s = np.expand_dims(np.vstack(p4s).flatten(),axis=-1)
+                           p5s = np.expand_dims(np.vstack(p5s).flatten(),axis=-1)
+                           p6s = np.expand_dims(np.vstack(p6s).flatten(),axis=-1)
+                           p7s = np.expand_dims(np.vstack(p7s).flatten(),axis=-1)
+                           p8s = np.expand_dims(np.vstack(p8s).flatten(),axis=-1)
+                        yield images,[p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s]
+
+                  else:
+                     if len(images) >= batch_size:
+                        yield np.array(images),[np.array(p1s), np.array(p2s), np.array(p3s),
+                              np.array(p4s), np.array(p5s), np.array(p6s),
+                              np.array(p7s), np.array(p8s)]
+                  images, p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s = \
+                  [], [], [], [], [], [], [], [], []
+
+               elif len(vars)==9:
+                  # if len(CS)==0:
+                  p1s = np.squeeze(np.array(p1s))
+                  p2s = np.squeeze(np.array(p2s))
+                  p3s = np.squeeze(np.array(p3s))
+                  p4s = np.squeeze(np.array(p4s))
+                  p5s = np.squeeze(np.array(p5s))
+                  p6s = np.squeeze(np.array(p6s))
+                  p7s = np.squeeze(np.array(p7s))
+                  p8s = np.squeeze(np.array(p8s))
+                  p9s = np.squeeze(np.array(p9s))
+                  # else:
+                  #     p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+                  #     p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1)))
+                  #     p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1)))
+                  #     p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1)))
+                  #     p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1)))
+                  #     p6s = np.squeeze(CS[5].transform(np.array(p6s).reshape(-1, 1)))
+                  #     p7s = np.squeeze(CS[6].transform(np.array(p7s).reshape(-1, 1)))
+                  #     p8s = np.squeeze(CS[7].transform(np.array(p8s).reshape(-1, 1)))
+                  #     p9s = np.squeeze(CS[8].transform(np.array(p9s).reshape(-1, 1)))
+
+                  try:
+                     if do_aug==True:
+                         if len(images) >= batch_size:
+                            if greyscale==False:
+                               images = np.array(np.vstack(images))
+                            else:
+                               images = np.expand_dims(np.array(np.vstack(images)), axis=-1)
+                               p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1)
+                               p2s = np.expand_dims(np.vstack(p2s).flatten(),axis=-1)
+                               p3s = np.expand_dims(np.vstack(p3s).flatten(),axis=-1)
+                               p4s = np.expand_dims(np.vstack(p4s).flatten(),axis=-1)
+                               p5s = np.expand_dims(np.vstack(p5s).flatten(),axis=-1)
+                               p6s = np.expand_dims(np.vstack(p6s).flatten(),axis=-1)
+                               p7s = np.expand_dims(np.vstack(p7s).flatten(),axis=-1)
+                               p8s = np.expand_dims(np.vstack(p8s).flatten(),axis=-1)
+                               p9s = np.expand_dims(np.vstack(p9s).flatten(),axis=-1)
+                            yield images,[p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s, p9s]
+                     else:
+                         if len(images) >= batch_size:
+                            yield np.array(images),[np.array(p1s), np.array(p2s), np.array(p3s),
+                                  np.array(p4s), np.array(p5s), np.array(p6s),
+                                  np.array(p7s), np.array(p8s), np.array(p9s)]
+                  except GeneratorExit:
+                      print("")
+                  images, p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s, p9s = \
+                  [], [], [], [], [], [], [], [], [], []
+
+        if not for_training:
+            break
+
+
+###===================================================
+def get_data_generator_1image(df, indices, for_training, ID_MAP,
+                              var, batch_size, greyscale, do_aug,
+                              standardize, tilesize):
+    """
+    This function creates a dataset generator consisting of batches of images
+    and corresponding one-hot-encoded labels describing the sediment in each image
+    """
+    try:
+        ID_MAP2 = dict((g, i) for i, g in ID_MAP.items())
+    except:
+        ID_MAP = dict(zip(np.arange(ID_MAP), [str(k) for k in range(ID_MAP)]))
+        ID_MAP2 = dict((g, i) for i, g in ID_MAP.items())
+
+    images, pops = [], []
+    while True:
+        for i in indices:
+            r = df.iloc[i]
+            file, pop = r['filenames'], r[var]
+
+            # if greyscale==True:
+            #    im = Image.open(file).convert('LA')
+            # else:
+            #    im = Image.open(file)
+            # im = im.resize((IM_HEIGHT, IM_HEIGHT))
+            # im = np.array(im) / 255.0
+            if greyscale==True:
+               im = Image.open(file).convert('LA')
+               #im = im.resize((IM_HEIGHT, IM_HEIGHT))
+               im = np.array(im)[:,:,0]
+               nx,ny = np.shape(im)
+               if (nx!=tilesize) or (ny!=tilesize):
+                   im = im[int(nx/2)-int(tilesize/2):int(nx/2)+int(tilesize/2), int(ny/2)-int(tilesize/2):int(ny/2)+int(tilesize/2)]
+
+            else:
+               im = Image.open(file)
+               #im = im.resize((IM_HEIGHT, IM_HEIGHT))
+               im = np.array(im)
+               nx,ny,nz = np.shape(im)
+               if (nx!=tilesize) or (ny!=tilesize):
+                   im = im[int(nx/2)-int(tilesize/2):int(nx/2)+int(tilesize/2), int(ny/2)-int(tilesize/2):int(ny/2)+int(tilesize/2)]
+
+            if standardize==True:
+               im = do_standardize(im)
+
+            # if np.ndim(im)==2:
+            #    im = np.dstack((im, im , im)) ##np.expand_dims(im[:,:,0], axis=2)
+            # im = im[:,:,:3]
+
+            if greyscale==True:
+               if do_aug==True:
+                   aug = apply_aug(im[:,:,0])
+                   images.append(aug)
+                   pops.append([to_categorical(pop, len(ID_MAP2)) for k in range(2)]) #3
+               else:
+                   images.append(np.expand_dims(im[:,:,0], axis=2))
+            else:
+               if do_aug==True:
+                   aug = apply_aug(im)
+                   images.append(aug)
+                   pops.append([to_categorical(pop, len(ID_MAP2)) for k in range(2)])
+               else:
+                   images.append(im)
+                   pops.append(to_categorical(pop, len(ID_MAP2)))
+
+            try:
+               if do_aug==True:
+                  if len(images) >= batch_size:
+                     if greyscale==False:
+                        images = np.array(np.vstack(images))
+                        pops = np.array(np.vstack(pops))
+                     else:
+                        images = np.expand_dims(np.array(np.vstack(images)), axis=-1)
+                        pops = np.array(np.vstack(pops))
+                     yield images, pops
+                     images, pops = [], []
+               else:
+                  if len(images) >= batch_size:
+                     yield np.squeeze(np.array(images)),np.array(pops) #[np.array(pops)]
+                     images, pops = [], []
+            except GeneratorExit:
+               print("") #pass
+
+        if not for_training:
+            break
+
+
+###===================================================
+### PLOT TRAINING HISTORY FUNCTIONS
+
+def  plot_train_history_1var(history):
+   """
+   This function plots loss and accuracy curves from the model training
+   """
+   fig, axes = plt.subplots(1, 2, figsize=(10, 10))
+
+   print(history.history.keys())
+
+   axes[0].plot(history.history['loss'], label='Training loss')
+   axes[0].plot(history.history['val_loss'], label='Validation loss')
+   axes[0].set_xlabel('Epochs')
+   axes[0].legend()
+   try:
+      axes[1].plot(history.history['acc'], label='pop train accuracy')
+      axes[1].plot(history.history['val_acc'], label='pop test accuracy')
+   except:
+      axes[1].plot(history.history['accuracy'], label='pop train accuracy')
+      axes[1].plot(history.history['val_accuracy'], label='pop test accuracy')
+   axes[1].set_xlabel('Epochs')
+   axes[1].legend()
+
+
+###===================================================
+def plot_train_history_Nvar(history, varuse, N):
+    """
+    This function makes a plot of error train/validation history for 9 variables,
+    plus overall loss functions
+    """
+    fig, axes = plt.subplots(1, N+1, figsize=(20, 5))
+    for k in range(N):
+       try:
+          axes[k].plot(history.history[varuse[k]+'_output_mean_absolute_error'],
+                       label=varuse[k]+' Train MAE')
+          axes[k].plot(history.history['val_'+varuse[k]+'_output_mean_absolute_error'],
+                       label=varuse[k]+' Val MAE')
+       except:
+          axes[k].plot(history.history[varuse[k]+'_output_mae'],
+                       label=varuse[k]+' Train MAE')
+          axes[k].plot(history.history['val_'+varuse[k]+'_output_mae'],
+                       label=varuse[k]+' Val MAE')
+       axes[k].set_xlabel('Epochs')
+       axes[k].legend()
+
+    axes[N].plot(history.history['loss'], label='Training loss')
+    axes[N].plot(history.history['val_loss'], label='Validation loss')
+    axes[N].set_xlabel('Epochs')
+    axes[N].legend()
+
+
+###===================================================
+def  plot_train_history_1var_mae(history):
+   """
+   This function plots loss and accuracy curves from the model training
+   """
+   print(history.history.keys())
+
+   fig, axes = plt.subplots(1, 2, figsize=(10, 10))
+
+   axes[0].plot(history.history['loss'], label='Training loss')
+   axes[0].plot(history.history['val_loss'],
+                label='Validation loss')
+   axes[0].set_xlabel('Epochs')
+   axes[0].legend()
+
+   try:
+      axes[1].plot(history.history['mean_absolute_error'],
+                   label='pop train MAE')
+      axes[1].plot(history.history['val_mean_absolute_error'],
+                   label='pop test MAE')
+   except:
+      axes[1].plot(history.history['mae'], label='pop train MAE')
+      axes[1].plot(history.history['val_mae'], label='pop test MAE')
+
+   axes[1].set_xlabel('Epochs')
+   axes[1].legend()
+
+###===================================================
+### PLOT CONFUSION MATRIX FUNCTIONS
+
+###===================================================
+def plot_confusion_matrix(cm, classes,
+                          normalize=False,
+                          cmap=plt.cm.Purples,
+                          dolabels=True):
+    """
+    This function prints and plots the confusion matrix.
+    Normalization can be applied by setting `normalize=True`.
+    """
+    if normalize:
+        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
+        cm[np.isnan(cm)] = 0
+
+    plt.imshow(cm, interpolation='nearest', cmap=cmap, vmax=1, vmin=0)
+    fmt = '.2f' if normalize else 'd'
+    thresh = cm.max() / 2.
+    if dolabels==True:
+       tick_marks = np.arange(len(classes))
+       plt.xticks(tick_marks, classes, fontsize=3) #, rotation=60
+       plt.yticks(tick_marks, classes, fontsize=3)
+
+       plt.ylabel('True label',fontsize=4)
+       plt.xlabel('Estimated label',fontsize=4)
+
+    else:
+       plt.axis('off')
+
+    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
+        if cm[i, j]>0:
+           plt.text(j, i, str(cm[i, j])[:4],fontsize=5,
+                    horizontalalignment="center",
+                    color="white" if cm[i, j] > 0.6 else "black")
+    #plt.tight_layout()
+
+    plt.xlim(-0.5, len(classes))
+    plt.ylim(-0.5, len(classes))
+    return cm
+
+###===================================================
+def plot_confmat(y_pred, y_true, prefix, classes):
+   """
+   This function generates and plots a confusion matrix
+   """
+   base = prefix+'_'
+
+   y = y_pred.copy()
+   del y_pred
+   l = y_true.copy()
+   del y_true
+
+   l = l.astype('float')
+   ytrue = l.flatten()
+   ypred = y.flatten()
+
+   ytrue = ytrue[~np.isnan(ytrue)]
+   ypred = ypred[~np.isnan(ypred)]
+
+   cm = confusion_matrix(ytrue, ypred)
+   cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
+   cm[np.isnan(cm)] = 0
+
+   fig=plt.figure()
+   plt.subplot(221)
+   plot_confusion_matrix(cm, classes=classes)
+
+
+
+###===================================================
+### PREDICTION FUNCTIONS
+
+def predict_test_train_cat(train_df, test_df, train_idx, test_idx, var, SM,
+                           classes, weights_path, greyscale, name, do_aug, tilesize):
+   """
+   This function creates makes predictions on test and train data,
+   prints a classification report, and prints confusion matrices
+   """
+   if type(SM) == list:
+      counter = 0
+      for s,wp in zip(SM, weights_path):
+         exec('SM[counter].load_weights(wp)')
+      counter += 1
+   else:
+     SM.load_weights(weights_path)
+
+   ##==============================================
+   ## make predictions on training data
+   for_training = False
+   train_gen = get_data_generator_1image(train_df, train_idx, for_training,
+               len(classes), var, len(train_idx), greyscale, #np.min((200, len(train_idx))),
+               do_aug, standardize, tilesize)
+   x_train, (trueT)= next(train_gen)
+
+   PT = []
+
+   if type(SM) == list:
+       #counter = 0
+       for s in SM:
+           tmp=s.predict(x_train, batch_size=8)
+           exec(
+              'PT.append(np.asarray(np.squeeze(tmp)))'
+           )
+           del tmp
+
+       predT = np.median(PT, axis=0)
+       #predT = np.squeeze(np.asarray(PT))
+       del PT
+       K.clear_session()
+       gc.collect()
+
+   else:
+     predT = SM.predict(x_train, batch_size=8)
+     #predT = np.asarray(predT).argmax(axis=-1)
+
+   del train_gen, x_train
+
+   if test_df is not None:
+       ## make predictions on testing data
+       for_training = False
+       do_aug = False
+       test_gen = get_data_generator_1image(test_df, test_idx, for_training,
+                  len(classes), var, len(test_idx), greyscale, #np.min((200, len(test_idx))),
+                  do_aug, standardize, tilesize) #no augmentation on validation data
+       x_test, (true)= next(test_gen)
+
+       PT = []
+
+       if type(SM) == list:
+           #counter = 0
+           for s in SM:
+               tmp=s.predict(x_test, batch_size=8)
+               exec(
+                  'PT.append(np.asarray(np.squeeze(tmp)))'
+               )
+               del tmp
+
+           pred = np.median(PT, axis=0)
+           #pred = np.squeeze(np.asarray(PT))
+           del PT
+           K.clear_session()
+           gc.collect()
+
+       else:
+
+           pred = SM.predict(x_test, batch_size=8) #1)
+           #pred = np.asarray(pred).argmax(axis=-1)
+
+       del test_gen, x_test
+
+   trueT = np.squeeze(np.asarray(trueT).argmax(axis=-1) )
+   predT = np.squeeze(np.asarray(predT).argmax(axis=-1))#[0])
+
+   if test_df is not None:
+       pred = np.squeeze(np.asarray(pred).argmax(axis=-1))#[0])
+       true = np.squeeze(np.asarray(true).argmax(axis=-1) )
+
+   ##==============================================
+   ## print a classification report to screen, showing f1, precision, recall and accuracy
+   print("==========================================")
+   print("Classification report for "+var)
+   print(classification_report(true, pred))
+
+   fig = plt.figure()
+   ##==============================================
+   ## create figures showing confusion matrices for train and test data sets
+   if type(SM) == list:
+      if test_df is not None:
+          plot_confmat(pred, true, var, classes)
+          plt.savefig(weights_path[0].replace('.hdf5','_cm.png').\
+                   replace('batch','_'.join(np.asarray(BATCH_SIZE, dtype='str'))),
+                   dpi=300, bbox_inches='tight')
+          plt.close('all')
+
+      plot_confmat(predT, trueT, var+'T',classes)
+      plt.savefig(weights_path[0].replace('.hdf5','_cmT.png').\
+               replace('batch','_'.join(np.asarray(BATCH_SIZE, dtype='str'))),
+               dpi=300, bbox_inches='tight')
+      plt.close('all')
+
+   else:
+      if test_df is not None:
+          plot_confmat(pred, true, var, classes)
+          plt.savefig(weights_path.replace('.hdf5','_cm.png'),
+                   dpi=300, bbox_inches='tight')
+          plt.close('all')
+
+      plot_confmat(predT, trueT, var+'T',classes)
+      plt.savefig(weights_path.replace('.hdf5','_cmT.png'),
+               dpi=300, bbox_inches='tight')
+      plt.close('all')
+
+   plt.close()
+   del fig
+
+
+###===================================================
+def predict_train_siso_simo(a, b, vars,
+                                 SM, weights_path, name, mode, greyscale,
+                                  dropout,do_aug, standardize,#CS,# scale,
+                                 count_in):
+    """
+    This function creates makes predcitions on test and train data
+    """
+    ##==============================================
+    ## make predictions on training data
+    if type(SM) == list:
+        counter = 0
+        for s,wp in zip(SM, weights_path):
+           exec('SM[counter].load_weights(wp)')
+        counter += 1
+    else:
+        SM.load_weights(weights_path)
+
+    # if scale == True:
+    #
+    #     if len(vars)>1:
+    #        counter = 0
+    #        for v in vars:
+    #           exec(
+    #           v+\
+    #           '_trueT = np.squeeze(CS[counter].inverse_transform(b[counter].reshape(-1,1)))'
+    #           )
+    #           counter +=1
+    #     else:
+    #        exec(
+    #        vars[0]+\
+    #        '_trueT = np.squeeze(CS[0].inverse_transform(b[0].reshape(-1,1)))'
+    #        )
+    #
+    # else:
+    if len(vars)>1:
+       counter = 0
+       for v in vars:
+          exec(
+          v+\
+          '_trueT = np.squeeze(b[counter])'
+          )
+          counter +=1
+    else:
+       exec(
+       vars[0]+\
+       '_trueT = np.squeeze(b)'
+       )
+
+    del b
+
+    for v in vars:
+       exec(v+'_PT = []')
+
+    # if scale == True:
+    #
+    #     if type(SM) == list:
+    #         counter = 0 #model iterator
+    #         for s in SM:
+    #             train_vals=s.predict(a, batch_size=8)
+    #
+    #             if len(vars)>1:
+    #                counter2 = 0 #variable iterator
+    #                for v in vars:
+    #                   exec(
+    #                   v+\
+    #                   '_PT.append(np.squeeze(CS[counter].inverse_transform(train_vals[counter2].reshape(-1,1))))'
+    #                   )
+    #                   counter2 +=1
+    #             else:
+    #                exec(
+    #                vars[0]+\
+    #                '_PT.append(np.asarray(np.squeeze(CS[0].inverse_transform(train_vals.reshape(-1,1)))))'
+    #                )
+    #
+    #             del train_vals
+    #
+    #         if len(vars)>1:
+    #            #counter = 0
+    #            for v in vars:
+    #               exec(
+    #               v+\
+    #               '_PT = np.median('+v+'_PT, axis=0)'
+    #               )
+    #               #counter +=1
+    #         else:
+    #            exec(
+    #            vars[0]+\
+    #            '_PT = np.median('+v+'_PT, axis=0)'
+    #            )
+    #
+    #     else:
+    #         train_vals = SM.predict(a, batch_size=8) #128)
+    #
+    #         if len(vars)>1:
+    #            counter = 0
+    #            for v in vars:
+    #               exec(
+    #               v+\
+    #               '_PT.append(np.squeeze(CS[counter].inverse_transform(train_vals[counter].reshape(-1,1))))'
+    #               )
+    #               counter +=1
+    #         else:
+    #            exec(
+    #            vars[0]+\
+    #            '_PT.append(np.asarray(np.squeeze(CS[0].inverse_transform(train_vals.reshape(-1,1)))))'
+    #            )
+    #
+    #         del train_vals
+    #
+    # else:
+
+    if type(SM) == list:
+        #counter = 0
+        for s in SM:
+            train_vals=s.predict(a, batch_size=8)
+
+            if len(vars)>1:
+               counter2 = 0
+               for v in vars:
+                  exec(
+                  v+\
+                  '_PT.append(np.squeeze(train_vals[counter2]))'
+                  )
+                  counter2 +=1
+            else:
+               exec(
+               vars[0]+\
+               '_PT.append(np.asarray(train_vals))'
+               )
+
+            del train_vals
+
+        if len(vars)>1:
+           #counter = 0
+           for v in vars:
+              exec(
+              v+\
+              '_PT = np.median('+v+'_PT, axis=0)'
+              )
+              #counter +=1
+        else:
+           exec(
+           vars[0]+\
+           '_PT = np.median('+v+'_PT, axis=0)'
+           )
+
+    else:
+        train_vals = SM.predict(a, batch_size=1)#8) #128)
+
+        if len(vars)>1:
+           counter = 0
+           for v in vars:
+              exec(
+              v+\
+              '_PT.append(np.squeeze(train_vals[counter]))'
+              )
+              counter +=1
+        else:
+           exec(
+           vars[0]+\
+           '_PT.append(np.asarray(np.squeeze(train_vals)))'
+           )
+
+        del train_vals
+
+
+
+    if len(vars)>1:
+       for k in range(len(vars)):
+          exec(vars[k]+'_predT = np.squeeze(np.asarray('+vars[k]+'_PT))')
+    else:
+       exec(vars[0]+'_predT = np.squeeze(np.asarray('+vars[0]+'_PT))')
+
+
+    for v in vars:
+       exec('del '+v+'_PT')
+
+    del a #train_gen,
+
+    if len(vars)==9:
+       nrows = 3; ncols = 3
+    elif len(vars)==8:
+       nrows = 2; ncols = 4
+    elif len(vars)==7:
+       nrows = 3; ncols = 3
+    elif len(vars)==6:
+       nrows = 3; ncols = 2
+    elif len(vars)==5:
+       nrows = 3; ncols = 2
+    elif len(vars)==4:
+       nrows = 2; ncols = 2
+    elif len(vars)==3:
+       nrows = 2; ncols = 2
+    elif len(vars)==2:
+       nrows = 1; ncols = 2
+    elif len(vars)==1:
+       nrows = 1; ncols = 1
+
+    out = dict()
+
+    ## make a plot
+    fig = plt.figure(figsize=(6*nrows,6*ncols))
+    labs = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+    for k in range(1,1+(nrows*ncols)):
+      # try:
+         plt.subplot(nrows,ncols,k)
+         x1 = eval(vars[k-1]+'_trueT')
+         y1 = eval(vars[k-1]+'_predT')
+         out[vars[k-1]+'_trueT'] = eval(vars[k-1]+'_trueT')
+         out[vars[k-1]+'_predT'] = eval(vars[k-1]+'_predT')
+
+         plt.plot(x1, y1, 'ko', markersize=5)
+         plt.plot([ np.min(np.hstack((x1,y1))),  np.max(np.hstack((x1,y1)))],
+                  [ np.min(np.hstack((x1,y1))),  np.max(np.hstack((x1,y1)))],
+                  'k', lw=2)
+         plt.text(np.nanmin(x1), 0.7*np.max(np.hstack((x1,y1))),'Train : '+\
+                  str(np.nanmean(100*(np.abs(y1-x1) / x1)))[:5]+\
+                  ' %', fontsize=10)
+
+         plt.title(r''+labs[k-1]+') '+vars[k-1], fontsize=8, loc='left')
+
+         #varstring = ''.join([str(k)+'_' for k in vars])
+         varstring = str(len(vars))+'vars'
+
+      # except:
+      #     pass
+    if type(SM) == list:
+       plt.savefig(weights_path[0].replace('.hdf5', '_skill_ensemble'+str(count_in)+'.png').\
+                replace('batch','_'.join(np.asarray(BATCH_SIZE, dtype='str'))),
+                dpi=300, bbox_inches='tight')
+
+    else:
+       plt.savefig(weights_path.replace('.hdf5', '_skill'+str(count_in)+'.png'),
+                dpi=300, bbox_inches='tight')
+
+    plt.close()
+    del fig
+
+    np.savez_compressed(weights_path.replace('.hdf5', '_out'+str(count_in)+'.npz'),**out)
+    del out
+
+    if len(vars)==9:
+       nrows = 3; ncols = 3
+    elif len(vars)==8:
+       nrows = 2; ncols = 4
+    elif len(vars)==7:
+       nrows = 3; ncols = 3
+    elif len(vars)==6:
+       nrows = 3; ncols = 2
+    elif len(vars)==5:
+       nrows = 3; ncols = 2
+    elif len(vars)==4:
+       nrows = 2; ncols = 2
+    elif len(vars)==3:
+       nrows = 2; ncols = 2
+    elif len(vars)==2:
+       nrows = 1; ncols = 2
+    elif len(vars)==1:
+       nrows = 1; ncols = 1
+
+    out = dict()
+
+    ## make a plot
+    fig = plt.figure(figsize=(6*nrows,6*ncols))
+    labs = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+    for k in range(1,1+(nrows*ncols)):
+      # try:
+         plt.subplot(nrows,ncols,k)
+         x1 = eval(vars[k-1]+'_trueT')
+         y1 = eval(vars[k-1]+'_predT')
+         out[vars[k-1]+'_trueT'] = eval(vars[k-1]+'_trueT')
+         out[vars[k-1]+'_predT'] = eval(vars[k-1]+'_predT')
+
+
+         plt.plot(x1, y1, 'ko', markersize=5)
+         plt.plot([ np.min(np.hstack((x1,y1))),  np.max(np.hstack((x1,y1)))],
+                  [ np.min(np.hstack((x1,y1))),  np.max(np.hstack((x1,y1)))],
+                  'k', lw=2)
+         plt.text(np.nanmin(x1), 0.7*np.max(np.hstack((x1,y1))),'Train : '+\
+                  str(np.nanmean(100*(np.abs(y1-x1) / x1)))[:5]+\
+                  ' %', fontsize=10)
+
+         plt.title(r''+labs[k-1]+') '+vars[k-1], fontsize=8, loc='left')
+
+         #varstring = ''.join([str(k)+'_' for k in vars])
+         varstring = str(len(vars))+'vars'
+
+      # except:
+      #     pass
+    if type(SM) == list:
+       plt.savefig(weights_path[0].replace('.hdf5', '_skill_ensemble'+str(count_in)+'.png').\
+                replace('batch','_'.join(np.asarray(BATCH_SIZE, dtype='str'))),
+                dpi=300, bbox_inches='tight')
+
+    else:
+       plt.savefig(weights_path.replace('.hdf5', '_skill'+str(count_in)+'.png'),
+                dpi=300, bbox_inches='tight')
+
+    plt.close()
+    del fig
+
+    np.savez_compressed(weights_path.replace('.hdf5', '_out'+str(count_in)+'.npz'),**out)
+    del out
+
+
+###============================================================
+def plot_all_save_all(weights_path, vars):
+
+    if type(weights_path) == list:
+        npz_files = glob(weights_path[0].replace('.hdf5', '*.npz'))
+    else:
+        npz_files = glob(weights_path.replace('.hdf5', '*.npz'))
+
+    npz_files = [n for n in npz_files if '_all.npz' not in n]
+
+    print("Found %i npz files "%(len(npz_files)))
+    if len(vars)==9:
+       nrows = 3; ncols = 3
+    elif len(vars)==8:
+       nrows = 2; ncols = 4
+    elif len(vars)==7:
+       nrows = 3; ncols = 3
+    elif len(vars)==6:
+       nrows = 3; ncols = 2
+    elif len(vars)==5:
+       nrows = 3; ncols = 2
+    elif len(vars)==4:
+       nrows = 2; ncols = 2
+    elif len(vars)==3:
+       nrows = 2; ncols = 2
+    elif len(vars)==2:
+       nrows = 1; ncols = 2
+    elif len(vars)==1:
+       nrows = 1; ncols = 1
+
+    ## make a plot
+    fig = plt.figure(figsize=(6*nrows,6*ncols))
+
+    for counter,file in enumerate(npz_files):
+        out = dict()
+        with np.load(file, allow_pickle=True) as dat:
+            for k in dat.keys():
+                out[k] = dat[k]
+
+        labs = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+        X = []; Y=[]
+        Xt = []; Yt=[]
+
+        for k in range(1,1+(nrows*ncols)):
+          # try:
+             plt.subplot(nrows,ncols,k)
+             x1 = out[vars[k-1]+'_trueT']
+             y1 = out[vars[k-1]+'_predT']
+
+             X.append(x1.flatten()); Y.append(y1.flatten())
+             del x1, y1
+
+             x1 = np.array(X)
+             y1 = np.array(Y)
+
+             plt.plot(x1, y1, 'ko', markersize=5)
+
+             if counter==len(npz_files)-1:
+                 plt.plot([ np.min(np.hstack((x1,y1))),  np.max(np.hstack((x1,y1)))],
+                          [ np.min(np.hstack((x1,y1))),  np.max(np.hstack((x1,y1)))],
+                          'k', lw=2)
+
+                 plt.text(np.nanmin(x1), 0.8*np.max(np.hstack((x1,y1))),'Train : '+\
+                          str(np.mean(100*(np.abs(y1-x1) / x1)))[:5]+\
+                          ' %', fontsize=12, color='r')
+
+                 plt.title(r''+labs[k-1]+') '+vars[k-1], fontsize=8, loc='left')
+
+                 out[vars[k-1]+'_trueT'] = x1 #eval(vars[k-1]+'_trueT')
+                 out[vars[k-1]+'_predT'] = y1 #eval(vars[k-1]+'_predT')
+
+
+             if counter==len(npz_files)-1:
+                 plt.plot([ np.min(np.hstack((x1,y1))),  np.max(np.hstack((x1,y1)))],
+                          [ np.min(np.hstack((x1,y1))),  np.max(np.hstack((x1,y1)))],
+                          'k', lw=2)
+
+                 plt.text(np.nanmin(x1), 0.8*np.max(np.hstack((x1,y1))),'Train : '+\
+                          str(np.mean(100*(np.abs(y1-x1) / x1)))[:5]+\
+                          ' %', fontsize=12, color='r')
+
+                 try:
+                    plt.text(np.nanmin(x2), 0.8*np.max(np.hstack((x2,y2))),'Test : '+\
+                           str(np.mean(100*(np.abs(y2-x2) / x2)))[:5]+\
+                           ' %', fontsize=12, color='r')
+                 except:
+                    pass
+
+                 plt.title(r''+labs[k-1]+') '+vars[k-1], fontsize=8, loc='left')
+
+
+    if type(weights_path) == list:
+       plt.savefig(weights_path[0].replace('.hdf5', '_skill_ensemble.png').\
+                replace('batch','_'.join(np.asarray(BATCH_SIZE, dtype='str'))),
+                dpi=300, bbox_inches='tight')
+
+    else:
+       plt.savefig(weights_path.replace('.hdf5', '_skill.png'),
+                dpi=300, bbox_inches='tight')
+
+    plt.close()
+    del fig
+    np.savez_compressed(weights_path.replace('.hdf5', '_out_all.npz'),**out)
+    del out
+
+
+###===================================================
+### MISC. UTILITIES
+
+def tidy(name,res_folder):
+    """
+    This function moves training outputs to a specific folder
+    """
+
+    pngfiles = glob('*'+name+'*.png')
+    jsonfiles = glob('*'+name+'*.json')
+    hfiles = glob('*'+name+'*.hdf5')
+    tfiles = glob('*'+name+'*.txt')
+    #pfiles = glob('*'+name+'*.pkl')
+    nfiles = glob('*'+name+'*.npz')
+
+    try:
+       [shutil.move(k, res_folder) for k in pngfiles]
+       [shutil.move(k, res_folder) for k in hfiles]
+       [shutil.move(k, res_folder) for k in jsonfiles]
+       [shutil.move(k, res_folder) for k in tfiles]
+       #[shutil.move(k, res_folder) for k in pfiles]
+       [shutil.move(k, res_folder) for k in nfiles]
+    except:
+       pass
+
+###===================================================
+def get_df(csvfile,fortrain=False):
+    """
+    This function reads a csvfile with image names and labels
+    and returns random indices
+    """
+    ###===================================================
+    ## read the data set in, clean and modify the pathnames so they are absolute
+    df = pd.read_csv(csvfile)
+
+    num_split = 50
+    if fortrain==False:
+        if len(df)>num_split:
+           #print('Spliting into  chunks')
+           df = np.array_split(df, int(np.round(len(df)/num_split)))
+           split = True
+        else:
+           split = False
+    else:
+        split = False
+
+    if split:
+        for k in range(len(df)):
+            df[k]['filenames'] = [k.strip() for k in df[k]['filenames']]
+    else:
+        df['filenames'] = [k.strip() for k in df['filenames']]
+
+    if split:
+        for k in range(len(df)):
+            df[k]['filenames'] = [os.getcwd()+os.sep+f.replace('\\',os.sep) for f in df[k]['filenames']]
+    else:
+        df['filenames'] = [os.getcwd()+os.sep+f.replace('\\',os.sep) for f in df['filenames']]
+
+    np.random.seed(2021)
+    if type(df)==list:
+        idx = [np.random.permutation(len(d)) for d in df]
+    else:
+        idx = np.random.permutation(len(df))
+
+    return idx, df, split
+
+
+
+
+#
+# ###===================================================
+# def predict_test_siso_simo(a, b, vars,
+#                                  SM, weights_path, name, mode, greyscale,
+#                                  CS, dropout, scale, do_aug, standardize,
+#                                  count_in):
+#
+#     #
+#     # ## make predictions on testing data
+#     # if d is not None:
+#     #     do_aug = False
+#     #     for_training = False
+#     #     # test_gen = get_data_generator_Nvars_siso_simo(test_df, test_idx, for_training,
+#     #     #            vars, len(test_idx), greyscale, CS, do_aug, standardize) #np.min((200, len(test_idx)))
+#     #     #
+#     #     # x_test, vals = next(test_gen)
+#     #
+#     #     if scale == True:
+#     #
+#     #         if len(vars)>1:
+#     #            counter = 0
+#     #            for v in vars:
+#     #               exec(
+#     #               v+\
+#     #               '_true = np.squeeze(CS[counter].inverse_transform(d[counter].reshape(-1,1)))'
+#     #               )
+#     #               counter +=1
+#     #         else:
+#     #            exec(
+#     #            vars[0]+\
+#     #            '_true = np.squeeze(CS[0].inverse_transform(d[0].reshape(-1,1)))'
+#     #            )
+#     #
+#     #     else:
+#     #         if len(vars)>1:
+#     #            counter = 0
+#     #            for v in vars:
+#     #               exec(
+#     #               v+\
+#     #               '_true = np.squeeze(d[counter])'
+#     #               )
+#     #               counter +=1
+#     #         else:
+#     #            exec(
+#     #            vars[0]+\
+#     #            '_true = np.squeeze(d)'
+#     #            )
+#     #
+#     #
+#     #     del d
+#     #
+#     #     for v in vars:
+#     #        exec(v+'_P = []')
+#     #
+#     #     if scale == True:
+#     #
+#     #         if type(SM) == list:
+#     #             #counter = 0
+#     #             for s in SM:
+#     #                 test_vals=s.predict(c, batch_size=8)
+#     #
+#     #                 if len(vars)>1:
+#     #                    counter = 0
+#     #                    for v in vars:
+#     #                       exec(
+#     #                       v+\
+#     #                       '_P.append(np.squeeze(CS[counter].inverse_transform(test_vals[counter].reshape(-1,1))))'
+#     #                       )
+#     #                       counter +=1
+#     #                 else:
+#     #                    exec(
+#     #                    vars[0]+\
+#     #                    '_P.append(np.asarray(np.squeeze(CS[0].inverse_transform(test_vals.reshape(-1,1)))))'
+#     #                    )
+#     #
+#     #                 del test_vals
+#     #
+#     #             if len(vars)>1:
+#     #                #counter = 0
+#     #                for v in vars:
+#     #                   exec(
+#     #                   v+\
+#     #                   '_P = np.median('+v+'_P, axis=0)'
+#     #                   )
+#     #                   #counter +=1
+#     #             else:
+#     #                exec(
+#     #                vars[0]+\
+#     #                '_P = np.median('+v+'_P, axis=0)'
+#     #                )
+#     #
+#     #         else:
+#     #
+#     #             test_vals = SM.predict(c, batch_size=8) #128)
+#     #             if len(vars)>1:
+#     #                counter = 0
+#     #                for v in vars:
+#     #                   exec(
+#     #                   v+\
+#     #                   '_P.append(np.squeeze(CS[counter].inverse_transform(test_vals[counter].reshape(-1,1))))'
+#     #                   )
+#     #                   counter +=1
+#     #             else:
+#     #                exec(
+#     #                vars[0]+\
+#     #                '_P.append(np.asarray(np.squeeze(CS[0].inverse_transform(test_vals.reshape(-1,1)))))'
+#     #                )
+#     #
+#     #             del test_vals
+#     #
+#     #
+#     #     else: #no scale
+#     #
+#     #         if type(SM) == list:
+#     #             counter = 0
+#     #             for s in SM:
+#     #                 test_vals=s.predict(c, batch_size=8)
+#     #
+#     #                 if len(vars)>1:
+#     #                    counter = 0
+#     #                    for v in vars:
+#     #                       exec(
+#     #                       v+\
+#     #                       '_P.append(np.squeeze(test_vals[counter]))'
+#     #                       )
+#     #                       counter +=1
+#     #                 else:
+#     #                    exec(
+#     #                    vars[0]+\
+#     #                    '_P.append(np.asarray(np.squeeze(test_vals)))'
+#     #                    )
+#     #
+#     #                 del test_vals
+#     #
+#     #             if len(vars)>1:
+#     #                #counter = 0
+#     #                for v in vars:
+#     #                   exec(
+#     #                   v+\
+#     #                   '_P = np.median('+v+'_P, axis=0)'
+#     #                   )
+#     #                   #counter +=1
+#     #             else:
+#     #                exec(
+#     #                vars[0]+\
+#     #                '_P = np.median('+v+'_P, axis=0)'
+#     #                )
+#     #
+#     #         else:
+#     #
+#     #             test_vals = SM.predict(c, batch_size=8) #128)
+#     #             if len(vars)>1:
+#     #                counter = 0
+#     #                for v in vars:
+#     #                   exec(
+#     #                   v+\
+#     #                   '_P.append(np.squeeze(test_vals[counter]))'
+#     #                   )
+#     #                   counter +=1
+#     #             else:
+#     #                exec(
+#     #                vars[0]+\
+#     #                '_P.append(np.asarray(np.squeeze(test_vals)))'
+#     #                )
+#     #
+#     #             # del test_vals
+#     #
+#     #
+#     #     del c #test_gen,
+#
+#         # if len(vars)>1:
+#         #    for k in range(len(vars)):
+#         #       exec(vars[k]+'_pred = np.squeeze(np.asarray('+vars[k]+'_P))')
+#         # else:
+#         #    exec(vars[0]+'_pred = np.squeeze(np.asarray('+vars[0]+'_P))')
+#         #
+#         # for v in vars:
+#         #    exec('del '+v+'_P')
+#
+#         # ## write out results to text files
+#         # if len(vars)>1:
+#         #    for k in range(len(vars)):
+#         #       exec('np.savetxt("'+name+'_test'+vars[k]+'.txt", ('+vars[k]+'_pred))') #','+vars[k]+'_true))')
+#         #       exec('np.savetxt("'+name+'_train'+vars[k]+'.txt", ('+vars[k]+'_predT))') #,'+vars[k]+'_trueT))')
+#         #    np.savetxt(name+"_test_files.txt", np.asarray(test_df.files.values), fmt="%s")
+#         #    np.savetxt(name+"_train_files.txt", np.asarray(train_df.files.values), fmt="%s")
+#         #
+#         # else:
+#         #    exec('np.savetxt("'+name+'_test'+vars[0]+'.txt", ('+vars[0]+'_pred))') #','+vars[k]+'_true))')
+#         #    exec('np.savetxt("'+name+'_train'+vars[0]+'.txt", ('+vars[0]+'_predT))') #,'+vars[k]+'_trueT))')
+#         #    np.savetxt(name+"_test_files.txt", np.asarray(test_df.files.values), fmt="%s")
+#         #    np.savetxt(name+"_train_files.txt", np.asarray(train_df.files.values), fmt="%s")
+#
+#     if len(vars)==9:
+#        nrows = 3; ncols = 3
+#     elif len(vars)==8:
+#        nrows = 2; ncols = 4
+#     elif len(vars)==7:
+#        nrows = 3; ncols = 3
+#     elif len(vars)==6:
+#        nrows = 3; ncols = 2
+#     elif len(vars)==5:
+#        nrows = 3; ncols = 2
+#     elif len(vars)==4:
+#        nrows = 2; ncols = 2
+#     elif len(vars)==3:
+#        nrows = 2; ncols = 2
+#     elif len(vars)==2:
+#        nrows = 1; ncols = 2
+#     elif len(vars)==1:
+#        nrows = 1; ncols = 1
+#
+#     out = dict()
+#
+#     ## make a plot
+#     fig = plt.figure(figsize=(6*nrows,6*ncols))
+#     labs = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+#     for k in range(1,1+(nrows*ncols)):
+#       # try:
+#          plt.subplot(nrows,ncols,k)
+#          x1 = eval(vars[k-1]+'_trueT')
+#          y1 = eval(vars[k-1]+'_predT')
+#          out[vars[k-1]+'_trueT'] = eval(vars[k-1]+'_trueT')
+#          out[vars[k-1]+'_predT'] = eval(vars[k-1]+'_predT')
+#
+#          # z = np.polyfit(y1,x1, 1)
+#          # Z.append(z)
+#          #
+#          # y1 = np.polyval(z,y1)
+#          # y1 = np.abs(y1)
+#
+#          plt.plot(x1, y1, 'ko', markersize=5)
+#          plt.plot([ np.min(np.hstack((x1,y1))),  np.max(np.hstack((x1,y1)))],
+#                   [ np.min(np.hstack((x1,y1))),  np.max(np.hstack((x1,y1)))],
+#                   'k', lw=2)
+#          plt.text(np.nanmin(x1), 0.7*np.max(np.hstack((x1,y1))),'Train : '+\
+#                   str(np.nanmean(100*(np.abs(y1-x1) / x1)))[:5]+\
+#                   ' %', fontsize=10)
+#
+#          # if test_vals is not None:
+#          #     x2 = eval(vars[k-1]+'_true')
+#          #     y2 = eval(vars[k-1]+'_pred')
+#          #     # y2 = np.abs(np.polyval(z,y2))
+#          #
+#          #     plt.plot(x2, y2, 'bx', markersize=5)
+#          #
+#          # if test_vals is not None:
+#          #     plt.text(np.nanmin(x2), 0.75*np.max(np.hstack((x2,y2))),'Test : '+\
+#          #              str(np.mean(100*(np.abs(y2-x2) / x2)))[:5]+\
+#          #              ' %',  fontsize=10, color='b')
+#          # else:
+#          # plt.text(np.nanmin(x1), 0.7*np.max(np.hstack((x1,y1))),''+\
+#          #          str(np.mean(100*(np.abs(y1-x1) / x1)))[:5]+\
+#          #          ' %', fontsize=10)
+#          plt.title(r''+labs[k-1]+') '+vars[k-1], fontsize=8, loc='left')
+#
+#          #varstring = ''.join([str(k)+'_' for k in vars])
+#          varstring = str(len(vars))+'vars'
+#
+#       # except:
+#       #     pass
+#     if type(SM) == list:
+#        plt.savefig(weights_path[0].replace('.hdf5', '_skill_ensemble'+str(count_in)+'.png').\
+#                 replace('batch','_'.join(np.asarray(BATCH_SIZE, dtype='str'))),
+#                 dpi=300, bbox_inches='tight')
+#
+#     else:
+#        plt.savefig(weights_path.replace('.hdf5', '_skill'+str(count_in)+'.png'),
+#                 dpi=300, bbox_inches='tight')
+#
+#     plt.close()
+#     del fig
+#
+#     np.savez_compressed(weights_path.replace('.hdf5', '_out'+str(count_in)+'.npz'),**out)
+#     del out
+
+#
+# ###===================================================
+# def predict_test_train_miso_mimo(train_df, test_df, train_idx, test_idx,
+#                                  vars, auxin, SM, weights_path, name, mode,
+#                                  greyscale, CS, CSaux):
+#     """
+#     This function creates makes predcitions on test and train data
+#     """
+#     ##==============================================
+#     ## make predictions on training data
+#
+#     SM.load_weights(weights_path)
+#
+#     train_gen = get_data_generator_Nvars_miso_mimo(train_df, train_idx, False,
+#                  vars, auxin,aux_mean, aux_std, len(train_idx), greyscale)
+#
+#     x_train, tmp = next(train_gen)
+#
+#     if len(vars)>1:
+#        counter = 0
+#        for v in vars:
+#           exec(
+#           v+\
+#           '_trueT = np.squeeze(CS[counter].inverse_transform(tmp[counter].reshape(-1,1)))'
+#           )
+#           counter +=1
+#     else:
+#        exec(
+#        vars[0]+\
+#        '_trueT = np.squeeze(CS[0].inverse_transform(tmp[0].reshape(-1,1)))'
+#        )
+#
+#     for v in vars:
+#        exec(v+'_PT = []')
+#
+#     del tmp
+#     tmp = SM.predict(x_train, batch_size=8) #128)
+#     if len(vars)>1:
+#        counter = 0
+#        for v in vars:
+#           exec(
+#               v+\
+#               '_PT.append(np.squeeze(CS[counter].inverse_transform(tmp[counter].reshape(-1,1))))'
+#               )
+#           counter +=1
+#     else:
+#        exec(
+#        vars[0]+\
+#        '_PT.append(np.asarray(np.squeeze(CS[0].inverse_transform(tmp.reshape(-1,1)))))'
+#        )
+#
+#
+#     if len(vars)>1:
+#        for k in range(len(vars)):
+#           exec(
+#           vars[k]+\
+#           '_predT = np.squeeze(np.mean(np.asarray('+vars[k]+'_PT), axis=0))'
+#           )
+#     else:
+#        exec(
+#        vars[0]+\
+#        '_predT = np.squeeze(np.mean(np.asarray('+vars[0]+'_PT), axis=0))'
+#        )
+#
+#     ## make predictions on testing data
+#     test_gen = get_data_generator_Nvars_miso_mimo(test_df, test_idx, False,
+#                  vars, auxin, aux_mean, aux_std, len(test_idx), greyscale)
+#
+#     del tmp
+#     x_test, tmp = next(test_gen)
+#     if len(vars)>1:
+#        counter = 0
+#        for v in vars:
+#           exec(v+\
+#                '_true = np.squeeze(CS[counter].inverse_transform(tmp[counter].reshape(-1,1)))'
+#                )
+#           counter +=1
+#     else:
+#        exec(vars[0]+\
+#        '_true = np.squeeze(CS[0].inverse_transform(tmp[0].reshape(-1,1)))'
+#        )
+#
+#     for v in vars:
+#        exec(v+'_P = []')
+#
+#     del tmp
+#     tmp = SM.predict(x_test, batch_size=8) #128)
+#     if len(vars)>1:
+#        counter = 0
+#        for v in vars:
+#           exec(
+#           v+\
+#           '_P.append(np.squeeze(CS[counter].inverse_transform(tmp[counter].reshape(-1,1))))'
+#           )
+#           counter +=1
+#     else:
+#        exec(
+#        vars[0]+\
+#        '_P.append(np.asarray(np.squeeze(CS[0].inverse_transform(tmp.reshape(-1,1)))))'
+#        )
+#
+#     if len(vars)>1:
+#        for k in range(len(vars)):
+#           exec(
+#           vars[k]+\
+#           '_pred = np.squeeze(np.mean(np.asarray('+vars[k]+'_P), axis=0))'
+#           )
+#     else:
+#        exec(
+#        vars[0]+\
+#        '_pred = np.squeeze(np.mean(np.asarray('+vars[0]+'_P), axis=0))'
+#        )
+#
+#
+#     if len(vars)==9:
+#        nrows = 3; ncols = 3
+#     elif len(vars)==8:
+#        nrows = 4; ncols = 2
+#     elif len(vars)==7:
+#        nrows = 4; ncols = 2
+#     elif len(vars)==6:
+#        nrows = 3; ncols = 2
+#     elif len(vars)==5:
+#        nrows = 3; ncols = 2
+#     elif len(vars)==4:
+#        nrows = 2; ncols = 2
+#     elif len(vars)==3:
+#        nrows = 3; ncols = 1
+#     elif len(vars)==2:
+#        nrows = 2; ncols = 1
+#     elif len(vars)==1:
+#        nrows = 1; ncols = 1
+#
+#     ## make a plot
+#     fig = plt.figure(figsize=(4*nrows,4*ncols))
+#     labs = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+#     for k in range(1,1+(nrows*ncols)):
+#       plt.subplot(nrows,ncols,k)
+#       x = eval(vars[k-1]+'_trueT')
+#       y = eval(vars[k-1]+'_predT')
+#       plt.plot(x, y, 'ko', markersize=5)
+#       plt.plot(eval(vars[k-1]+'_true'), eval(vars[k-1]+'_pred'),
+#                'bx', markersize=5)
+#       plt.plot([ np.min(np.hstack((x,y))),  np.max(np.hstack((x,y)))],
+#                [ np.min(np.hstack((x,y))),  np.max(np.hstack((x,y)))], 'k', lw=2)
+#
+#       plt.text(np.nanmin(x), 0.96*np.max(np.hstack((x,y))),'Test : '+\
+#                str(np.mean(100*(np.abs(eval(vars[k-1]+'_pred') -\
+#                 eval(vars[k-1]+'_true')) / eval(vars[k-1]+'_true'))))[:5]+\
+#                ' %',  fontsize=8, color='b')
+#       plt.text(np.nanmin(x), np.max(np.hstack((x,y))),'Train : '+\
+#                str(np.mean(100*(np.abs(eval(vars[k-1]+'_predT') -\
+#                 eval(vars[k-1]+'_trueT')) / eval(vars[k-1]+'_trueT'))))[:5]+\
+#                ' %', fontsize=8)
+#       plt.title(r''+labs[k-1]+') '+vars[k-1], fontsize=8, loc='left')
+#
+#     varstring = ''.join([str(k)+'_' for k in vars])
+#
+#     plt.savefig(weights_path.replace('.hdf5', '_skill.png'),
+#                 dpi=300, bbox_inches='tight')
+#     plt.close()
+#     del fig
+#
+
+#
+# ###===================================================
+# def get_data_generator_Nvars_miso_mimo(df, indices, for_training, vars, auxin,
+#                                        batch_size, greyscale, CS, CSaux): ##BATCH_SIZE
+#     """
+#     This function generates data for a batch of images and 1 auxilliary variable,
+#     and N associated output metrics
+#     """
+#     if len(vars)==1:
+#        images, a, p1s = [], [], []
+#     elif len(vars)==2:
+#        images, a, p1s, p2s = [], [], [], []
+#     elif len(vars)==3:
+#        images, a, p1s, p2s, p3s = [], [], [], [], []
+#     elif len(vars)==4:
+#        images, a, p1s, p2s, p3s, p4s = [], [], [], [], [], []
+#     elif len(vars)==5:
+#        images, a, p1s, p2s, p3s, p4s, p5s = [], [], [], [], [], [], []
+#     elif len(vars)==6:
+#        images, a, p1s, p2s, p3s, p4s, p5s, p6s = \
+#        [], [], [], [], [], [], [], []
+#     elif len(vars)==7:
+#        images, a, p1s, p2s, p3s, p4s, p5s, p6s, p7s = \
+#        [], [], [], [], [], [], [], [], []
+#     elif len(vars)==8:
+#        images, a, p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s = \
+#        [], [], [], [], [], [], [], [], [], []
+#     elif len(vars)==9:
+#        images, a, p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s, p9s = \
+#        [], [], [], [], [], [], [], [], [], [], []
+#
+#     while True:
+#         for i in indices:
+#             r = df.iloc[i]
+#             if len(vars)==1:
+#                file, p1, aa = r['files'], r[vars[0]], r[auxin]
+#             if len(vars)==2:
+#                file, p1, p2, aa = \
+#                r['files'], r[vars[0]], r[vars[1]], r[auxin]
+#             if len(vars)==3:
+#                file, p1, p2, p3, aa = \
+#                r['files'], r[vars[0]], r[vars[1]], r[vars[2]], r[auxin]
+#             if len(vars)==4:
+#                file, p1, p2, p3, p4, aa = \
+#                r['files'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[auxin]
+#             if len(vars)==5:
+#                file, p1, p2, p3, p4, p5, aa = \
+#                r['files'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[auxin]
+#             if len(vars)==6:
+#                file, p1, p2, p3, p4, p5, p6, aa = \
+#                r['files'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[vars[5]], r[auxin]
+#             if len(vars)==7:
+#                file, p1, p2, p3, p4, p5, p6, p7, aa =\
+#                r['files'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[vars[5]], r[vars[6]], r[auxin]
+#             if len(vars)==8:
+#                file, p1, p2, p3, p4, p5, p6, p7, p8, aa = \
+#                r['files'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[vars[5]], r[vars[6]], r[vars[7]], r[auxin]
+#             elif len(vars)==9:
+#                file, p1, p2, p3, p4, p5, p6, p7, p8, p9, aa = \
+#                r['files'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[vars[5]], r[vars[6]], r[vars[7]], r[vars[8]], r[auxin]
+#
+#             if greyscale==True:
+#                im = Image.open(file).convert('LA')
+#             else:
+#                im = Image.open(file)
+#             im = im.resize((IM_HEIGHT, IM_HEIGHT))
+#             im = np.array(im) / 255.0
+#
+#             if np.ndim(im)==2:
+#                im = np.dstack((im, im , im)) ##np.expand_dims(im[:,:,0], axis=2)
+#
+#             im = im[:,:,:3]
+#
+#             if greyscale==True:
+#                images.append(np.expand_dims(im, axis=2))
+#             else:
+#                images.append(im)
+#
+#             if len(vars)==1:
+#                p1s.append(p1); a.append(aa)
+#             elif len(vars)==2:
+#                p1s.append(p1); p2s.append(p2); a.append(aa)
+#             elif len(vars)==3:
+#                p1s.append(p1); p2s.append(p2); a.append(aa)
+#                p3s.append(p3);
+#             elif len(vars)==4:
+#                p1s.append(p1); p2s.append(p2); a.append(aa)
+#                p3s.append(p3); p4s.append(p4)
+#             elif len(vars)==5:
+#                p1s.append(p1); p2s.append(p2); a.append(aa)
+#                p3s.append(p3); p4s.append(p4)
+#                p5s.append(p5);
+#             elif len(vars)==6:
+#                p1s.append(p1); p2s.append(p2); a.append(aa)
+#                p3s.append(p3); p4s.append(p4)
+#                p5s.append(p5); p6s.append(p6)
+#             elif len(vars)==7:
+#                p1s.append(p1); p2s.append(p2); a.append(aa)
+#                p3s.append(p3); p4s.append(p4)
+#                p5s.append(p5); p6s.append(p6)
+#                p7s.append(p7);
+#             elif len(vars)==8:
+#                p1s.append(p1); p2s.append(p2); a.append(aa)
+#                p3s.append(p3); p4s.append(p4)
+#                p5s.append(p5); p6s.append(p6)
+#                p7s.append(p7); p8s.append(p8)
+#             elif len(vars)==9:
+#                p1s.append(p1); p2s.append(p2); a.append(aa)
+#                p3s.append(p3); p4s.append(p4)
+#                p5s.append(p5); p6s.append(p6)
+#                p7s.append(p7); p8s.append(p8)
+#                p9s.append(p9)
+#
+#
+#             if len(images) >= batch_size:
+#                if len(vars)==1:
+#                   p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+#                   a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1)))
+#                   yield [np.array(a), np.array(images)], [np.array(p1s)]
+#                   images, a, p1s = [], [], []
+#                elif len(vars)==2:
+#                   p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+#                   p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1)))
+#                   a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1)))
+#                   yield [np.array(a), np.array(images)],[np.array(p1s), np.array(p2s)]
+#                   images, a, p1s, p2s = [], [], [], []
+#                elif len(vars)==3:
+#                   p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+#                   p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1)))
+#                   p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1)))
+#                   a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1)))
+#                   yield [np.array(a), np.array(images)],[np.array(p1s), np.array(p2s), np.array(p3s)]
+#                   images, a, p1s, p2s, p3s = [], [], [], [], []
+#                elif len(vars)==4:
+#                   p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+#                   p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1)))
+#                   p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1)))
+#                   p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1)))
+#                   a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1)))
+#                   yield [np.array(a), np.array(images)],[np.array(p1s), np.array(p2s), np.array(p3s), np.array(p4s)]
+#                   images, a, p1s, p2s, p3s, p4s = [], [], [], [], [], []
+#                elif len(vars)==5:
+#                   p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+#                   p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1)))
+#                   p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1)))
+#                   p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1)))
+#                   p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1)))
+#                   a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1)))
+#                   yield [np.array(a), np.array(images)],[np.array(p1s), np.array(p2s), np.array(p3s),
+#                         np.array(p4s), np.array(p5s)]
+#                   images, a, p1s, p2s, p3s, p4s, p5s = \
+#                   [], [], [], [], [], [], []
+#                elif len(vars)==6:
+#                   p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+#                   p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1)))
+#                   p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1)))
+#                   p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1)))
+#                   p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1)))
+#                   p6s = np.squeeze(CS[5].transform(np.array(p6s).reshape(-1, 1)))
+#                   a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1)))
+#                   yield [np.array(a), np.array(images)],[np.array(p1s), np.array(p2s), np.array(p3s),
+#                         np.array(p4s), np.array(p5s), np.array(p6s)]
+#                   images, a, p1s, p2s, p3s, p4s, p5s, p6s = \
+#                   [], [], [], [], [], [], [], []
+#                elif len(vars)==7:
+#                   p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+#                   p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1)))
+#                   p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1)))
+#                   p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1)))
+#                   p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1)))
+#                   p6s = np.squeeze(CS[5].transform(np.array(p6s).reshape(-1, 1)))
+#                   p7s = np.squeeze(CS[6].transform(np.array(p7s).reshape(-1, 1)))
+#                   a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1)))
+#                   yield [np.array(a), np.array(images)],[np.array(p1s), np.array(p2s), np.array(p3s),
+#                         np.array(p4s), np.array(p5s), np.array(p6s), np.array(p7s)]
+#                   images, a, p1s, p2s, p3s, p4s, p5s, p6s, p7s = \
+#                   [], [], [], [], [], [], [], [], []
+#                elif len(vars)==8:
+#                   p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+#                   p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1)))
+#                   p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1)))
+#                   p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1)))
+#                   p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1)))
+#                   p6s = np.squeeze(CS[5].transform(np.array(p6s).reshape(-1, 1)))
+#                   p7s = np.squeeze(CS[6].transform(np.array(p7s).reshape(-1, 1)))
+#                   p8s = np.squeeze(CS[7].transform(np.array(p8s).reshape(-1, 1)))
+#                   a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1)))
+#                   yield [np.array(a), np.array(images)],[np.array(p1s), np.array(p2s), np.array(p3s),
+#                         np.array(p4s), np.array(p5s), np.array(p6s),
+#                         np.array(p7s), np.array(p8s)]
+#                   images, a, p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s = \
+#                   [], [], [], [], [], [], [], [], [], []
+#                elif len(vars)==9:
+#                   p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1)))
+#                   p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1)))
+#                   p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1)))
+#                   p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1)))
+#                   p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1)))
+#                   p6s = np.squeeze(CS[5].transform(np.array(p6s).reshape(-1, 1)))
+#                   p7s = np.squeeze(CS[6].transform(np.array(p7s).reshape(-1, 1)))
+#                   p8s = np.squeeze(CS[7].transform(np.array(p8s).reshape(-1, 1)))
+#                   p9s = np.squeeze(CS[8].transform(np.array(p9s).reshape(-1, 1)))
+#                   a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1)))
+#                   try:
+#                      yield [np.array(a), np.array(images)],[np.array(p1s), np.array(p2s), np.array(p3s),
+#                            np.array(p4s), np.array(p5s), np.array(p6s),
+#                            np.array(p7s), np.array(p8s), np.array(p9s)]
+#                   except GeneratorExit:
+#                      print(" ") #pass
+#                   images, a, p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s, p9s = \
+#                   [], [], [], [], [], [], [], [], [], [], []
+#         if not for_training:
+#             break

requirements.txt

@@ -0,0 +1,4 @@
+tensorflow
+numpy
+matplotlib
+scikit-image

weights/config_usace_combined2021_2022_v12.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54a50d316b63ca29d01277eadbaa098ee71435f69e7b57f5a9c0d2be80c8b282
+size 597

weights/sandsnap_merged_1024_modelrevOct2022_v12_simo_batch10_im1024_9vars_mse_noaug.hdf5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6383827d51f659cb4fc433c9c2d75594708456a5e3fd15906be8821454c0c74e
+size 1010376

weights/sandsnap_merged_1024_modelrevOct2022_v12_simo_batch10_im1024_9vars_mse_noaug.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:927bf368c23ef8c512e5c563bf953b0b1b04950d3eb6f2c69f4e72d32b1d0cfe
+size 17855