Script showing issue with Keras 2.2.3 that occurs when trying to save best model via ModelCheckpoint callback.

keras_h5py_issue.py

import numpy as np
import os
import pandas as pd
import sys
import uuid
import sys

import tensorflow as tf
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
import keras
import keras.backend as K
import gc

def main(**kwargs):
    model_dir = os.path.join("temp/", str(uuid.uuid1())+"/")
    os.makedirs(model_dir, exist_ok=True)
    model_name = os.path.join(model_dir, "model.best.hdf5")
    
    trainX = np.random.rand(1024, 24, 10)
    trainY = np.random.rand(1024, 1)
    testX = np.random.rand(256, 24, 10)
    testY = np.random.rand(256, 1)
    
    try:
        input_layer = keras.layers.Input(shape=trainX.shape[1:])

        conv_layers = []
        conv_net_out = []
        conv_skip_out = []
        
        conv_layers.append(keras.layers.Conv1D(kwargs['conv_layer_filter_num'], 
                                               kwargs['conv_layer_size'], 
                                               activation=kwargs['conv_layer_activation'], 
                                               padding='causal', 
                                               dilation_rate=1,
                                               activity_regularizer=keras.regularizers.l1(kwargs['conv_l1_reg_parameter']),
                                               kernel_regularizer=keras.regularizers.l2(kwargs['conv_l2_reg_parameter']))(input_layer))
        
        intermediate_sum = keras.layers.BatchNormalization()(keras.layers.Activation("selu")(conv_layers[-1]))
        
        conv_skip_out.append(keras.layers.Conv1D(1,1)(intermediate_sum))
        intermediate_sum = keras.layers.Conv1D(1,1)(intermediate_sum)
        conv_net_out.append(keras.layers.Add()([intermediate_sum, input_layer]))

        for conv_idx in range(kwargs['conv_layer_count'] - 1):
            conv_layers.append(keras.layers.Conv1D(kwargs['conv_layer_filter_num'], 
                                                   kwargs['conv_layer_size'], 
                                                   activation=kwargs['conv_layer_activation'], 
                                                   padding='causal', 
                                                   dilation_rate=kwargs['conv_layer_dilation_rate'] * (conv_idx + 1),
                                                   activity_regularizer=keras.regularizers.l1(kwargs['conv_l1_reg_parameter']),
                                                   kernel_regularizer=keras.regularizers.l2(kwargs['conv_l2_reg_parameter']))(conv_net_out[-1]))
            
            intermediate_sum = keras.layers.BatchNormalization()(keras.layers.Activation("selu")(conv_layers[-1]))
            
            conv_skip_out.append(keras.layers.Conv1D(1,1)(intermediate_sum))
            intermediate_sum = keras.layers.Conv1D(1,1)(intermediate_sum)
            conv_net_out.append(keras.layers.Add()([intermediate_sum, conv_net_out[-1]]))

        conv_sum_layer = keras.layers.Add()(conv_net_out)
        csum_out = keras.layers.Activation('selu')(conv_sum_layer)
        oxo_layer1 = keras.layers.Conv1D(1,1)(csum_out)
        relu_layer = keras.layers.Activation('selu')(oxo_layer1)
        oxo_layer2 = keras.layers.Conv1D(1,1)(relu_layer)
        flat_oxo_layer = keras.layers.Flatten()(oxo_layer2)
        output_layer = keras.layers.Dense(trainY.shape[-1], activation="linear")(flat_oxo_layer)
        model = keras.models.Model(inputs=input_layer, outputs=output_layer)
        model.compile(loss=kwargs['loss'], optimizer=kwargs['optimizer'])
        test_csv_cb = keras.callbacks.CSVLogger(os.path.join(model_dir, 'progress.csv'), separator=',', append=False)
        early_stopping_cb = keras.callbacks.EarlyStopping(patience=15)
        model_saver_cb = keras.callbacks.ModelCheckpoint(model_name, monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=False, mode='auto', period=1)
        model.fit(trainX, trainY, validation_split=0.1, 
                  epochs=kwargs['training_epochs'], batch_size=kwargs['batch_size'],
                  shuffle=True, verbose=0, callbacks=[early_stopping_cb, model_saver_cb, test_csv_cb])
        model.save(model_name)

        try: 
            model = keras.models.load_model(model_name)
            
            predictions = model.predict(testX) - testY
            
            gc.collect()
            
            return np.percentile(np.square(predictions), 75)
        except Exception as e:
            raise e 

        
    except Exception as e:
        raise e

if __name__ == "__main__":
    CNN_params = {'optimizer': 'adam', 
                    'loss': 'mse', 
                    'conv_layer_count': 8,
                    'conv_layer_filter_num': 16, 
                    'conv_layer_size': 2, 
                    'conv_layer_dilation_rate': 2,
                    'conv_layer_activation': 'selu',
                    'conv_l2_reg_parameter': 5e-4,
                    'conv_l1_reg_parameter': 5e-5,
                    'training_epochs': 100,
                    'batch_size': 256}
                    
    print("75th percentile of test predictions is: {:.2e}".format(main(**CNN_params)))