Merging Conv2D and Dense models results in “RuntimeError: You must compile your model before using it.”, despite having compiled the merged model

Question

I'm attempting to train an AI to identify lesions based off of images and patient info. I'm using Keras' Sequential model to do so. I make two sequential models, then merge them and compile the merged model. When I try to fit the model I get the error RuntimeError: You must compile your model before using it. even though my models have input shapes defined.

I've tried switching input_dim=dim to input_shape=(dim,). The only things I can find on the issue such as this post or  this one input_shape, which mine have. I can't imagine you'd have to do that for the Concatenate layer as well.

I first create the dense layers for the patient info:

metadata_model = Sequential()

metadata_model.add(Dense(32, input_dim=X_train.iloc[:, L*W:].shape[1], activation="relu"))

metadata_model.add(Dense(64))

Then the model for the images:

model = Sequential()

model.add(Conv2D(32, (3, 3), padding="same", input_shape=(W, L, 3)))

model.add(Activation("relu"))

model.add(BatchNormalization(axis=-1))

model.add(MaxPooling2D(pool_size=(3,3)))

model.add(Dropout(rate = 0.25))

model.add(Conv2D(64, (3, 3), padding="same"))

model.add(Activation("relu"))

model.add(BatchNormalization(axis=-1))

model.add(Conv2D(64, (3, 3), padding="same"))

model.add(Activation("relu"))

model.add(BatchNormalization(axis=-1))

model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Dropout(0.25))

model.add(Conv2D(128, (3, 3), padding="same"))

model.add(Activation("relu"))

model.add(BatchNormalization(axis=-1))

model.add(Conv2D(128, (3, 3), padding="same"))

model.add(Activation("relu"))

model.add(BatchNormalization(axis=-1))

model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Dropout(0.25))

model.add(Flatten())

model.add(Dense(1024))

model.add(Activation("relu"))

model.add(BatchNormalization())

model.add(Dropout(0.5))

Then I merge them:

merged_model = Sequential()

merged_model.add(Concatenate([model, metadata_model]))

merged_model.add(Dense(7)) #7 lesion classes

merged_model.add(Activation("softmax"))

compile and create an ImageDataGenerator:

opt = Adam(lr=INIT_LR, decay=INIT_LR/EPOCHS)

merged_model.compile(loss="categorical_crossentropy", optimizer = opt, metrics=["accuracy"])

aug = ImageDataGenerator(rotation_range=25, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.2, zoom_range=0.2, horizontal_flip=True, fill_mode="nearest")

and try to train it:

train = merged_model.fit_generator(

aug.flow([trainInput, X_train.iloc[:, L*W:]], labels, batch_size=BS),

validation_data=([testInput, X_test.iloc[:, L*W:]], labels_test),

steps_per_epoch=500,

epochs=EPOCHS,

verbose=1)

This line results in the following error:

RuntimeError                              Traceback (most recent call last)

<ipython-input-114-fc6c254db390> in <module>

      4 steps_per_epoch=500,

      5 epochs=EPOCHS,

----> 6 verbose=1)

c:\users\megag\appdata\local\programs\python\python37\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)

     89                 warnings.warn('Update your `' + object_name + '` call to the ' +

     90                               'Keras 2 API: ' + signature, stacklevel=2)

---> 91             return func(*args, **kwargs)

     92         wrapper._original_function = func

     93         return wrapper

c:\users\megag\appdata\local\programs\python\python37\lib\site-packages\keras\engine\training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)

   1416             use_multiprocessing=use_multiprocessing,

   1417             shuffle=shuffle,

-> 1418             initial_epoch=initial_epoch)

   1419 

   1420     @interfaces.legacy_generator_methods_support

c:\users\megag\appdata\local\programs\python\python37\lib\site-packages\keras\engine\training_generator.py in fit_generator(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)

     38 

     39     do_validation = bool(validation_data)

---> 40     model._make_train_function()

     41     if do_validation:

     42         model._make_test_function()

c:\users\megag\appdata\local\programs\python\python37\lib\site-packages\keras\engine\training.py in _make_train_function(self)

    494     def _make_train_function(self):

    495         if not hasattr(self, 'train_function'):

--> 496             raise RuntimeError('You must compile your model before using it.')

    497         self._check_trainable_weights_consistency()

    498         if self.train_function is None:

RuntimeError: You must compile your model before using it.

Answer 1

Since, your merged model has two input layers, so it is no longer sequential.So, here Sequential API cannot be used.

You should use the Functional API of Keras to merge your models:

from keras.models import Model

x = Concatenate()([model.output, metadata_model.output])

x = Dense(7)(x)

out = Activation("softmax")(x)

merged_model = Model([model.input, metadata_model.input], out)

# the rest is the same...