diff --git a/donkeycar/parts/keras.py b/donkeycar/parts/keras.py index 918c9604b..1b3b28fb2 100644 --- a/donkeycar/parts/keras.py +++ b/donkeycar/parts/keras.py @@ -15,12 +15,14 @@ from tensorflow import keras from tensorflow.keras.layers import Input, Dense from tensorflow.keras.layers import Convolution2D, MaxPooling2D, BatchNormalization +from tensorflow.keras.layers import GlobalAveragePooling2D from tensorflow.keras.layers import Activation, Dropout, Flatten from tensorflow.keras.layers import LSTM from tensorflow.keras.layers import TimeDistributed as TD from tensorflow.keras.layers import Conv3D, MaxPooling3D, Conv2DTranspose from tensorflow.keras.backend import concatenate from tensorflow.keras.models import Model, Sequential +from donkeycar.parts.keras_resnet18 import identity_block,conv_block,ResNet18 import donkeycar as dk from donkeycar.utils import normalize_image @@ -698,3 +700,111 @@ def default_latent(num_outputs, input_shape): model = Model(inputs=[img_in], outputs=outputs) return model + +# ResNet18 pre-requesite parameters +backend = tf.compat.v1.keras.backend +layers = tf.keras.layers +models = tf.keras.models +utils = tf.keras.utils + + +def resnet18_default_n_linear(num_outputs, input_shape=(120,60,3)): + + # Instantiate a ResNet18 model + resnet18 = ResNet18(include_top=False,weights='cifar100_coarse',input_shape=input_shape,backend=backend,layers=layers,models=models,utils=utils) + for layer in resnet18.layers: + layer.trainable=False + resnet18_preprocess = tf.keras.applications.resnet.preprocess_input + + # Transfer learning with Resnet18 + drop = 0.2 + img_in = Input(shape=input_shape,name='img_in') + x = resnet18_preprocess(img_in) + x = resnet18(img_in,training=True) + x = GlobalAveragePooling2D()(x) + # Classifier + x = Dense(128,activation='relu',name='dense_1')(x) + x = Dropout(drop)(x) + x = Dense(64,activation='relu',name='dense_2')(x) + x = Dropout(drop)(x) + + outputs = [] + for i in range(num_outputs): + outputs.append( + Dense(1, activation='linear', name='n_outputs' + str(i))(x)) + + model = Model(inputs=[img_in],outputs=outputs) + + return model + + +def resnet18_default_categorical(input_shape=(120, 60, 3)): + # Instantiate a ResNet18 model + resnet18 = ResNet18(include_top=False,weights='cifar100_coarse',input_shape=input_shape,backend=backend,layers=layers,models=models,utils=utils) + for layer in resnet18.layers: + layer.trainable=False + resnet18_preprocess = tf.keras.applications.resnet.preprocess_input + + # Transfer learning with Resnet18 + drop = 0.2 + img_in = Input(shape=input_shape,name='img_in') + x = resnet18_preprocess(img_in) + x = resnet18(img_in,training=True) + x = GlobalAveragePooling2D()(x) + # Classifier + x = Dense(128,activation='relu',name='dense_1')(x) + x = Dropout(drop)(x) + x = Dense(64,activation='relu',name='dense_2')(x) + x = Dropout(drop)(x) + + # Categorical output of the angle into 15 bins + angle_out = Dense(15, activation='softmax', name='angle_out')(x) + # categorical output of throttle into 20 bins + throttle_out = Dense(20, activation='softmax', name='throttle_out')(x) + + model = Model(inputs=[img_in], outputs=[angle_out, throttle_out]) + return model + + +class Resnet18LinearKeras(KerasPilot): + def __init__(self, num_outputs=2, input_shape=(120, 160, 3)): + super().__init__() + self.model = resnet18_default_n_linear(num_outputs, input_shape) + self.optimizer = 'adam' + + def compile(self): + self.model.compile(optimizer=self.optimizer, loss='mse',metrics='mse') + + def inference(self, img_arr, other_arr): + img_arr = img_arr.reshape((1,) + img_arr.shape) + outputs = self.model.predict(img_arr) + steering = outputs[0] + return steering[0] , dk.utils.throttle(steering[0]) + + +class Resnet18CategoricalKeras(KerasPilot): + def __init__(self, input_shape=(120, 160, 3), throttle_range=0.5): + super().__init__() + self.model = resnet18_default_categorical(input_shape) + self.optimizer = 'adam' + self.compile() + self.throttle_range = throttle_range + + def compile(self): + self.model.compile(optimizer=self.optimizer, metrics=['accuracy'], + loss={'angle_out': 'categorical_crossentropy', + 'throttle_out': 'categorical_crossentropy'}, + loss_weights={'angle_out': 0.5, 'throttle_out': 0.5}) + + def inference(self, img_arr, other_arr): + if img_arr is None: + print('no image') + return 0.0, 0.0 + + img_arr = img_arr.reshape((1,) + img_arr.shape) + angle_binned, throttle_binned = self.model.predict(img_arr) + N = len(throttle_binned[0]) + throttle = dk.utils.linear_unbin(throttle_binned, N=N, + offset=0.0, R=self.throttle_range) + angle = dk.utils.linear_unbin(angle_binned) + return angle, throttle diff --git a/donkeycar/parts/keras_resnet18.py b/donkeycar/parts/keras_resnet18.py new file mode 100644 index 000000000..37552f742 --- /dev/null +++ b/donkeycar/parts/keras_resnet18.py @@ -0,0 +1,191 @@ +import tensorflow as tf +from tensorflow.keras.layers import ZeroPadding2D, Input, GlobalAveragePooling2D,GlobalMaxPooling2D,Dense +from tensorflow.keras.layers import Convolution2D,MaxPooling2D,BatchNormalization +from tensorflow.keras.layers import Activation,Dropout,Flatten +from tensorflow.keras.models import Model,Sequential +from keras_applications.imagenet_utils import _obtain_input_shape, get_submodules_from_kwargs +import os +import warnings + +# Those are mandatory for ResNet function to work +backend = tf.compat.v1.keras.backend +layers = tf.keras.layers +models = tf.keras.models +utils = tf.keras.utils + +WEIGHTS_PATH = 'https://raw.githubusercontent.com/cl3m3nt/resnet/master/resnet18_cifar100_top.h5' +WEIGHTS_PATH_NO_TOP = 'https://raw.githubusercontent.com/cl3m3nt/resnet/master/resnet18_cifar100_no_top.h5' + + +def identity_block(input_tensor, kernel_size, filters, stage, block): + filters1, filters2 = filters + if backend.image_data_format() == 'channels_last': + bn_axis = 3 + + else: + bn_axis = 1 + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block +'_branch' + + x = Convolution2D(filters1,(1,1), + kernel_initializer='he_normal', + name = conv_name_base + '2a')(input_tensor) + x = BatchNormalization(axis=bn_axis,name=bn_name_base + '2a')(x) + x = Activation('relu')(x) + + x = Convolution2D(filters2, kernel_size, + padding='same', + kernel_initializer='he_normal', + name = conv_name_base + '2b')(x) + x = BatchNormalization(axis=bn_axis,name=bn_name_base+'2b')(x) + x = Activation('relu')(x) + + x = tf.keras.layers.add([x,input_tensor]) + x = Activation('relu')(x) + return x + + +def conv_block(input_tensor,kernel_size,filters,stage,block,strides=(2,2)): + + filters1, filters2 = filters + if backend.image_data_format() == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + + x = Convolution2D(filters1,(1,1),strides=strides, + kernel_initializer='he_normal', + name = conv_name_base + '2a')(input_tensor) + x = BatchNormalization(bn_axis,name=bn_name_base + '2a')(x) + x = Activation('relu')(x) + + + x = Convolution2D(filters2, kernel_size, + padding='same', + kernel_initializer='he_normal', + name = conv_name_base + '2b')(x) + x = BatchNormalization(axis=bn_axis,name=bn_name_base+'2b')(x) + x = Activation('relu')(x) + + + shortcut = Convolution2D(filters2,(1,1),strides=strides, + kernel_initializer='he_normal', + name=conv_name_base+'1')(input_tensor) + + shortcut = BatchNormalization( + axis=bn_axis,name=bn_name_base+'1')(shortcut) + + x = tf.keras.layers.add([x,shortcut]) + x = Activation('relu')(x) + return x + + +# ResnNet18 +def ResNet18(include_top=True, + weights='cifar100_coarse', + input_tensor=None, + input_shape=None, + pooling=None, + classes=20, + **kwargs): + global backend, layers, models, keras_utils + backend, layers, models, keras_utils = get_submodules_from_kwargs(kwargs) + + # Check Weights + if not (weights in {'cifar100_coarse', None} or os.path.exists(weights)): + raise ValueError('The `weights` argument should be either ' + '`None` (random initialization), `cifar100_coarse` ' + '(pre-training on cifar100 coarse (super) classes), ' + 'or the path to the weights file to be loaded.') + + if weights == 'cifar100_coarse' and include_top and classes != 20: + raise ValueError('If using `weights` as `"cifar100_coarse"` with `include_top`' + ' as true, `classes` should be 20') + + # Determine proper input shape + input_shape = _obtain_input_shape(input_shape, + default_size=224, + min_size=32, + data_format=backend.image_data_format(), + require_flatten=include_top, + weights=weights) + + if input_tensor is None: + img_input = layers.Input(shape=input_shape) + else: + if not tf.keras.backend.is_keras_tensor(input_tensor): + img_input = layers.Input(tensor=input_tensor, shape=input_shape) + else: + img_input = input_tensor + if backend.image_data_format() == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + + # Build ResNet18 architecture + x = ZeroPadding2D(padding=(3,3),name='conv1_pad')(img_input) + x = Convolution2D(64,(7,7), + strides=(2,2), + padding='valid', + kernel_initializer='he_normal', + name='conv1')(x) + x = BatchNormalization(axis=bn_axis,name='bn_conv1')(x) + x = Activation('relu')(x) + x = ZeroPadding2D(padding=(1,1),name='pool1_pad')(x) + x = MaxPooling2D((3,3),strides=(2,2))(x) + + x = identity_block(x,3,[64,64],stage=2,block='a') + x = identity_block(x,3,[64,64],stage=2,block='b') + + x = conv_block(x,3,[128,128],stage=3,block='a') + x = identity_block(x,3,[128,128],stage=3,block='b') + + x = conv_block(x,3,[256,256],stage=4,block='a') + x = identity_block(x,3,[256,256],stage=4,block='b') + + x = conv_block(x,3,[512,512],stage=5,block='a') + x = identity_block(x,3,[512,512],stage=5,block='b') + + # Managing Top + if include_top: + x = layers.GlobalAveragePooling2D(name='avg_pool')(x) + x = layers.Dense(classes, activation='softmax', name='fc20')(x) + else: + if pooling == 'avg': + x = layers.GlobalAveragePooling2D()(x) + elif pooling == 'max': + x = layers.GlobalMaxPooling2D()(x) + else: + warnings.warn('No flattenting layer operation like AveragePooling2D or MaxPooling2D has been added' + 'whereas there are not top. You will need to apply AveragePooling2D or MaxPooling2D in case of' + 'doing transfer learning') + + # Ensure that the model takes into account + # any potential predecessors of `input_tensor`. + if input_tensor is not None: + inputs = keras_utils.get_source_inputs(input_tensor) + else: + inputs = img_input + # Create model + model = Model(inputs, x, name='resnet18') + + # Load weights + if weights == 'cifar100_coarse': + if include_top: + weights_path = keras_utils.get_file( + 'resnet18_cifar100_top.h5', + WEIGHTS_PATH, + cache_subdir='models', + md5_hash='e0798dd90ac7e0498cbdea853bd3ed7f') + else: + weights_path = keras_utils.get_file( + 'resnet18_cifar100_no_top.h5', + WEIGHTS_PATH_NO_TOP, + cache_subdir='models', + md5_hash='bfeace78cec55f2b0401c1f41c81e1dd') + model.load_weights(weights_path) + + + return model \ No newline at end of file diff --git a/donkeycar/parts/tflite.py b/donkeycar/parts/tflite.py index e776ca7f6..139af12eb 100755 --- a/donkeycar/parts/tflite.py +++ b/donkeycar/parts/tflite.py @@ -46,6 +46,11 @@ def load(self, model_path): 'TFlitePilot should load only .tflite files' # Load TFLite model and allocate tensors. self.interpreter = tf.lite.Interpreter(model_path=model_path) + ''' + #Uncomment below self.interpreter and comment above in case you have TPU edge on your donkeycar to accelerate inference + #You need tpu edge runtime installed as pre-requesite: https://coral.ai/docs/accelerator/get-started + self.interpreter = tf.lite.Interpreter(model_path=model_path,experimental_delegates=[tflite.load_delegate('libedgetpu.so.1')]) + ''' self.interpreter.allocate_tensors() # Get input and output tensors. diff --git a/donkeycar/templates/train.py b/donkeycar/templates/train.py index 573cf9218..e177faddd 100644 --- a/donkeycar/templates/train.py +++ b/donkeycar/templates/train.py @@ -4,136 +4,117 @@ Basic usage should feel familiar: python train_v2.py --model models/mypilot Usage: - train.py [--tubs=tubs] (--model=) [--type=(linear|inferred|tensorrt_linear|tflite_linear)] + train.py [--tubs=tubs] (--model=) [--type=(linear|inferred|tensorrt_linear|tflite_linear|tflite_r18_lin|tflite_r18_cat)] Options: -h --help Show this screen. """ -import math import os +import random from pathlib import Path -from typing import Any, List, Optional, Tuple, cast -import donkeycar +import cv2 import numpy as np from docopt import docopt -from donkeycar.parts.keras import KerasCategorical, KerasInferred, KerasLinear -from donkeycar.parts.tflite import keras_model_to_tflite -from donkeycar.parts.tub_v2 import Tub -from donkeycar.pipeline.sequence import TubRecord -from donkeycar.pipeline.sequence import TubSequence as PipelineSequence -from donkeycar.utils import get_model_by_type, linear_bin, train_test_split +from PIL import Image from tensorflow.python.keras.callbacks import EarlyStopping, ModelCheckpoint from tensorflow.python.keras.utils.data_utils import Sequence +import donkeycar +from donkeycar.parts.keras import KerasInferred, KerasCategorical, Resnet18LinearKeras, Resnet18CategoricalKeras +from donkeycar.parts.tflite import keras_model_to_tflite +from donkeycar.parts.tub_v2 import Tub +from donkeycar.utils import get_model_by_type, load_image_arr, \ + train_test_split, linear_bin, normalize_image + class TubDataset(object): ''' Loads the dataset, and creates a train/test split. ''' - def __init__(self, config, tub_paths, test_size=0.2, shuffle=True): - self.config = config + def __init__(self, tub_paths, test_size=0.2, shuffle=True): self.tub_paths = tub_paths self.test_size = test_size self.shuffle = shuffle - self.tubs = [Tub(tub_path, read_only=True) - for tub_path in self.tub_paths] - self.records: List[TubRecord] = list() + self.tubs = [Tub(tub_path, read_only=True) for tub_path in + self.tub_paths] + self.records = list() - def train_test_split(self) -> Tuple[List[TubRecord], List[TubRecord]]: + def train_test_split(self): print('Loading tubs from paths %s' % (self.tub_paths)) for tub in self.tubs: - for underlying in tub: - record = TubRecord(self.config, tub.base_path, - underlying=underlying) + for record in tub: + record['_image_base_path'] = tub.images_base_path self.records.append(record) return train_test_split(self.records, shuffle=self.shuffle, test_size=self.test_size) class TubSequence(Sequence): - # Improve batched_pipeline to make most of this go away as well. - # The idea is to have a shallow sequence with types that can hydrate themselves to an ndarray - - def __init__(self, keras_model, config, records: List[TubRecord] = list()): + def __init__(self, keras_model, config, records=list()): self.keras_model = keras_model self.config = config self.records = records - self.sequence = PipelineSequence(self.records) self.batch_size = self.config.BATCH_SIZE - self.consumed = 0 - - # Keep track of model type - # Eventually move this part into the model itself. - self.is_linear = type(self.keras_model) is KerasLinear - self.is_inferred = type(self.keras_model) is KerasInferred - self.is_categorical = type(self.keras_model) is KerasCategorical - - # Define transformations - def x_transform(record: TubRecord): - # Using an identity transform to delay image loading - return record - - def y_categorical(record: TubRecord): - angle: np.ndarray = record.underlying['user/angle'] - throttle: np.ndarray = record.underlying['user/throttle'] - R = self.config.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE - angle = linear_bin(angle, N=15, offset=1, R=2.0) - throttle = linear_bin(throttle, N=20, offset=0.0, R=R) - return angle, throttle - - def y_inferred(record: TubRecord): - return record.underlying['user/angle'] - - def y_linear(record: TubRecord): - angle: float = record.underlying['user/angle'] - throttle: float = record.underlying['user/throttle'] - return angle, throttle - - if self.is_linear: - self.pipeline = list(self.sequence.build_pipeline(x_transform=x_transform, y_transform=y_linear)) - elif self.is_categorical: - self.pipeline = list(self.sequence.build_pipeline(x_transform=x_transform, y_transform=y_categorical)) - else: - self.pipeline = list(self.sequence.build_pipeline(x_transform=x_transform, y_transform=y_inferred)) def __len__(self): - if not self.pipeline: - raise RuntimeError('Pipeline is not initialized') - - return math.ceil(len(self.pipeline) / self.batch_size) + return len(self.records) // self.batch_size def __getitem__(self, index): count = 0 + records = [] images = [] angles = [] throttles = [] + + is_inferred = type(self.keras_model) is KerasInferred + is_categorical = type(self.keras_model) is KerasCategorical + is_resnet18_categorical = type(self.keras_model) is Resnet18CategoricalKeras + while count < self.batch_size: i = (index * self.batch_size) + count - if i >= len(self.pipeline): + if i >= len(self.records): break - record, r = self.pipeline[i] - images.append(record.image(cached=False, normalize=True)) + record = self.records[i] + record = self._transform_record(record) + records.append(record) + count += 1 - if isinstance(r, tuple): - angle, throttle = r - angles.append(angle) - throttles.append(throttle) - else: - angles.append(r) + for record in records: + image = record['cam/image_array'] + angle = record['user/angle'] + throttle = record['user/throttle'] - count += 1 + images.append(image) + # for categorical convert to one-hot vector + if is_categorical or is_resnet18_categorical: + R = self.config.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE + angle = linear_bin(angle, N=15, offset=1, R=2.0) + throttle = linear_bin(throttle, N=20, offset=0.0, R=R) + angles.append(angle) + throttles.append(throttle) X = np.array(images) - if self.is_inferred: + + if is_inferred: Y = np.array(angles) else: Y = [np.array(angles), np.array(throttles)] + return X, Y + def _transform_record(self, record): + for key, value in record.items(): + if key == 'cam/image_array' and isinstance(value, str): + image_path = os.path.join(record['_image_base_path'], value) + image = load_image_arr(image_path, self.config) + record[key] = normalize_image(image) + + return record + class ImagePreprocessing(Sequence): ''' @@ -162,6 +143,10 @@ def train(cfg, tub_paths, output_path, model_type): if 'linear' in model_type: train_type = 'linear' + elif 'tflite_r18_lin' in model_type: + train_type = 'resnet18_lin' + elif 'tflite_r18_cat' in model_type: + train_type = 'resnet18_cat' else: train_type = model_type @@ -172,7 +157,7 @@ def train(cfg, tub_paths, output_path, model_type): print(kl.model.summary()) batch_size = cfg.BATCH_SIZE - dataset = TubDataset(cfg, tub_paths, test_size=(1. - cfg.TRAIN_TEST_SPLIT)) + dataset = TubDataset(tub_paths, test_size=(1. - cfg.TRAIN_TEST_SPLIT)) training_records, validation_records = dataset.train_test_split() print('Records # Training %s' % len(training_records)) print('Records # Validation %s' % len(validation_records)) @@ -225,7 +210,15 @@ def main(): tubs = tubs.split(',') data_paths = [Path(os.path.expanduser(tub)).absolute().as_posix() for tub in tubs] output_path = os.path.expanduser(model) - history = train(cfg, data_paths, output_path, model_type) + if is_tflite: + tflite_model_path = f'{os.path.splitext(output_path)[0]}.tflite' + + try: + history = train(cfg, data_paths, output_path, model_type) + except KeyboardInterrupt: + print("There was a keyboard interuption during training") + pass + if is_tflite: tflite_model_path = f'{os.path.splitext(output_path)[0]}.tflite' keras_model_to_tflite(output_path, tflite_model_path) diff --git a/donkeycar/utils.py b/donkeycar/utils.py index f24860358..cd05fb523 100644 --- a/donkeycar/utils.py +++ b/donkeycar/utils.py @@ -409,7 +409,7 @@ def get_model_by_type(model_type, cfg): ''' from donkeycar.parts.keras import KerasRNN_LSTM, KerasBehavioral, \ KerasCategorical, KerasIMU, KerasLinear, Keras3D_CNN, \ - KerasLocalizer, KerasLatent + KerasLocalizer, KerasLatent, Resnet18LinearKeras, Resnet18CategoricalKeras from donkeycar.parts.tflite import TFLitePilot if model_type is None: @@ -419,11 +419,20 @@ def get_model_by_type(model_type, cfg): input_shape = (cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH) if model_type == "linear": kl = KerasLinear(input_shape=input_shape) + elif model_type == "resnet18_lin": + kl = Resnet18LinearKeras(input_shape=input_shape) elif model_type == "categorical": kl = KerasCategorical(input_shape=input_shape, throttle_range=cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE) + elif model_type == 'resnet18_cat': + kl = Resnet18CategoricalKeras(input_shape=input_shape, + throttle_range=cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE) elif model_type == "tflite_linear": kl = TFLitePilot() + elif model_type == "tflite_r18_lin": + kl = TFLitePilot() + elif model_type == "tflite_r18_cat": + kl = TFLitePilot() elif model_type == "tensorrt_linear": # Aggressively lazy load this. This module imports pycuda.autoinit # which causes a lot of unexpected things to happen when using TF-GPU @@ -432,7 +441,7 @@ def get_model_by_type(model_type, cfg): kl = TensorRTLinear(cfg=cfg) else: raise Exception("Unknown model type {:}, supported types are " - "linear, categorical, tflite_linear, tensorrt_linear" + "linear,resnet18_lin, categorical, resnet18_cat, tflite_linear, tensorrt_linear" .format(model_type)) return kl