oneAPI RNN State and Activation Quantizers (#1195)

* RNN Accum type to QKeras state quantizer and RNN QActivation parsing

* pre-commit hook

* QKeras recurrent activation

* Cast array size to int

* QRNN fix

Author: laurilaatu

hls4ml/backends/oneapi/passes/reshaping_templates.py

@@ -188,7 +188,7 @@ def format(self, node):
         new_shape, perm_strides = node.model.config.backend.permute_config_gen(name, shape, perm)
         return transpose_config_template.format(
             dims=len(shape),
-            N=np.prod(shape),
+            N=int(np.prod(shape)),
             from_shape=', '.join(str(x) for x in shape),
             perm=', '.join(str(x) for x in perm),
             perm_strides=', '.join(str(x) for x in perm_strides),
@@ -251,5 +251,5 @@ def __init__(self):
 
     def format(self, node):
         params = self._default_function_params(node)
-        params['size'] = np.prod(node.get_output_variable().shape)
+        params['size'] = int(np.prod(node.get_output_variable().shape))
         return self.template.format(**params)

hls4ml/converters/keras/qkeras.py

@@ -80,15 +80,30 @@ def parse_qrnn_layer(keras_layer, input_names, input_shapes, data_reader):
     layer['weight_quantizer'] = get_quantizer_from_config(keras_layer, 'kernel')
     layer['recurrent_weight_quantizer'] = get_quantizer_from_config(keras_layer, 'recurrent')
     layer['bias_quantizer'] = get_quantizer_from_config(keras_layer, 'bias')
+    layer['accum_quantizer'] = get_quantizer_from_config(keras_layer, 'state')
+
+    if not isinstance(keras_layer['config']['activation'], str):
+        activation = get_activation_quantizer(keras_layer, input_names)
+
+        assert activation['class_name'] != 'HardActivation', 'Hard activation not supported'
+
+        layer['activation'] = activation['activation']
+        layer['activation_quantizer'] = activation['activation_quantizer']
+
+    if keras_layer['class_name'] in ['QLSTM', 'QGRU'] and not isinstance(keras_layer['config']['recurrent_activation'], str):
+        recurrent_activation = get_activation_quantizer(keras_layer, input_names, activation_name='recurrent_activation')
+
+        assert recurrent_activation['class_name'] != 'HardActivation', 'Hard activation not supported'
+
+        layer['recurrent_activation'] = recurrent_activation['recurrent_activation']
+        layer['recurrent_activation_config'] = recurrent_activation
 
     return layer, output_shape
 
 
-@keras_handler('QActivation')
-def parse_qactivation_layer(keras_layer, input_names, input_shapes, data_reader):
+def get_activation_quantizer(keras_layer, input_names, activation_name='activation'):
     from qkeras.quantizers import get_quantizer
 
-    assert keras_layer['class_name'] == 'QActivation'
     supported_activations = [
         'quantized_relu',
         'quantized_tanh',
@@ -102,7 +117,7 @@ def parse_qactivation_layer(keras_layer, input_names, input_shapes, data_reader)
 
     layer = parse_default_keras_layer(keras_layer, input_names)
 
-    activation_config = keras_layer['config']['activation']
+    activation_config = keras_layer['config'][activation_name]
     quantizer_obj = get_quantizer(activation_config)
     activation_config = {}
     # some activations are classes
@@ -136,7 +151,7 @@ def parse_qactivation_layer(keras_layer, input_names, input_shapes, data_reader)
         layer['threshold'] = activation_config.get('config', {}).get('threshold', 0.33)
         if layer['threshold'] is None:
             layer['threshold'] = 0.33  # the default ternary tanh threshold for QKeras
-        layer['activation'] = 'ternary_tanh'
+        layer[activation_name] = 'ternary_tanh'
     elif (
         activation_config['class_name'] == 'quantized_sigmoid'
         and not activation_config['config'].get('use_real_sigmoid', False)
@@ -149,16 +164,27 @@ def parse_qactivation_layer(keras_layer, input_names, input_shapes, data_reader)
         # Quartus seems to have trouble if the width is 1.
         layer['slope_prec'] = FixedPrecisionType(width=2, integer=0, signed=False)
         layer['shift_prec'] = FixedPrecisionType(width=2, integer=0, signed=False)
-        layer['activation'] = activation_config['class_name'].replace('quantized_', 'hard_')
+        layer[activation_name] = activation_config['class_name'].replace('quantized_', 'hard_')
     elif activation_config['class_name'] == 'quantized_relu' and activation_config['config']['negative_slope'] != 0:
         layer['class_name'] = 'LeakyReLU'
-        layer['activation'] = activation_config['class_name'].replace('quantized_', 'leaky_')
+        layer[activation_name] = activation_config['class_name'].replace('quantized_', 'leaky_')
         layer['activ_param'] = activation_config['config']['negative_slope']
     else:
         layer['class_name'] = 'Activation'
-        layer['activation'] = activation_config['class_name'].replace('quantized_', '')
+        layer[activation_name] = activation_config['class_name'].replace('quantized_', '')
+
+    layer[f'{activation_name}_quantizer'] = activation_config
+
+    return layer
+
+
+@keras_handler('QActivation')
+def parse_qactivation_layer(keras_layer, input_names, input_shapes, data_reader):
+
+    assert keras_layer['class_name'] == 'QActivation'
+
+    layer = get_activation_quantizer(keras_layer, input_names)
 
-    layer['activation_quantizer'] = activation_config
     return layer, [shape for shape in input_shapes[0]]
 
 

hls4ml/converters/keras_to_hls.py

@@ -209,7 +209,7 @@ def parse_keras_model(model_arch, reader):
         'HGQ>UnaryLUT',
     ]
     # Recurrent layers
-    recurrent_layers = ['SimpleRNN', 'LSTM', 'GRU']
+    recurrent_layers = ['SimpleRNN', 'LSTM', 'GRU', 'QSimpleRNN', 'QLSTM', 'QGRU']
     # All supported layers
     supported_layers = get_supported_keras_layers() + skip_layers
 

hls4ml/model/types.py

@@ -437,7 +437,7 @@ def __init__(self, var_name, type_name, precision, data, quantizer=None, **kwarg
         self.data = data
         self.nzeros = -1
         self.shape = list(self.data.shape)
-        self.data_length = np.prod(self.data.shape)
+        self.data_length = int(np.prod(self.data.shape))
         self.nonzeros = np.count_nonzero(self.data)
         self.nzeros = self.data_length - self.nonzeros
         self.min = np.min(self.data)

hls4ml/templates/oneapi/firmware/nnet_utils/nnet_recurrent.h

@@ -165,7 +165,7 @@ void gru(const data_T &data, res_T &res, const typename CONFIG_T::weight_t &weig
          const typename CONFIG_T::recurrent_weight_t &recurrent_weights, const typename CONFIG_T::bias_t &bias,
          const typename CONFIG_T::recurrent_bias_t &recurrent_bias) {
 
-    using h_T = array<typename res_T::value_type, CONFIG_T::n_units>;
+    using h_T = array<typename CONFIG_T::accum_t, CONFIG_T::n_units>;
     [[intel::fpga_register]] data_T x;
     [[intel::fpga_register]] h_T h;
 
@@ -259,7 +259,7 @@ void simple_rnn(const data_T &data, res_T &res, const typename CONFIG_T::weight_
                 const typename CONFIG_T::recurrent_weight_t &rec_kernel, const typename CONFIG_T::bias_t &bias) {
 
     using in_T = array<typename data_T::value_type, CONFIG_T::n_in>;
-    using h_T = array<typename res_T::value_type, CONFIG_T::n_out>;
+    using h_T = array<typename CONFIG_T::accum_t, CONFIG_T::n_out>;
 
     [[intel::fpga_register]] h_T hidden_state[CONFIG_T::n_timesteps + 1];
     [[intel::fpga_register]] h_T hidden_state_temp;
@@ -500,7 +500,7 @@ void lstm(const data_T &data, res_T &res, const typename CONFIG_T::weight_i_t &W
     // Note:  currently this does not support recurrent bias
 
     using in_T = array<typename data_T::value_type, CONFIG_T::n_in>;
-    using h_T = array<typename res_T::value_type, CONFIG_T::n_out>;
+    using h_T = array<typename CONFIG_T::accum_t, CONFIG_T::n_out>;
 
     [[intel::fpga_register]] h_T hidden_state[CONFIG_T::n_timesteps + 1];
     [[intel::fpga_register]] h_T hidden_state_temp;