Skip to content

Commit 454d556

Browse files
authored
Merge branch 'main' into oneapi_backend/experiment
2 parents c307715 + c2a75fd commit 454d556

36 files changed

+1702
-253
lines changed

.pre-commit-config.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ repos:
1010
'--skip-string-normalization']
1111

1212
- repo: https://github.com/tox-dev/pyproject-fmt
13-
rev: v2.5.0
13+
rev: v2.5.1
1414
hooks:
1515
- id: pyproject-fmt
1616

@@ -30,7 +30,7 @@ repos:
3030
- id: trailing-whitespace
3131

3232
- repo: https://github.com/PyCQA/isort
33-
rev: 6.0.0
33+
rev: 6.0.1
3434
hooks:
3535
- id: isort
3636

docs/advanced/fifo_depth.rst

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,28 +5,29 @@ FIFO Buffer Depth Optimization
55
With the ``io_stream`` IO type, each layer is connected with the subsequent layer through first-in first-out (FIFO) buffers.
66
The implementation of the FIFO buffers contribute to the overall resource utilization of the design, impacting in particular the BRAM or LUT utilization.
77
Because the neural networks can have complex architectures generally, it is hard to know a priori the correct depth of each FIFO buffer.
8-
By default ``hls4ml`` choses the most conservative possible depth for each FIFO buffer, which can result in a an unnecessary overutilization of resources.
8+
By default ``hls4ml`` choses the most conservative possible depth for each FIFO buffer, which can result in a an unnecessary over-utilization of resources.
99

10-
In order to reduce the impact on the resources used for FIFO buffer implementation, an optimization has been developed in `#509 <https://github.com/fastmachinelearning/hls4ml/pull/509>`_ that correctly sizes the depth of the FIFO buffers by analyzing the RTL cosimulation.
11-
We implemented this FIFO buffer resizing as a :py:class:`~hls4ml.backends.vivado.passes.fifo_depth_optimization` optimizer pass.
10+
In order to reduce the impact on the resources used for FIFO buffer implementation, an optimization flow has been developed that correctly sizes the depth
11+
of the FIFO buffers by analyzing the RTL co-simulation. This feature is currently available in ``Vitis`` and ``Vivado`` backends.
12+
13+
In ``Vivado`` backend, FIFO buffer resizing is implemented as a :py:class:`~hls4ml.backends.vivado.passes.fifo_depth_optimization` optimizer pass.
1214
Through RTL simulation with large FIFO buffers (by default set to a depth of 100,000), we estimate the maximum occupation of each FIFO.
1315
Once the maximum depth is determined, the optimizer pass sets the FIFO buffer depth to that value plus 1.
1416

15-
As an example, we show below how to use the optimizer pass, inspired by this `GitHub Gist <https://gist.github.com/nicologhielmetti/3a268be32755448920e9f7d5c78a76d8>`_.
16-
First, we can define a simple neural network in Keras
17+
Below we show an example of the use of the FIFO depth optimization. First, we can define a simple neural network in Keras:
1718

1819
.. code-block:: Python
1920
2021
from tensorflow.keras.layers import Dense
2122
from tensorflow.keras.models import Sequential
2223
2324
model = Sequential()
24-
model.add(Dense(64, input_shape=(16,), name='fc1', activation='relu')
25+
model.add(Dense(64, input_shape=(16,), name='fc1', activation='relu'))
2526
model.add(Dense(32, name='fc2', activation='relu'))
2627
model.add(Dense(32, name='fc3', activation='relu'))
27-
model.add(Dense(5, name='fc3', activation='softmax'))
28+
model.add(Dense(5, name='fc4', activation='softmax'))
2829
29-
Then, we can convert the model, including the flow
30+
Then, we can convert the model, including the flow:
3031

3132
.. code-block:: Python
3233
@@ -47,3 +48,17 @@ Then, we can convert the model, including the flow
4748
hls_model.build(reset=False, csim=True, synth=True, cosim=True)
4849
4950
For more details and results, see `H. Borras et al., "Open-source FPGA-ML codesign for the MLPerf Tiny Benchmark" (2022) <https://arxiv.org/abs/2206.11791>`_.
51+
52+
Similarly, the FIFO buffers can be optimized while using the ``Vitis`` backend with the following changes:
53+
54+
.. code-block:: Python
55+
56+
config['Flows'] = ['vitis:fifo_depth_optimization']
57+
hls4ml.model.optimizer.get_optimizer('vitis:fifo_depth_optimization').configure(profiling_fifo_depth=100_000)
58+
59+
hls_model = hls4ml.converters.convert_from_keras_model(model,
60+
io_type='io_stream',
61+
hls_config=config,
62+
output_dir='hls4mlprj_fifo_depth_opt',
63+
part='xc7z020clg400-1',
64+
backend='Vitis')

hls4ml/backends/oneapi/passes/recurrent_templates.py

Lines changed: 54 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -92,10 +92,14 @@
9292
using activation_recr = nnet::activation::{recurrent_activation}<x_T, y_T, config_T>;
9393
9494
static const unsigned reuse_factor = {reuse};
95+
static const unsigned pytorch_order = {pytorch};
9596
static const bool store_weights_in_bram = false;
9697
}};\n'''
9798

9899
gru_function_template = 'nnet::gru<{input_t}, {output_t}, {config}>({input}, {output}, {w}, {wr}, {b}, {br});'
100+
gru_function_initial_state_template = (
101+
'nnet::gru_init_state<{input_t}, {h_t}, {output_t}, {config}>({input}, {init_state}, {output}, {w}, {wr}, {b}, {br});'
102+
)
99103
gru_task_sequence_template = 'task_sequence<nnet::gru_stream<{input_pipe}, {output_pipe}, {config}>> {name};'
100104
gru_stream_function_template = '{name}.async({w}, {wr}, {b}, {br});'
101105

@@ -120,6 +124,7 @@ def format(self, node):
120124
params['config_mult_h'] = f'config{node.index}_h_mult'
121125
params['act_t'] = '{}_config{}'.format(node.get_attr('activation'), str(node.index) + '_act')
122126
params['act_recurrent_t'] = '{}_config{}'.format(node.get_attr('recurrent_activation'), str(node.index) + '_rec_act')
127+
params['pytorch'] = 'true' if node.get_attr('pytorch', False) else 'false'
123128
gru_config = self.gru_template.format(**params)
124129

125130
# Activation is on candidate hidden state, dimensionality (1, n_units)
@@ -163,15 +168,23 @@ def format(self, node):
163168
class GRUFunctionTemplate(FunctionCallTemplate):
164169
def __init__(self):
165170
super().__init__(GRU, include_header=recurrent_include_list)
166-
self.template = gru_function_template
167171

168172
def format(self, node):
169173
params = self._default_function_params(node)
174+
if params['pass_initial_states'] == 'true':
175+
params['h_t'] = node.get_input_variable(node.inputs[1]).type.name
176+
params['init_state'] = node.get_input_variable(node.inputs[1]).name
170177
params['w'] = node.get_weights('weight').name
171178
params['b'] = node.get_weights('bias').name
172179
params['wr'] = node.get_weights('recurrent_weight').name
173180
params['br'] = node.get_weights('recurrent_bias').name
174-
return self.template.format(**params)
181+
182+
if params['pass_initial_states'] == 'true':
183+
template = gru_function_initial_state_template
184+
else:
185+
template = gru_function_template
186+
187+
return template.format(**params)
175188

176189

177190
class GRUTaskSequenceTemplate(TaskSequenceTemplate):
@@ -235,6 +248,10 @@ def format(self, node):
235248
}};\n"""
236249

237250
lstm_function_template = 'nnet::lstm<{input_t}, {output_t}, {config}>({input}, {output}, {weights});'
251+
lstm_function_initial_state_template = (
252+
'nnet::lstm_init_state<{input_t}, {h_t}, {hc_t}, {output_t}, {config}>'
253+
'({input}, {init_state}, {init_cell}, {output}, {weights});'
254+
)
238255

239256

240257
class LSTMConfigTemplate(LayerConfigTemplate):
@@ -275,11 +292,16 @@ def format(self, node):
275292
class LSTMFunctionTemplate(FunctionCallTemplate):
276293
def __init__(self):
277294
super().__init__(LSTM, include_header=recurrent_include_list)
278-
self.template = lstm_function_template
279295

280296
def format(self, node):
281297
params = self._default_function_params(node)
282298

299+
if params['pass_initial_states'] == 'true':
300+
params['h_t'] = node.get_input_variable(node.inputs[1]).type.name
301+
params['init_state'] = node.get_input_variable(node.inputs[1]).name
302+
params['init_cell'] = node.get_input_variable(node.inputs[2]).name
303+
params['hc_t'] = node.get_input_variable(node.inputs[2]).type.name
304+
283305
types = ['i', 'f', 'c', 'o']
284306
params['weights'] = ''
285307
for t in types:
@@ -289,13 +311,18 @@ def format(self, node):
289311
for t in types:
290312
params['weights'] += 'bias_{}_{}{}'.format(t, str(node.index), ',' if t != 'o' else '')
291313

292-
return self.template.format(**params)
314+
if params['pass_initial_states'] == 'true':
315+
template = lstm_function_initial_state_template
316+
else:
317+
template = lstm_function_template
318+
319+
return template.format(**params)
293320

294321

295322
################################################
296323
# SimpleRNN Template
297324
################################################
298-
simple_rnn_config_template = """struct config{index} : nnet::simpleRNN_config {{
325+
simple_rnn_config_template = """struct config{index} : nnet::simple_rnn_config {{
299326
static const unsigned n_in = {n_in};
300327
static const unsigned n_out = {n_out};
301328
static const unsigned n_outputs = {n_outputs};
@@ -306,6 +333,7 @@ def format(self, node):
306333
typedef {weight_t.name} weight_t;
307334
typedef {bias_t.name} bias_t;
308335
typedef {recurrent_weight_t.name} recurrent_weight_t;
336+
typedef {recurrent_bias_t.name} recurrent_bias_t;
309337
310338
typedef {act_t} ACT_CONFIG_T;
311339
template<class x_T, class y_T, class config_T>
@@ -320,6 +348,10 @@ def format(self, node):
320348
}};\n"""
321349

322350
simple_rnn_function_template = 'nnet::simple_rnn<{input_t}, {output_t}, {config}>({input}, {output}, {weights});'
351+
simple_rnn_pytorch_function_template = (
352+
'nnet::simple_rnn_pytorch<{input_t}, {output_t}, {config}>({input}, {output}, {weights});'
353+
)
354+
simple_rnn_pytorch_function_initial_state_template = 'nnet::simple_rnn_pytorch_init_state<{input_t}, {h_t}, {output_t}, {config}>({input}, {init_state}, {output}, {weights});' # noqa E501
323355

324356

325357
class SimpleRNNConfigTemplate(LayerConfigTemplate):
@@ -341,6 +373,9 @@ def format(self, node):
341373
)
342374
simple_rnn_params['recurrent_activation'] = 'relu'
343375

376+
# In Keras there is no recurrent bias, so put a placeholder
377+
simple_rnn_params.setdefault('recurrent_bias_t', simple_rnn_params['bias_t'])
378+
344379
simple_rnn_config = self.template.format(**simple_rnn_params)
345380

346381
act_params = self._default_config_params(node)
@@ -365,5 +400,17 @@ def __init__(self):
365400

366401
def format(self, node):
367402
params = self._default_function_params(node)
368-
params['weights'] = 'w{0}, wr{0}, b{0}'.format(str(node.index))
369-
return self.template.format(**params)
403+
if params['pass_initial_states'] == 'true':
404+
params['h_t'] = node.get_input_variable(node.inputs[1]).type.name
405+
params['init_state'] = node.get_input_variable(node.inputs[1]).name
406+
407+
if node.get_attr('pytorch', False):
408+
if params['pass_initial_states'] == 'true':
409+
template = simple_rnn_pytorch_function_initial_state_template
410+
else:
411+
template = simple_rnn_pytorch_function_template
412+
params['weights'] = 'w{0}, wr{0}, b{0}, br{0}'.format(str(node.index))
413+
else:
414+
template = simple_rnn_function_template
415+
params['weights'] = 'w{0}, wr{0}, b{0}'.format(str(node.index))
416+
return template.format(**params)

hls4ml/backends/quartus/passes/recurrent_templates.py

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@
7171
}};\n'''
7272

7373
gru_function_template = 'nnet::gru<{input_t}, {output_t}, {config}>({input}, {output}, {w}, {wr}, {b}, {br});'
74+
gru_function_initial_state_template = (
75+
'nnet::gru<{input_t}, {input2_t}, {output_t}, {config}>({input}, {input2}, {output}, {w}, {wr}, {b}, {br});'
76+
)
7477

7578

7679
class GRUConfigTemplate(LayerConfigTemplate):
@@ -137,15 +140,23 @@ def format(self, node):
137140
class GRUFunctionTemplate(FunctionCallTemplate):
138141
def __init__(self):
139142
super().__init__(GRU, include_header=recurrent_include_list)
140-
self.template = gru_function_template
141143

142144
def format(self, node):
143145
params = self._default_function_params(node)
146+
if params['pass_initial_states'] == 'true':
147+
params['input2_t'] = node.get_input_variable(node.inputs[1]).type.name
148+
params['input2'] = node.get_input_variable(node.inputs[1]).name
144149
params['w'] = node.get_weights('weight').name
145150
params['b'] = node.get_weights('bias').name
146151
params['wr'] = node.get_weights('recurrent_weight').name
147152
params['br'] = node.get_weights('recurrent_bias').name
148-
return self.template.format(**params)
153+
154+
if params['pass_initial_states'] == 'true':
155+
template = gru_function_initial_state_template
156+
else:
157+
template = gru_function_template
158+
159+
return template.format(**params)
149160

150161

151162
################################################
@@ -174,6 +185,9 @@ def format(self, node):
174185
}};\n"""
175186

176187
lstm_function_template = 'nnet::lstm<{input_t}, {output_t}, {config}>({input}, {output}, {weights});'
188+
lstm_function_initial_state_template = (
189+
'nnet::lstm<{input_t}, {input2_t}, {input3_t}, {output_t}, {config}>({input}, {input2}, {input3}, {output}, {weights});'
190+
)
177191

178192

179193
class LSTMConfigTemplate(LayerConfigTemplate):
@@ -214,11 +228,16 @@ def format(self, node):
214228
class LSTMFunctionTemplate(FunctionCallTemplate):
215229
def __init__(self):
216230
super().__init__(LSTM, include_header=recurrent_include_list)
217-
self.template = lstm_function_template
218231

219232
def format(self, node):
220233
params = self._default_function_params(node)
221234

235+
if params['pass_initial_states'] == 'true':
236+
params['input2_t'] = node.get_input_variable(node.inputs[1]).type.name
237+
params['input2'] = node.get_input_variable(node.inputs[1]).name
238+
params['input3'] = node.get_input_variable(node.inputs[2]).name
239+
params['input3_t'] = node.get_input_variable(node.inputs[2]).type.name
240+
222241
types = ['i', 'f', 'c', 'o']
223242
params['weights'] = ''
224243
for t in types:
@@ -228,13 +247,18 @@ def format(self, node):
228247
for t in types:
229248
params['weights'] += 'bias_{}_{}{}'.format(t, str(node.index), ',' if t != 'o' else '')
230249

231-
return self.template.format(**params)
250+
if params['pass_initial_states'] == 'true':
251+
template = lstm_function_initial_state_template
252+
else:
253+
template = lstm_function_template
254+
255+
return template.format(**params)
232256

233257

234258
################################################
235259
# SimpleRNN Template
236260
################################################
237-
simple_rnn_config_template = """struct config{index} : nnet::simpleRNN_config {{
261+
simple_rnn_config_template = """struct config{index} : nnet::simple_rnn_config {{
238262
static const unsigned n_in = {n_in};
239263
static const unsigned n_out = {n_out};
240264
static const unsigned n_outputs = {n_outputs};
@@ -261,6 +285,9 @@ def format(self, node):
261285
simple_rnn_pytorch_function_template = (
262286
'nnet::simple_rnn_pytorch<{input_t}, {output_t}, {config}>({input}, {output}, {weights});'
263287
)
288+
simple_rnn_pytorch_function_initial_state_template = (
289+
'nnet::simple_rnn_pytorch<{input_t}, {input2_t}, {output_t}, {config}>({input}, {input2}, {output}, {weights});'
290+
)
264291

265292

266293
class SimpleRNNConfigTemplate(LayerConfigTemplate):
@@ -302,13 +329,20 @@ def format(self, node):
302329
class SimpleRNNFunctionTemplate(FunctionCallTemplate):
303330
def __init__(self):
304331
super().__init__(SimpleRNN, include_header=recurrent_include_list)
305-
self.template = simple_rnn_function_template
306332

307333
def format(self, node):
308334
params = self._default_function_params(node)
335+
if params['pass_initial_states'] == 'true':
336+
params['input2_t'] = node.get_input_variable(node.inputs[1]).type.name
337+
params['input2'] = node.get_input_variable(node.inputs[1]).name
338+
309339
if node.get_attr('pytorch', False):
310-
self.template = simple_rnn_pytorch_function_template
340+
if params['pass_initial_states'] == 'true':
341+
template = simple_rnn_pytorch_function_initial_state_template
342+
else:
343+
template = simple_rnn_pytorch_function_template
311344
params['weights'] = 'w{0}, wr{0}, b{0}, br{0}'.format(str(node.index))
312345
else:
346+
template = simple_rnn_function_template
313347
params['weights'] = 'w{0}, wr{0}, b{0}'.format(str(node.index))
314-
return self.template.format(**params)
348+
return template.format(**params)

hls4ml/backends/symbolic/passes/expr_templates.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ def __init__(self, layer, lut_functions, use_built_in_luts=False, settings=None)
3333
user_functions = settings.get('user_functions', {})
3434
user_functions.update(lut_functions)
3535
settings['user_functions'] = user_functions
36+
settings['strict'] = False
3637

3738
super().__init__(settings)
3839
self.layer = layer

0 commit comments

Comments
 (0)