Merge branch 'feat/add_merged_bin_cmd' into 'master'

feat(tools): Add idf.py merge-bin command and cmake target

See merge request espressif/esp-idf!29996

Author: dobairoland

components/esptool_py/project_include.cmake

@@ -230,6 +230,35 @@ add_custom_target(uf2-app
     VERBATIM
     )
 
+set(MERGE_BIN_ARGS merge_bin)
+if(DEFINED ENV{ESP_MERGE_BIN_OUTPUT})
+    list(APPEND MERGE_BIN_ARGS "-o" "$ENV{ESP_MERGE_BIN_OUTPUT}")
+else()
+    if(DEFINED ENV{ESP_MERGE_BIN_FORMAT} AND "$ENV{ESP_MERGE_BIN_FORMAT}" STREQUAL "hex")
+        list(APPEND MERGE_BIN_ARGS "-o" "${CMAKE_CURRENT_BINARY_DIR}/merged-binary.hex")
+    else()
+        list(APPEND MERGE_BIN_ARGS "-o" "${CMAKE_CURRENT_BINARY_DIR}/merged-binary.bin")
+    endif()
+endif()
+
+if(DEFINED ENV{ESP_MERGE_BIN_FORMAT})
+    list(APPEND MERGE_BIN_ARGS "-f" "$ENV{ESP_MERGE_BIN_FORMAT}")
+endif()
+
+list(APPEND MERGE_BIN_ARGS "@${CMAKE_CURRENT_BINARY_DIR}/flash_args")
+
+add_custom_target(merge-bin
+    COMMAND ${CMAKE_COMMAND}
+    -D "IDF_PATH=${idf_path}"
+    -D "SERIAL_TOOL=${ESPTOOLPY}"
+    -D "SERIAL_TOOL_ARGS=${MERGE_BIN_ARGS}"
+    -D "WORKING_DIRECTORY=${CMAKE_CURRENT_BINARY_DIR}"
+    -P run_serial_tool.cmake
+    WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+    DEPENDS gen_project_binary bootloader
+    USES_TERMINAL
+    VERBATIM
+    )
 
 set(MONITOR_ARGS "")
 

docs/en/api-guides/tools/idf-py.rst

@@ -124,6 +124,37 @@ This command automatically builds the project if necessary, and then flash it to
 
 Similarly to the ``build`` command, the command can be run with ``app``, ``bootloader`` and ``partition-table`` arguments to flash only the app, bootloader or partition table as applicable.
 
+.. _merging-binaries:
+
+Merge binaries: ``merge-bin``
+-----------------------------
+
+.. code-block:: bash
+
+  idf.py merge-bin [-o output-file] [-f format] [<format-specific-options>]
+
+There are some situations, e.g. transferring the file to another machine and flashing it without ESP-IDF, where it is convenient to have only one file for flashing instead the several file output of ``idf.py build``.
+
+The command ``idf.py merge-bin`` will merge the bootloader, partition table, the application itself, and other partitions (if there are any) according to the project configuration and create a single binary file ``merged-binary.[bin|hex]`` in the build folder, which can then be flashed later.
+
+It is possible to output merged file in binary (raw), IntelHex (hex) and UF2 (uf2) formats.
+
+The uf2 binary can also be generated by :ref:`idf.py uf2 <generate-uf2-binary>`. The ``idf.py uf2`` is functionally equivalent to ``idf.py merge-bin -f uf2``. However, the ``idf.py merge-bin`` command provides more flexibility and options for merging binaries into various formats described above.
+
+Example usage:
+
+.. code-block:: bash
+
+  idf.py merge-bin -o my-merged-binary.bin -f raw
+
+There are also some format specific options, which are listed below:
+
+- Only for raw format:
+  - ``--flash-offset``: This option will create a merged binary that should be flashed at the specified offset, instead of at the standard offset of 0x0.
+  - ``--fill-flash-size``: If set, the final binary file will be padded with FF bytes up to this flash size in order to fill the full flash content with the image and re-write the whole flash chip upon flashing.
+- Only for uf2 format:
+  - ``--md5-disable``: This option will disable MD5 checksums at the end of each block. This can be useful for integration with e.g. `tinyuf2 <https://github.com/adafruit/tinyuf2>`__.
+
 Hints on How to Resolve Errors
 ==============================
 
@@ -203,6 +234,8 @@ Clean the Python Byte Code: ``python-clean``
 
 This command deletes generated python byte code from the ESP-IDF directory. The byte code may cause issues when switching between ESP-IDF and Python versions. It is advised to run this target after switching versions of Python.
 
+.. _generate-uf2-binary:
+
 Generate a UF2 Binary: ``uf2``
 ------------------------------
 
@@ -216,6 +249,8 @@ This UF2 file can be copied to a USB mass storage device exposed by another ESP
 
 To generate a UF2 binary for the application only (not including the bootloader and partition table), use the ``uf2-app`` command.
 
+The ``idf.py uf2`` command is functionally equivalent to ``idf.py merge-bin -f uf2`` described :ref:`above <merging-binaries>`. However, the ``idf.py merge-bin`` command provides more flexibility and options for merging binaries into various formats, not only uf2.
+
 .. code-block:: bash
 
   idf.py uf2-app

tools/idf_py_actions/serial_ext.py

@@ -1,17 +1,23 @@
-# SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import json
 import os
 import shlex
 import signal
 import sys
-from typing import Any, Dict, List, Optional
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
 
 import click
 from idf_py_actions.global_options import global_options
-from idf_py_actions.tools import (PropertyDict, RunTool, ensure_build_directory, get_default_serial_port,
-                                  get_sdkconfig_value, run_target)
+from idf_py_actions.tools import ensure_build_directory
+from idf_py_actions.tools import get_default_serial_port
+from idf_py_actions.tools import get_sdkconfig_value
+from idf_py_actions.tools import PropertyDict
+from idf_py_actions.tools import run_target
+from idf_py_actions.tools import RunTool
 
 PYTHON = sys.executable
 
@@ -34,7 +40,7 @@
 }
 
 
-def yellow_print(message, newline='\n'):  # type: (str, Optional[str]) -> None
+def yellow_print(message: str, newline: Optional[str]='\n') -> None:
     """Print a message to stderr with yellow highlighting """
     sys.stderr.write('%s%s%s%s' % ('\033[0;33m', message, '\033[0m', newline))
     sys.stderr.flush()
@@ -212,6 +218,47 @@ def ota_targets(target_name: str, ctx: click.core.Context, args: PropertyDict) -
         ensure_build_directory(args, ctx.info_name)
         run_target(target_name, args, {'ESPBAUD': str(args.baud), 'ESPPORT': args.port})
 
+    def merge_bin(action: str,
+                  ctx: click.core.Context,
+                  args: PropertyDict,
+                  output: str,
+                  format: str,
+                  md5_disable: str,
+                  flash_offset: str,
+                  fill_flash_size: str) -> None:
+        ensure_build_directory(args, ctx.info_name)
+        project_desc = _get_project_desc(ctx, args)
+        merge_bin_args = [PYTHON, '-m', 'esptool']
+        target = project_desc['target']
+        merge_bin_args += ['--chip', target]
+        merge_bin_args += ['merge_bin']  # needs to be after the --chip option
+        if not output:
+            if format in ('raw', 'uf2'):
+                output = 'merged-binary.bin'
+            elif format == 'hex':
+                output = 'merged-binary.hex'
+        merge_bin_args += ['-o', output]
+        if format:
+            merge_bin_args += ['-f', format]
+        if md5_disable:
+            if format != 'uf2':
+                yellow_print('idf.py merge-bin: --md5-disable is only valid for UF2 format. Option will be ignored.')
+            else:
+                merge_bin_args += ['--md5-disable']
+        if flash_offset:
+            if format != 'raw':
+                yellow_print('idf.py merge-bin: --flash-offset is only valid for RAW format. Option will be ignored.')
+            else:
+                merge_bin_args += ['-t', flash_offset]
+        if fill_flash_size:
+            if format != 'raw':
+                yellow_print('idf.py merge-bin: --fill-flash-size is only valid for RAW format, option will be ignored.')
+            else:
+                merge_bin_args += ['--fill-flash-size', fill_flash_size]
+        merge_bin_args += ['@flash_args']
+        print(f'Merged binary {output} will be created in the build directory...')
+        RunTool('merge_bin', merge_bin_args, args.build_dir, build_dir=args.build_dir, hints=not args.no_hints)()
+
     BAUD_AND_PORT = [BAUD_RATE, PORT]
     flash_options = BAUD_AND_PORT + [
         {
@@ -252,6 +299,37 @@ def ota_targets(target_name: str, ctx: click.core.Context, args: PropertyDict) -
                 'help': 'Erase entire flash chip.',
                 'options': BAUD_AND_PORT,
             },
+            'merge-bin': {
+                'callback': merge_bin,
+                'options': [
+                    {
+                        'names': ['--output', '-o'],
+                        'help': ('Output filename'),
+                        'type': click.Path(),
+                    },
+                    {
+                        'names': ['--format', '-f'],
+                        'help': ('Format of the output file'),
+                        'type': click.Choice(['hex', 'uf2', 'raw']),
+                        'default': 'raw',
+                    },
+                    {
+                        'names': ['--md5-disable'],
+                        'is_flag': True,
+                        'help': ('[ONLY UF2] Disable MD5 checksum in UF2 output.'),
+                    },
+                    {
+                        'names': ['--flash-offset', '-t'],
+                        'help': ('[ONLY RAW] Flash offset where the output file will be flashed.'),
+                    },
+                    {
+                        'names': ['--fill-flash-size'],
+                        'help': ('[ONLY RAW] If set, the final binary file will be padded with FF bytes up to this flash size.'),
+                        'type': click.Choice(['256KB', '512KB', '1MB', '2MB', '4MB', '8MB', '16MB', '32MB', '64MB', '128MB']),
+                    },
+                ],
+                'dependencies': ['all'],  # all = build
+            },
             'monitor': {
                 'callback':
                 monitor,

tools/test_build_system/test_common.py

@@ -305,3 +305,13 @@ def test_save_defconfig_check(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
         'Missing CONFIG_IDF_TARGET="esp32c3" in sdkconfig.defaults'
     assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_PARTITION_TABLE_OFFSET=0x8001'), \
         'Missing CONFIG_PARTITION_TABLE_OFFSET=0x8001 in sdkconfig.defaults'
+
+
+def test_merge_bin_cmd(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('Test if merge-bin command works correctly')
+    idf_py('merge-bin')
+    assert (test_app_copy / 'build' / 'merged-binary.bin').is_file()
+    idf_py('merge-bin', '--output', 'merged-binary-2.bin')
+    assert (test_app_copy / 'build' / 'merged-binary-2.bin').is_file()
+    idf_py('merge-bin', '--format', 'hex')
+    assert (test_app_copy / 'build' / 'merged-binary.hex').is_file()