From 3688b7b79dddd944f463fc9077e183bb112be24b Mon Sep 17 00:00:00 2001 From: valeros Date: Tue, 15 Sep 2020 13:12:59 +0300 Subject: [PATCH 01/11] Allow custom offsets for bootloader and partitions in ESP-IDF Issue #11 --- builder/frameworks/espidf.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 9197f3127..cb7487e7d 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -1065,8 +1065,15 @@ def _skip_prj_source_files(node): LINKFLAGS=extra_flags, LIBS=libs, FLASH_EXTRA_IMAGES=[ - ("0x1000", os.path.join("$BUILD_DIR", "bootloader.bin")), - ("0x8000", os.path.join("$BUILD_DIR", "partitions.bin")), + ( + board.get("upload.bootloader_offset", "0x1000"), + os.path.join("$BUILD_DIR", "bootloader.bin") + ), + ( + board.get("upload.partition_table_offset", hex( + sdk_config.get("PARTITION_TABLE_OFFSET", 0x8000))), + os.path.join("$BUILD_DIR", "partitions.bin") + ), ], ) From f5ffb5896c691634998a3c02eded961840a0bf66 Mon Sep 17 00:00:00 2001 From: valeros Date: Mon, 26 Oct 2020 16:02:27 +0200 Subject: [PATCH 02/11] Initial support for new OpenOCD package --- platform.json | 2 +- platform.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/platform.json b/platform.json index 6f309b654..dab586886 100644 --- a/platform.json +++ b/platform.json @@ -114,7 +114,7 @@ "type": "debugger", "optional": true, "owner": "platformio", - "version": "~1.1000.20181026" + "version": "~2.1000.0" }, "tool-mkspiffs": { "type": "uploader", diff --git a/platform.py b/platform.py index 8f8561692..13af01377 100644 --- a/platform.py +++ b/platform.py @@ -122,7 +122,8 @@ def _add_dynamic_options(self, board): server_args = [ "-s", "$PACKAGE_DIR/share/openocd/scripts", "-f", "interface/%s.cfg" % openocd_interface, - "-f", "board/%s" % debug.get("openocd_board") + "-f", "board/%s" % debug.get("openocd_board"), + "-c", "adapter_khz %d" % debug.get("adapter_speed", 20000) ] debug['tools'][link] = { From dedfed0b85c991d4d007b40f9da21a67d36995da Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 26 Oct 2020 18:44:10 +0200 Subject: [PATCH 03/11] Extend debugging options with flash images --- platform.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/platform.py b/platform.py index 8f8561692..e037b618e 100644 --- a/platform.py +++ b/platform.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os.path import isdir +import copy +import os +from platformio import fs from platformio.managers.platform import PlatformBase from platformio.util import get_systype @@ -32,7 +34,7 @@ def configure_default_packages(self, variables, targets): self.packages['tool-mkspiffs']['optional'] = False if variables.get("upload_protocol"): self.packages['tool-openocd-esp32']['optional'] = False - if isdir("ulp"): + if os.path.isdir("ulp"): self.packages['toolchain-esp32ulp']['optional'] = False if "espidf" in frameworks: for p in self.packages: @@ -152,3 +154,27 @@ def _add_dynamic_options(self, board): board.manifest['debug'] = debug return board + + def configure_debug_options(self, initial_debug_options, ide_data): + flash_images = ide_data.get("extra", {}).get("flash_images") + ignore_conds = [ + initial_debug_options["load_cmds"] != ["load"], + not flash_images, + not all([os.path.isfile(item["path"]) for item in flash_images]), + ] + if any(ignore_conds): + return initial_debug_options + + debug_options = copy.deepcopy(initial_debug_options) + load_cmds = [ + 'monitor program_esp32 "{{{path}}}" {offset} verify'.format( + path=fs.to_unix_path(item["path"]), offset=item["offset"] + ) + for item in flash_images + ] + load_cmds.append( + 'monitor program_esp32 "{%s.bin}" 0x10000 verify' + % fs.to_unix_path(ide_data["prog_path"][:-4]) + ) + debug_options["load_cmds"] = load_cmds + return debug_options From d10bbbe7dcaab73aa7d7089d16f6d1dd58dc12db Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 26 Oct 2020 18:44:10 +0200 Subject: [PATCH 04/11] Extend debugging options with flash images --- platform.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/platform.py b/platform.py index 13af01377..eb5e8fe51 100644 --- a/platform.py +++ b/platform.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os.path import isdir +import copy +import os +from platformio import fs from platformio.managers.platform import PlatformBase from platformio.util import get_systype @@ -32,7 +34,7 @@ def configure_default_packages(self, variables, targets): self.packages['tool-mkspiffs']['optional'] = False if variables.get("upload_protocol"): self.packages['tool-openocd-esp32']['optional'] = False - if isdir("ulp"): + if os.path.isdir("ulp"): self.packages['toolchain-esp32ulp']['optional'] = False if "espidf" in frameworks: for p in self.packages: @@ -153,3 +155,27 @@ def _add_dynamic_options(self, board): board.manifest['debug'] = debug return board + + def configure_debug_options(self, initial_debug_options, ide_data): + flash_images = ide_data.get("extra", {}).get("flash_images") + ignore_conds = [ + initial_debug_options["load_cmds"] != ["load"], + not flash_images, + not all([os.path.isfile(item["path"]) for item in flash_images]), + ] + if any(ignore_conds): + return initial_debug_options + + debug_options = copy.deepcopy(initial_debug_options) + load_cmds = [ + 'monitor program_esp32 "{{{path}}}" {offset} verify'.format( + path=fs.to_unix_path(item["path"]), offset=item["offset"] + ) + for item in flash_images + ] + load_cmds.append( + 'monitor program_esp32 "{%s.bin}" 0x10000 verify' + % fs.to_unix_path(ide_data["prog_path"][:-4]) + ) + debug_options["load_cmds"] = load_cmds + return debug_options From f2df24ff088e8218acaaaef20ca81d7a21026d0f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 26 Oct 2020 19:24:18 +0200 Subject: [PATCH 05/11] Use "program_esp" command for new openOCD --- platform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platform.py b/platform.py index eb5e8fe51..67a00a4b4 100644 --- a/platform.py +++ b/platform.py @@ -168,13 +168,13 @@ def configure_debug_options(self, initial_debug_options, ide_data): debug_options = copy.deepcopy(initial_debug_options) load_cmds = [ - 'monitor program_esp32 "{{{path}}}" {offset} verify'.format( + 'monitor program_esp "{{{path}}}" {offset} verify'.format( path=fs.to_unix_path(item["path"]), offset=item["offset"] ) for item in flash_images ] load_cmds.append( - 'monitor program_esp32 "{%s.bin}" 0x10000 verify' + 'monitor program_esp "{%s.bin}" 0x10000 verify' % fs.to_unix_path(ide_data["prog_path"][:-4]) ) debug_options["load_cmds"] = load_cmds From 7b932d161e0fe6da3ed35686e2d3deceb9b93c4a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 26 Oct 2020 22:19:01 +0200 Subject: [PATCH 06/11] Use "program_esp" command for new openOCD --- builder/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/main.py b/builder/main.py index bff5d70a0..e894d5623 100644 --- a/builder/main.py +++ b/builder/main.py @@ -371,13 +371,13 @@ def __fetch_spiffs_size(target, source, env): debug_tools.get(upload_protocol).get("server").get("arguments", [])) openocd_args.extend([ "-c", - "program_esp32 {{$SOURCE}} %s verify" % + "program_esp {{$SOURCE}} %s verify" % board.get("upload.offset_address", "0x10000") ]) for image in env.get("FLASH_EXTRA_IMAGES", []): openocd_args.extend([ "-c", - 'program_esp32 {{%s}} %s verify' % + 'program_esp {{%s}} %s verify' % (_to_unix_slashes(image[1]), image[0]) ]) openocd_args.extend(["-c", "reset run; shutdown"]) From 7ee2d88153699e88f29b24fc51e6e0ace491c4d6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 18 Nov 2020 21:49:40 +0200 Subject: [PATCH 07/11] Move "meteca" packages to its organization --- platform.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platform.json b/platform.json index dab586886..77781306f 100644 --- a/platform.json +++ b/platform.json @@ -79,7 +79,7 @@ "framework-arduino-mbcwb": { "type": "framework", "optional": true, - "owner": "platformio", + "owner": "meteca", "version": ">=2.1.1" }, "framework-espidf": { @@ -107,7 +107,7 @@ }, "tool-mbctool": { "optional": true, - "owner": "platformio", + "owner": "meteca", "version": ">=2.0.0" }, "tool-openocd-esp32": { From 25e09897368e7781e14d3c256c6d7367935a57ce Mon Sep 17 00:00:00 2001 From: valeros Date: Thu, 19 Nov 2020 18:26:22 +0200 Subject: [PATCH 08/11] Update esptoolpy to v3.0 // Resolve #401 --- platform.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform.json b/platform.json index 77781306f..22928511d 100644 --- a/platform.json +++ b/platform.json @@ -103,7 +103,7 @@ "tool-esptoolpy": { "type": "uploader", "owner": "platformio", - "version": "~1.20600.0" + "version": "~1.30000.0" }, "tool-mbctool": { "optional": true, From cd0af53d4bb8773756107a12d8aee12c8b8c0bc8 Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 27 Nov 2020 15:26:55 +0200 Subject: [PATCH 09/11] Force ESP-IDF build system to use proper Python // Issue #11 --- builder/frameworks/espidf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index cb7487e7d..7868a251b 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -913,6 +913,7 @@ def create_verion_file(): [ "-DIDF_TARGET=" + idf_variant, "-DEXTRA_COMPONENT_DIRS:PATH=" + ";".join(extra_components), + "-DPYTHON=" + env.subst("$PYTHONEXE"), ] + click.parser.split_arg_string(board.get("build.cmake_extra_args", "")), ) From 115f98151ed2e8328ee392c62752bd4678b8b5e5 Mon Sep 17 00:00:00 2001 From: valeros Date: Wed, 2 Dec 2020 13:10:37 +0200 Subject: [PATCH 10/11] Dynamically configure partition offsets for ESP-IDF --- builder/frameworks/espidf.py | 131 ++++++++++++++++++++++++++++++++--- builder/main.py | 5 +- platform.py | 10 ++- 3 files changed, 132 insertions(+), 14 deletions(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 7868a251b..494139a82 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -499,7 +499,9 @@ def generate_project_ld_script(sdk_config, ignore_targets=None): "env_file": os.path.join("$BUILD_DIR", "config.env"), "libraries_list": libraries_list, "objdump": os.path.join( - TOOLCHAIN_DIR, "bin", env.subst("$CC").replace("-gcc", "-objdump"), + TOOLCHAIN_DIR, + "bin", + env.subst("$CC").replace("-gcc", "-objdump"), ), } @@ -803,19 +805,90 @@ def find_default_component(target_configs): return "" -def create_verion_file(): +def create_version_file(): version_file = os.path.join(FRAMEWORK_DIR, "version.txt") if not os.path.isfile(version_file): with open(version_file, "w") as fp: fp.write(platform.get_package_version("framework-espidf")) -# +def generate_empty_partition_image(binary_path, image_size): + empty_partition = env.Command( + binary_path, + None, + env.VerboseAction( + '"$PYTHONEXE" "%s" %s $TARGET' + % ( + os.path.join( + FRAMEWORK_DIR, + "components", + "partition_table", + "gen_empty_partition.py", + ), + image_size, + ), + "Generating an empty partition $TARGET", + ), + ) + + env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", empty_partition) + + +def get_partition_info(pt_path, pt_offset, pt_params): + assert os.path.isfile(pt_path) + cmd = [ + env.subst("$PYTHONEXE"), + os.path.join(FRAMEWORK_DIR, "components", "partition_table", "parttool.py"), + "-q", + "--partition-table-offset", + hex(pt_offset), + "--partition-table-file", + pt_path, + "get_partition_info", + "--info", + "size", + "offset", + ] + + if pt_params["name"] == "boot": + cmd.append("--partition-boot-default") + else: + cmd.extend( + [ + "--partition-type", + pt_params["type"], + "--partition-subtype", + pt_params["subtype"], + ] + ) + + result = exec_command(cmd) + if result["returncode"] != 0: + sys.stderr.write( + "Couldn't extract information for %s/%s from the partition table\n" + % (pt_params["type"], pt_params["subtype"]) + ) + sys.stderr.write(result["out"] + "\n") + sys.stderr.write(result["err"] + "\n") + env.Exit(1) + + size = offset = 0 + if result["out"].strip(): + size, offset = result["out"].strip().split(" ", 1) + + return {"size": size, "offset": offset} + + +def get_app_partition_offset(pt_table, pt_offset): + # Get the default boot partition offset + app_params = get_partition_info(pt_table, pt_offset, {"name": "boot"}) + return app_params.get("offset", "0x10000") + + # ESP-IDF package doesn't contain .git folder, instead package version is specified # in a special file "version.h" in the root folder of the package -# -create_verion_file() +create_version_file() # # Generate final linker script @@ -845,6 +918,7 @@ def create_verion_file(): fwpartitions_dir = os.path.join(FRAMEWORK_DIR, "components", "partition_table") partitions_csv = board.get("build.partitions", "partitions_singleapp.csv") + env.Replace( PARTITIONS_TABLE_CSV=os.path.abspath( os.path.join(fwpartitions_dir, partitions_csv) @@ -1058,6 +1132,7 @@ def _skip_prj_source_files(node): ) ) +partition_table_offset = sdk_config.get("PARTITION_TABLE_OFFSET", 0x8000) project_flags.update(link_args) env.MergeFlags(project_flags) env.Prepend( @@ -1068,12 +1143,11 @@ def _skip_prj_source_files(node): FLASH_EXTRA_IMAGES=[ ( board.get("upload.bootloader_offset", "0x1000"), - os.path.join("$BUILD_DIR", "bootloader.bin") + os.path.join("$BUILD_DIR", "bootloader.bin"), ), ( - board.get("upload.partition_table_offset", hex( - sdk_config.get("PARTITION_TABLE_OFFSET", 0x8000))), - os.path.join("$BUILD_DIR", "partitions.bin") + board.get("upload.partition_table_offset", hex(partition_table_offset)), + os.path.join("$BUILD_DIR", "partitions.bin"), ), ], ) @@ -1095,3 +1169,42 @@ def _skip_prj_source_files(node): ulp_dir = os.path.join(env.subst("$PROJECT_DIR"), "ulp") if os.path.isdir(ulp_dir) and os.listdir(ulp_dir): env.SConscript("ulp.py", exports="env project_config idf_variant") + +# +# Process OTA partition and image +# + +ota_partition_params = get_partition_info( + env.subst("$PARTITIONS_TABLE_CSV"), + partition_table_offset, + {"name": "ota", "type": "data", "subtype": "ota"}, +) + +if ota_partition_params["size"] and ota_partition_params["offset"]: + # Generate an empty image if OTA is enabled in partition table + ota_partition_image = os.path.join("$BUILD_DIR", "ota_data_initial.bin") + generate_empty_partition_image(ota_partition_image, ota_partition_params["size"]) + + env.Append( + FLASH_EXTRA_IMAGES=[ + ( + board.get( + "upload.ota_partition_offset", ota_partition_params["offset"] + ), + ota_partition_image, + ) + ] + ) + +# +# Configure application partition offset +# + +env.Replace( + ESP32_APP_OFFSET=get_app_partition_offset( + env.subst("$PARTITIONS_TABLE_CSV"), partition_table_offset + ) +) + +# Propagate application offset to debug configurations +env["IDE_EXTRA_DATA"].update({"application_offset": env.subst("$ESP32_APP_OFFSET")}) diff --git a/builder/main.py b/builder/main.py index e894d5623..6faed8e2c 100644 --- a/builder/main.py +++ b/builder/main.py @@ -163,6 +163,7 @@ def __fetch_spiffs_size(target, source, env): MKSPIFFSTOOL="mkspiffs_${PIOPLATFORM}_" + ("espidf" if "espidf" in env.subst( "$PIOFRAMEWORK") else "${PIOFRAMEWORK}"), ESP32_SPIFFS_IMAGE_NAME=env.get("ESP32_SPIFFS_IMAGE_NAME", "spiffs"), + ESP32_APP_OFFSET="0x10000", PROGSUFFIX=".elf" ) @@ -310,7 +311,7 @@ def __fetch_spiffs_size(target, source, env): "--flash_freq", "${__get_board_f_flash(__env__)}", "--flash_size", "detect" ], - UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS 0x10000 $SOURCE' + UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS $ESP32_APP_OFFSET $SOURCE' ) for image in env.get("FLASH_EXTRA_IMAGES", []): env.Append(UPLOADERFLAGS=[image[0], env.subst(image[1])]) @@ -372,7 +373,7 @@ def __fetch_spiffs_size(target, source, env): openocd_args.extend([ "-c", "program_esp {{$SOURCE}} %s verify" % - board.get("upload.offset_address", "0x10000") + board.get("upload.offset_address", "$ESP32_APP_OFFSET") ]) for image in env.get("FLASH_EXTRA_IMAGES", []): openocd_args.extend([ diff --git a/platform.py b/platform.py index 67a00a4b4..81af44c8e 100644 --- a/platform.py +++ b/platform.py @@ -157,7 +157,8 @@ def _add_dynamic_options(self, board): return board def configure_debug_options(self, initial_debug_options, ide_data): - flash_images = ide_data.get("extra", {}).get("flash_images") + ide_extra_data = ide_data.get("extra", {}) + flash_images = ide_extra_data.get("flash_images", []) ignore_conds = [ initial_debug_options["load_cmds"] != ["load"], not flash_images, @@ -174,8 +175,11 @@ def configure_debug_options(self, initial_debug_options, ide_data): for item in flash_images ] load_cmds.append( - 'monitor program_esp "{%s.bin}" 0x10000 verify' - % fs.to_unix_path(ide_data["prog_path"][:-4]) + 'monitor program_esp "{%s.bin}" %s verify' + % ( + fs.to_unix_path(ide_data["prog_path"][:-4]), + ide_extra_data.get("application_offset", "0x10000"), + ) ) debug_options["load_cmds"] = load_cmds return debug_options From f32e3ec3f339f9e05927535ef47d82cc0c03d251 Mon Sep 17 00:00:00 2001 From: valeros Date: Wed, 2 Dec 2020 16:41:35 +0200 Subject: [PATCH 11/11] Bump version to 2.1.0 --- platform.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform.json b/platform.json index 22928511d..61a5f83de 100644 --- a/platform.json +++ b/platform.json @@ -18,7 +18,7 @@ "type": "git", "url": "https://github.com/platformio/platform-espressif32.git" }, - "version": "2.0.0", + "version": "2.1.0", "frameworks": { "arduino": { "package": "framework-arduinoespressif32",