From 671234e19374c90de494f8c56299ebd440151939 Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 18 Oct 2019 18:19:53 +0300 Subject: [PATCH 1/7] Update support for embedding files // Issue #161, Resolves #220 --- .../{_embedtxt_files.py => _embed_files.py} | 47 ++++++++++++------- builder/frameworks/arduino.py | 2 +- builder/frameworks/espidf.py | 2 +- 3 files changed, 33 insertions(+), 18 deletions(-) rename builder/frameworks/{_embedtxt_files.py => _embed_files.py} (67%) diff --git a/builder/frameworks/_embedtxt_files.py b/builder/frameworks/_embed_files.py similarity index 67% rename from builder/frameworks/_embedtxt_files.py rename to builder/frameworks/_embed_files.py index 1be72aa9c..1e2ccfff8 100644 --- a/builder/frameworks/_embedtxt_files.py +++ b/builder/frameworks/_embed_files.py @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os import SEEK_CUR, SEEK_END -from os.path import basename, isfile, join +import shutil +from os import SEEK_CUR, SEEK_END, makedirs +from os.path import basename, isfile, isdir, join from SCons.Script import Builder @@ -21,9 +22,8 @@ Import("env") - # -# TXT files helpers +# Embedded files helpers # @@ -31,28 +31,38 @@ def prepare_files(files): if not files: return + fixed_files = [] + build_dir = env.subst("$BUILD_DIR") + if not isdir(build_dir): + makedirs(build_dir) for f in files: - with open(env.subst(f), "rb+") as fp: + fixed_file = join(build_dir, basename(f)) + shutil.copy(env.subst(f), fixed_file) + with open(fixed_file, "rb+") as fp: fp.seek(-1, SEEK_END) if fp.read(1) != '\0': fp.seek(0, SEEK_CUR) fp.write(b'\0') + fixed_files.append(fixed_file) + + return fixed_files + -def extract_files(cppdefines): +def extract_files(cppdefines, files_type): for define in cppdefines: - if "COMPONENT_EMBED_TXTFILES" not in define: + if files_type not in define: continue if not isinstance(define, tuple): - print("Warning! COMPONENT_EMBED_TXTFILES macro cannot be empty!") + print("Warning! %s macro cannot be empty!" % files_type) return [] with cd(env.subst("$PROJECT_DIR")): value = define[1] if not isinstance(value, str): - print("Warning! COMPONENT_EMBED_TXTFILES macro must contain " - "a list of files separated by ':'") + print("Warning! %s macro must contain " + "a list of files separated by ':'" % files_type) return [] result = [] @@ -65,9 +75,9 @@ def extract_files(cppdefines): return result -def remove_config_define(cppdefines): +def remove_config_define(cppdefines, files_type): for define in cppdefines: - if "COMPONENT_EMBED_TXTFILES" in define: + if files_type in define: env.ProcessUnFlags("-D%s" % "=".join(str(d) for d in define)) return @@ -94,9 +104,14 @@ def embed_files(files): suffix=".txt.o")) ) + + flags = env.get("CPPDEFINES") -if "COMPONENT_EMBED_TXTFILES" in env.Flatten(flags): - files = extract_files(flags) - prepare_files(files) +for component_files in ("COMPONENT_EMBED_TXTFILES", "COMPONENT_EMBED_FILES"): + if component_files not in env.Flatten(flags): + continue + files = extract_files(flags, component_files) + if component_files == "COMPONENT_EMBED_TXTFILES": + files = prepare_files(files) embed_files(files) - remove_config_define(flags) + remove_config_define(flags, component_files) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index eab296d47..0f58a9d79 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -28,7 +28,7 @@ env = DefaultEnvironment() -SConscript("_embedtxt_files.py", exports="env") +SConscript("_embed_files.py", exports="env") if "espidf" not in env.subst("$PIOFRAMEWORK"): SConscript( diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 965260f11..eee3467a5 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -33,7 +33,7 @@ platform = env.PioPlatform() env.SConscript("_bare.py", exports="env") -env.SConscript("_embedtxt_files.py", exports="env") +env.SConscript("_embed_files.py", exports="env") ulp_lib = None ulp_dir = join(env.subst("$PROJECT_DIR"), "ulp") From d03fafac19f2f890cff7ece1bde2835a54362d68 Mon Sep 17 00:00:00 2001 From: Florian <1technophile@users.noreply.github.com> Date: Mon, 21 Oct 2019 09:38:59 +0200 Subject: [PATCH 2/7] Correct upload speed for m5stick-c board (#240) The current upload speed gives an error when trying to upload on m5stick-c: ``` A fatal error occurred: Timed out waiting for packet header ``` With 1500000 the upload works correctly --- boards/m5stick-c.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boards/m5stick-c.json b/boards/m5stick-c.json index 2fcf22072..b5d7425c6 100644 --- a/boards/m5stick-c.json +++ b/boards/m5stick-c.json @@ -25,7 +25,7 @@ "maximum_ram_size": 327680, "maximum_size": 4194304, "require_upload_port": true, - "speed": 460800 + "speed": 1500000 }, "url": "http://www.m5stack.com", "vendor": "M5Stack" From 31f86244734fc2ae12ab18428133bb293ea0eb4a Mon Sep 17 00:00:00 2001 From: valeros Date: Mon, 21 Oct 2019 17:06:00 +0300 Subject: [PATCH 3/7] Improve support for embedding external files --- builder/frameworks/_embed_files.py | 64 ++++++++++++++---------------- 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/builder/frameworks/_embed_files.py b/builder/frameworks/_embed_files.py index 1e2ccfff8..8117aa8c8 100644 --- a/builder/frameworks/_embed_files.py +++ b/builder/frameworks/_embed_files.py @@ -13,8 +13,8 @@ # limitations under the License. import shutil -from os import SEEK_CUR, SEEK_END, makedirs -from os.path import basename, isfile, isdir, join +from os import SEEK_CUR, SEEK_END +from os.path import basename, isfile, join from SCons.Script import Builder @@ -22,33 +22,11 @@ Import("env") + # # Embedded files helpers # - -def prepare_files(files): - if not files: - return - - fixed_files = [] - build_dir = env.subst("$BUILD_DIR") - if not isdir(build_dir): - makedirs(build_dir) - for f in files: - fixed_file = join(build_dir, basename(f)) - shutil.copy(env.subst(f), fixed_file) - with open(fixed_file, "rb+") as fp: - fp.seek(-1, SEEK_END) - if fp.read(1) != '\0': - fp.seek(0, SEEK_CUR) - fp.write(b'\0') - - fixed_files.append(fixed_file) - - return fixed_files - - def extract_files(cppdefines, files_type): for define in cppdefines: if files_type not in define: @@ -82,11 +60,31 @@ def remove_config_define(cppdefines, files_type): return -def embed_files(files): +def prepare_file(source, target, env): + filepath = source[0].get_abspath() + shutil.copy(filepath, filepath + ".piobkp") + + with open(filepath, "rb+") as fp: + fp.seek(-1, SEEK_END) + if fp.read(1) != '\0': + fp.seek(0, SEEK_CUR) + fp.write(b'\0') + + +def revert_original_file(source, target, env): + filepath = source[0].get_abspath() + if isfile(filepath + ".piobkp"): + shutil.move(filepath + ".piobkp", filepath) + + +def embed_files(files, files_type): for f in files: filename = basename(f) + ".txt.o" file_target = env.TxtToBin(join("$BUILD_DIR", filename), f) env.Depends("$PIOMAINPROG", file_target) + if files_type == "COMPONENT_EMBED_TXTFILES": + env.AddPreAction(file_target, prepare_file) + env.AddPostAction(file_target, revert_original_file) env.Append(PIOBUILDFILES=[env.File(join("$BUILD_DIR", filename))]) @@ -104,14 +102,10 @@ def embed_files(files): suffix=".txt.o")) ) - - flags = env.get("CPPDEFINES") -for component_files in ("COMPONENT_EMBED_TXTFILES", "COMPONENT_EMBED_FILES"): - if component_files not in env.Flatten(flags): +for files_type in ("COMPONENT_EMBED_TXTFILES", "COMPONENT_EMBED_FILES"): + if files_type not in env.Flatten(flags): continue - files = extract_files(flags, component_files) - if component_files == "COMPONENT_EMBED_TXTFILES": - files = prepare_files(files) - embed_files(files) - remove_config_define(flags, component_files) + files = extract_files(flags, files_type) + embed_files(files, files_type) + remove_config_define(flags, files_type) From 11ee6b38cb5428e549b6db769ff9b804650c52c9 Mon Sep 17 00:00:00 2001 From: valeros Date: Mon, 28 Oct 2019 15:28:29 +0200 Subject: [PATCH 4/7] Allow specifying embeddable files in a special platformio.ini section --- builder/frameworks/_embed_files.py | 45 +++++++++++++++++++----------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/builder/frameworks/_embed_files.py b/builder/frameworks/_embed_files.py index 8117aa8c8..1df8b02a4 100644 --- a/builder/frameworks/_embed_files.py +++ b/builder/frameworks/_embed_files.py @@ -22,35 +22,44 @@ Import("env") +board = env.BoardConfig() # # Embedded files helpers # def extract_files(cppdefines, files_type): - for define in cppdefines: - if files_type not in define: - continue - - if not isinstance(define, tuple): - print("Warning! %s macro cannot be empty!" % files_type) - return [] + files = [] + if "build." + files_type in board: + files.extend( + [join("$PROJECT_DIR", f) for f in board.get( + "build." + files_type, "").split() if f]) + else: + files_define = "COMPONENT_" + files_type.upper() + for define in cppdefines: + if files_define not in define: + continue - with cd(env.subst("$PROJECT_DIR")): value = define[1] + if not isinstance(define, tuple): + print("Warning! %s macro cannot be empty!" % files_define) + return [] + if not isinstance(value, str): print("Warning! %s macro must contain " - "a list of files separated by ':'" % files_type) + "a list of files separated by ':'" % files_define) return [] - result = [] for f in value.split(':'): - if not isfile(f): - print("Warning! Could not find file %s" % f) + if not f: continue - result.append(join("$PROJECT_DIR", f)) + files.append(join("$PROJECT_DIR", f)) - return result + for f in files: + if not isfile(env.subst(f)): + print("Warning! Could not find file \"%s\"" % basename(f)) + + return files def remove_config_define(cppdefines, files_type): @@ -82,7 +91,7 @@ def embed_files(files, files_type): filename = basename(f) + ".txt.o" file_target = env.TxtToBin(join("$BUILD_DIR", filename), f) env.Depends("$PIOMAINPROG", file_target) - if files_type == "COMPONENT_EMBED_TXTFILES": + if files_type == "embed_txtfiles": env.AddPreAction(file_target, prepare_file) env.AddPostAction(file_target, revert_original_file) env.Append(PIOBUILDFILES=[env.File(join("$BUILD_DIR", filename))]) @@ -103,9 +112,11 @@ def embed_files(files, files_type): ) flags = env.get("CPPDEFINES") -for files_type in ("COMPONENT_EMBED_TXTFILES", "COMPONENT_EMBED_FILES"): - if files_type not in env.Flatten(flags): +for files_type in ("embed_txtfiles", "embed_files"): + if "COMPONENT_" + files_type.upper() not in env.Flatten( + flags) and "build." + files_type not in board: continue + files = extract_files(flags, files_type) embed_files(files, files_type) remove_config_define(flags, files_type) From 685e78f9c3e89d71d29286a923c6ee46e998a081 Mon Sep 17 00:00:00 2001 From: valeros Date: Mon, 28 Oct 2019 15:42:14 +0200 Subject: [PATCH 5/7] Update aws-iot example with new configuration // Resolve #242 --- examples/espidf-aws-iot/platformio.ini | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/espidf-aws-iot/platformio.ini b/examples/espidf-aws-iot/platformio.ini index ec18ab75b..2d9863d0a 100644 --- a/examples/espidf-aws-iot/platformio.ini +++ b/examples/espidf-aws-iot/platformio.ini @@ -13,6 +13,10 @@ framework = espidf board = esp32dev monitor_speed = 115200 build_flags = - -DCOMPONENT_EMBED_TXTFILES=src/private.pem.key:src/certificate.pem.crt:src/aws-root-ca.pem -DCONFIG_WIFI_SSID=\"ESP_AP\" -DCONFIG_WIFI_PASSWORD=\"MYPASS\" + +board_build.embed_txtfiles = + src/private.pem.key + src/certificate.pem.crt + src/aws-root-ca.pem From 1fadd964e9e3ca89aaf08cb95753d5050848fa41 Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 1 Nov 2019 18:40:03 +0200 Subject: [PATCH 6/7] Fix mkspiffs binary name when both arduino and esp-idf frameworks selected // Resolve #244 --- builder/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builder/main.py b/builder/main.py index f00e8136b..9516c4885 100644 --- a/builder/main.py +++ b/builder/main.py @@ -158,7 +158,8 @@ def __fetch_spiffs_size(target, source, env): ], ERASECMD='"$PYTHONEXE" "$OBJCOPY" $ERASEFLAGS erase_flash', - MKSPIFFSTOOL="mkspiffs_${PIOPLATFORM}_${PIOFRAMEWORK}", + MKSPIFFSTOOL="mkspiffs_${PIOPLATFORM}_" + ("espidf" if "espidf" in env.subst( + "$PIOFRAMEWORK") else "${PIOFRAMEWORK}"), PROGSUFFIX=".elf" ) From afa7d2fab6a3006ae357f2ac018a39ec9e7a7e42 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 1 Nov 2019 18:42:16 +0200 Subject: [PATCH 7/7] Bump version to 1.11.1 --- platform.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform.json b/platform.json index df0a4f52a..c319a1120 100644 --- a/platform.json +++ b/platform.json @@ -12,7 +12,7 @@ "type": "git", "url": "https://github.com/platformio/platform-espressif32.git" }, - "version": "1.11.0", + "version": "1.11.1", "packageRepositories": [ "https://dl.bintray.com/platformio/dl-packages/manifest.json", "http://dl.platformio.org/packages/manifest.json",