diff --git a/.gitignore b/.gitignore index 03f4a3c..0d20b64 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -.pio +*.pyc diff --git a/boards/d1_mini_pro_ota.json b/boards/d1_mini_pro_ota.json new file mode 100644 index 0000000..abce8e0 --- /dev/null +++ b/boards/d1_mini_pro_ota.json @@ -0,0 +1,33 @@ +{ + "build": { + "arduino": { + "ldscript": "eagle.flash.16m14m.ld" + }, + "core": "esp8266", + "extra_flags": "-DESP8266 -DARDUINO_ARCH_ESP8266 -DARDUINO_ESP8266_WEMOS_D1MINIPRO", + "f_cpu": "80000000L", + "f_flash": "40000000L", + "flash_mode": "dio", + "mcu": "esp8266", + "partitions": "partitions_two_ota.csv", + "variant": "d1_mini" + }, + "connectivity": [ + "wifi" + ], + "frameworks": [ + "arduino", + "esp8266-rtos-sdk", + "esp8266-nonos-sdk" + ], + "name": "WeMos D1 mini Pro", + "upload": { + "maximum_ram_size": 81920, + "maximum_size": 16777216, + "require_upload_port": true, + "resetmethod": "nodemcu", + "speed": 115200 + }, + "url": "https://wiki.wemos.cc/products:d1:d1_mini", + "vendor": "WEMOS" +} diff --git a/boards/esp12e_ota.json b/boards/esp12e_ota.json new file mode 100644 index 0000000..4a6e67a --- /dev/null +++ b/boards/esp12e_ota.json @@ -0,0 +1,33 @@ +{ + "build": { + "arduino": { + "ldscript": "eagle.flash.4m1m.ld" + }, + "core": "esp8266", + "extra_flags": "-DESP8266 -DARDUINO_ARCH_ESP8266 -DARDUINO_ESP8266_ESP12", + "f_cpu": "80000000L", + "f_flash": "40000000L", + "flash_mode": "dio", + "mcu": "esp8266", + "partitions": "partitions_two_ota.csv", + "variant": "nodemcu" + }, + "connectivity": [ + "wifi" + ], + "frameworks": [ + "arduino", + "esp8266-rtos-sdk", + "esp8266-nonos-sdk" + ], + "name": "Espressif ESP8266 ESP-12E", + "upload": { + "maximum_ram_size": 81920, + "maximum_size": 4194304, + "require_upload_port": true, + "resetmethod": "nodemcu", + "speed": 115200 + }, + "url": "http://www.esp8266.com/wiki/doku.php?id=esp8266-module-family", + "vendor": "Espressif" +} diff --git a/builder/frameworks/_embed_files.py b/builder/frameworks/_embed_files.py new file mode 100644 index 0000000..d2770d6 --- /dev/null +++ b/builder/frameworks/_embed_files.py @@ -0,0 +1,179 @@ +# Copyright 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import shutil +from os import SEEK_CUR, SEEK_END +from os.path import basename, isfile, join + +from SCons.Script import Builder + +Import("env") + +board = env.BoardConfig() + +# +# Embedded files helpers +# + + +def extract_files(cppdefines, files_type): + result = [] + files = env.GetProjectOption("board_build.%s" % files_type, "").splitlines() + if files: + result.extend([join("$PROJECT_DIR", f.strip()) for f in files if f]) + else: + files_define = "COMPONENT_" + files_type.upper() + for define in cppdefines: + if files_define not in define: + continue + + value = define[1] + if not isinstance(define, tuple): + print("Warning! %s macro cannot be empty!" % files_define) + return [] + + if not isinstance(value, str): + print( + "Warning! %s macro must contain " + "a list of files separated by ':'" % files_define + ) + return [] + + for f in value.split(":"): + if not f: + continue + result.append(join("$PROJECT_DIR", f)) + + for f in result: + if not isfile(env.subst(f)): + print('Warning! Could not find file "%s"' % basename(f)) + + return result + + +def remove_config_define(cppdefines, files_type): + for define in cppdefines: + if files_type in define: + env.ProcessUnFlags("-D%s" % "=".join(str(d) for d in define)) + return + + +def prepare_file(source, target, env): + filepath = source[0].get_abspath() + shutil.copy(filepath, filepath + ".piobkp") + + with open(filepath, "rb+") as fp: + fp.seek(-1, SEEK_END) + if fp.read(1) != "\0": + fp.seek(0, SEEK_CUR) + fp.write(b"\0") + + +def revert_original_file(source, target, env): + filepath = source[0].get_abspath() + if isfile(filepath + ".piobkp"): + shutil.move(filepath + ".piobkp", filepath) + + +def embed_files(files, files_type): + for f in files: + filename = basename(f) + ".txt.o" + file_target = env.TxtToBin(join("$BUILD_DIR", filename), f) + env.Depends("$PIOMAINPROG", file_target) + if files_type == "embed_txtfiles": + env.AddPreAction(file_target, prepare_file) + env.AddPostAction(file_target, revert_original_file) + env.AppendUnique(PIOBUILDFILES=[env.File(join("$BUILD_DIR", filename))]) + + +def transform_to_asm(target, source, env): + files = [join("$BUILD_DIR", s.name + ".S") for s in source] + return files, source + + +env.Append( + BUILDERS=dict( + TxtToBin=Builder( + action=env.VerboseAction( + " ".join( + [ + "xtensa-lx106-elf-objcopy", + "--input-target", + "binary", + "--output-target", + "elf32-xtensa-le", + "--binary-architecture", + "xtensa", + "--rename-section", + ".data=.rodata.embedded", + "$SOURCE", + "$TARGET", + ] + ), + "Converting $TARGET", + ), + suffix=".txt.o", + ), + FileToAsm=Builder( + action=env.VerboseAction( + " ".join( + [ + join( + env.PioPlatform().get_package_dir("tool-cmake") or "", + "bin", + "cmake", + ), + "-DDATA_FILE=$SOURCE", + "-DSOURCE_FILE=$TARGET", + "-DFILE_TYPE=$FILE_TYPE", + "-P", + join( + env.PioPlatform().get_package_dir("framework-esp8266-rtos-sdk") or "", + "tools", + "cmake", + "scripts", + "data_file_embed_asm.cmake", + ), + ] + ), + "Generating assembly for $TARGET", + ), + emitter=transform_to_asm, + single_source=True, + ), + ) +) + + +flags = env.get("CPPDEFINES") +for files_type in ("embed_txtfiles", "embed_files"): + if ( + "COMPONENT_" + files_type.upper() not in env.Flatten(flags) + and "build." + files_type not in board + ): + continue + + files = extract_files(flags, files_type) + if "esp8266-rtos-sdk" in env.subst("$PIOFRAMEWORK"): + env.Requires( + join("$BUILD_DIR", "${PROGNAME}.elf"), + env.FileToAsm( + files, + FILE_TYPE="TEXT" if files_type == "embed_txtfiles" else "BINARY", + ), + ) + else: + embed_files(files, files_type) + remove_config_define(flags, files_type) + \ No newline at end of file diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index e979a97..67fbd4e 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -26,6 +26,8 @@ from SCons.Script import COMMAND_LINE_TARGETS, DefaultEnvironment, SConscript +env = DefaultEnvironment() +SConscript("_embed_files.py", exports="env") if "nobuild" not in COMMAND_LINE_TARGETS: SConscript( diff --git a/builder/frameworks/esp8266-nonos-sdk.py b/builder/frameworks/esp8266-nonos-sdk.py index bac9915..9cd62f9 100644 --- a/builder/frameworks/esp8266-nonos-sdk.py +++ b/builder/frameworks/esp8266-nonos-sdk.py @@ -20,11 +20,12 @@ https://github.com/espressif/ESP8266_NONOS_SDK """ -from os.path import isdir, join +from os.path import isdir, join, isfile from SCons.Script import Builder, DefaultEnvironment env = DefaultEnvironment() +SConscript("_embed_files.py", exports="env") platform = env.PioPlatform() FRAMEWORK_DIR = platform.get_package_dir("framework-esp8266-nonos-sdk") @@ -61,7 +62,8 @@ CXXFLAGS=[ "-fno-rtti", "-fno-exceptions", - "-std=c++11" + "-std=c++11", + "-Wno-literal-suffix" ], LINKFLAGS=[ @@ -83,16 +85,8 @@ CPPPATH=[ join(FRAMEWORK_DIR, "include"), - join(FRAMEWORK_DIR, "extra_include"), join(FRAMEWORK_DIR, "driver_lib", "include"), - join(FRAMEWORK_DIR, "include", "espressif"), - join(FRAMEWORK_DIR, "include", "lwip"), - join(FRAMEWORK_DIR, "include", "lwip", "ipv4"), - join(FRAMEWORK_DIR, "include", "lwip", "ipv6"), - join(FRAMEWORK_DIR, "include", "nopoll"), - join(FRAMEWORK_DIR, "include", "ssl"), - join(FRAMEWORK_DIR, "include", "json"), - join(FRAMEWORK_DIR, "include", "openssl") + join(FRAMEWORK_DIR, "third_party", "include") ], LIBPATH=[ @@ -102,61 +96,112 @@ LIBS=[ "airkiss", "at", "c", "crypto", "driver", "espnow", "gcc", "json", - "lwip", "main", "mbedtls", "mesh", "net80211", "phy", "pp", "pwm", + "lwip", "main", "mbedtls", "net80211", "phy", "pp", "pwm", "smartconfig", "ssl", "upgrade", "wpa", "wpa2", "wps" - ], - - BUILDERS=dict( - ElfToBin=Builder( - action=env.VerboseAction(" ".join([ - '"%s"' % join(platform.get_package_dir("tool-esptool"), "esptool"), - "-eo", "$SOURCE", - "-bo", "${TARGET}", - "-bm", "$BOARD_FLASH_MODE", - "-bf", "${__get_board_f_flash(__env__)}", - "-bz", "${__get_flash_size(__env__)}", - "-bs", ".text", - "-bs", ".data", - "-bs", ".rodata", - "-bc", "-ec", - "-eo", "$SOURCE", - "-es", ".irom0.text", "${TARGET}.irom0text.bin", - "-ec", "-v" - ]), "Building $TARGET"), - suffix=".bin" - ) - ) + ] ) -if not env.BoardConfig().get("build.ldscript", ""): - env.Replace( - LDSCRIPT_PATH=join(FRAMEWORK_DIR, "ld", "eagle.app.v6.ld") - ) -board_flash_size = int(env.BoardConfig().get("upload.maximum_size", 0)) -if board_flash_size > 8388608: - init_data_flash_address = 0xffc000 # for 16 MB -elif board_flash_size > 4194304: - init_data_flash_address = 0x7fc000 # for 8 MB -elif board_flash_size > 2097152: - init_data_flash_address = 0x3fc000 # for 4 MB -elif board_flash_size > 1048576: - init_data_flash_address = 0x1fc000 # for 2 MB -elif board_flash_size > 524288: - init_data_flash_address = 0xfc000 # for 1 MB -else: - init_data_flash_address = 0x7c000 # for 512 kB +################################################################################### +# OTA support + +board = env.BoardConfig() +partitions_csv = board.get("build.partitions", "partitions_singleapp.csv") + +# choose LDSCRIPT_PATH based on OTA +if not board.get("build.ldscript", ""): + if "ota" in partitions_csv: # flash map size >= 5 only!!! + LDSCRIPT_PATH=join(FRAMEWORK_DIR, "ld", "eagle.app.v6.new.2048.ld") + else: + LDSCRIPT_PATH=join(FRAMEWORK_DIR, "ld", "eagle.app.v6.ld") + env.Replace(LDSCRIPT_PATH=LDSCRIPT_PATH) + + +# evaluate SPI_FLASH_SIZE_MAP flag for NONOS_SDK 3.x and set CCFLAG +board_flash_size = int(board.get("upload.maximum_size", 524288)) +flash_size_maps = [0.5, 0.25, 1.0, 0.0, 0.0, 2.0, 4.0, 0.0, 8.0, 16.0] # ignore maps 3 and 4.prefer 5 and 6 +flash_sizes_str = ['512KB','256KB','1MB','2MB','4MB','2MB-c1','4MB-c1','4MB-c2','8MB','16MB'] +try: + flash_size_map = flash_size_maps.index(board_flash_size/1048576) + flash_size_str = flash_sizes_str[flash_size_map] +except: + flash_size_map = 6 + flash_size_str = '4MB-c1' +# for OTA, only size maps 5, 6, 8 and 9 are supported to avoid linking twice for user1 and user2 + +env.Append(CCFLAGS=["-DSPI_FLASH_SIZE_MAP="+str(flash_size_map)]) # NONOS-SDK 3.x user_main.c need it +env.Append(FLASH_SIZE_STR=flash_size_str) # required for custom uploader + + +# create binaries list to upload + +if "ota" in partitions_csv: # if OTA, flash user1 but generate user1 and user2 + boot_bin = join(FRAMEWORK_DIR, "bin", "boot_v1.7.bin") + user_bin = join("$BUILD_DIR", "${PROGNAME}.bin.user1.bin") # firmware.bin.user1.bin # user1.4096.new.6.bin + user_addr = 0x1000 +else: # non ota + boot_bin = join("$BUILD_DIR", "${PROGNAME}.bin") # firmware.bin # eagle.flash.bin + user_bin = join("$BUILD_DIR", "${PROGNAME}.bin.irom0text.bin") # firmware.bin.irom0text.bin # eagle.irom0text.bin + user_addr = 0x10000 + + +# check the init_data_default file to use +esp_init_data_default_file = "esp_init_data_default_v08.bin" # new in NONOS 3.04 +if not isfile(join(FRAMEWORK_DIR, "bin", esp_init_data_default_file)): + esp_init_data_default_file = "esp_init_data_default.bin" + +data_bin = join(FRAMEWORK_DIR, "bin", esp_init_data_default_file) +blank_bin = join(FRAMEWORK_DIR, "bin", "blank.bin") +rf_cal_addr = board_flash_size-0x5000 # 3fb000 for 4M board blank_bin +phy_data_addr = board_flash_size-0x4000 # 3fc000 for 4M board data_bin +sys_param_addr = board_flash_size-0x2000 # 3fe000 for 4M board blank_bin env.Append( FLASH_EXTRA_IMAGES=[ - ("0x10000", join("$BUILD_DIR", "${PROGNAME}.bin.irom0text.bin")), - (hex(init_data_flash_address), - join(FRAMEWORK_DIR, "bin", "esp_init_data_default.bin")), - (hex(init_data_flash_address + 0x2000), - join(FRAMEWORK_DIR, "bin", "blank.bin")) + (hex(0), boot_bin), + (hex(user_addr), user_bin), + (hex(phy_data_addr), data_bin), + (hex(sys_param_addr), blank_bin), + (hex(rf_cal_addr), blank_bin), + ("--flash_mode", "$BOARD_FLASH_MODE"), + ("--flash_freq", "$${__get_board_f_flash(__env__)}m"), + ("--flash_size", "$FLASH_SIZE_STR") # required by NONOS 3.0.4 ] ) +# register genbin.py BUILDER which allows to create OTA files +if "ota" in partitions_csv: # if OTA, flash user1 but generate user1 and user2 + env.Append( + BUILDERS=dict( + ElfToBin=Builder( + action=env.VerboseAction(" ".join([ + '"%s"' % env.subst("$PYTHONEXE"), join(platform.get_package_dir("tool-genbin-esp8266"), "genbin.py"), + "12", # create firmware.bin.user1.bin and firmware.bin.user2.bin + "$BOARD_FLASH_MODE", "${__get_board_f_flash(__env__)}m", "$FLASH_SIZE_STR", + "$SOURCE", "${TARGET}.user1.bin", "${TARGET}.user2.bin" + # could have used espressif naming: user1.4096.new.6.bin or user1.16384.new.9.bin + ]), "Building $TARGET"), + suffix=".bin" + ) + ) + ) +else: + env.Append( + BUILDERS=dict( + ElfToBin=Builder( + action=env.VerboseAction(" ".join([ + '"%s"' % env.subst("$PYTHONEXE"), join(platform.get_package_dir("tool-genbin-esp8266"), "genbin.py"), + "0", # create firmware.bin and firmware.bin.irom0text.bin + "$BOARD_FLASH_MODE", "${__get_board_f_flash(__env__)}m", "$FLASH_SIZE_STR", + "$SOURCE", "${TARGET}", "${TARGET}.irom0text.bin" + ]), "Building $TARGET"), + suffix=".bin" + ) + ) + ) + +################################################################################### + # # Target: Build Driver Library @@ -164,9 +209,10 @@ libs = [] -libs.append(env.BuildLibrary( - join(FRAMEWORK_DIR, "lib", "driver"), - join(FRAMEWORK_DIR, "driver_lib") -)) +if False: + libs.append(env.BuildLibrary( + join(FRAMEWORK_DIR, "lib", "driver"), + join(FRAMEWORK_DIR, "driver_lib") + )) env.Prepend(LIBS=libs) diff --git a/builder/frameworks/esp8266-rtos-sdk.py b/builder/frameworks/esp8266-rtos-sdk.py index 0065c52..192e370 100644 --- a/builder/frameworks/esp8266-rtos-sdk.py +++ b/builder/frameworks/esp8266-rtos-sdk.py @@ -1,173 +1,1367 @@ -# Copyright 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Copyright 2020-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Espressif RTOS-SDK + +Espressif IoT Development Framework for ESP8266 MCU + +https://github.com/espressif/eSP8266_RTOS_SDK +https://docs.espressif.com/projects/esp8266-rtos-sdk/en/latest/ + +Code adapted from Wallace William https://github.com/WallaceWilliam +""" + +import copy +import json +import subprocess +import sys +import os + +import click +import semantic_version + +from SCons.Script import ( + ARGUMENTS, + COMMAND_LINE_TARGETS, + DefaultEnvironment, +) + +from platformio import fs +from platformio.proc import exec_command +from platformio.util import get_systype +from platformio.builder.tools.piolib import ProjectAsLibBuilder +from platformio.package.version import get_original_version, pepver_to_semver + +env = DefaultEnvironment() +env.SConscript("_embed_files.py", exports="env") + +platform = env.PioPlatform() +board = env.BoardConfig() +mcu = board.get("build.mcu", "esp8266") +idf_variant = mcu.lower() + +FRAMEWORK_DIR = platform.get_package_dir("framework-esp8266-rtos-sdk") +TOOLCHAIN_DIR = platform.get_package_dir("toolchain-xtensa") + + +assert os.path.isdir(FRAMEWORK_DIR) +assert os.path.isdir(TOOLCHAIN_DIR) + + +BUILD_DIR = env.subst("$BUILD_DIR") +PROJECT_DIR = env.subst("$PROJECT_DIR") +PROJECT_SRC_DIR = env.subst("$PROJECT_SRC_DIR") +CMAKE_API_REPLY_PATH = os.path.join(".cmake", "api", "v1", "reply") +SDKCONFIG_PATH = board.get( + "build.rtos-sdk.sdkconfig_path", + os.path.join(PROJECT_DIR, "sdkconfig.%s" % env.subst("$PIOENV")), +) + +# replace paths for windows +SDKCONFIG_PATH = SDKCONFIG_PATH.replace("\\","/") + +def set_elftobin(): + env.Append( + # copy CCFLAGS to ASFLAGS (-x assembler-with-cpp mode) + ASFLAGS=env.get("CCFLAGS", [])[:], + + BUILDERS=dict( + ElfToBin=Builder( + action=env.VerboseAction(" ".join([ + '"$PYTHONEXE" "$OBJCOPY"', + "--chip", mcu, + "elf2image", + "--version", "3", + "--flash_mode", "$BOARD_FLASH_MODE", + "--flash_freq", "${__get_board_f_flash(__env__)}m", + "--flash_size", board.get("upload.flash_size", "4MB"), + "-o", "$TARGET", "$SOURCES" + ]), "Building $TARGET"), + suffix=".bin" + ) + ) + ) + +set_elftobin() + + + +def get_project_lib_includes(env): + project = ProjectAsLibBuilder(env, "$PROJECT_DIR") + project.install_dependencies() + project.search_deps_recursive() + + paths = [] + for lb in env.GetLibBuilders(): + if not lb.dependent: + continue + lb.env.PrependUnique(CPPPATH=lb.get_include_dirs()) + paths.extend(lb.env["CPPPATH"]) + + DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=None) + + return paths + + +def is_cmake_reconfigure_required(cmake_api_reply_dir): + cmake_cache_file = os.path.join(BUILD_DIR, "CMakeCache.txt") + cmake_txt_files = [ + os.path.join(PROJECT_DIR, "CMakeLists.txt"), + os.path.join(PROJECT_SRC_DIR, "CMakeLists.txt"), + ] + cmake_preconf_dir = os.path.join(BUILD_DIR, "config") + deafult_sdk_config = os.path.join(PROJECT_DIR, "sdkconfig.defaults") + + for d in (cmake_api_reply_dir, cmake_preconf_dir): + if not os.path.isdir(d) or not os.listdir(d): + return True + if not os.path.isfile(cmake_cache_file): + return True + if not os.path.isfile(os.path.join(BUILD_DIR, "build.ninja")): + return True + if not os.path.isfile(SDKCONFIG_PATH) or os.path.getmtime( + SDKCONFIG_PATH + ) > os.path.getmtime(cmake_cache_file): + return True + if os.path.isfile(deafult_sdk_config) and os.path.getmtime( + deafult_sdk_config + ) > os.path.getmtime(cmake_cache_file): + return True + if any( + os.path.getmtime(f) > os.path.getmtime(cmake_cache_file) + for f in cmake_txt_files + [cmake_preconf_dir, FRAMEWORK_DIR] + ): + return True + + return False + + + +def is_proper_idf_project(): + return all( + os.path.isfile(path) + for path in ( + os.path.join(PROJECT_DIR, "CMakeLists.txt"), + os.path.join(PROJECT_SRC_DIR, "CMakeLists.txt"), + ) + ) + + +def collect_src_files(): + return [ + f + for f in env.MatchSourceFiles("$PROJECT_SRC_DIR", env.get("SRC_FILTER")) + if not f.endswith((".h", ".hpp")) + ] + + +def normalize_path(path): + if PROJECT_DIR in path: + path = path.replace(PROJECT_DIR, "${CMAKE_SOURCE_DIR}") + return fs.to_unix_path(path) + + +def create_default_project_files(): + root_cmake_tpl = """cmake_minimum_required(VERSION 3.16.0) +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(%s) +""" + prj_cmake_tpl = """# This file was automatically generated for projects +# without default 'CMakeLists.txt' file. + +FILE(GLOB_RECURSE app_sources %s/*.*) + +idf_component_register(SRCS ${app_sources}) +""" + + if not os.listdir(PROJECT_SRC_DIR): + # create a default main file to make CMake happy during first init + with open(os.path.join(PROJECT_SRC_DIR, "main.c"), "w") as fp: + fp.write("void app_main() {}") + + project_dir = PROJECT_DIR + if not os.path.isfile(os.path.join(project_dir, "CMakeLists.txt")): + with open(os.path.join(project_dir, "CMakeLists.txt"), "w") as fp: + fp.write(root_cmake_tpl % os.path.basename(project_dir)) + + project_src_dir = PROJECT_SRC_DIR + if not os.path.isfile(os.path.join(project_src_dir, "CMakeLists.txt")): + with open(os.path.join(project_src_dir, "CMakeLists.txt"), "w") as fp: + fp.write(prj_cmake_tpl % normalize_path(PROJECT_SRC_DIR)) + + +def get_cmake_code_model(src_dir, build_dir, extra_args=None): + cmake_api_dir = os.path.join(build_dir, ".cmake", "api", "v1") + cmake_api_query_dir = os.path.join(cmake_api_dir, "query") + cmake_api_reply_dir = os.path.join(cmake_api_dir, "reply") + query_file = os.path.join(cmake_api_query_dir, "codemodel-v2") + + if not os.path.isfile(query_file): + os.makedirs(os.path.dirname(query_file)) + open(query_file, "a").close() # create an empty file + + if not is_proper_idf_project(): + create_default_project_files() + + if is_cmake_reconfigure_required(cmake_api_reply_dir): + run_cmake(src_dir, build_dir, extra_args) + + if not os.path.isdir(cmake_api_reply_dir) or not os.listdir(cmake_api_reply_dir): + sys.stderr.write("Error: Couldn't find CMake API response file\n") + env.Exit(1) + + codemodel = {} + for target in os.listdir(cmake_api_reply_dir): + if target.startswith("codemodel-v2"): + with open(os.path.join(cmake_api_reply_dir, target), "r") as fp: + codemodel = json.load(fp) + + assert codemodel["version"]["major"] == 2 + return codemodel + + +def populate_idf_env_vars(idf_env): + idf_env["IDF_PATH"] = FRAMEWORK_DIR + additional_packages = [ + os.path.join(TOOLCHAIN_DIR, "bin"), + platform.get_package_dir("tool-ninja"), + os.path.join(platform.get_package_dir("tool-cmake"), "bin"), + os.path.dirname(env.subst("$PYTHONEXE")), + ] + + if "windows" in get_systype(): + additional_packages.append(platform.get_package_dir("tool-mconf")) + + idf_env["PATH"] = os.pathsep.join(additional_packages + [idf_env["PATH"]]) + + # Some users reported that the `IDF_TOOLS_PATH` var can seep into the + # underlying build system. Unsetting it is a safe workaround. + if "IDF_TOOLS_PATH" in idf_env: + del idf_env["IDF_TOOLS_PATH"] + +def get_target_config(project_configs, target_index, cmake_api_reply_dir): + target_json = project_configs.get("targets")[target_index].get("jsonFile", "") + target_config_file = os.path.join(cmake_api_reply_dir, target_json) + if not os.path.isfile(target_config_file): + sys.stderr.write("Error: Couldn't find target config %s\n" % target_json) + env.Exit(1) + + with open(target_config_file) as fp: + return json.load(fp) + + +def load_target_configurations(cmake_codemodel, cmake_api_reply_dir): + configs = {} + project_configs = cmake_codemodel.get("configurations")[0] + for config in project_configs.get("projects", []): + for target_index in config.get("targetIndexes", []): + target_config = get_target_config( + project_configs, target_index, cmake_api_reply_dir + ) + configs[target_config["name"]] = target_config + + return configs + + +def build_library( + default_env, lib_config, project_src_dir, prepend_dir=None, debug_allowed=True +): + lib_name = lib_config["nameOnDisk"] + lib_path = lib_config["paths"]["build"] + if prepend_dir: + lib_path = os.path.join(prepend_dir, lib_path) + lib_objects = compile_source_files( + lib_config, default_env, project_src_dir, prepend_dir, debug_allowed + ) + return default_env.Library( + target=os.path.join("$BUILD_DIR", lib_path, lib_name), source=lib_objects + ) + + +def get_app_includes(app_config): + plain_includes = [] + sys_includes = [] + cg = app_config["compileGroups"][0] + for inc in cg.get("includes", []): + inc_path = inc["path"] + if inc.get("isSystem", False): + sys_includes.append(inc_path) + else: + plain_includes.append(inc_path) + + return {"plain_includes": plain_includes, "sys_includes": sys_includes} + + +def extract_defines(compile_group): + result = [] + result.extend( + [ + d.get("define").replace('"', '\\"').strip() + for d in compile_group.get("defines", []) + ] + ) + for f in compile_group.get("compileCommandFragments", []): + if f.get("fragment", "").startswith("-D"): + result.append(f["fragment"][2:]) + return result + + +def get_app_defines(app_config): + return extract_defines(app_config["compileGroups"][0]) + + +def extract_link_args(target_config): + def _add_to_libpath(lib_path, link_args): + if lib_path not in link_args["LIBPATH"]: + link_args["LIBPATH"].append(lib_path) + + def _add_archive(archive_path, link_args): + archive_name = os.path.basename(archive_path) + if archive_name not in link_args["LIBS"]: + _add_to_libpath(os.path.dirname(archive_path), link_args) + link_args["LIBS"].append(archive_name) + + link_args = {"LINKFLAGS": [], "LIBS": [], "LIBPATH": [], "__LIB_DEPS": []} + + for f in target_config.get("link", {}).get("commandFragments", []): + fragment = f.get("fragment", "").strip() + fragment_role = f.get("role", "").strip() + if not fragment or not fragment_role: + continue + args = click.parser.split_arg_string(fragment) + if fragment_role == "flags": + link_args["LINKFLAGS"].extend(args) + elif fragment_role == "libraries": + if fragment.startswith("-l"): + link_args["LIBS"].extend(args) + elif fragment.startswith("-L"): + lib_path = fragment.replace("-L", "").strip(" '\"") + _add_to_libpath(lib_path, link_args) + elif fragment.startswith("-") and not fragment.startswith("-l"): + # CMake mistakenly marks LINKFLAGS as libraries + link_args["LINKFLAGS"].extend(args) + elif fragment.endswith(".a"): + archive_path = fragment + # process static archives + if archive_path.startswith(FRAMEWORK_DIR): + # In case of precompiled archives from framework package + _add_archive(archive_path, link_args) + else: + # In case of archives within project + if archive_path.startswith(".."): + # Precompiled archives from project component + _add_archive( + os.path.normpath(os.path.join(BUILD_DIR, archive_path)), + link_args, + ) + else: + # Internally built libraries used for dependency resolution + link_args["__LIB_DEPS"].append(os.path.basename(archive_path)) + + return link_args + + + +def filter_args(args, allowed, ignore=None): + if not allowed: + return [] + + ignore = ignore or [] + result = [] + i = 0 + length = len(args) + while i < length: + if any(args[i].startswith(f) for f in allowed) and not any( + args[i].startswith(f) for f in ignore + ): + result.append(args[i]) + if i + 1 < length and not args[i + 1].startswith("-"): + i += 1 + result.append(args[i]) + i += 1 + return result + + +def get_app_flags(app_config, default_config): + def _extract_flags(config): + flags = {} + for cg in config["compileGroups"]: + flags[cg["language"]] = [] + for ccfragment in cg["compileCommandFragments"]: + fragment = ccfragment.get("fragment", "") + if not fragment.strip() or fragment.startswith("-D"): + continue + flags[cg["language"]].extend( + click.parser.split_arg_string(fragment.strip()) + ) + + return flags + + app_flags = _extract_flags(app_config) + default_flags = _extract_flags(default_config) + + # Flags are sorted because CMake randomly populates build flags in code model + return { + "ASFLAGS": sorted(app_flags.get("ASM", default_flags.get("ASM"))), + "CFLAGS": sorted(app_flags.get("C", default_flags.get("C"))), + "CXXFLAGS": sorted(app_flags.get("CXX", default_flags.get("CXX"))), + } + + +def get_sdk_configuration(): + config_path = os.path.join(BUILD_DIR, "config", "sdkconfig.json") + if not os.path.isfile(config_path): + print('Warning: Could not find "sdkconfig.json" file\n') + + try: + with open(config_path, "r") as fp: + return json.load(fp) + except: + return {} + + + +def load_component_paths(framework_components_dir, ignored_component_prefixes=None): + def _scan_components_from_framework(): + result = [] + for component in os.listdir(framework_components_dir): + component_path = os.path.join(framework_components_dir, component) + if component.startswith(ignored_component_prefixes) or not os.path.isdir( + component_path + ): + continue + result.append(component_path) + + return result + + # First of all, try to load the list of used components from the project description + components = [] + ignored_component_prefixes = ignored_component_prefixes or [] + project_description_file = os.path.join(BUILD_DIR, "project_description.json") + if os.path.isfile(project_description_file): + with open(project_description_file) as fp: + try: + data = json.load(fp) + for path in data.get("build_component_paths", []): + if not os.path.basename(path).startswith( + ignored_component_prefixes + ): + components.append(path) + except: + print( + "Warning: Could not find load components from project description!\n" + ) + + return components or _scan_components_from_framework() + + -""" -ESP8266 RTOS SDK - -ESP8266 SDK based on FreeRTOS, a truly free professional grade RTOS for -microcontrollers - -https://github.com/espressif/ESP8266_RTOS_SDK -""" - -from os.path import isdir, join - -from SCons.Script import Builder, DefaultEnvironment - -env = DefaultEnvironment() -platform = env.PioPlatform() - -FRAMEWORK_DIR = platform.get_package_dir("framework-esp8266-rtos-sdk") -assert isdir(FRAMEWORK_DIR) - -env.Append( - ASFLAGS=[ - "-mlongcalls", - ], - ASPPFLAGS=[ - "-x", "assembler-with-cpp", - ], - - CFLAGS=[ - "-std=gnu99", - "-Wpointer-arith", - "-Wno-implicit-function-declaration", - "-Wl,-EL", - "-fno-inline-functions", - "-nostdlib" - ], - - CCFLAGS=[ - "-Os", # optimize for size - "-mlongcalls", - "-mtext-section-literals", - "-falign-functions=4", - "-U__STRICT_ANSI__", - "-ffunction-sections", - "-fdata-sections" - ], - - CXXFLAGS=[ - "-fno-rtti", - "-fno-exceptions", - "-std=c++11" - ], - - LINKFLAGS=[ - "-Os", - "-nostdlib", - "-Wl,--no-check-sections", - "-Wl,-static", - "-Wl,--gc-sections", - "-u", "call_user_start", - "-u", "_printf_float", - "-u", "_scanf_float" - ], - - CPPDEFINES=[ - ("F_CPU", "$BOARD_F_CPU"), - "__ets__", - "ICACHE_FLASH" - ], - - CPPPATH=[ - join(FRAMEWORK_DIR, "include"), - join(FRAMEWORK_DIR, "extra_include"), - join(FRAMEWORK_DIR, "driver_lib", "include"), - join(FRAMEWORK_DIR, "include", "espressif"), - join(FRAMEWORK_DIR, "include", "lwip"), - join(FRAMEWORK_DIR, "include", "lwip", "ipv4"), - join(FRAMEWORK_DIR, "include", "lwip", "ipv6"), - join(FRAMEWORK_DIR, "include", "nopoll"), - join(FRAMEWORK_DIR, "include", "spiffs"), - join(FRAMEWORK_DIR, "include", "ssl"), - join(FRAMEWORK_DIR, "include", "json"), - join(FRAMEWORK_DIR, "include", "openssl"), - ], - - LIBPATH=[ - join(FRAMEWORK_DIR, "lib"), - join(FRAMEWORK_DIR, "ld") - ], - - LIBS=[ - "cirom", "crypto", "driver", "espconn", "espnow", "freertos", "gcc", - "json", "hal", "lwip", "main", "mesh", "mirom", "net80211", "nopoll", - "phy", "pp", "pwm", "smartconfig", "spiffs", "ssl", "wpa", "wps" - ], - - BUILDERS=dict( - ElfToBin=Builder( - action=env.VerboseAction(" ".join([ - '"%s"' % join(platform.get_package_dir("tool-esptool"), "esptool"), - "-eo", "$SOURCE", - "-bo", "${TARGET}", - "-bm", "$BOARD_FLASH_MODE", - "-bf", "${__get_board_f_flash(__env__)}", - "-bz", "${__get_flash_size(__env__)}", - "-bs", ".text", - "-bs", ".data", - "-bs", ".rodata", - "-bc", "-ec", - "-eo", "$SOURCE", - "-es", ".irom0.text", "${TARGET}.irom0text.bin", - "-ec", "-v" - ]), "Building $TARGET"), - suffix=".bin" - ) - ) -) - -if not env.BoardConfig().get("build.ldscript", ""): - env.Replace( - LDSCRIPT_PATH=join(FRAMEWORK_DIR, "ld", "eagle.app.v6.ld"), - ) - -# Extra flash images -board_flash_size = int(env.BoardConfig().get("upload.maximum_size", 0)) -if board_flash_size > 8388608: - init_data_flash_address = 0xffc000 # for 16 MB -elif board_flash_size > 4194304: - init_data_flash_address = 0x7fc000 # for 8 MB -elif board_flash_size > 2097152: - init_data_flash_address = 0x3fc000 # for 4 MB -elif board_flash_size > 1048576: - init_data_flash_address = 0x1fc000 # for 2 MB -elif board_flash_size > 524288: - init_data_flash_address = 0xfc000 # for 1 MB -else: - init_data_flash_address = 0x7c000 # for 512 kB - -env.Append( - FLASH_EXTRA_IMAGES=[ - ("0x20000", join("$BUILD_DIR", "${PROGNAME}.bin.irom0text.bin")), - (hex(init_data_flash_address), - join(FRAMEWORK_DIR, "bin", "esp_init_data_default.bin")), - (hex(init_data_flash_address + 0x2000), - join(FRAMEWORK_DIR, "bin", "blank.bin")) - ] -) - -# -# Target: Build Driver Library -# - -libs = [] - -libs.append(env.BuildLibrary( - join(FRAMEWORK_DIR, "lib", "driver"), - join(FRAMEWORK_DIR, "driver_lib") -)) - -env.Prepend(LIBS=libs) +def extract_linker_script_fragments(framework_components_dir, sdk_config): + # Hardware-specific components are excluded from search and added manually below + project_components = load_component_paths( + framework_components_dir, ignored_component_prefixes=("...") # ignored_component_prefixes can't be None + ) + result = [] + for component_path in project_components: + linker_fragment = os.path.join(component_path, "linker.lf") + if os.path.isfile(linker_fragment): + result.append(linker_fragment) + + if not result: + sys.stderr.write("Error: Failed to extract paths to linker script fragments\n") + env.Exit(1) + + # Add extra linker fragments + for fragment in ( + os.path.join("esp8266", "ld", "esp8266_fragments.lf"), + os.path.join("esp8266", "ld", "esp8266_bss_fragments.lf"), + ): + result.append(os.path.join(framework_components_dir, fragment)) + + if sdk_config.get("SPIRAM_CACHE_WORKAROUND", False): + result.append( + os.path.join( + framework_components_dir, "newlib", "esp32-spiram-rom-functions-c.lf" + ) + ) + + if board.get("build.rtos-sdk.extra_lf_files", ""): + result.extend( + [ + lf if os.path.isabs(lf) else os.path.join(PROJECT_DIR, lf) + for lf in board.get("build.rtos-sdk.extra_lf_files").splitlines() + if lf.strip() + ] + ) + + return result + + +def create_custom_libraries_list(ldgen_libraries_file, ignore_targets): + if not os.path.isfile(ldgen_libraries_file): + sys.stderr.write("Error: Couldn't find the list of framework libraries\n") + env.Exit(1) + + pio_libraries_file = ldgen_libraries_file + "_pio" + + if os.path.isfile(pio_libraries_file): + return pio_libraries_file + + lib_paths = [] + with open(ldgen_libraries_file, "r") as fp: + lib_paths = fp.readlines() + + with open(pio_libraries_file, "w") as fp: + for lib_path in lib_paths: + if all( + "lib%s.a" % t.replace("__idf_", "") not in lib_path + for t in ignore_targets + ): + fp.write(lib_path) + + return pio_libraries_file + + +def generate_project_ld_script(sdk_config, ignore_targets=None): + ignore_targets = ignore_targets or [] + linker_script_fragments = extract_linker_script_fragments( + os.path.join(FRAMEWORK_DIR, "components"), sdk_config + ) + + # Create a new file to avoid automatically generated library entry as files from + # this library are built internally by PlatformIO + libraries_list = create_custom_libraries_list( + os.path.join(BUILD_DIR, "ldgen_libraries"), ignore_targets + ) + + args = { + "script": os.path.join(FRAMEWORK_DIR, "tools", "ldgen", "ldgen.py"), + "config": SDKCONFIG_PATH, + "fragments": " ".join( + ['"%s"' % fs.to_unix_path(f) for f in linker_script_fragments] + ), + "kconfig": os.path.join(FRAMEWORK_DIR, "Kconfig"), + "env_file": os.path.join("$BUILD_DIR", "config.env"), + "libraries_list": libraries_list, + "objdump": os.path.join( + TOOLCHAIN_DIR, + "bin", + env.subst("$CC").replace("-gcc", "-objdump"), + ), + } + + cmd = ( + '"$PYTHONEXE" "{script}" --input $SOURCE ' + '--config "{config}" --fragments {fragments} --output $TARGET ' + '--kconfig "{kconfig}" --env-file "{env_file}" ' + '--libraries-file "{libraries_list}" ' + '--objdump "{objdump}"' + ).format(**args) + + os.environ["IDF_PATH"] = FRAMEWORK_DIR + return env.Command( + os.path.join("$BUILD_DIR", "esp8266.project.ld"), + os.path.join( + FRAMEWORK_DIR, + "components", + "esp8266", + "ld", + "esp8266.project.ld.in" + ), + env.VerboseAction(cmd, "Generating project linker script $TARGET"), + ) + + +def prepare_build_envs(config, default_env, debug_allowed=True): + build_envs = [] + target_compile_groups = config.get("compileGroups") + + is_build_type_debug = "debug" in env.GetBuildType() and debug_allowed + for cg in target_compile_groups: + includes = [] + sys_includes = [] + for inc in cg.get("includes", []): + inc_path = inc["path"] + if inc.get("isSystem", False): + sys_includes.append(inc_path) + else: + includes.append(inc_path) + + defines = extract_defines(cg) + compile_commands = cg.get("compileCommandFragments", []) + build_env = default_env.Clone() + for cc in compile_commands: + build_flags = cc.get("fragment") + if not build_flags.startswith("-D"): + build_env.AppendUnique(**build_env.ParseFlags(build_flags)) + build_env.AppendUnique(CPPDEFINES=defines, CPPPATH=includes) + if sys_includes: + build_env.Append(CCFLAGS=[("-isystem", inc) for inc in sys_includes]) + build_env.ProcessUnFlags(default_env.get("BUILD_UNFLAGS")) + if is_build_type_debug: + build_env.ConfigureDebugFlags() + build_envs.append(build_env) + + return build_envs + + +def compile_source_files( + config, default_env, project_src_dir, prepend_dir=None, debug_allowed=True +): + build_envs = prepare_build_envs(config, default_env, debug_allowed) + objects = [] + components_dir = fs.to_unix_path(os.path.join(FRAMEWORK_DIR, "components")) + for source in config.get("sources", []): + if source["path"].endswith(".rule"): + continue + compile_group_idx = source.get("compileGroupIndex") + if compile_group_idx is not None: + src_dir = config["paths"]["source"] + if not os.path.isabs(src_dir): + src_dir = os.path.join(project_src_dir, config["paths"]["source"]) + src_path = source.get("path") + if not os.path.isabs(src_path): + # For cases when sources are located near CMakeLists.txt + src_path = os.path.join(project_src_dir, src_path) + + obj_path = os.path.join("$BUILD_DIR", prepend_dir or "") + if src_path.startswith(components_dir): + obj_path = os.path.join( + obj_path, os.path.relpath(src_path, components_dir) + ) + else: + if not os.path.isabs(source["path"]): + obj_path = os.path.join(obj_path, source["path"]) + else: + obj_path = os.path.join(obj_path, os.path.basename(src_path)) + + objects.append( + build_envs[compile_group_idx].StaticObject( + target=os.path.splitext(obj_path)[0] + ".o", + source=os.path.realpath(src_path), + ) + ) + + return objects + + +def run_tool(cmd): + idf_env = os.environ.copy() + populate_idf_env_vars(idf_env) + + result = exec_command(cmd, env=idf_env) + if result["returncode"] != 0: + sys.stderr.write(result["out"] + "\n") + sys.stderr.write(result["err"] + "\n") + env.Exit(1) + + if int(ARGUMENTS.get("PIOVERBOSE", 0)): + print(result["out"]) + print(result["err"]) + + +def RunMenuconfig(target, source, env): + idf_env = os.environ.copy() + populate_idf_env_vars(idf_env) + + rc = subprocess.call( + [ + os.path.join(platform.get_package_dir("tool-cmake"), "bin", "cmake"), + "--build", + BUILD_DIR, + "--target", + "menuconfig", + ], + env=idf_env, + ) + + if rc != 0: + sys.stderr.write("Error: Couldn't execute 'menuconfig' target.\n") + env.Exit(1) + + +def run_cmake(src_dir, build_dir, extra_args=None): + cmd = [ + os.path.join(platform.get_package_dir("tool-cmake") or "", "bin", "cmake"), + "-S", + src_dir, + "-B", + build_dir, + "-G", + "Ninja", + ] + + if extra_args: + cmd.extend(extra_args) + + run_tool(cmd) + + +def find_lib_deps(components_map, elf_config, link_args, ignore_components=None): + ignore_components = ignore_components or [] + result = [ + components_map[d["id"]]["lib"] + for d in elf_config.get("dependencies", []) + if components_map.get(d["id"], {}) + and not d["id"].startswith(tuple(ignore_components)) + ] + + implicit_lib_deps = link_args.get("__LIB_DEPS", []) + for component in components_map.values(): + component_config = component["config"] + if ( + component_config["type"] not in ("STATIC_LIBRARY", "OBJECT_LIBRARY") + or component_config["name"] in ignore_components + ): + continue + if ( + component_config["nameOnDisk"] in implicit_lib_deps + and component["lib"] not in result + ): + result.append(component["lib"]) + + return result + + +def build_bootloader(sdk_config): + bootloader_src_dir = os.path.join( + FRAMEWORK_DIR, "components", "bootloader", "subproject" + ) + code_model = get_cmake_code_model( + bootloader_src_dir, + os.path.join(BUILD_DIR, "bootloader"), + [ + "-DIDF_TARGET=" + idf_variant, + "-DPYTHON_DEPS_CHECKED=1", + "-DPYTHON=" + env.subst("$PYTHONEXE"), + "-DIDF_PATH=" + FRAMEWORK_DIR, + "-DSDKCONFIG=" + SDKCONFIG_PATH, + "-DLEGACY_INCLUDE_COMMON_HEADERS=", + "-DEXTRA_COMPONENT_DIRS=" + + os.path.join(FRAMEWORK_DIR, "components", "bootloader"), + ], + ) + + if not code_model: + sys.stderr.write("Error: Couldn't find code model for bootloader\n") + env.Exit(1) + + target_configs = load_target_configurations( + code_model, + os.path.join(BUILD_DIR, "bootloader", ".cmake", "api", "v1", "reply"), + ) + + elf_config = get_project_elf(target_configs) + if not elf_config: + sys.stderr.write( + "Error: Couldn't load the main firmware target of the project\n" + ) + env.Exit(1) + + bootloader_env = env.Clone() + components_map = get_components_map( + target_configs, ["STATIC_LIBRARY", "OBJECT_LIBRARY"] + ) + + build_components( + bootloader_env, + components_map, + bootloader_src_dir, + "bootloader", + debug_allowed=sdk_config.get("BOOTLOADER_COMPILER_OPTIMIZATION_DEBUG", False), + ) + link_args = extract_link_args(elf_config) + extra_flags = filter_args(link_args["LINKFLAGS"], ["-T", "-u"]) + link_args["LINKFLAGS"] = sorted( + list(set(link_args["LINKFLAGS"]) - set(extra_flags)) + ) + + bootloader_env.MergeFlags(link_args) + bootloader_env.Append(LINKFLAGS=extra_flags) + bootloader_libs = find_lib_deps(components_map, elf_config, link_args) + + bootloader_env.Prepend(__RPATH="-Wl,--start-group ") + bootloader_env.Append( + CPPDEFINES=["__BOOTLOADER_BUILD"], _LIBDIRFLAGS=" -Wl,--end-group" + ) + + return bootloader_env.ElfToBin( + os.path.join("$BUILD_DIR", "bootloader"), + bootloader_env.Program( + os.path.join("$BUILD_DIR", "bootloader.elf"), bootloader_libs + ), + ) + + +def get_targets_by_type(target_configs, target_types, ignore_targets=None): + ignore_targets = ignore_targets or [] + result = [] + for target_config in target_configs.values(): + if ( + target_config["type"] in target_types + and target_config["name"] not in ignore_targets + ): + result.append(target_config) + + return result + + +def get_components_map(target_configs, target_types, ignore_components=None): + result = {} + for config in get_targets_by_type(target_configs, target_types, ignore_components): + if "nameOnDisk" not in config: + config["nameOnDisk"] = "lib%s.a" % config["name"] + result[config["id"]] = {"config": config} + + return result + + +def build_components( + env, components_map, project_src_dir, prepend_dir=None, debug_allowed=True +): + for k, v in components_map.items(): + components_map[k]["lib"] = build_library( + env, v["config"], project_src_dir, prepend_dir, debug_allowed + ) + + +def get_project_elf(target_configs): + exec_targets = get_targets_by_type(target_configs, ["EXECUTABLE"]) + if len(exec_targets) > 1: + print( + "Warning: Multiple elf targets found. The %s will be used!" + % exec_targets[0]["name"] + ) + + return exec_targets[0] + + +def generate_default_component(): + # Used to force CMake generate build environments for all supported languages + + prj_cmake_tpl = """# Warning! Do not delete this auto-generated file. +file(GLOB component_sources *.c* *.S) +idf_component_register(SRCS ${component_sources}) +""" + dummy_component_path = os.path.join(FRAMEWORK_DIR, "components", "__pio_env") + if os.path.isdir(dummy_component_path): + return dummy_component_path + + os.makedirs(dummy_component_path) + + for ext in (".cpp", ".c", ".S"): + dummy_file = os.path.join(dummy_component_path, "__dummy" + ext) + if not os.path.isfile(dummy_file): + open(dummy_file, "a").close() + + component_cmake = os.path.join(dummy_component_path, "CMakeLists.txt") + if not os.path.isfile(component_cmake): + with open(component_cmake, "w") as fp: + fp.write(prj_cmake_tpl) + + return dummy_component_path + + +def find_default_component(target_configs): + for config in target_configs: +# if "__pio_env" in config: + if "__idf___pio_env" == config: + return config + sys.stderr.write( + "Error! Failed to find the default IDF component with build information for " + "generic files.\nCheck that the `EXTRA_COMPONENT_DIRS` option is not overridden " + "in your CMakeLists.txt.\nSee an example with an extra component here " + "https://docs.platformio.org/en/latest/frameworks/espidf.html#esp-idf-components\n" + ) + env.Exit(1) + + +def create_version_file(): + version_file = os.path.join(FRAMEWORK_DIR, "version.txt") + if not os.path.isfile(version_file): + with open(version_file, "w") as fp: + package_version = platform.get_package_version("framework-esp8266-rtos-sdk") + fp.write(get_original_version(package_version) or package_version) + + + +def generate_empty_partition_image(binary_path, image_size): + empty_partition = env.Command( + binary_path, + None, + env.VerboseAction( + '"$PYTHONEXE" "%s" %s $TARGET' + % ( + os.path.join( + FRAMEWORK_DIR, + "components", + "partition_table", + "gen_empty_partition.py", + ), + image_size, + ), + "Generating an empty partition $TARGET", + ), + ) + + env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", empty_partition) + + +def get_partition_info(pt_path, pt_offset, pt_params): + if not os.path.isfile(pt_path): + sys.stderr.write( + "Missing partition table file `%s`\n" % os.path.basename(pt_path) + ) + env.Exit(1) + + cmd = [ + env.subst("$PYTHONEXE"), + os.path.join(FRAMEWORK_DIR, "components", "partition_table", "parttool.py"), + "-q", + "--partition-table-offset", + hex(pt_offset), + "--partition-table-file", + pt_path, + "get_partition_info", + "--info", + "size", + "offset", + ] + + if pt_params["name"] == "boot": + cmd.append("--partition-boot-default") + else: + cmd.extend( + [ + "--partition-type", + pt_params["type"], + "--partition-subtype", + pt_params["subtype"], + ] + ) + + result = exec_command(cmd) + if result["returncode"] != 0: + sys.stderr.write( + "Couldn't extract information for %s/%s from the partition table\n" + % (pt_params["type"], pt_params["subtype"]) + ) + sys.stderr.write(result["out"] + "\n") + sys.stderr.write(result["err"] + "\n") + env.Exit(1) + + size = offset = 0 + if result["out"].strip(): + size, offset = result["out"].strip().split(" ", 1) + + return {"size": size, "offset": offset} + + +def get_app_partition_offset(pt_table, pt_offset): + # Get the default boot partition offset + app_params = get_partition_info(pt_table, pt_offset, {"name": "boot"}) + return app_params.get("offset", "0x10000") + + +def install_python_deps(): + def _get_installed_pip_packages(): + result = {} + packages = {} + pip_output = subprocess.check_output( + [env.subst("$PYTHONEXE"), "-m", "pip", "list", "--format=json"] + ) + try: + packages = json.loads(pip_output) + except: + print("Warning! Couldn't extract the list of installed Python packages.") + return {} + for p in packages: + result[p["name"]] = pepver_to_semver(p["version"]) + + return result + + deps = { + # https://github.com/platformio/platform-espressif32/issues/635 + "cryptography": ">=2.1.4,<35.0.0", + "future": ">=0.15.2", + "pyparsing": ">=2.0.3,<2.4.0", + "setuptools": ">0" + } + + installed_packages = _get_installed_pip_packages() + packages_to_install = [] + for package, spec in deps.items(): + if package not in installed_packages: + packages_to_install.append(package) + else: + version_spec = semantic_version.Spec(spec) + if not version_spec.match(installed_packages[package]): + packages_to_install.append(package) + + if packages_to_install: + env.Execute( + env.VerboseAction( + ( + '"$PYTHONEXE" -m pip install -U --force-reinstall ' + + " ".join(['"%s%s"' % (p, deps[p]) for p in packages_to_install]) + ), + "Installing RTOS-SDK's Python dependencies", + ) + ) + + # a special "esp-windows-curses" python package is required on Windows for Menuconfig + if "windows" in get_systype(): + if "esp-windows-curses" not in installed_packages and False: + env.Execute( + env.VerboseAction( + '$PYTHONEXE -m pip install "file://%s/tools/kconfig_new/esp-windows-curses" windows-curses' + % FRAMEWORK_DIR, + "Installing windows-curses package", + ) + ) + + +# +# RTOS-SDK requires Python packages with specific versions +# + +install_python_deps() + +# RTOS-SDK package doesn't contain .git folder, instead package version is specified +# in a special file "version.h" in the root folder of the package + +create_version_file() + +# Generate a default component with dummy C/C++/ASM source files in the framework +# folder. This component is used to force the IDF build system generate build +# information for generic C/C++/ASM sources regardless of whether such files are used in project + +generate_default_component() + + +# +# Generate final linker script +# + +if not board.get("build.ldscript", "") or True: + linker_script = env.Command( + os.path.join("$BUILD_DIR", "esp8266_out.ld"), + board.get( + "build.rtos-sdk.ldscript", + os.path.join( + FRAMEWORK_DIR, + "components", + "esp8266", + "ld", + "esp8266.ld"), + ), + env.VerboseAction( + '$CC -I"$BUILD_DIR/config" -C -P -x c -E $SOURCE -DAPP_OFFSET=$ENV_APP_OFFSET -DAPP_SIZE=$ENV_APP_SIZE -o $TARGET', + "Generating LD script $TARGET", + ), + ) + env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", linker_script) + env.Replace(LDSCRIPT_PATH="esp8266_out.ld") + + +# +# Current build script limitations +# + +if any(" " in p for p in (FRAMEWORK_DIR, BUILD_DIR)): + sys.stderr.write("Error: Detected a whitespace character in project paths.\n") + env.Exit(1) + + +if not os.path.isdir(PROJECT_SRC_DIR): + sys.stderr.write( + "Error: Missing the `%s` folder with project sources.\n" + % os.path.basename(PROJECT_SRC_DIR) + ) + env.Exit(1) + +if env.subst("$SRC_FILTER"): + print( + ( + "Warning: the 'src_filter' option cannot be used with RTOS-SDK. Select source " + "files to build in the project CMakeLists.txt file.\n" + ) + ) + +if os.path.isfile(os.path.join(PROJECT_SRC_DIR, "sdkconfig.h")): + print( + "Warning! Starting with RTOS-SDK v3.0, new project structure is required: \n" + "https://docs.platformio.org/en/latest/frameworks/espidf.html#project-structure" + ) + + +# +# Initial targets loading +# + +# By default 'main' folder is used to store source files. In case when a user has +# default 'src' folder we need to add this as an extra component. If there is no 'main' +# folder CMake won't generate dependencies properly +extra_components = [] +if PROJECT_SRC_DIR != os.path.join(PROJECT_DIR, "main"): + extra_components.append(PROJECT_SRC_DIR) + +print("Reading CMake configuration...") +project_codemodel = get_cmake_code_model( + PROJECT_DIR, + BUILD_DIR, + [ + "-DIDF_TARGET=" + idf_variant, + "-DPYTHON_DEPS_CHECKED=1", + "-DEXTRA_COMPONENT_DIRS:PATH=" + ";".join(extra_components), + "-DPYTHON=" + env.subst("$PYTHONEXE"), + "-DSDKCONFIG=" + SDKCONFIG_PATH, + ] + + click.parser.split_arg_string(board.get("build.cmake_extra_args", "")), +) + +# At this point the sdkconfig file should be generated by the underlying build system +assert os.path.isfile(SDKCONFIG_PATH), ( + "Missing auto-generated SDK configuration file `%s`" % SDKCONFIG_PATH +) + +if not project_codemodel: + sys.stderr.write("Error: Couldn't find code model generated by CMake\n") + env.Exit(1) + +target_configs = load_target_configurations( + project_codemodel, os.path.join(BUILD_DIR, CMAKE_API_REPLY_PATH) +) + +sdk_config = get_sdk_configuration() + +project_target_name = "__idf_%s" % os.path.basename(PROJECT_SRC_DIR) +if project_target_name not in target_configs: + sys.stderr.write("Error: Couldn't find the main target of the project!\n") + env.Exit(1) + +if project_target_name != "__idf_main" and "__idf_main" in target_configs: + sys.stderr.write( + ( + "Warning! Detected two different targets with project sources. Please use " + "either %s or specify 'main' folder in 'platformio.ini' file.\n" + % project_target_name + ) + ) + env.Exit(1) + +project_ld_scipt = generate_project_ld_script( + sdk_config, [project_target_name, "__pio_env"] +) +env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", project_ld_scipt) + +elf_config = get_project_elf(target_configs) +default_config_name = find_default_component(target_configs) +framework_components_map = get_components_map( + target_configs, + ["STATIC_LIBRARY", "OBJECT_LIBRARY"], + [project_target_name, default_config_name], +) + +build_components(env, framework_components_map, PROJECT_DIR) + +if not elf_config: + sys.stderr.write("Error: Couldn't load the main firmware target of the project\n") + env.Exit(1) + +for component_config in framework_components_map.values(): + env.Depends(project_ld_scipt, component_config["lib"]) + +project_config = target_configs.get(project_target_name, {}) +default_config = target_configs.get(default_config_name, {}) +project_defines = get_app_defines(project_config) +project_flags = get_app_flags(project_config, default_config) +link_args = extract_link_args(elf_config) +app_includes = get_app_includes(elf_config) + +# +# Compile bootloader +# + +env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", build_bootloader(sdk_config)) + +# +# Target: RTOS-SDK menuconfig +# + +env.AddPlatformTarget( + "menuconfig", + None, + [env.VerboseAction(RunMenuconfig, "Running menuconfig...")], + "Run Menuconfig", +) + +# +# Process main parts of the framework +# + +libs = find_lib_deps( + framework_components_map, elf_config, link_args, [project_target_name] +) + +# Extra flags which need to be explicitly specified in LINKFLAGS section because SCons +# cannot merge them correctly +extra_flags = filter_args(link_args["LINKFLAGS"], ["-T", "-u"]) +link_args["LINKFLAGS"] = sorted(list(set(link_args["LINKFLAGS"]) - set(extra_flags))) + +# remove the main linker script flags '-T esp32_out.ld' +try: + ld_index = extra_flags.index("esp8266_out.ld") + extra_flags.pop(ld_index) + extra_flags.pop(ld_index - 1) +except: + print("Warning! Couldn't find the main linker script in the CMake code model.") + +# +# Process project sources +# + + +# Remove project source files from following build stages as they're +# built as part of the framework +def _skip_prj_source_files(node): + if node.srcnode().get_path().lower().startswith(PROJECT_SRC_DIR.lower()): + return None + return node + + +env.AddBuildMiddleware(_skip_prj_source_files) + +# +# Generate partition table +# + +fwpartitions_dir = os.path.join(FRAMEWORK_DIR, "components", "partition_table") +partitions_csv = board.get("build.partitions", "partitions_singleapp.csv") +partition_table_offset = sdk_config.get("PARTITION_TABLE_OFFSET", 0x8000) + +env.Replace( + PARTITIONS_TABLE_CSV=os.path.abspath( + os.path.join(fwpartitions_dir, partitions_csv) + if os.path.isfile(os.path.join(fwpartitions_dir, partitions_csv)) + else partitions_csv + ) +) + +partition_table = env.Command( + os.path.join("$BUILD_DIR", "partitions.bin"), + "$PARTITIONS_TABLE_CSV", + env.VerboseAction( + '"$PYTHONEXE" "%s" -q --flash-size "%s" $SOURCE $TARGET' + % ( + os.path.join(FRAMEWORK_DIR, "components", "partition_table", "gen_esp32part.py"), + board.get("upload.flash_size", "4MB"), + ), + "Generating partitions $TARGET", + ), +) + +env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", partition_table) + +# linker needs APP_OFFSE and APP_SIZE +app_params = get_partition_info(env.subst("$PARTITIONS_TABLE_CSV"), partition_table_offset, {"name": "boot"}) +app_offset = app_params.get("offset") +app_size = app_params.get("size") +env.Replace( + ENV_APP_OFFSET=app_offset, + ENV_APP_SIZE=app_size, +) + +# +# Main environment configuration +# + +project_flags.update(link_args) +env.MergeFlags(project_flags) +env.Prepend( + CPPPATH=app_includes["plain_includes"], + CPPDEFINES=project_defines, + LINKFLAGS=extra_flags, + LIBS=libs, + FLASH_EXTRA_IMAGES=[ + ("0x0000", os.path.join("$BUILD_DIR", "bootloader.bin")), + ("0x8000", os.path.join("$BUILD_DIR", "partitions.bin")), + ], +) + + +# Project files should be compiled only when a special +# option is enabled when running 'test' command +if "__test" not in COMMAND_LINE_TARGETS or env.GetProjectOption( + "test_build_project_src" +): + project_env = env.Clone() + if project_target_name != "__idf_main": + # Manually add dependencies to CPPPATH since RTOS-SDK build system doesn't generate + # this info if the folder with sources is not named 'main' + # https://docs.espressif.com/projects/esp8266-rtos-sdk/en/latest/api-guides/build-system.html?highlight=main#example-project + project_env.AppendUnique(CPPPATH=app_includes["plain_includes"]) + + # Add include dirs from PlatformIO build system to project CPPPATH so + # they're visible to PIOBUILDFILES + project_env.Append( + CPPPATH=["$PROJECT_INCLUDE_DIR", "$PROJECT_SRC_DIR"] + + get_project_lib_includes(env) + ) + + env.Append( + PIOBUILDFILES=compile_source_files( + target_configs.get(project_target_name), + project_env, + project_env.subst("$PROJECT_DIR"), + ) + ) + + +# +# To embed firmware checksum a special argument for esptool.py is required +# + +action = copy.deepcopy(env["BUILDERS"]["ElfToBin"].action) +#action.cmd_list = env["BUILDERS"]["ElfToBin"].action.cmd_list.replace( +# "-o", "--elf-sha256-offset 0xb0 -o") +env["BUILDERS"]["ElfToBin"].action = action + + +# +# Process OTA partition and image +# + +ota_partition_params = get_partition_info( + env.subst("$PARTITIONS_TABLE_CSV"), + partition_table_offset, + {"name": "ota", "type": "data", "subtype": "ota"}, +) + +if ota_partition_params["size"] and ota_partition_params["offset"]: + # Generate an empty image if OTA is enabled in partition table + ota_partition_image = os.path.join("$BUILD_DIR", "ota_data_initial.bin") + generate_empty_partition_image(ota_partition_image, ota_partition_params["size"]) + + env.Append( + FLASH_EXTRA_IMAGES=[ + ( + board.get( + "upload.ota_partition_offset", ota_partition_params["offset"] + ), + ota_partition_image, + ) + ] + ) diff --git a/builder/main.py b/builder/main.py index fa32383..b24ab90 100644 --- a/builder/main.py +++ b/builder/main.py @@ -33,6 +33,9 @@ def _get_board_f_flash(env): frequency = str(frequency).replace("L", "") return int(int(frequency) / 1000000) +def _get_board_flash_mode(env): + mode = env.subst("$BOARD_FLASH_MODE") + return mode def _parse_size(value): if isinstance(value, int): @@ -149,6 +152,7 @@ def get_esptoolpy_reset_flags(resetmethod): env.Replace( __get_flash_size=_get_flash_size, __get_board_f_flash=_get_board_f_flash, + __get_board_flash_mode=_get_board_flash_mode, AR="xtensa-lx106-elf-ar", AS="xtensa-lx106-elf-as", @@ -182,6 +186,8 @@ def get_esptoolpy_reset_flags(resetmethod): "--chip", "esp8266", "--port", '"$UPLOAD_PORT"' ], + ESPTOOL=join( + platform.get_package_dir("tool-esptoolpy") or "", "esptool.py"), ERASETOOL=join( platform.get_package_dir("tool-esptoolpy") or "", "esptool.py"), ERASECMD='"$PYTHONEXE" "$ERASETOOL" $ERASEFLAGS erase_flash', @@ -189,6 +195,13 @@ def get_esptoolpy_reset_flags(resetmethod): PROGSUFFIX=".elf" ) +if "esp8266-rtos-sdk" in env.subst("$PIOFRAMEWORK"): + env.Replace( +# OBJCOPY=join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py") + OBJCOPY=join(platform.get_package_dir("framework-esp8266-rtos-sdk"), "components", "esptool_py", "esptool", "esptool.py"), #"xtensa-lx106-elf-objcopy", + ) + + # Allow user to override via pre:script if env.get("PROGNAME", "program") == "program": env.Replace(PROGNAME="firmware") @@ -312,10 +325,9 @@ def get_esptoolpy_reset_flags(resetmethod): env.Append(UPLOADERFLAGS=["-s"]) upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")] -elif upload_protocol == "esptool": +elif upload_protocol == "esptool" and not "esp8266-rtos-sdk" in env.subst("$PIOFRAMEWORK"): env.Replace( - UPLOADER=join( - platform.get_package_dir("tool-esptoolpy") or "", "esptool.py"), + UPLOADER="$ESPTOOL", UPLOADERFLAGS=[ "--chip", "esp8266", "--port", '"$UPLOAD_PORT"', @@ -325,6 +337,10 @@ def get_esptoolpy_reset_flags(resetmethod): UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS 0x0 $SOURCE' ) for image in env.get("FLASH_EXTRA_IMAGES", []): + if image[0]=='0x0': # a bootloader is furnished. clean the UPLOADCMD + env.Replace( + UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS' + ) env.Append(UPLOADERFLAGS=[image[0], env.subst(image[1])]) if "uploadfs" in COMMAND_LINE_TARGETS: @@ -349,6 +365,48 @@ def get_esptoolpy_reset_flags(resetmethod): env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE") ] +elif upload_protocol == "esptool" and "esp8266-rtos-sdk" in env.subst("$PIOFRAMEWORK"): + env.Replace( + UPLOADER="$ESPTOOL", +# platform.get_package_dir("tool-esptoolpy") or "", "esptool.py"), + UPLOADERFLAGS=[ + "--chip", "esp8266", + "--port", '"$UPLOAD_PORT"', + "--baud", "$UPLOAD_SPEED", + "--before", "default_reset", + "--after", "hard_reset", + "write_flash", "-z", + "--flash_mode", "${__get_board_flash_mode(__env__)}", + "--flash_freq", "${__get_board_f_flash(__env__)}m", + "--flash_size", "detect" + ], + UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS 0x10000 $SOURCE' + ) + for image in env.get("FLASH_EXTRA_IMAGES", []): + env.Append(UPLOADERFLAGS=[image[0], env.subst(image[1])]) + + if "uploadfs" in COMMAND_LINE_TARGETS: + env.Replace( + UPLOADERFLAGS=[ + "--chip", mcu, + "--port", '"$UPLOAD_PORT"', + "--baud", "$UPLOAD_SPEED", + "--before", "default_reset", + "--after", "hard_reset", + "write_flash", "-z", + "--flash_mode", "$BOARD_FLASH_MODE", + "--flash_size", "detect", + "$SPIFFS_START" + ], + UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS $SOURCE', + ) + + upload_actions = [ + env.VerboseAction(env.AutodetectUploadPort, + "Looking for upload port..."), + env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE") + ] + # custom upload tool elif upload_protocol == "custom": upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")] diff --git a/platform.json b/platform.json index 9a006c3..5e78ef1 100644 --- a/platform.json +++ b/platform.json @@ -55,13 +55,18 @@ "type": "framework", "optional": true, "owner": "platformio", - "version": ">=1.5.0-beta" + "version": "~3.4" }, "framework-esp8266-nonos-sdk": { "type": "framework", "optional": true, "owner": "platformio", - "version": ">=2.1.0" + "version": "~3.0.5" + }, + "tool-genbin-esp8266": { + "type": "uploader", + "owner": "platformio", + "version": "~1.0.0" }, "tool-esptool": { "type": "uploader", @@ -71,7 +76,7 @@ "tool-esptoolpy": { "type": "uploader", "owner": "platformio", - "version": "~1.30000.0" + "version": "~1.40501.0" }, "tool-mkspiffs": { "type": "uploader", @@ -84,6 +89,21 @@ "optional": true, "owner": "platformio", "version": "~1.203.0" + }, + "tool-cmake": { + "optional": true, + "owner": "platformio", + "version": "~3.16.0" + }, + "tool-ninja": { + "optional": true, + "owner": "platformio", + "version": "^1.7.0" + }, + "tool-mconf": { + "optional": true, + "owner": "platformio", + "version": "~1.4060000.0" } } } diff --git a/platform.py b/platform.py index df10412..a18a43b 100644 --- a/platform.py +++ b/platform.py @@ -13,17 +13,22 @@ # limitations under the License. from platformio.public import PlatformBase - +import sys class Espressif8266Platform(PlatformBase): def configure_default_packages(self, variables, targets): framework = variables.get("pioframework", []) - if "arduino" not in framework: - self.packages['toolchain-xtensa']['version'] = "~1.40802.0" if "buildfs" in targets: self.packages['tool-mkspiffs']['optional'] = False self.packages['tool-mklittlefs']['optional'] = False + if "esp8266-rtos-sdk" in framework: + self.packages['toolchain-xtensa']['version'] = '~8.4.0' + for p in self.packages: + if p in ('tool-cmake', 'tool-ninja'): + self.packages[p]['optional'] = False + elif p in ('tool-mconf') and sys.platform.startswith("win"): + self.packages[p]['optional'] = False return super().configure_default_packages(variables, targets) def get_boards(self, id_=None):