Fix build system on Windows and test it deliberately

.github/workflows/windows.yml

@@ -93,14 +93,15 @@ jobs:
       - name: Install Python packages
         if: always()
         run: |
-          pip install --upgrade --upgrade-strategy=eager modm scons future
+          pip install --upgrade --upgrade-strategy=eager modm scons future protobuf grpcio-tools
 
       - name: Show Version Information
         if: always()
         run: |
           gcc --version
           g++ --version
           make --version
+          cmake --version
           arm-none-eabi-g++ --version
           avr-g++ --version
           lbuild --version
@@ -115,6 +116,7 @@ jobs:
       - name: Hosted Examples
         if: always()
         shell: bash
+        # Compiled and run, build and git information is generated while building.
         run: |
           (cd examples && python ../tools/scripts/examples_compile.py linux/assert linux/block_device/ram linux/build_info linux/git linux/logger linux/printf linux/etl linux/fiber)
 
@@ -127,13 +129,25 @@ jobs:
       - name: Compile STM32 Examples
         if: always()
         shell: bash
+        # These examples are selected by what can break on Windows.
         run: |
-          (cd examples && python ../tools/scripts/examples_compile.py nucleo_f031k6 nucleo_f103rb nucleo_f303re nucleo_f411re nucleo_f746zg)
-          (cd examples && python ../tools/scripts/examples_compile.py nucleo_g071rb nucleo_l031k6 nucleo_l152re nucleo_l432kc nucleo_g474re)
+          # Preprocessed assembly, which SCons only recognizes as `.sx` there:
+          # RTT, CrashCatcher and LVGL without its Helium assembly. Fibers
+          # only use inline assembly.
+          (cd examples && python ../tools/scripts/examples_compile.py blue_pill_f103/flash stm32f469_discovery/hard_fault nucleo_h563zi/fiber nucleo_l452re/lvgl)
+          # Paths in generated build scripts, where a backslash is an escape
+          # character: Nanopb and image sources. See AVR for XPCC.
+          (cd examples && python ../tools/scripts/examples_compile.py nucleo_f429zi/nanopb stm32_f4ve/gui)
+          # Many object files: LVGL, CMSIS-DSP and Ethernet exceed the command
+          # line length of cmd.exe when archiving, FreeRTOS and TinyUSB do not.
+          (cd examples && python ../tools/scripts/examples_compile.py stm32f469_discovery/lvgl nucleo_f429zi/ethernet nucleo_f429zi/freertos_static nucleo_f429zi/usb_freertos)
+          # CMake and CompilationDB generators
+          (cd examples && python ../tools/scripts/examples_compile.py stm32f469_discovery/assert nucleo_f429zi/cmsis_dsp/convolution stm32_f4ve/flash)
 
       - name: Compile AVR Examples
         if: always()
         shell: bash
+        # Includes the XPCC generator, image sources and CMake.
         run: |
           (cd examples && ../tools/scripts/examples_compile.py avr arduino_nano arduino_uno srxe)
 

examples/avr/xpcc/receiver/project.xml

@@ -1,5 +1,4 @@
 <library>
-  <!-- CI: enable "Windows" not in platform.platform() -->
   <options>
     <option name="modm:target">atmega644-20au</option>
     <option name="modm:platform:core:f_cpu">14.7456M</option>

examples/avr/xpcc/sender/project.xml

@@ -1,5 +1,4 @@
 <library>
-  <!-- CI: enable "Windows" not in platform.platform() -->
   <options>
     <option name="modm:target">atmega644-20au</option>
     <option name="modm:platform:core:f_cpu">14.7456M</option>

ext/lvgl/lvgl.lb

@@ -88,7 +88,8 @@ def build(env):
     env.copy("lvgl/lvgl_private.h")
     env.copy("lvgl/include")
     env.template("lv_conf.h.in")
-    files = env.copy("lvgl/src")
+    # The Helium assembly is for Cortex-M55 only and not preprocessed on Windows
+    files = env.copy("lvgl/src", ignore=env.ignore_files("*.S"))
     env.collect(":build:ccflags", "-Wno-maybe-uninitialized", operations=files)
     env.substitutions = {"has_debug": env.has_module(":debug")}
     env.template("lv_modm_port.cpp.in")

repo.lb

@@ -27,7 +27,7 @@ def StrictVersion(v):
 
 # Check for miminum required lbuild version
 import lbuild
-min_lbuild_version = "1.21.6"
+min_lbuild_version = "1.23.0"
 if StrictVersion(getattr(lbuild, "__version__", "0.1.0")) < StrictVersion(min_lbuild_version):
     print("modm requires at least lbuild v{}, please upgrade!\n"
           "    pip3 install -U lbuild".format(min_lbuild_version))
@@ -193,19 +193,6 @@ def init(repo):
                                          config, host if host in config else "windows"))
 
     # Add Jinja2 filters commonly used in this repository
-    if 'Windows' in platform.platform():
-        def windowsify(path, escape_level):
-            escaped = "\\" * (2 ** escape_level)
-            return path.replace("\\", escaped)
-        def posixify(path):
-            return path.replace("\\", "/")
-    else:
-        def windowsify(path, escape_level):
-            return path
-        def posixify(path):
-            return path
-    repo.add_filter("modm.windowsify", windowsify)
-    repo.add_filter("modm.posixify", posixify)
     repo.add_filter("modm.ord", lambda letter: ord(letter[0].lower()) - ord("a"))
     repo.add_filter("modm.chr", lambda num: chr(num + ord("A")))
     repo.add_filter("modm.digsep", lambda num: f"{num:,}".replace(",", "'"))

src/modm/architecture/interface.hpp.in

@@ -15,7 +15,7 @@
 #include <stdint.h>
 
 %% for header in headers
-#include "{{ header | modm.posixify }}"
+#include "{{ header }}"
 %% endfor
 #include "interface/peripheral.hpp"
 

src/modm/math/math.hpp.in

@@ -13,7 +13,7 @@
 #define	MODM_MATH_HPP
 
 %% for header in headers
-#include "{{ header | modm.posixify }}"
+#include "{{ header }}"
 %% endfor
 
 #endif	// MODM_MATH_HPP

src/modm/platform/platform.hpp.in

@@ -13,5 +13,5 @@
 #include <modm/architecture.hpp>
 
 %% for header in headers
-#include "{{ header | modm.posixify }}"
+#include "{{ header }}"
 %% endfor

src/modm/processing/module.lb

@@ -11,7 +11,6 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 # -----------------------------------------------------------------------------
 
-import os
 
 def init(module):
     module.name = ":processing"
@@ -36,5 +35,5 @@ def build(env):
         env.template("fiber.hpp.in")
 
     env.outbasepath = "modm/src/modm"
-    headers = env.generated_local_files(filterfunc=lambda path: ".hpp" in path and path.count(os.path.sep) == 1)
+    headers = env.generated_local_files(filterfunc=lambda path: ".hpp" in path and path.replace("\\", "/").count("/") == 1)
     env.template("processing.hpp.in", substitutions={"headers": sorted(headers)})

src/modm/processing/processing.hpp.in

@@ -15,7 +15,7 @@
 #define	MODM_PROCESSING_HPP
 
 %% for header in headers
-#include "{{ header | modm.posixify }}"
+#include "{{ header }}"
 %% endfor
 
 #endif	// MODM_PROCESSING_HPP

tools/build_script_generator/cmake/resources/CMakeLists.txt.in

@@ -17,7 +17,7 @@ include(cmake/ModmConfiguration.cmake)
 %% if asm_sources | length
 set_source_files_properties(
 %% for file in asm_sources | sort
-  {{ file | relocate | modm.posixify  }}
+  {{ file | relocate }}
 %% endfor
   PROPERTIES LANGUAGE CXX
 )
@@ -27,7 +27,7 @@ set_source_files_properties(
 %% if sources | length
 add_library({{ repo }} OBJECT
 %% for file, flags in sources | sort
-  {{ file | relocate | modm.posixify  }}
+  {{ file | relocate }}
 %% endfor
 )
 %% else
@@ -36,7 +36,7 @@ add_library({{ repo }} INTERFACE )
 
 %% if per_file_attr | length
 %% for section in per_file_attr | sort
-{{ section | modm.posixify }}
+{{ section }}
 %% endfor
 
 %% endif
@@ -57,12 +57,12 @@ target_include_directories({{ repo }} SYSTEM
   INTERFACE
 %% endif
 %% for path in include_paths | sort
-  {{ path | relocate | modm.posixify  }}
+  {{ path | relocate }}
 %% endfor
 %% if sources | length
   PRIVATE
 %% for path in private_include_paths | sort
-  {{ path | relocate | modm.posixify  }}
+  {{ path | relocate }}
 %% endfor
 %% endif
 )

tools/build_script_generator/cmake/resources/ModmConfiguration.cmake.in

@@ -99,7 +99,7 @@ function(modm_target_config_create target target_arch target_options target_warn
 %% macro generate_flags_for_profile(name, profile)
 set({{ name | upper }}{{ "_" ~ (profile | upper) if profile | length else "" }}
 %% for flag in flags[name][profile] | sort
-    {{ flag | flags_format | modm.posixify }}
+    {{ flag | flags_format }}
 %% endfor
   )
 %% endmacro
@@ -169,10 +169,12 @@ function(modm_targets_create project_name)
   set_target_properties(${project_name}
     PROPERTIES SUFFIX ".elf")
 
+%% if platform != "hosted"
   add_custom_command(TARGET ${project_name}
     POST_BUILD
     COMMAND ${CMAKE_OBJCOPY} -Obinary ${project_name}.elf ${project_name}.bin
     COMMAND ${CMAKE_OBJCOPY} -Oihex ${project_name}.elf ${project_name}.hex)
+%% endif
 
 %% if core.startswith("cortex-m")
   add_custom_command(TARGET ${project_name}

tools/build_script_generator/compilation_db/resources/builder_compilation_db.py.in

@@ -13,7 +13,7 @@ from pathlib import Path
 env = defaultdict(list)
 
 env["CONFIG_PROJECT_NAME"] = "{{ options[":build:project.name"] }}"
-env["CONFIG_BUILD_BASE"] = os.path.abspath("{{ build_path | modm.windowsify(escape_level=1)}}")
+env["CONFIG_BUILD_BASE"] = os.path.abspath("{{ build_path }}")
 env["CONFIG_BUILD_PROFILE"] = "debug" if "--debug" in sys.argv else "release"
 
 env["BUILDPATH"] = os.path.join(env["CONFIG_BUILD_BASE"], env["CONFIG_BUILD_PROFILE"])

tools/build_script_generator/compilation_db/resources/compilation_db.py.in

@@ -103,13 +103,13 @@ if profile == "{{ profile }}":
 
 env["CPPPATH"].extend([
 %% for path in include_paths | sort
-    abspath(r"{{ path | modm.windowsify(escape_level=0) }}"),
+    abspath(r"{{ path }}"),
 %% endfor
 ])
 
 env["FILES"].extend([
 %% for file, flags in sources if not flags | length
-    (r"{{ file | modm.windowsify(escape_level=0) }}", {}),
+    (r"{{ file }}", {}),
 %% endfor
 ])
 %% for file, flags in sources
@@ -120,14 +120,14 @@ flags = { {%- for key, profiles in flags.items() if "" in profiles %}"{{ key | u
 if profile == "{{profile}}": flags["{{ key | upper }}"].extend({{flags}});
         %% endfor
     %% endfor
-env["FILES"].append( (r"{{ file | modm.windowsify(escape_level=0) }}", flags) )
+env["FILES"].append( (r"{{ file }}", flags) )
     %% endif
 %% endfor
 
 env["STATIC_LIBS"].append(
     ("{{repo}}/lib{{repo}}.a", [
 %% for file, flags in sources
-        r"{{ file | modm.windowsify(escape_level=0) }}",
+        r"{{ file }}",
 %% endfor
     ])
 )
@@ -141,6 +141,6 @@ env["LIBS"].extend([
 env["LIBPATH"].extend([
     abspath(join(env["BUILDPATH"], "{{repo}}")),
 %% for library in library_paths | sort
-    abspath(r"{{ library | modm.windowsify(escape_level=0) }}"),
+    abspath(r"{{ library }}"),
 %% endfor
 ])

tools/build_script_generator/openocd.cfg.in

@@ -1,12 +1,12 @@
 %% for path in openocd_search_dirs | sort
-add_script_search_dir {{ path | modm.windowsify(escape_level=1) }}
+add_script_search_dir {{ path }}
 %% endfor
 %# Include the users config before the modm config
 %% if openocd_user_path is defined
-source [find {{ openocd_user_path | modm.windowsify(escape_level=1) }}]
+source [find {{ openocd_user_path }}]
 %% endif
 %% for file in openocd_sources | sort
-source [find {{ file | modm.windowsify(escape_level=1) }}]
+source [find {{ file }}]
 %% endfor
 
 proc modm_itm_log { OUTPUT F_CPU {BAUDRATE 2000000} } {

tools/build_script_generator/scons/module.lb

@@ -153,7 +153,7 @@ def post_build(env):
     has_nanopb = env.has_module(":nanopb")
     repositories = [p for p in env.buildlog.repositories if isdir(env.real_outpath(p, basepath="."))]
     repositories.sort(key=lambda name: "0" if name == "modm" else name)
-    generated_paths = [join(env.relcwdoutpath(""), r) for r in repositories]
+    generated_paths = [env.relcwdoutpath("") + "/" + r for r in repositories]
 
     subs = env.query("::device")
     cache_dir = env["cache_dir"]
@@ -192,8 +192,7 @@ def post_build(env):
         })
     if has_nanopb:
         subs["nanopb_source"] = [env.relcwdoutpath(p) for p in env.get(":nanopb:sources", [])]
-        # forward slashes, since a Windows path like generated\nanopb contains an escape sequence
-        subs["nanopb_path"] = env.relcwdoutpath(env.get(":nanopb:path", ".")).replace("\\", "/")
+        subs["nanopb_path"] = env.relcwdoutpath(env.get(":nanopb:path", "."))
     if subs["platform"] == "avr":
         subs.update(env.query("::avrdude_options"))
     if subs["platform"] == "sam":

tools/build_script_generator/scons/resources/SConscript.in

@@ -33,7 +33,7 @@ env["COMPILER"] = "{{ options.get(":build:compiler", "gcc") }}"
 # SCons tools
 env.Append(toolpath=[
 %% for toolpath in toolpaths | sort
-    abspath("{{ toolpath | modm.windowsify(escape_level=1) }}"),
+    abspath("{{ toolpath }}"),
 %% endfor
 ])
 %% endif
@@ -44,7 +44,7 @@ env.Tool("{{tool}}")
 %% macro generate_flags_for_profile(name, profile, append=False)
 env{% if append %}.Append({{name | upper}}{% else %}["{{name | upper}}"]{% endif %} = [
 %% for flag in flags[name][profile] | sort
-    {{ flag | flags_format | modm.windowsify(escape_level=1) }},
+    {{ flag | flags_format }},
 %% endfor
 ]{% if append %}){% endif -%}
 %% endmacro
@@ -157,22 +157,22 @@ env.AppendUnique(CPPFLAGS=[
 
 env.AppendUnique(CPPPATH=[
 %% for path in (include_paths + system_include_paths) | sort
-    abspath("{{ path | relocate | modm.windowsify(escape_level=1) }}"),
+    abspath("{{ path | relocate }}"),
 %% endfor
 ])
 %% if system_include_paths | length
 # The paths are also in CPPPATH so that SCons scans them for dependencies. The
 # compiler ignores -I for paths that are also passed as -isystem.
 env.AppendUnique(CPPFLAGS=[
 %% for path in system_include_paths | sort
-    "-isystem" + abspath("{{ path | relocate | modm.windowsify(escape_level=1) }}"),
+    "-isystem" + abspath("{{ path | relocate }}"),
 %% endfor
 ])
 %% endif
 
 files = [
 %% for file, flags in sources if not flags | length
-    env.File("{{ file | relocate | modm.windowsify(escape_level=1) }}"),
+    env.File("{{ file | relocate }}"),
 %% endfor
 ]
 %% for file, flags in sources
@@ -183,7 +183,7 @@ flags = { {%- for key, profiles in flags.items() if "" in profiles %}"{{ key | u
 if profile == "{{profile}}": flags["{{ key | upper }}"].extend({{flags}});
         %% endfor
     %% endfor
-files.append(env.Object("{{ file | relocate | modm.windowsify(escape_level=1) }}", **flags))
+files.append(env.Object("{{ file | relocate }}", **flags))
     %% endif
 %% endfor
 
@@ -201,7 +201,7 @@ env.AppendUnique(LIBPATH=[
 %% endif
     abspath(str(library[0].get_dir())),
 %% for library in library_paths | sort
-    abspath("{{ library | relocate | modm.windowsify(escape_level=1) }}"),
+    abspath("{{ library | relocate }}"),
 %% endfor
 ])
 %% if packages | length

tools/build_script_generator/scons/resources/SConstruct.in

@@ -25,12 +25,12 @@ import os
 from os.path import join, abspath
 
 %% if cache_dir | length
-CacheDir(abspath("{{ cache_dir | modm.windowsify(escape_level=1)}}"))
+CacheDir(abspath("{{ cache_dir }}"))
 %% endif
 env = DefaultEnvironment(tools=[], ENV=os.environ)
 env["CONFIG_PROJECT_NAME"] = "{{ options[":build:project.name"] }}"
-env["CONFIG_BUILD_BASE"] = abspath("{{ build_path | modm.windowsify(escape_level=1)}}")
-env["CONFIG_ARTIFACT_PATH"] = abspath("{{ artifact_path | modm.windowsify(escape_level=1)}}")
+env["CONFIG_BUILD_BASE"] = abspath("{{ build_path }}")
+env["CONFIG_ARTIFACT_PATH"] = abspath("{{ artifact_path }}")
 generated_paths = {{ generated_paths }}
 
 # Building all libraries