Release engineering improvements (wled#1844)

* version set from package.json and release bin names

* support direnv virtualenv

* versioned PlatformIO environment

* matrix support for parralel CI

* gather artifacts

* release on tagging

* minor scripts formatting

Author: andyshinn

.envrc

@@ -0,0 +1 @@
+layout python-venv python3

.github/workflows/wled-ci.yml

@@ -3,10 +3,37 @@ name: PlatformIO CI
 on: [push, pull_request]
 
 jobs:
-  build:
 
+  get_default_envs:
+    name: Gather Environments
     runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Cache pip
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - uses: actions/setup-python@v2
+    - name: Install PlatformIO
+      run: pip install -r requirements.txt
+    - name: Get default environments
+      id: envs
+      run: |
+        echo "::set-output name=environments::$(pio project config --json-output | jq -cr '.[0][1][0][1]')"
+    outputs:
+      environments: ${{ steps.envs.outputs.environments }}
+
 
+  build:
+    name: Build Enviornments
+    runs-on: ubuntu-latest
+    needs: get_default_envs
+    strategy:
+      matrix:
+        environment: ${{ fromJSON(needs.get_default_envs.outputs.environments) }}
     steps:
     - uses: actions/checkout@v2
     - name: Cache pip
@@ -24,8 +51,36 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
     - name: Install PlatformIO
-      run: |
-        python -m pip install --upgrade pip
-        pip install --upgrade platformio
-    - name: Run PlatformIO
-      run: pio run
+      run: pip install -r requirements.txt
+    - name: Build firmware
+      env:
+        WLED_RELEASE: True
+      run: pio run -e ${{ matrix.environment }}
+    - uses: actions/upload-artifact@v2
+      with:
+        name: firmware-${{ matrix.environment }}
+        path: |
+          build_output/firmware/*.bin
+          build_output/firmware/*.gz
+    - uses: actions/upload-artifact@v2
+      if: startsWith(github.ref, 'refs/tags/')
+      with:
+        name: firmware-release
+        path: build_output/release/*.bin
+  release:
+    name: Create Release
+    runs-on: ubuntu-latest
+    needs: [get_default_envs, build]
+    if: startsWith(github.ref, 'refs/tags/')
+    steps:
+    - uses: actions/download-artifact@v2
+      with:
+        name: firmware-release
+    - name: Create draft release
+      uses: softprops/action-gh-release@v1
+      with:
+        draft: True
+        files: |
+          *.bin
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -14,3 +14,4 @@
 .clang-format
 node_modules
 .idea
+.direnv

pio-scripts/gzip-firmware.py

@@ -0,0 +1,69 @@
+Import('env')
+import os
+import shutil
+import gzip
+
+OUTPUT_DIR = "build_output{}".format(os.path.sep)
+
+def _get_cpp_define_value(env, define):
+    define_list = [item[-1] for item in env["CPPDEFINES"] if item[0] == define]
+
+    if define_list:
+        return define_list[0]
+
+    return None
+
+def _create_dirs(dirs=["firmware", "map"]):
+    # check if output directories exist and create if necessary
+    if not os.path.isdir(OUTPUT_DIR):
+        os.mkdir(OUTPUT_DIR)
+
+    for d in dirs:
+        if not os.path.isdir("{}{}".format(OUTPUT_DIR, d)):
+            os.mkdir("{}{}".format(OUTPUT_DIR, d))
+
+def bin_rename_copy(source, target, env):
+    _create_dirs()
+    variant = env["PIOENV"]
+
+    # create string with location and file names based on variant
+    map_file = "{}map{}{}.map".format(OUTPUT_DIR, os.path.sep, variant)
+    bin_file = "{}firmware{}{}.bin".format(OUTPUT_DIR, os.path.sep, variant)
+
+    release_name = _get_cpp_define_value(env, "WLED_RELEASE_NAME")
+
+    if release_name and os.getenv("WLED_RELEASE"):
+        _create_dirs(["release"])
+        version = _get_cpp_define_value(env, "WLED_VERSION")
+        release_file = "{}release{}WLED_{}_{}.bin".format(OUTPUT_DIR, os.path.sep, version, release_name)
+        shutil.copy(str(target[0]), release_file)
+
+    # check if new target files exist and remove if necessary
+    for f in [map_file, bin_file]:
+        if os.path.isfile(f):
+            os.remove(f)
+
+    # copy firmware.bin to firmware/<variant>.bin
+    shutil.copy(str(target[0]), bin_file)
+
+    # copy firmware.map to map/<variant>.map
+    if os.path.isfile("firmware.map"):
+        shutil.move("firmware.map", map_file)
+
+def bin_gzip(source, target, env):
+    _create_dirs()
+    variant = env["PIOENV"]
+
+    # create string with location and file names based on variant
+    bin_file = "{}firmware{}{}.bin".format(OUTPUT_DIR, os.path.sep, variant)
+    gzip_file = "{}firmware{}{}.bin.gz".format(OUTPUT_DIR, os.path.sep, variant)
+
+    # check if new target files exist and remove if necessary
+    if os.path.isfile(gzip_file): os.remove(gzip_file)
+
+    # write gzip firmware file
+    with open(bin_file,"rb") as fp:
+        with gzip.open(gzip_file, "wb", compresslevel = 9) as f:
+            shutil.copyfileobj(fp, f)
+
+env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", [bin_rename_copy, bin_gzip])

pio-scripts/name-firmware.py

@@ -0,0 +1,8 @@
+Import('env')
+import json
+
+PACKAGE_FILE = "package.json"
+
+with open(PACKAGE_FILE, "r") as package:
+    version = json.load(package)["version"]
+    env.Append(BUILD_FLAGS=[f"-DWLED_VERSION={version}"])

pio-scripts/output_bins.py

@@ -151,10 +151,11 @@ build_flags = -g
   -DCONFIG_LITTLEFS_FOR_IDF_3_2
 
 [scripts_defaults]
-extra_scripts             = pio-scripts/name-firmware.py
-                            pio-scripts/gzip-firmware.py
-                            pio-scripts/strip-floats.py
-                            pio-scripts/user_config_copy.py
+extra_scripts =
+  pre:pio-scripts/set_version.py
+  post:pio-scripts/output_bins.py
+  post:pio-scripts/strip-floats.py
+  pre:pio-scripts/user_config_copy.py
 
 # ------------------------------------------------------------------------------
 # COMMON SETTINGS:
@@ -213,15 +214,15 @@ platform = ${common.platform_wled_default}
 platform_packages = ${common.platform_packages}
 board_build.ldscript = ${common.ldscript_4m1m}
 build_unflags = ${common.build_unflags}
-build_flags = ${common.build_flags_esp8266}
+build_flags = ${common.build_flags_esp8266} -D WLED_RELEASE_NAME=ESP8266
 
 [env:esp01_1m_full]
 board = esp01_1m
 platform = ${common.platform_wled_default}
 platform_packages = ${common.platform_packages}
 board_build.ldscript = ${common.ldscript_1m128k}
 build_unflags = ${common.build_unflags}
-build_flags = ${common.build_flags_esp8266} -D WLED_DISABLE_OTA
+build_flags = ${common.build_flags_esp8266} -D WLED_RELEASE_NAME=ESP01 -D WLED_DISABLE_OTA
 
 [env:esp07]
 board = esp07
@@ -261,7 +262,7 @@ build_flags = ${common.build_flags_esp8266} -D LEDPIN=1 -D WLED_DISABLE_INFRARED
 board = esp32dev
 platform = [email protected]
 build_unflags = ${common.build_unflags}
-build_flags = ${common.build_flags_esp32}
+build_flags = ${common.build_flags_esp32} -D WLED_RELEASE_NAME=ESP32
 lib_ignore =
   ESPAsyncTCP
   ESPAsyncUDP
@@ -271,7 +272,7 @@ board = esp32-poe
 platform = [email protected]
 upload_speed = 921600
 build_unflags = ${common.build_unflags}
-build_flags = ${common.build_flags_esp32} -D RLYPIN=-1 -D WLED_USE_ETHERNET -D BTNPIN=-1
+build_flags = ${common.build_flags_esp32} -D WLED_RELEASE_NAME=ESP32_Ethernet -D RLYPIN=-1 -D WLED_USE_ETHERNET -D BTNPIN=-1
 lib_ignore =
   ESPAsyncTCP
   ESPAsyncUDP

pio-scripts/set_version.py

@@ -0,0 +1 @@
+platformio

platformio.ini

@@ -0,0 +1,54 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile
+#
+aiofiles==0.6.0
+    # via platformio
+ajsonrpc==1.1.0
+    # via platformio
+bottle==0.12.19
+    # via platformio
+certifi==2020.12.5
+    # via requests
+chardet==4.0.0
+    # via requests
+click==7.1.2
+    # via
+    #   platformio
+    #   uvicorn
+colorama==0.4.4
+    # via platformio
+h11==0.12.0
+    # via
+    #   uvicorn
+    #   wsproto
+idna==2.10
+    # via requests
+ifaddr==0.1.7
+    # via zeroconf
+marshmallow==3.11.1
+    # via platformio
+platformio==5.1.1
+    # via -r requirements.in
+pyelftools==0.27
+    # via platformio
+pyserial==3.5
+    # via platformio
+requests==2.25.1
+    # via platformio
+semantic-version==2.8.5
+    # via platformio
+starlette==0.14.2
+    # via platformio
+tabulate==0.8.9
+    # via platformio
+urllib3==1.26.4
+    # via requests
+uvicorn==0.13.4
+    # via platformio
+wsproto==1.0.0
+    # via platformio
+zeroconf==0.28.8
+    # via platformio

requirements.in

@@ -51,7 +51,7 @@
 //This is generally a terrible idea, but improves boot success on boards with a 3.3v regulator + cap setup that can't provide 400mA peaks
 //#define WLED_DISABLE_BROWNOUT_DET
 
-// Library inclusions. 
+// Library inclusions.
 #include <Arduino.h>
 #ifdef ESP8266
   #include <ESP8266WiFi.h>
@@ -171,8 +171,15 @@
 # define _INIT_N(x) UNPACK x
 #endif
 
+#define STRINGIFY(X) #X
+#define TOSTRING(X) STRINGIFY(X)
+
+#ifndef WLED_VERSION
+  #define WLED_VERSION "dev"
+#endif
+
 // Global Variable definitions
-WLED_GLOBAL char versionString[] _INIT("0.12.0");
+WLED_GLOBAL char versionString[] _INIT(TOSTRING(WLED_VERSION));
 #define WLED_CODENAME "Hikari"
 
 // AP and OTA default passwords (for maximum security change them!)
@@ -203,7 +210,7 @@ WLED_GLOBAL int8_t irPin _INIT(4);
 WLED_GLOBAL int8_t irPin _INIT(IRPIN);
 #endif
 
-//WLED_GLOBAL byte presetToApply _INIT(0); 
+//WLED_GLOBAL byte presetToApply _INIT(0);
 
 WLED_GLOBAL char ntpServerName[33] _INIT("0.wled.pool.ntp.org");   // NTP server to use
 
@@ -224,7 +231,7 @@ WLED_GLOBAL bool noWifiSleep _INIT(false);                         // disabling
     WLED_GLOBAL int ethernetType _INIT(WLED_ETH_DEFAULT);          // ethernet board type
   #else
     WLED_GLOBAL int ethernetType _INIT(WLED_ETH_NONE);             // use none for ethernet board type if default not defined
-  #endif               
+  #endif
 #endif
 
 // LED CONFIG
@@ -548,7 +555,7 @@ WLED_GLOBAL bool e131NewData _INIT(false);
 // led fx library object
 WLED_GLOBAL BusManager busses _INIT(BusManager());
 WLED_GLOBAL WS2812FX strip _INIT(WS2812FX());
-WLED_GLOBAL BusConfig* busConfigs[WLED_MAX_BUSSES] _INIT({nullptr}); //temporary, to remember values from network callback until after 
+WLED_GLOBAL BusConfig* busConfigs[WLED_MAX_BUSSES] _INIT({nullptr}); //temporary, to remember values from network callback until after
 WLED_GLOBAL bool doInitBusses _INIT(false);
 
 // Usermod manager