From d9de4bfa1c781eb42b7d87391c6b8242e8d27f7a Mon Sep 17 00:00:00 2001 From: Jorge Zapata Date: Wed, 8 May 2024 19:10:05 +0200 Subject: [PATCH 1/3] Draft: Implement the emscripten bootstrapper --- cerbero/bootstrap/bootstrapper.py | 3 +- cerbero/bootstrap/emscripten.py | 65 ++++++++++++++++++++++++++ cerbero/build/filesprovider.py | 5 +- cerbero/enums.py | 5 ++ config/cross-emscripten-wasm.cbc | 10 ++++ config/cross-web-emscripten-wasm32.cbc | 10 ++++ config/cross-web-emscripten-wasm64.cbc | 10 ++++ config/emscripten.config | 43 +++++++++++++++++ config/web.config | 41 ++++++++++++++++ 9 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 cerbero/bootstrap/emscripten.py create mode 100644 config/cross-emscripten-wasm.cbc create mode 100644 config/cross-web-emscripten-wasm32.cbc create mode 100644 config/cross-web-emscripten-wasm64.cbc create mode 100644 config/emscripten.config create mode 100644 config/web.config diff --git a/cerbero/bootstrap/bootstrapper.py b/cerbero/bootstrap/bootstrapper.py index 365b7f2dc..73221747b 100644 --- a/cerbero/bootstrap/bootstrapper.py +++ b/cerbero/bootstrap/bootstrapper.py @@ -86,10 +86,11 @@ def __new__(klass, config, system, toolchains, build_tools, offline, assume_yes) return bs -from cerbero.bootstrap import linux, windows, android, osx, ios # noqa: E402 +from cerbero.bootstrap import linux, windows, android, osx, ios, emscripten # noqa: E402 linux.register_all() windows.register_all() android.register_all() osx.register_all() ios.register_all() +emscripten.register_all() diff --git a/cerbero/bootstrap/emscripten.py b/cerbero/bootstrap/emscripten.py new file mode 100644 index 000000000..7d4c21e1d --- /dev/null +++ b/cerbero/bootstrap/emscripten.py @@ -0,0 +1,65 @@ +# cerbero - a multi-platform build system for Open Source software +# Copyright (C) 2024 Fluendo S.A +# Authors: Maxim Dementyev +# Andoni Morales +# Jorge Zapata +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Library General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Library General Public License for more details. +# +# You should have received a copy of the GNU Library General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +from cerbero.bootstrap import BootstrapperBase +from cerbero.bootstrap.bootstrapper import register_toolchain_bootstrapper +from cerbero.enums import Distro +from cerbero.utils import messages as m +from cerbero.utils import shell +import os.path +import shutil + + +EMSDK_VERSION = '3.1.58' +EMSDK_BUNDLE_EXT = '.tar.gz' +EMSDK_BASE_URL = 'https://github.com/emscripten-core/emsdk/archive/refs/tags/%s' + EMSDK_BUNDLE_EXT +EMSDK_CHECKSUMS = {'3.1.58' + EMSDK_BUNDLE_EXT: '6d860f7ae4bd16bfc18f732200e6608cb2aa739f2ddf9c522755cb0d4037f025'} + + +class EmscriptenToolchainBootstrapper(BootstrapperBase): + """ + Bootstrapper for Emscripten builds. + Installs the Emscripten SDK + """ + + def __init__(self, config, offline, assume_yes): + super().__init__(config, offline) + url = EMSDK_BASE_URL % (EMSDK_VERSION) + bundle_name = 'emsdk-' + EMSDK_VERSION + EMSDK_BUNDLE_EXT + urls = (url, bundle_name, EMSDK_CHECKSUMS[os.path.basename(url)]) + self.fetch_urls.append(urls) + self.extract_steps.append((url, True, self.config.home_dir)) + + async def start(self, jobs=0): + sdk_path_after_extract = os.path.join(self.config.home_dir, 'emsdk-' + EMSDK_VERSION) + shutil.rmtree(self.config.toolchain_prefix) # to be sure it's rename and not a move + shutil.move(sdk_path_after_extract, self.config.toolchain_prefix) + m.message('Install Emscripten SDK...') + await shell.async_call_output( + ['./emsdk', 'install', EMSDK_VERSION], cmd_dir=self.config.toolchain_prefix, cpu_bound=False + ) + m.message('Activate Emscripten SDK...') + await shell.async_call_output( + ['./emsdk', 'activate', EMSDK_VERSION], cmd_dir=self.config.toolchain_prefix, cpu_bound=False + ) + + +def register_all(): + register_toolchain_bootstrapper(Distro.EMSCRIPTEN, EmscriptenToolchainBootstrapper) diff --git a/cerbero/build/filesprovider.py b/cerbero/build/filesprovider.py index f9ae30373..68c1097ba 100644 --- a/cerbero/build/filesprovider.py +++ b/cerbero/build/filesprovider.py @@ -149,7 +149,10 @@ class FilesProvider(object): Platform.DARWIN: {'bext': '', 'sregex': _DYLIB_REGEX, 'mext': '.so', 'smext': '.a', 'pext': '.so', 'srext': '.dylib'}, Platform.IOS: {'bext': '', 'sregex': _DYLIB_REGEX, - 'mext': '.so', 'smext': '.a', 'pext': '.so', 'srext': '.dylib'}} + 'mext': '.so', 'smext': '.a', 'pext': '.so', 'srext': '.dylib'}, + Platform.WEB: {'bext': '.js', 'sregex': _DYLIB_REGEX, + 'mext': '.so', 'smext': '.a', 'pext': '.so', 'srext': '.dylib'}, + } # Match static gstreamer plugins, GIO modules, etc. _FILES_STATIC_PLUGIN_REGEX = re.compile(r'lib/.+/lib(gst|)([^/.]+)\.a') diff --git a/cerbero/enums.py b/cerbero/enums.py index fb91189fa..70f0d3726 100644 --- a/cerbero/enums.py +++ b/cerbero/enums.py @@ -31,6 +31,7 @@ class Platform: DARWIN = 'darwin' ANDROID = 'android' IOS = 'ios' + WEB = 'web' class Architecture: @@ -43,6 +44,9 @@ class Architecture: ARMv7 = 'armv7' ARMv7S = 'armv7s' ARM64 = 'arm64' + WASM = 'wasm' + WASM32 = 'wasm32' + WASM64 = 'wasm64' @staticmethod def is_arm(arch): @@ -72,6 +76,7 @@ class Distro: ANDROID = 'android' GENTOO = 'gentoo' NONE = 'none' + EMSCRIPTEN = 'emscripten' class DistroVersion: diff --git a/config/cross-emscripten-wasm.cbc b/config/cross-emscripten-wasm.cbc new file mode 100644 index 000000000..503e0df09 --- /dev/null +++ b/config/cross-emscripten-wasm.cbc @@ -0,0 +1,10 @@ +# FLUENDO S.A. +# Copyright (C) <2024> +# Cross-compilation config file for Emscripten WASM. + +from cerbero.config import Platform, Architecture, Distro + +target_platform = Platform.EMSCRIPTEN +target_distro = Distro.EMSCRIPTEN +target_distro_version = None +target_arch = Architecture.WASM diff --git a/config/cross-web-emscripten-wasm32.cbc b/config/cross-web-emscripten-wasm32.cbc new file mode 100644 index 000000000..82b5244f5 --- /dev/null +++ b/config/cross-web-emscripten-wasm32.cbc @@ -0,0 +1,10 @@ +# FLUENDO S.A. +# Copyright (C) <2024> +# Cross-compilation config file for Emscripten WASM 32 bits. + +from cerbero.config import Platform, Architecture, Distro + +target_platform = Platform.WEB +target_distro = Distro.EMSCRIPTEN +target_distro_version = None +target_arch = Architecture.WASM32 diff --git a/config/cross-web-emscripten-wasm64.cbc b/config/cross-web-emscripten-wasm64.cbc new file mode 100644 index 000000000..ebb96c2b7 --- /dev/null +++ b/config/cross-web-emscripten-wasm64.cbc @@ -0,0 +1,10 @@ +# FLUENDO S.A. +# Copyright (C) <2024> +# Cross-compilation config file for Emscripten WASM 64 bits. + +from cerbero.config import Platform, Architecture, Distro + +target_platform = Platform.WEB +target_distro = Distro.EMSCRIPTEN +target_distro_version = None +target_arch = Architecture.WASM64 diff --git a/config/emscripten.config b/config/emscripten.config new file mode 100644 index 000000000..6bf0d7ea5 --- /dev/null +++ b/config/emscripten.config @@ -0,0 +1,43 @@ +# FLUENDO S.A. +# Copyright (C) <2024> +# Platform config file for Emscripten. + +import os +import os.path + +# These variants are enabled by default, but not available on our platform +variants.rust = False +variants.mingw = False + +# unfortunately, variants.rust = False is not enough, there is no check to use cargo +# in "cerbero/build/build.py", _get_meson_target_file_contents() +#cargo_home = None + +if not toolchain_prefix: + # the bundle contains 'emsdk-X.Y.Z' as a root folder + # we need to unify it, because have no that info on config level + toolchain_prefix = os.path.join(home_dir, "emsdk") + +# after emsdk activate, we've got .emscripten config file +dot_emscripten_config = os.path.join(toolchain_prefix, '.emscripten') +if os.path.exists(dot_emscripten_config): + # after execution, this config file defines: + # BINARYEN_ROOT, EMSCRIPTEN_ROOT, LLVM_ROOT, NODE_JS + os.environ['EM_CONFIG'] = dot_emscripten_config + exec(open(dot_emscripten_config).read(), globals()) + + # emconfigure and emmake prepare the environment for build, + # see get_building_env() from emsdk/upstream/emscripten/tools/building.py + # that is used in those wrappers. + var_path_execs = ( + ('CC', EMSCRIPTEN_ROOT, 'emcc'), + ('CXX', EMSCRIPTEN_ROOT, 'em++'), + ('AR', EMSCRIPTEN_ROOT, 'emar'), + ('LD', EMSCRIPTEN_ROOT, 'emcc'), + ('NM', LLVM_ROOT, 'llvm-nm'), + ('LDSHARED', EMSCRIPTEN_ROOT, 'emcc'), + ('RANLIB', EMSCRIPTEN_ROOT, 'emranlib'), + ) + + for var,path,exe in var_path_execs: + env[var] = path + '/' + exe diff --git a/config/web.config b/config/web.config new file mode 100644 index 000000000..7f2122546 --- /dev/null +++ b/config/web.config @@ -0,0 +1,41 @@ +# FLUENDO S.A. +# Copyright (C) <2024> +# Platform config file for the Web platform. +# It contains sensitive enviroment configuration that +# shouldn't be modified unless you know what you are doing. +# PLEASE, DO NOT EDIT THIS FILE + +import os +import os.path +from cerbero.config import Distro, FatalError +from cerbero.utils import EnvValueCmd + +if target_distro != Distro.EMSCRIPTEN: + raise FatalError('EMSCRIPTEN is the only Distro supported for the WEB platform') + +# These variants are enabled by default, but not available on our platform +variants.rust = False + +if not toolchain_prefix: + toolchain_prefix = os.path.join(home_dir, "emsdk") + +dot_emscripten_config = os.path.join(toolchain_prefix, '.emscripten') +# This file might be missing if we are bootstrapping +if os.path.exists(dot_emscripten_config): + # Parse the .emscripten config file to fetch the installed + # sdk enviroment. + # after execution, this config file defines: + # BINARYEN_ROOT, EMSCRIPTEN_ROOT, LLVM_ROOT, NODE_JS + os.environ['EM_CONFIG'] = dot_emscripten_config + exec(open(dot_emscripten_config).read(), globals()) + + def add_cmd(var, prefix, command): + env[var] = os.path.join(prefix, command) + + add_cmd('CC', EMSCRIPTEN_ROOT, 'emcc') + add_cmd('CXX', EMSCRIPTEN_ROOT, 'em++') + add_cmd('AR', EMSCRIPTEN_ROOT, 'emar') + add_cmd('LD', EMSCRIPTEN_ROOT, 'emcc') + add_cmd('NM', LLVM_ROOT, 'llvm-nm') + add_cmd('LDSHARED', EMSCRIPTEN_ROOT, 'emcc'), + add_cmd('RANLIB', EMSCRIPTEN_ROOT, 'emranlib') \ No newline at end of file From 977100864b073f4540b68bd0697e6448d4f9e1f6 Mon Sep 17 00:00:00 2001 From: "Maxim P. DEMENTYEV" Date: Tue, 11 Jun 2024 18:33:28 +0200 Subject: [PATCH 2/3] meson: don't try to setup rust if the variant is disabled --- cerbero/build/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cerbero/build/build.py b/cerbero/build/build.py index 0247dff13..a0e970327 100644 --- a/cerbero/build/build.py +++ b/cerbero/build/build.py @@ -948,7 +948,7 @@ def merge_env(old_env, new_env): binaries['qmake6'] = [self.config.qt6_qmake_path] # Point meson to rustc with correct arguments to ensure it's detected when cross-compiling - if self.config.cargo_home: + if self.config.variants.rust and self.config.cargo_home: target_triple = self.config.rust_triple(self.config.target_arch, self.config.target_platform, self.using_msvc()) binaries['rust'] = [self.config.cargo_home + '/bin/rustc', '--target', target_triple] From dbf63904641fd2f2f1f3d6f0cd1f4cb02e53287b Mon Sep 17 00:00:00 2001 From: Andoni Morales Alastruey Date: Wed, 12 Jun 2024 11:48:05 +0200 Subject: [PATCH 3/3] web: force static libraries for all recipes --- cerbero/build/build.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cerbero/build/build.py b/cerbero/build/build.py index a0e970327..8aba03c9f 100644 --- a/cerbero/build/build.py +++ b/cerbero/build/build.py @@ -322,6 +322,8 @@ class Build(object): def __init__(self): self._properties_keys = [] + if self.config.target_platform == Platform.WEB: + self.library_type = LibraryType.STATIC @modify_environment def get_env(self, var, default=None):