diff --git a/build_modules/CHANGELOG.md b/build_modules/CHANGELOG.md index 1a4826b345..b7a73f4327 100644 --- a/build_modules/CHANGELOG.md +++ b/build_modules/CHANGELOG.md @@ -1,6 +1,11 @@ +## 5.1.0 + +- Add drivers and state resources required for DDC + Frontend Server compilation. +- Add an option to disable strongly connected components for determining module boundaries. + ## 5.0.18 -- Remove unused dev depencency: `build_runner_core`. +- Remove unused dev dependency: `build_runner_core`. - Allow Dart SDK 3.10.x and 3.11 prerelease. ## 5.0.17 diff --git a/build_modules/lib/build_modules.dart b/build_modules/lib/build_modules.dart index 834333bb4d..37ae14a0ad 100644 --- a/build_modules/lib/build_modules.dart +++ b/build_modules/lib/build_modules.dart @@ -2,17 +2,25 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +export 'src/ddc_names.dart'; export 'src/errors.dart' show MissingModulesException, UnsupportedModules; -export 'src/kernel_builder.dart' - show KernelBuilder, multiRootScheme, reportUnusedKernelInputs; +export 'src/frontend_server_resources.dart' + show frontendServerState, frontendServerStateResource; +export 'src/kernel_builder.dart' show KernelBuilder, reportUnusedKernelInputs; export 'src/meta_module_builder.dart' show MetaModuleBuilder, metaModuleExtension; export 'src/meta_module_clean_builder.dart' show MetaModuleCleanBuilder, metaModuleCleanExtension; export 'src/module_builder.dart' show ModuleBuilder, moduleExtension; +export 'src/module_library.dart' show ModuleLibrary; export 'src/module_library_builder.dart' show ModuleLibraryBuilder, moduleLibraryExtension; export 'src/modules.dart'; export 'src/platform.dart' show DartPlatform; export 'src/scratch_space.dart' show scratchSpace, scratchSpaceResource; -export 'src/workers.dart' show dartdevkDriverResource, maxWorkersPerTask; +export 'src/workers.dart' + show + dartdevkDriverResource, + frontendServerProxyDriverResource, + maxWorkersPerTask, + persistentFrontendServerResource; diff --git a/build_modules/lib/src/common.dart b/build_modules/lib/src/common.dart index 1d90fdcf1e..7c8c8792b1 100644 --- a/build_modules/lib/src/common.dart +++ b/build_modules/lib/src/common.dart @@ -2,9 +2,17 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:io'; + import 'package:build/build.dart'; +import 'package:path/path.dart' as p; import 'package:scratch_space/scratch_space.dart'; +const multiRootScheme = 'org-dartlang-app'; +const webHotReloadOption = 'web-hot-reload'; +final sdkDir = p.dirname(p.dirname(Platform.resolvedExecutable)); +final packagesFilePath = p.join('.dart_tool', 'package_config.json'); + final defaultAnalysisOptionsId = AssetId( 'build_modules', 'lib/src/analysis_options.default.yaml', @@ -16,6 +24,12 @@ String defaultAnalysisOptionsArg(ScratchSpace scratchSpace) => enum ModuleStrategy { fine, coarse } ModuleStrategy moduleStrategy(BuilderOptions options) { + // DDC's Library Bundle module system only supports fine modules since it must + // align with the Frontend Server's library management scheme. + final usesWebHotReload = options.config[webHotReloadOption] as bool? ?? false; + if (usesWebHotReload) { + return ModuleStrategy.fine; + } final config = options.config['strategy'] as String? ?? 'coarse'; switch (config) { case 'coarse': diff --git a/build_web_compilers/lib/src/ddc_names.dart b/build_modules/lib/src/ddc_names.dart similarity index 84% rename from build_web_compilers/lib/src/ddc_names.dart rename to build_modules/lib/src/ddc_names.dart index 5664eaf0c8..c660e8d7af 100644 --- a/build_web_compilers/lib/src/ddc_names.dart +++ b/build_modules/lib/src/ddc_names.dart @@ -1,14 +1,14 @@ -// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. import 'package:path/path.dart' as p; +/// Logic in this file must be synchronized with their namesakes in DDC at: +/// pkg/dev_compiler/lib/src/compiler/js_names.dart + /// Transforms a path to a valid JS identifier. /// -/// This logic must be synchronized with [pathToJSIdentifier] in DDC at: -/// pkg/dev_compiler/lib/src/compiler/module_builder.dart -/// /// For backwards compatibility, if this pattern is changed, /// dev_compiler_bootstrap.dart must be updated to accept both old and new /// patterns. @@ -35,12 +35,11 @@ String toJSIdentifier(String name) { for (var i = 0; i < name.length; i++) { final ch = name[i]; final needsEscape = ch == r'$' || _invalidCharInIdentifier.hasMatch(ch); - if (needsEscape && buffer == null) { - buffer = StringBuffer(name.substring(0, i)); - } - if (buffer != null) { - buffer.write(needsEscape ? '\$${ch.codeUnits.join("")}' : ch); + if (needsEscape) { + buffer ??= StringBuffer(name.substring(0, i)); } + + buffer?.write(needsEscape ? '\$${ch.codeUnits.join("")}' : ch); } final result = buffer != null ? '$buffer' : name; @@ -56,7 +55,11 @@ String toJSIdentifier(String name) { /// Also handles invalid variable names in strict mode, like "arguments". bool invalidVariableName(String keyword, {bool strictMode = true}) { switch (keyword) { - // http://www.ecma-international.org/ecma-262/6.0/#sec-future-reserved-words + // https://262.ecma-international.org/6.0/#sec-reserved-words + case 'true': + case 'false': + case 'null': + // https://262.ecma-international.org/6.0/#sec-keywords case 'await': case 'break': case 'case': @@ -79,7 +82,6 @@ bool invalidVariableName(String keyword, {bool strictMode = true}) { case 'import': case 'in': case 'instanceof': - case 'let': case 'new': case 'return': case 'super': @@ -99,6 +101,7 @@ bool invalidVariableName(String keyword, {bool strictMode = true}) { // http://www.ecma-international.org/ecma-262/6.0/#sec-identifiers-static-semantics-early-errors case 'implements': case 'interface': + case 'let': case 'package': case 'private': case 'protected': diff --git a/build_modules/lib/src/frontend_server_driver.dart b/build_modules/lib/src/frontend_server_driver.dart new file mode 100644 index 0000000000..aa3f7eb655 --- /dev/null +++ b/build_modules/lib/src/frontend_server_driver.dart @@ -0,0 +1,556 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// ignore_for_file: constant_identifier_names + +import 'dart:async'; +import 'dart:collection'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:logging/logging.dart'; +import 'package:path/path.dart' as p; +import 'package:uuid/uuid.dart'; + +import 'common.dart'; + +final _log = Logger('FrontendServerProxy'); +final dartaotruntimePath = p.join(sdkDir, 'bin', 'dartaotruntime'); +final frontendServerSnapshotPath = p.join( + sdkDir, + 'bin', + 'snapshots', + 'frontend_server_aot.dart.snapshot', +); + +/// A driver that proxies build requests to a [PersistentFrontendServer] +/// instance. +class FrontendServerProxyDriver { + PersistentFrontendServer? _frontendServer; + final _requestQueue = Queue<_CompilationRequest>(); + bool _isProcessing = false; + CompilerOutput? _cachedOutput; + + void init(PersistentFrontendServer frontendServer) { + _frontendServer = frontendServer; + } + + /// Sends a recompile request to the Frontend Server at [entrypoint] with + /// [invalidatedFiles]. + /// + /// The initial recompile request is treated as a full compile. + /// + /// [filesToWrite] contains JS files that should be written to the filesystem. + /// If left empty, all files are written. + Future recompileAndRecord( + String entrypoint, + List invalidatedFiles, + Iterable filesToWrite, + ) async { + final compilerOutput = await recompile(entrypoint, invalidatedFiles); + if (compilerOutput == null || + compilerOutput.errorCount != 0 || + compilerOutput.errorMessage != null) { + // Don't update the Frontend Server's state if an error occurred. + return compilerOutput; + } + _frontendServer!.recordFiles(); + if (filesToWrite.isEmpty) { + _frontendServer!.writeAllFiles(); + } else { + for (final file in filesToWrite) { + _frontendServer!.writeFile(file); + } + } + return compilerOutput; + } + + Future compile(String entrypoint) async { + final completer = Completer(); + _requestQueue.add(_CompileRequest(entrypoint, completer)); + if (!_isProcessing) _processQueue(); + return completer.future; + } + + Future recompile( + String entrypoint, + List invalidatedFiles, + ) async { + final completer = Completer(); + _requestQueue.add( + _RecompileRequest(entrypoint, invalidatedFiles, completer), + ); + if (!_isProcessing) _processQueue(); + return completer.future; + } + + void _processQueue() async { + if (_isProcessing || _requestQueue.isEmpty) return; + + _isProcessing = true; + final request = _requestQueue.removeFirst(); + CompilerOutput? output; + try { + if (request is _CompileRequest) { + _cachedOutput = + output = await _frontendServer!.compile(request.entrypoint); + } else if (request is _RecompileRequest) { + // Compile the first [_RecompileRequest] as a [_CompileRequest] to warm + // up the Frontend Server. + if (_cachedOutput == null) { + output = + _cachedOutput = await _frontendServer!.compile( + request.entrypoint, + ); + } else { + output = await _frontendServer!.recompile( + request.entrypoint, + request.invalidatedFiles, + ); + } + } + if (output != null && output.errorCount == 0) { + _frontendServer!.accept(); + } else { + // We must await [reject]'s output, but we swallow the output since it + // doesn't provide useful information. + await _frontendServer!.reject(); + } + request.completer.complete(output); + } catch (e, s) { + request.completer.completeError(e, s); + } + + _isProcessing = false; + if (_requestQueue.isNotEmpty) { + _processQueue(); + } + } + + /// Clears the proxy driver's state between invocations of separate apps. + void reset() { + _requestQueue.clear(); + _isProcessing = false; + _cachedOutput = null; + _frontendServer?.reset(); + } + + Future terminate() async { + await _frontendServer!.shutdown(); + _frontendServer = null; + } +} + +abstract class _CompilationRequest { + final Completer completer; + _CompilationRequest(this.completer); +} + +class _CompileRequest extends _CompilationRequest { + final String entrypoint; + _CompileRequest(this.entrypoint, super.completer); +} + +class _RecompileRequest extends _CompilationRequest { + final String entrypoint; + final List invalidatedFiles; + _RecompileRequest(this.entrypoint, this.invalidatedFiles, super.completer); +} + +/// A single instance of the Frontend Server that persists across +/// compile/recompile requests. +class PersistentFrontendServer { + Process? _server; + final StdoutHandler _stdoutHandler; + final StreamController _stdinController; + final Uri outputDillUri; + final WebMemoryFilesystem _fileSystem; + + PersistentFrontendServer._( + this._server, + this._stdoutHandler, + this._stdinController, + this.outputDillUri, + this._fileSystem, + ); + + static Future start({ + required String sdkRoot, + required Uri fileSystemRoot, + required Uri packagesFile, + }) async { + final outputDillUri = fileSystemRoot.resolve('output.dill'); + // [platformDill] must be passed to the Frontend Server with a 'file:' + // prefix to pass schema checks for Windows drive letters. + final platformDill = Uri.file( + p.join(sdkDir, 'lib', '_internal', 'ddc_outline.dill'), + ); + final args = [ + frontendServerSnapshotPath, + '--sdk-root=$sdkRoot', + '--incremental', + '--target=dartdevc', + '--dartdevc-module-format=ddc', + '--dartdevc-canary', + '--no-js-strongly-connected-components', + '--packages=${packagesFile.toFilePath()}', + '--experimental-emit-debug-metadata', + '--filesystem-scheme=$multiRootScheme', + '--filesystem-root=${fileSystemRoot.toFilePath()}', + '--platform=$platformDill', + '--output-dill=${outputDillUri.toFilePath()}', + '--output-incremental-dill=${outputDillUri.toFilePath()}', + ]; + final process = await Process.start(dartaotruntimePath, args); + final fileSystem = WebMemoryFilesystem(fileSystemRoot); + final stdoutHandler = StdoutHandler(logger: _log); + process.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen(stdoutHandler.handler); + process.stderr.transform(utf8.decoder).listen(stderr.writeln); + + final stdinController = StreamController(); + stdinController.stream.listen(process.stdin.writeln); + + return PersistentFrontendServer._( + process, + stdoutHandler, + stdinController, + outputDillUri, + fileSystem, + ); + } + + Future compile(String entrypoint) async { + _stdoutHandler.reset(); + _stdinController.add('compile $entrypoint'); + return await _stdoutHandler.compilerOutput!.future; + } + + /// Either [accept] or [reject] should be called after every [recompile] call. + Future recompile( + String entrypoint, + List invalidatedFiles, + ) async { + _stdoutHandler.reset(); + final inputKey = const Uuid().v4(); + _stdinController.add('recompile $entrypoint $inputKey'); + for (final file in invalidatedFiles) { + _stdinController.add(file.toString()); + } + _stdinController.add(inputKey); + return await _stdoutHandler.compilerOutput!.future; + } + + void accept() { + _stdinController.add('accept'); + } + + Future reject() async { + _stdoutHandler.reset(expectSources: false); + _stdinController.add('reject'); + return await _stdoutHandler.compilerOutput!.future; + } + + void reset() { + _stdoutHandler.reset(); + _stdinController.add('reset'); + } + + /// Records all modified files into the in-memory filesystem. + void recordFiles() { + final outputDillPath = outputDillUri.toFilePath(); + final codeFile = File('$outputDillPath.sources'); + final manifestFile = File('$outputDillPath.json'); + final sourcemapFile = File('$outputDillPath.map'); + final metadataFile = File('$outputDillPath.metadata'); + _fileSystem.update(codeFile, manifestFile, sourcemapFile, metadataFile); + } + + void writeAllFiles() { + _fileSystem.writeAllFilesToDisk(_fileSystem.jsRootUri); + } + + void writeFile(String fileName) { + _fileSystem.writeFileToDisk(_fileSystem.jsRootUri, fileName); + } + + Future shutdown() async { + _stdinController.add('quit'); + await _server?.exitCode; + await _stdinController.close(); + _server = null; + } +} + +class CompilerOutput { + const CompilerOutput( + this.outputFilename, + this.errorCount, + this.sources, { + this.expressionData, + this.errorMessage, + }); + + final String outputFilename; + final int errorCount; + final List sources; + + /// Non-null for expression compilation requests. + final Uint8List? expressionData; + + /// Non-null when a compilation error was encountered. + final String? errorMessage; +} + +// A simple in-memory filesystem for handling the Frontend Server's compiled +// output. +class WebMemoryFilesystem { + /// The root directory's URI from which JS file are being served. + final Uri jsRootUri; + + final Map files = {}; + final Map sourcemaps = {}; + final Map metadata = {}; + + WebMemoryFilesystem(this.jsRootUri); + + /// Clears all files registered in the filesystem. + void clearWritableState() { + files.clear(); + sourcemaps.clear(); + metadata.clear(); + } + + /// Writes the entirety of this filesystem to [outputDirectoryUri]. + void writeAllFilesToDisk(Uri outputDirectoryUri) { + assert( + Directory.fromUri(outputDirectoryUri).existsSync(), + '$outputDirectoryUri does not exist.', + ); + final filesToWrite = {...files, ...sourcemaps, ...metadata}; + _writeToDisk(outputDirectoryUri, filesToWrite); + } + + /// Writes [fileName] and its associated sourcemap and metadata files to + /// [outputDirectoryUri]. + /// + /// [fileName] is the file path of the compiled JS file being requested. + /// Source maps and metadata files will be pulled in based on this name. + void writeFileToDisk(Uri outputDirectoryUri, String fileName) { + assert( + Directory.fromUri(outputDirectoryUri).existsSync(), + '$outputDirectoryUri does not exist.', + ); + var sourceFile = fileName; + if (sourceFile.startsWith('package:')) { + sourceFile = + 'packages/${sourceFile.substring('package:'.length, sourceFile.length)}'; + } else if (sourceFile.startsWith('$multiRootScheme:///')) { + sourceFile = sourceFile.substring( + '$multiRootScheme:///'.length, + sourceFile.length, + ); + } + final sourceMapFile = '$sourceFile.map'; + final metadataFile = '$sourceFile.metadata'; + final filesToWrite = { + sourceFile: files[sourceFile]!, + sourceMapFile: sourcemaps[sourceMapFile]!, + metadataFile: metadata[metadataFile]!, + }; + + _writeToDisk(outputDirectoryUri, filesToWrite); + } + + /// Writes [rawFilesToWrite] to [outputDirectoryUri], where [rawFilesToWrite] + /// is a map of file paths to their contents. + void _writeToDisk( + Uri outputDirectoryUri, + Map rawFilesToWrite, + ) { + rawFilesToWrite.forEach((path, content) { + final outputFileUri = outputDirectoryUri.resolve(path); + final outputFilePath = outputFileUri.toFilePath().replaceFirst( + '.dart.lib.js', + '.ddc.js', + ); + final outputFile = File(outputFilePath); + outputFile.createSync(recursive: true); + outputFile.writeAsBytesSync(content); + }); + } + + /// Update the filesystem with the provided source and manifest files. + /// + /// Returns the list of updated files. + List update( + File codeFile, + File manifestFile, + File sourcemapFile, + File metadataFile, + ) { + final updatedFiles = []; + final codeBytes = codeFile.readAsBytesSync(); + final sourcemapBytes = sourcemapFile.readAsBytesSync(); + final manifest = Map.castFrom( + json.decode(manifestFile.readAsStringSync()) as Map, + ); + final metadataBytes = metadataFile.readAsBytesSync(); + + for (final filePath in manifest.keys) { + final Map offsets = + Map.castFrom( + manifest[filePath] as Map, + ); + final codeOffsets = (offsets['code'] as List).cast(); + final sourcemapOffsets = + (offsets['sourcemap'] as List).cast(); + final metadataOffsets = + (offsets['metadata'] as List).cast(); + + if (codeOffsets.length != 2 || + sourcemapOffsets.length != 2 || + metadataOffsets.length != 2) { + _log.severe('Invalid manifest byte offsets: $offsets'); + continue; + } + + final codeStart = codeOffsets[0]; + final codeEnd = codeOffsets[1]; + if (codeStart < 0 || codeEnd > codeBytes.lengthInBytes) { + _log.severe('Invalid byte index: [$codeStart, $codeEnd]'); + continue; + } + + final byteView = Uint8List.view( + codeBytes.buffer, + codeStart, + codeEnd - codeStart, + ); + final fileName = + filePath.startsWith('/') ? filePath.substring(1) : filePath; + files[fileName] = byteView; + updatedFiles.add(fileName); + + final sourcemapStart = sourcemapOffsets[0]; + final sourcemapEnd = sourcemapOffsets[1]; + if (sourcemapStart < 0 || sourcemapEnd > sourcemapBytes.lengthInBytes) { + continue; + } + final sourcemapView = Uint8List.view( + sourcemapBytes.buffer, + sourcemapStart, + sourcemapEnd - sourcemapStart, + ); + final sourcemapName = '$fileName.map'; + sourcemaps[sourcemapName] = sourcemapView; + + final metadataStart = metadataOffsets[0]; + final metadataEnd = metadataOffsets[1]; + if (metadataStart < 0 || metadataEnd > metadataBytes.lengthInBytes) { + _log.severe('Invalid byte index: [$metadataStart, $metadataEnd]'); + continue; + } + final metadataView = Uint8List.view( + metadataBytes.buffer, + metadataStart, + metadataEnd - metadataStart, + ); + metadata['$fileName.metadata'] = metadataView; + } + return updatedFiles; + } +} + +enum StdoutState { CollectDiagnostic, CollectDependencies } + +/// Handles stdin/stdout communication with the Frontend Server. +class StdoutHandler { + StdoutHandler({required Logger logger}) : _logger = logger { + reset(); + } + final Logger _logger; + + String? boundaryKey; + StdoutState state = StdoutState.CollectDiagnostic; + Completer? compilerOutput; + final sources = []; + + var _suppressCompilerMessages = false; + var _expectSources = true; + var _errorBuffer = StringBuffer(); + + void handler(String message) { + const kResultPrefix = 'result '; + if (boundaryKey == null && message.startsWith(kResultPrefix)) { + boundaryKey = message.substring(kResultPrefix.length); + return; + } + final messageBoundaryKey = boundaryKey; + if (messageBoundaryKey != null && message.startsWith(messageBoundaryKey)) { + if (_expectSources) { + if (state == StdoutState.CollectDiagnostic) { + state = StdoutState.CollectDependencies; + return; + } + } + if (message.length <= messageBoundaryKey.length) { + compilerOutput?.complete(); + return; + } + final spaceDelimiter = message.lastIndexOf(' '); + final fileName = message.substring( + messageBoundaryKey.length + 1, + spaceDelimiter, + ); + final errorCount = int.parse( + message.substring(spaceDelimiter + 1).trim(), + ); + + final output = CompilerOutput( + fileName, + errorCount, + sources, + expressionData: null, + errorMessage: _errorBuffer.isNotEmpty ? _errorBuffer.toString() : null, + ); + compilerOutput?.complete(output); + return; + } + switch (state) { + case StdoutState.CollectDiagnostic when _suppressCompilerMessages: + _logger.info(message); + _errorBuffer.writeln(message); + case StdoutState.CollectDiagnostic: + _logger.warning(message); + _errorBuffer.writeln(message); + case StdoutState.CollectDependencies: + switch (message[0]) { + case '+': + sources.add(Uri.parse(message.substring(1))); + case '-': + sources.remove(Uri.parse(message.substring(1))); + default: + _logger.warning('Ignoring unexpected prefix for $message uri'); + } + } + } + + // This is needed to get ready to process next compilation result output, + // with its own boundary key and new completer. + void reset({ + bool suppressCompilerMessages = false, + bool expectSources = true, + }) { + boundaryKey = null; + compilerOutput = Completer(); + _suppressCompilerMessages = suppressCompilerMessages; + _expectSources = expectSources; + state = StdoutState.CollectDiagnostic; + _errorBuffer = StringBuffer(); + } +} diff --git a/build_modules/lib/src/frontend_server_resources.dart b/build_modules/lib/src/frontend_server_resources.dart new file mode 100644 index 0000000000..17b5899e4c --- /dev/null +++ b/build_modules/lib/src/frontend_server_resources.dart @@ -0,0 +1,22 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:build/build.dart'; + +/// A persistent shared [FrontendServerState] for DDC workers that interact with +/// the Frontend Server. +final frontendServerState = FrontendServerState(); + +class FrontendServerState { + /// The built app's main entrypoint file. + /// + /// This must be set before any asset builders run when compiling with DDC and + /// hot reload. + late AssetId entrypointAssetId; +} + +/// A shared [Resource] for a [FrontendServerState]. +final frontendServerStateResource = Resource(() async { + return frontendServerState; +}); diff --git a/build_modules/lib/src/kernel_builder.dart b/build_modules/lib/src/kernel_builder.dart index 80644ecc88..222bb5bddb 100644 --- a/build_modules/lib/src/kernel_builder.dart +++ b/build_modules/lib/src/kernel_builder.dart @@ -17,12 +17,11 @@ import 'package:scratch_space/scratch_space.dart'; import 'package:stream_transform/stream_transform.dart'; import '../build_modules.dart'; +import 'common.dart'; import 'errors.dart'; import 'module_cache.dart'; import 'workers.dart'; -const multiRootScheme = 'org-dartlang-app'; - /// A builder which can output kernel files for a given sdk. /// /// This creates kernel files based on [moduleExtension] files, which are what @@ -431,7 +430,7 @@ Future _addRequestArguments( request.arguments.addAll([ '--dart-sdk-summary=${Uri.file(p.join(sdkDir, sdkKernelPath))}', '--output=${outputFile.path}', - '--packages-file=$multiRootScheme:///${p.join('.dart_tool', 'package_config.json')}', + '--packages-file=$multiRootScheme:///$packagesFilePath', '--multi-root-scheme=$multiRootScheme', '--exclude-non-sources', summaryOnly ? '--summary-only' : '--no-summary-only', diff --git a/build_modules/lib/src/module_builder.dart b/build_modules/lib/src/module_builder.dart index 85a621b5b6..f421ff3136 100644 --- a/build_modules/lib/src/module_builder.dart +++ b/build_modules/lib/src/module_builder.dart @@ -7,12 +7,14 @@ import 'dart:async'; import 'package:build/build.dart'; import 'package:collection/collection.dart'; +import 'meta_module_builder.dart'; import 'meta_module_clean_builder.dart'; import 'module_cache.dart'; import 'module_library.dart'; import 'module_library_builder.dart' show moduleLibraryExtension; import 'modules.dart'; import 'platform.dart'; +import 'scratch_space.dart'; /// The extension for serialized module assets. String moduleExtension(DartPlatform platform) => '.${platform.name}.module'; @@ -22,7 +24,15 @@ String moduleExtension(DartPlatform platform) => '.${platform.name}.module'; class ModuleBuilder implements Builder { final DartPlatform _platform; - ModuleBuilder(this._platform) + /// Emits DDC code with the Library Bundle module system, which supports hot + /// reload. + /// + /// If set, this builder will consume raw meta modules (instead of clean). + /// Clean meta modules are only used for DDC's AMD module system due its + /// requirement that self-referential libraries be bundled. + final bool usesWebHotReload; + + ModuleBuilder(this._platform, {this.usesWebHotReload = false}) : buildExtensions = { '.dart': [moduleExtension(_platform)], }; @@ -32,31 +42,31 @@ class ModuleBuilder implements Builder { @override Future build(BuildStep buildStep) async { - final cleanMetaModules = await buildStep.fetchResource(metaModuleCache); + final metaModules = await buildStep.fetchResource(metaModuleCache); + final metaModuleExtensionString = + usesWebHotReload + ? metaModuleExtension(_platform) + : metaModuleCleanExtension(_platform); final metaModule = - (await cleanMetaModules.find( - AssetId( - buildStep.inputId.package, - 'lib/${metaModuleCleanExtension(_platform)}', - ), + (await metaModules.find( + AssetId(buildStep.inputId.package, 'lib/$metaModuleExtensionString'), buildStep, ))!; var outputModule = metaModule.modules.firstWhereOrNull( (m) => m.primarySource == buildStep.inputId, ); - if (outputModule == null) { - final serializedLibrary = await buildStep.readAsString( - buildStep.inputId.changeExtension(moduleLibraryExtension), - ); - final libraryModule = ModuleLibrary.deserialize( - buildStep.inputId, - serializedLibrary, + final serializedLibrary = await buildStep.readAsString( + buildStep.inputId.changeExtension(moduleLibraryExtension), + ); + final libraryModule = ModuleLibrary.deserialize( + buildStep.inputId, + serializedLibrary, + ); + final scratchSpace = await buildStep.fetchResource(scratchSpaceResource); + if (outputModule == null && libraryModule.hasMain) { + outputModule = metaModule.modules.firstWhere( + (m) => m.sources.contains(buildStep.inputId), ); - if (libraryModule.hasMain) { - outputModule = metaModule.modules.firstWhere( - (m) => m.sources.contains(buildStep.inputId), - ); - } } if (outputModule == null) return; final modules = await buildStep.fetchResource(moduleCache); @@ -65,5 +75,13 @@ class ModuleBuilder implements Builder { buildStep, outputModule, ); + if (usesWebHotReload) { + // All sources must be declared before the Frontend Server is invoked, as + // it only accepts the main entrypoint as its compilation target. + await buildStep.trackStage( + 'EnsureAssets', + () => scratchSpace.ensureAssets(outputModule!.sources, buildStep), + ); + } } } diff --git a/build_modules/lib/src/modules.dart b/build_modules/lib/src/modules.dart index 59163ec069..a7e86449e7 100644 --- a/build_modules/lib/src/modules.dart +++ b/build_modules/lib/src/modules.dart @@ -148,6 +148,7 @@ class Module { Future> computeTransitiveDependencies( BuildStep buildStep, { bool throwIfUnsupported = false, + bool computeStronglyConnectedComponents = true, }) async { final modules = await buildStep.fetchResource(moduleCache); final transitiveDeps = {}; @@ -183,13 +184,67 @@ class Module { if (throwIfUnsupported && unsupportedModules.isNotEmpty) { throw UnsupportedModules(unsupportedModules); } - final orderedModules = stronglyConnectedComponents( - transitiveDeps.values, - (m) => m.directDependencies.map((s) => transitiveDeps[s]!), - equals: (a, b) => a.primarySource == b.primarySource, - hashCode: (m) => m.primarySource.hashCode, - ); - return orderedModules.map((c) => c.single).toList(); + + if (computeStronglyConnectedComponents) { + final orderedModules = stronglyConnectedComponents( + transitiveDeps.values, + (m) => m.directDependencies.map((s) => transitiveDeps[s]!), + equals: (a, b) => a.primarySource == b.primarySource, + hashCode: (m) => m.primarySource.hashCode, + ); + return orderedModules.map((c) => c.single).toList(); + } + return transitiveDeps.values.toList(); + } + + /// Returns all [AssetId]s in the transitive dependencies of this module in + /// no specific order. + /// + /// Throws a [MissingModulesException] if there are any missing modules. This + /// typically means that somebody is trying to import a non-existing file. + /// + /// If [throwIfUnsupported] is `true`, then an [UnsupportedModules] + /// will be thrown if there are any modules that are not supported. + Future> computeTransitiveAssets( + BuildStep buildStep, { + bool throwIfUnsupported = false, + }) async { + final modules = await buildStep.fetchResource(moduleCache); + final transitiveDeps = {}; + final modulesToCrawl = {primarySource}; + final missingModuleSources = {}; + final unsupportedModules = {}; + final seenSources = {}; + + while (modulesToCrawl.isNotEmpty) { + final next = modulesToCrawl.last; + modulesToCrawl.remove(next); + if (transitiveDeps.containsKey(next)) continue; + final nextModuleId = next.changeExtension(moduleExtension(platform)); + final module = await modules.find(nextModuleId, buildStep); + if (module == null || module.isMissing) { + missingModuleSources.add(next); + continue; + } + if (throwIfUnsupported && !module.isSupported) { + unsupportedModules.add(module); + } + transitiveDeps[next] = module; + modulesToCrawl.addAll(module.directDependencies); + seenSources.addAll(module.sources); + } + + if (missingModuleSources.isNotEmpty) { + throw await MissingModulesException.create( + missingModuleSources, + transitiveDeps.values.toList()..add(this), + buildStep, + ); + } + if (throwIfUnsupported && unsupportedModules.isNotEmpty) { + throw UnsupportedModules(unsupportedModules); + } + return seenSources; } } diff --git a/build_modules/lib/src/scratch_space.dart b/build_modules/lib/src/scratch_space.dart index 9ef56e5bfe..6c8bf35f32 100644 --- a/build_modules/lib/src/scratch_space.dart +++ b/build_modules/lib/src/scratch_space.dart @@ -44,11 +44,15 @@ final scratchSpaceResource = Resource( } return scratchSpace; }, + dispose: (scratchSpace) { + scratchSpace.dispose(); + }, beforeExit: () async { // The workers are running inside the scratch space, so wait for them to // shut down before deleting it. await dartdevkWorkersAreDone; await frontendWorkersAreDone; + await frontendServerProxyWorkersAreDone; // Attempt to clean up the scratch space. Even after waiting for the workers // to shut down we might get file system exceptions on windows for an // arbitrary amount of time, so do retries with an exponential backoff. @@ -83,7 +87,7 @@ final scratchSpaceResource = Resource( }, ); -/// Modifies all package uris in [rootConfig] to work with the sctrach_space +/// Modifies all package uris in [rootConfig] to work with the scratch_space /// layout. These are uris of the form `../packages/`. /// /// Also modifies the `packageUri` for each package to be empty since the @@ -109,12 +113,12 @@ String _scratchSpacePackageConfig(String rootConfig, Uri packageConfigUri) { rootUri = rootUri.replace(path: '${rootUri.path}/'); } // We expect to see exactly one package where the root uri is equal to - // the current directory, and that is the current packge. + // the current directory, and that is the current package. if (rootUri == _currentDirUri) { assert(!foundRoot); foundRoot = true; - package['rootUri'] = '../'; - package['packageUri'] = '../packages/${package['name']}/'; + package['packageUri'] = ''; + package['rootUri'] = '../packages/${package['name']}/'; } else { package['rootUri'] = '../packages/${package['name']}/'; package.remove('packageUri'); diff --git a/build_modules/lib/src/workers.dart b/build_modules/lib/src/workers.dart index ba99eaf1d3..6278c13391 100644 --- a/build_modules/lib/src/workers.dart +++ b/build_modules/lib/src/workers.dart @@ -10,10 +10,10 @@ import 'package:bazel_worker/driver.dart'; import 'package:build/build.dart'; import 'package:path/path.dart' as p; +import 'common.dart'; +import 'frontend_server_driver.dart'; import 'scratch_space.dart'; -final sdkDir = p.dirname(p.dirname(Platform.resolvedExecutable)); - // If no terminal is attached, prevent a new one from launching. final _processMode = stdin.hasTerminal @@ -108,3 +108,43 @@ final frontendDriverResource = Resource( __frontendDriver = null; }, ); + +/// Completes once the Frontend Service proxy workers have been shut down. +Future get frontendServerProxyWorkersAreDone => + _frontendServerProxyWorkersAreDoneCompleter?.future ?? Future.value(); +Completer? _frontendServerProxyWorkersAreDoneCompleter; + +FrontendServerProxyDriver get _frontendServerProxyDriver { + _frontendServerProxyWorkersAreDoneCompleter ??= Completer(); + return __frontendServerProxyDriver ??= FrontendServerProxyDriver(); +} + +FrontendServerProxyDriver? __frontendServerProxyDriver; + +/// Manages a shared set of workers that proxy requests to a single +/// [persistentFrontendServerResource]. +final frontendServerProxyDriverResource = Resource( + () async => _frontendServerProxyDriver, + beforeExit: () async { + _frontendServerProxyWorkersAreDoneCompleter?.complete(); + await __frontendServerProxyDriver?.terminate(); + _frontendServerProxyWorkersAreDoneCompleter = null; + __frontendServerProxyDriver = null; + }, +); + +PersistentFrontendServer? __persistentFrontendServer; + +/// Manages a single persistent instance of the Frontend Server targeting DDC. +final persistentFrontendServerResource = Resource( + () async => + __persistentFrontendServer ??= await PersistentFrontendServer.start( + sdkRoot: sdkDir, + fileSystemRoot: scratchSpace.tempDir.uri, + packagesFile: scratchSpace.tempDir.uri.resolve(packagesFilePath), + ), + beforeExit: () async { + await __persistentFrontendServer?.shutdown(); + __persistentFrontendServer = null; + }, +); diff --git a/build_modules/pubspec.yaml b/build_modules/pubspec.yaml index 2bb7fffdd8..d11df4bb76 100644 --- a/build_modules/pubspec.yaml +++ b/build_modules/pubspec.yaml @@ -1,5 +1,5 @@ name: build_modules -version: 5.0.18 +version: 5.1.0 description: >- Builders to analyze and split Dart code into individually compilable modules based on imports. @@ -16,13 +16,15 @@ dependencies: build: '>=2.0.0 <5.0.0' collection: ^1.15.0 crypto: ^3.0.0 + file: ^7.0.1 glob: ^2.0.0 graphs: ^2.0.0 json_annotation: ^4.3.0 logging: ^1.0.0 path: ^1.8.0 - scratch_space: ^1.0.0 + scratch_space: ^1.2.0 stream_transform: ^2.0.0 + uuid: ^4.4.2 dev_dependencies: a: diff --git a/build_modules/test/frontend_server_driver_test.dart b/build_modules/test/frontend_server_driver_test.dart new file mode 100644 index 0000000000..0105b7271e --- /dev/null +++ b/build_modules/test/frontend_server_driver_test.dart @@ -0,0 +1,179 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:build_modules/src/common.dart'; +import 'package:build_modules/src/frontend_server_driver.dart'; +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; + +void main() { + group('Frontend Server', () { + late PersistentFrontendServer server; + late Directory tempDir; + + setUp(() async { + tempDir = await Directory.systemTemp.createTemp('fes-test'); + final packageConfig = tempDir.uri.resolve( + '.dart_tool/package_config.json', + ); + File(packageConfig.toFilePath()) + ..createSync(recursive: true) + ..writeAsStringSync( + jsonEncode({'configVersion': 2, 'packages': []}), + ); + server = await PersistentFrontendServer.start( + sdkRoot: sdkDir, + fileSystemRoot: tempDir.uri, + packagesFile: packageConfig, + ); + }); + + tearDown(() async { + await server.shutdown(); + await tempDir.delete(recursive: true); + }); + + test('can compile a simple dart file', () async { + final entrypoint = tempDir.uri.resolve('entry.dart'); + File(entrypoint.toFilePath()).writeAsStringSync('void main() {}'); + + final output = await server.compile(entrypoint.toString()); + expect(output, isNotNull); + expect(output!.errorCount, 0); + expect(output.outputFilename, endsWith('output.dill')); + expect(output.sources, contains(entrypoint)); + server.accept(); + }); + + test('can handle compilation errors', () async { + final entrypoint = tempDir.uri.resolve('entry.dart'); + File(entrypoint.toFilePath()).writeAsStringSync('void main() {'); + + final output = await server.compile(entrypoint.toString()); + expect(output, isNotNull); + expect(output!.errorCount, greaterThan(0)); + expect(output.errorMessage, isNotNull); + expect( + output.errorMessage, + contains("Error: Can't find '}' to match '{'"), + ); + await server.reject(); + }); + + test('can reject and recompile with an invalidated file', () async { + final entrypoint = tempDir.uri.resolve('entrypoint.dart'); + final dep = tempDir.uri.resolve('dep.dart'); + File( + entrypoint.toFilePath(), + ).writeAsStringSync("import 'dep.dart'; void main() {}"); + File(dep.toFilePath()).writeAsStringSync('// empty'); + + var output = await server.compile(entrypoint.toString()); + expect(output, isNotNull); + expect(output!.errorCount, 0); + server.accept(); + + File(dep.toFilePath()).writeAsStringSync('invalid dart code'); + output = await server.recompile(entrypoint.toString(), [dep]); + expect(output, isNotNull); + expect(output!.errorCount, greaterThan(0)); + expect(output.errorMessage, contains("Error: Expected ';' after this.")); + await server.reject(); + + File(dep.toFilePath()).writeAsStringSync('// empty'); + output = await server.recompile(entrypoint.toString(), [dep]); + expect(output, isNotNull); + expect(output!.errorCount, 0); + server.accept(); + }); + }); + + group('Frontend Server Proxy', () { + late FrontendServerProxyDriver driver; + late PersistentFrontendServer server; + late Directory tempDir; + + setUp(() async { + tempDir = await Directory.systemTemp.createTemp('fes-test'); + final packageConfig = tempDir.uri.resolve( + '.dart_tool/package_config.json', + ); + File(packageConfig.toFilePath()) + ..createSync(recursive: true) + ..writeAsStringSync( + jsonEncode({'configVersion': 2, 'packages': []}), + ); + server = await PersistentFrontendServer.start( + sdkRoot: sdkDir, + fileSystemRoot: tempDir.uri, + packagesFile: packageConfig, + ); + driver = FrontendServerProxyDriver(); + driver.init(server); + }); + + tearDown(() async { + await driver.terminate(); + await tempDir.delete(recursive: true); + }); + + test('can recompile after valid edits', () async { + final file1 = tempDir.uri.resolve('file1.dart'); + File(file1.toFilePath()).writeAsStringSync('void main() {}'); + final file2 = tempDir.uri.resolve('file2.dart'); + File(file2.toFilePath()).writeAsStringSync('void main() {}'); + + var future1 = driver.recompile(file1.toString(), []); + var future2 = driver.recompile(file2.toString(), []); + + var results = await Future.wait([future1, future2]); + expect(results[0], isNotNull); + expect(results[0]!.errorCount, 0); + expect(results[1], isNotNull); + expect(results[1]!.errorCount, 0); + + File( + file1.toFilePath(), + ).writeAsStringSync("void main() { print('edit!'); }"); + File( + file2.toFilePath(), + ).writeAsStringSync("void main() { print('edit!'); }"); + + future1 = driver.recompile(file1.toString(), [file1]); + future2 = driver.recompile(file2.toString(), [file2]); + + results = await Future.wait([future1, future2]); + expect(results[0], isNotNull); + expect(results[0]!.errorCount, 0); + expect(results[1], isNotNull); + expect(results[1]!.errorCount, 0); + }); + + test('recompileAndRecord successfully records and writes files', () async { + final entrypoint = tempDir.uri.resolve('entrypoint.dart'); + File(entrypoint.toFilePath()).writeAsStringSync('void main() {}'); + + final jsFESOutputPath = p.join(tempDir.path, 'entrypoint.dart.lib.js'); + final output = await driver.recompileAndRecord( + '$multiRootScheme:///entrypoint.dart', + [entrypoint], + ['entrypoint.dart.lib.js'], + ); + + expect(output, isNotNull); + expect(output!.errorCount, 0); + + final jsOutputFile = File( + jsFESOutputPath.replaceFirst('.dart.lib.js', '.ddc.js'), + ); + expect(jsOutputFile.existsSync(), isTrue); + final content = jsOutputFile.readAsStringSync(); + expect(content, contains('function main()')); + }); + }); +} diff --git a/build_modules/test/meta_module_builder_test.dart b/build_modules/test/meta_module_builder_test.dart index 56b11dc779..c208399ee6 100644 --- a/build_modules/test/meta_module_builder_test.dart +++ b/build_modules/test/meta_module_builder_test.dart @@ -5,7 +5,6 @@ import 'package:build/build.dart'; import 'package:build_modules/build_modules.dart'; import 'package:build_modules/src/meta_module.dart'; -import 'package:build_modules/src/module_library.dart'; import 'package:build_test/build_test.dart'; import 'package:test/test.dart'; diff --git a/build_modules/test/meta_module_test.dart b/build_modules/test/meta_module_test.dart index 1917d48e54..5b0aa9ce34 100644 --- a/build_modules/test/meta_module_test.dart +++ b/build_modules/test/meta_module_test.dart @@ -8,7 +8,6 @@ import 'package:build/build.dart'; import 'package:build_modules/build_modules.dart'; import 'package:build_modules/src/common.dart'; import 'package:build_modules/src/meta_module.dart'; -import 'package:build_modules/src/module_library.dart'; import 'package:build_test/build_test.dart'; import 'package:test/test.dart'; diff --git a/build_modules/test/module_builder_test.dart b/build_modules/test/module_builder_test.dart index 065ffd9b05..3a684b5cfe 100644 --- a/build_modules/test/module_builder_test.dart +++ b/build_modules/test/module_builder_test.dart @@ -7,7 +7,6 @@ import 'dart:convert'; import 'package:build/build.dart'; import 'package:build_modules/build_modules.dart'; import 'package:build_modules/src/meta_module.dart'; -import 'package:build_modules/src/module_library.dart'; import 'package:build_test/build_test.dart'; import 'package:test/test.dart'; @@ -49,8 +48,14 @@ void main() { 'a|lib/${metaModuleCleanExtension(platform)}': jsonEncode( metaModule.toJson(), ), + 'a|lib/a$moduleLibraryExtension': + ModuleLibrary.fromSource(assetA, '').serialize(), + 'a|lib/b$moduleLibraryExtension': + ModuleLibrary.fromSource(assetB, '').serialize(), 'a|lib/c$moduleLibraryExtension': ModuleLibrary.fromSource(assetC, '').serialize(), + 'a|lib/d$moduleLibraryExtension': + ModuleLibrary.fromSource(assetD, '').serialize(), 'a|lib/e$moduleLibraryExtension': ModuleLibrary.fromSource(assetE, '').serialize(), }, @@ -61,4 +66,67 @@ void main() { }, ); }); + + test('can serialize modules and output all modules when strongly connected ' + 'components are disabled', () async { + final assetA = AssetId('a', 'lib/a.dart'); + final assetB = AssetId('a', 'lib/b.dart'); + final assetC = AssetId('a', 'lib/c.dart'); + final assetD = AssetId('a', 'lib/d.dart'); + final assetE = AssetId('a', 'lib/e.dart'); + final moduleA = Module(assetA, [assetA], [], platform, true); + final moduleB = Module( + assetB, + [assetB, assetC], + [], + platform, + true, + ); + final moduleC = Module(assetC, [assetC], [], platform, true); + final moduleD = Module( + assetD, + [assetD, assetE], + [], + platform, + false, + ); + final moduleE = Module(assetE, [assetE], [], platform, true); + final metaModule = MetaModule([ + moduleA, + moduleB, + moduleC, + moduleD, + moduleE, + ]); + await testBuilder( + ModuleBuilder(platform, usesWebHotReload: true), + { + 'a|lib/a.dart': '', + 'a|lib/b.dart': '', + 'a|lib/c.dart': '', + 'a|lib/d.dart': '', + 'a|lib/e.dart': '', + 'a|lib/${metaModuleExtension(platform)}': jsonEncode( + metaModule.toJson(), + ), + 'a|lib/a$moduleLibraryExtension': + ModuleLibrary.fromSource(assetA, '').serialize(), + 'a|lib/b$moduleLibraryExtension': + ModuleLibrary.fromSource(assetB, '').serialize(), + 'a|lib/c$moduleLibraryExtension': + ModuleLibrary.fromSource(assetC, '').serialize(), + 'a|lib/d$moduleLibraryExtension': + ModuleLibrary.fromSource(assetD, '').serialize(), + 'a|lib/e$moduleLibraryExtension': + ModuleLibrary.fromSource(assetE, '').serialize(), + }, + outputs: { + 'a|lib/a${moduleExtension(platform)}': encodedMatchesModule(moduleA), + 'a|lib/b${moduleExtension(platform)}': encodedMatchesModule(moduleB), + 'a|lib/c${moduleExtension(platform)}': encodedMatchesModule(moduleC), + 'a|lib/d${moduleExtension(platform)}': encodedMatchesModule(moduleD), + 'a|lib/e${moduleExtension(platform)}': encodedMatchesModule(moduleE), + }, + ); + }); } diff --git a/build_runner/test/common/build_runner_tester.dart b/build_runner/test/common/build_runner_tester.dart index 1cb3810ce8..52ea72b90c 100644 --- a/build_runner/test/common/build_runner_tester.dart +++ b/build_runner/test/common/build_runner_tester.dart @@ -100,7 +100,7 @@ class BuildRunnerTester { /// Deletes the workspace-relative [path]. void delete(String path) { final file = File(p.join(tempDirectory.path, path)); - file.deleteSync(); + file.deleteSync(recursive: true); } /// Reads the tree of files at the workspace-relative [path]. @@ -239,32 +239,63 @@ class BuildRunnerProcess { /// defaults to [BuildLog.failurePattern] so that `expect` will stop if the /// process reports a build failure. /// + /// if [expectFailure] is set, then both [pattern] and [failOn] must be + /// encountered for the test to pass. + /// /// If the process exits instead, the test fails immediately. /// /// Otherwise, waits until [pattern] appears, returns the matching line. /// /// Throws if the process appears to be stuck or done: if it outputs nothing /// for 30s. - Future expect(Pattern pattern, {Pattern? failOn}) async { + Future expect( + Pattern pattern, { + Pattern? failOn, + bool expectFailure = false, + }) async { printOnFailure( '--- $_testLine expects `$pattern`' - '${failOn == null ? '' : ', failOn: `$failOn`'}', + '${failOn == null ? '' : ', failOn: `$failOn`'}' + '${expectFailure ? ', expectFailure: true' : ''}', ); failOn ??= BuildLog.failurePattern; + + final expectsMessage = + expectFailure + ? '`$pattern` with failure matching (`$failOn`)' + : '`$pattern`'; + + var failureSeen = false; + var patternSeen = false; while (true) { String? line; try { line = await _outputs.next.timeout(const Duration(seconds: 30)); } on TimeoutException catch (_) { - throw fail('While expecting `$pattern`, timed out after 30s.'); + throw fail('While expecting $expectsMessage, timed out after 30s.'); } catch (_) { - throw fail('While expecting `$pattern`, process exited.'); + throw fail('While expecting $expectsMessage, process exited.'); } printOnFailure(line); if (line.contains(failOn)) { - fail('While expecting `$pattern`, got `$failOn`.'); + failureSeen = true; + } + if (line.contains(pattern)) { + patternSeen = true; + } + + if (expectFailure) { + if (patternSeen && failureSeen) { + return line; + } + } else { + if (failureSeen) { + fail('While expecting $expectsMessage, got `$failOn`.'); + } + if (patternSeen) { + return line; + } } - if (line.contains(pattern)) return line; } } @@ -389,7 +420,6 @@ dependencies: ..writeln(' $package:') ..writeln(' path: ../$package'); } - return result.toString(); } } diff --git a/build_runner/test/integration_tests/daemon_command_test.dart b/build_runner/test/integration_tests/daemon_command_test.dart index b465df60ff..bdffcba138 100644 --- a/build_runner/test/integration_tests/daemon_command_test.dart +++ b/build_runner/test/integration_tests/daemon_command_test.dart @@ -18,6 +18,8 @@ import 'package:test/test.dart'; import '../common/common.dart'; +const defaultTimeout = Timeout(Duration(seconds: 90)); + void main() async { final webTarget = DefaultBuildTarget((b) { b.target = 'web'; @@ -40,6 +42,7 @@ void main() async { 'build_runner', 'build_web_compilers', 'build_test', + 'scratch_space', ], pathDependencies: ['builder_pkg'], files: { @@ -212,5 +215,5 @@ void main() { // Check the second client was notified too. expect((await results2.next).results.single.status, BuildStatus.started); expect((await results2.next).results.single.status, BuildStatus.succeeded); - }); + }, timeout: defaultTimeout); } diff --git a/build_runner/test/integration_tests/web_compilers_test.dart b/build_runner/test/integration_tests/web_compilers_test.dart index 8de66d8277..6bdaf689a3 100644 --- a/build_runner/test/integration_tests/web_compilers_test.dart +++ b/build_runner/test/integration_tests/web_compilers_test.dart @@ -10,6 +10,8 @@ import 'package:test/test.dart'; import '../common/common.dart'; +const defaultTimeout = Timeout(Duration(seconds: 90)); + void main() async { test('web compilers', () async { final pubspecs = await Pubspecs.load(); @@ -27,6 +29,7 @@ void main() async { 'build_runner', 'build_web_compilers', 'build_test', + 'scratch_space', ], pathDependencies: ['builder_pkg'], files: { @@ -148,5 +151,334 @@ void main() { '--define=build_web_compilers:dart_source_cleanup=enabled=true', ); expect(tester.read('root_pkg/build/unused.dart'), null); - }); + }, timeout: defaultTimeout); + + // TODO(davidmorgan): the remaining tests are integration tests of + // the web compilers themselves, support testing like this outside the + // `build_runner` package. + test('DDC compiled with the Frontend Server', () async { + final pubspecs = await Pubspecs.load(); + final tester = BuildRunnerTester(pubspecs); + + tester.writeFixturePackage(FixturePackages.copyBuilder()); + + tester.writePackage( + name: 'root_pkg', + dependencies: [ + 'build', + 'build_config', + 'build_daemon', + 'build_modules', + 'build_runner', + 'build_web_compilers', + 'build_test', + 'scratch_space', + ], + pathDependencies: ['builder_pkg'], + files: { + 'build.yaml': r''' + targets: + $default: + builders: + build_web_compilers:entrypoint: + generate_for: + - web/main.dart + + global_options: + build_web_compilers|sdk_js: + options: + web-hot-reload: true + build_web_compilers|entrypoint: + options: + web-hot-reload: true + build_web_compilers|entrypoint_marker: + options: + web-hot-reload: true + build_web_compilers|ddc: + options: + web-hot-reload: true + build_web_compilers|ddc_modules: + options: + web-hot-reload: true + ''', + 'web/main.dart': ''' + import 'package:root_pkg/a.dart'; + + void main() { + print(helloWorld); + } + ''', + 'lib/a.dart': "const helloWorld = 'Hello World!';", + 'web/unused.dart': '', + }, + ); + + // Initial build. + await tester.run( + 'root_pkg', + 'dart run build_runner build --output web:build', + ); + expect( + tester.readFileTree('root_pkg/build')!.keys, + containsAll([ + 'main.dart.js', + 'main.digests', + 'main.dart.ddc_merged_metadata', + 'main.dart.bootstrap.js', + 'main.ddc.js.metadata', + 'main.ddc.js.map', + 'main.ddc.js', + 'main.dart', + r'packages/$sdk/dev_compiler/amd/require.js', + r'packages/$sdk/dev_compiler/ddc/ddc_module_loader.js', + r'packages/$sdk/dev_compiler/web/dart_stack_trace_mapper.js', + ]), + ); + // DDC by default. Does not compile submodules into the root module. + expect( + tester.read('root_pkg/build/main.dart.js'), + isNot(contains('// Generated by dart2js')), + ); + expect( + tester.read('root_pkg/build/main.dart.js'), + isNot(contains('Hello World!')), + ); + + // Introduce an error, build fails. + tester.write('root_pkg/lib/a.dart', 'error'); + var output = await tester.run( + 'root_pkg', + 'dart run build_runner build --output web:build', + expectExitCode: 1, + ); + expect(output, contains(BuildLog.failurePattern)); + + // Stop importing the file with the error, build succeeds. + tester.write('root_pkg/web/main.dart', 'void main() {}'); + output = await tester.run( + 'root_pkg', + 'dart run build_runner build --output web:build', + ); + expect(output, contains(BuildLog.successPattern)); + + // With dart_source_cleanup unused source is removed. + expect(tester.read('root_pkg/build/unused.dart'), ''); + await tester.run( + 'root_pkg', + 'dart run build_runner build --output web:build ' + '--define=build_web_compilers:dart_source_cleanup=enabled=true', + ); + expect(tester.read('root_pkg/build/unused.dart'), null); + }, timeout: defaultTimeout); + + test('DDC compiled with the Frontend Server ' + 'can recompile incrementally after valid edits', () async { + final pubspecs = await Pubspecs.load(); + final tester = BuildRunnerTester(pubspecs); + + tester.writeFixturePackage(FixturePackages.copyBuilder()); + tester.writePackage( + name: 'root_pkg', + dependencies: [ + 'build', + 'build_config', + 'build_daemon', + 'build_modules', + 'build_runner', + 'build_web_compilers', + 'build_test', + 'scratch_space', + ], + pathDependencies: ['builder_pkg', 'pkg_a'], + files: { + 'build.yaml': r''' + targets: + $default: + builders: + build_web_compilers:entrypoint: + generate_for: + - web/**.dart + + global_options: + build_web_compilers|sdk_js: + options: + web-hot-reload: true + build_web_compilers|entrypoint: + options: + web-hot-reload: true + build_web_compilers|entrypoint_marker: + options: + web-hot-reload: true + build_web_compilers|ddc: + options: + web-hot-reload: true + build_web_compilers|ddc_modules: + options: + web-hot-reload: true + ''', + 'web/main.dart': ''' + import 'package:pkg_a/a.dart'; + + void main() { + print(helloWorld); + } + ''', + }, + ); + tester.writePackage( + name: 'pkg_a', + dependencies: ['build_runner'], + pathDependencies: ['builder_pkg'], + files: {'lib/a.dart': "String helloWorld = 'Hello World!';"}, + ); + final generatedDirRoot = 'root_pkg/.dart_tool/build/generated'; + final watch = await tester.start('root_pkg', 'dart run build_runner watch'); + await watch.expect(BuildLog.successPattern); + expect( + tester.read('$generatedDirRoot/pkg_a/lib/a.ddc.js'), + contains('Hello World!'), + ); + expect( + tester.read('$generatedDirRoot/root_pkg/web/main.ddc.js'), + isNot(contains('Hello')), + ); + + // Make a simple edit, rebuild succeeds. + tester.write('pkg_a/lib/a.dart', "String helloWorld = 'Hello Dash!';"); + await watch.expect(BuildLog.successPattern); + expect( + tester.read('$generatedDirRoot/pkg_a/lib/a.ddc.js'), + contains('Hello Dash!'), + ); + expect( + tester.read('$generatedDirRoot/root_pkg/web/main.ddc.js'), + isNot(contains('Hello')), + ); + + // Make another simple edit, rebuild succeeds. + tester.write('pkg_a/lib/a.dart', "String helloWorld = 'Hello Dart!';"); + await watch.expect(BuildLog.successPattern); + expect( + tester.read('$generatedDirRoot/pkg_a/lib/a.ddc.js'), + contains('Hello Dart!'), + ); + expect( + tester.read('$generatedDirRoot/root_pkg/web/main.ddc.js'), + isNot(contains('Hello')), + ); + }, timeout: defaultTimeout); + + test('DDC compiled with the Frontend Server ' + 'can recompile incrementally after invalid edits', () async { + final pubspecs = await Pubspecs.load(); + final tester = BuildRunnerTester(pubspecs); + + tester.writeFixturePackage(FixturePackages.copyBuilder()); + tester.writePackage( + name: 'root_pkg', + dependencies: [ + 'build', + 'build_config', + 'build_daemon', + 'build_modules', + 'build_runner', + 'build_web_compilers', + 'build_test', + 'scratch_space', + ], + pathDependencies: ['builder_pkg', 'pkg_a'], + files: { + 'build.yaml': r''' + targets: + $default: + builders: + build_web_compilers:entrypoint: + generate_for: + - web/**.dart + + global_options: + build_web_compilers|sdk_js: + options: + web-hot-reload: true + build_web_compilers|entrypoint: + options: + web-hot-reload: true + build_web_compilers|entrypoint_marker: + options: + web-hot-reload: true + build_web_compilers|ddc: + options: + web-hot-reload: true + build_web_compilers|ddc_modules: + options: + web-hot-reload: true + ''', + 'web/main.dart': ''' + import 'package:pkg_a/a.dart'; + + void main() { + print(helloWorld); + } + ''', + }, + ); + tester.writePackage( + name: 'pkg_a', + dependencies: ['build_runner'], + pathDependencies: ['builder_pkg'], + files: {'lib/a.dart': "String helloWorld = 'Hello World!';"}, + ); + final generatedDirRoot = 'root_pkg/.dart_tool/build/generated'; + final watch = await tester.start('root_pkg', 'dart run build_runner watch'); + await watch.expect(BuildLog.successPattern); + expect( + tester.read('$generatedDirRoot/pkg_a/lib/a.ddc.js'), + contains('Hello World!'), + ); + expect( + tester.read('$generatedDirRoot/root_pkg/web/main.ddc.js'), + isNot(contains('Hello')), + ); + + // Introduce a generic class, rebuild succeeds. + tester.write('pkg_a/lib/a.dart', ''' +class Foo{} +String helloWorld = 'Hello Dash!'; +'''); + await watch.expect(BuildLog.successPattern); + expect( + tester.read('$generatedDirRoot/pkg_a/lib/a.ddc.js'), + contains('Hello Dash!'), + ); + expect( + tester.read('$generatedDirRoot/root_pkg/web/main.ddc.js'), + isNot(contains('Hello')), + ); + + // Perform an invalid edit (such as changing the number of generic + // parameters of a class), rebuild succeeds. + tester.write('pkg_a/lib/a.dart', ''' +class Foo{} +String helloWorld = 'Hello Dash!'; +'''); + await watch.expect( + 'Hot reload rejected due to unsupported changes', + expectFailure: true, + ); + + // Revert the invalid edit, rebuild succeeds. + tester.write('pkg_a/lib/a.dart', ''' +class Foo{} +String helloWorld = 'Hello Dash!'; +'''); + await watch.expect(BuildLog.successPattern); + expect( + tester.read('$generatedDirRoot/pkg_a/lib/a.ddc.js'), + contains('Hello Dash!'), + ); + expect( + tester.read('$generatedDirRoot/root_pkg/web/main.ddc.js'), + isNot(contains('Hello')), + ); + }, timeout: defaultTimeout); } diff --git a/build_web_compilers/CHANGELOG.md b/build_web_compilers/CHANGELOG.md index 32f02dd8b6..48586ecc03 100644 --- a/build_web_compilers/CHANGELOG.md +++ b/build_web_compilers/CHANGELOG.md @@ -1,3 +1,8 @@ +## 4.4.0 + +- Add DDC + Frontend Server compilation support to existing builders. +- Add new builders: `DdcFrontendServerBuilder` and `WebEntrypointMarkerBuilder`. + ## 4.3.2 - Fix Dart2JS adding extraneous sourcemaps to its archive when both wasm and js are enabled. diff --git a/build_web_compilers/build.yaml b/build_web_compilers/build.yaml index 3f9c2eb887..95718f9f81 100644 --- a/build_web_compilers/build.yaml +++ b/build_web_compilers/build.yaml @@ -1,6 +1,8 @@ targets: $default: builders: + build_web_compilers:entrypoint_marker: + enabled: true build_web_compilers:entrypoint: options: compiler: dart2js @@ -24,6 +26,7 @@ builders: - lib/src/dev_compiler/dart_sdk.js - lib/src/dev_compiler/dart_sdk.js.map - lib/src/dev_compiler/require.js + - lib/src/dev_compiler/ddc_module_loader.js is_optional: True auto_apply: none runs_before: ["build_web_compilers:entrypoint"] @@ -145,6 +148,18 @@ builders: compiler: dart2js applies_builders: - build_web_compilers:dart2js_archive_extractor + entrypoint_marker: + import: "package:build_web_compilers/builders.dart" + builder_factories: + - webEntrypointMarkerBuilder + build_extensions: + $web$: + - .web.entrypoint.json + required_inputs: + - .dart + build_to: cache + auto_apply: root_package + runs_before: ["build_web_compilers:ddc", "build_web_compilers:ddc_modules"] _stack_trace_mapper_copy: import: "tool/copy_builder.dart" builder_factories: diff --git a/build_web_compilers/lib/build_web_compilers.dart b/build_web_compilers/lib/build_web_compilers.dart index 92e101ead4..8044292333 100644 --- a/build_web_compilers/lib/build_web_compilers.dart +++ b/build_web_compilers/lib/build_web_compilers.dart @@ -3,15 +3,15 @@ // BSD-style license that can be found in the LICENSE file. export 'src/archive_extractor.dart' show Dart2JsArchiveExtractor; -export 'src/dev_compiler_builder.dart' +export 'src/common.dart' show - DevCompilerBuilder, fullKernelExtension, jsModuleErrorsExtension, jsModuleExtension, jsSourceMapExtension, metadataExtension, symbolsExtension; +export 'src/dev_compiler_builder.dart' show DevCompilerBuilder; export 'src/platforms.dart' show dart2jsPlatform, dart2wasmPlatform, ddcPlatform; export 'src/web_entrypoint_builder.dart' diff --git a/build_web_compilers/lib/builders.dart b/build_web_compilers/lib/builders.dart index cf8e66f207..f11ddac3e8 100644 --- a/build_web_compilers/lib/builders.dart +++ b/build_web_compilers/lib/builders.dart @@ -8,29 +8,58 @@ import 'package:collection/collection.dart'; import 'build_web_compilers.dart'; import 'src/common.dart'; +import 'src/ddc_frontend_server_builder.dart'; import 'src/sdk_js_compile_builder.dart'; import 'src/sdk_js_copy_builder.dart'; +import 'src/web_entrypoint_marker_builder.dart'; // Shared entrypoint builder -Builder webEntrypointBuilder(BuilderOptions options) => - WebEntrypointBuilder.fromOptions(options); +Builder webEntrypointBuilder(BuilderOptions options) { + _ensureSameDdcHotReloadOptions(options); + return WebEntrypointBuilder.fromOptions(options); +} + +Builder webEntrypointMarkerBuilder(BuilderOptions options) { + _ensureSameDdcHotReloadOptions(options); + return WebEntrypointMarkerBuilder( + usesWebHotReload: _readWebHotReloadOption(options), + ); +} + +// DDC related builders +Builder ddcMetaModuleBuilder(BuilderOptions options) { + _ensureSameDdcHotReloadOptions(options); + return MetaModuleBuilder.forOptions(ddcPlatform, options); +} -// Ddc related builders -Builder ddcMetaModuleBuilder(BuilderOptions options) => - MetaModuleBuilder.forOptions(ddcPlatform, options); -Builder ddcMetaModuleCleanBuilder(BuilderOptions _) => - MetaModuleCleanBuilder(ddcPlatform); -Builder ddcModuleBuilder(BuilderOptions _) => ModuleBuilder(ddcPlatform); +Builder ddcMetaModuleCleanBuilder(BuilderOptions options) { + _ensureSameDdcHotReloadOptions(options); + return MetaModuleCleanBuilder(ddcPlatform); +} + +Builder ddcModuleBuilder(BuilderOptions options) { + _ensureSameDdcHotReloadOptions(options); + return ModuleBuilder( + ddcPlatform, + usesWebHotReload: _readWebHotReloadOption(options), + ); +} Builder ddcBuilder(BuilderOptions options) { validateOptions(options.config, _supportedOptions, 'build_web_compilers:ddc'); + _ensureSameDdcHotReloadOptions(options); _ensureSameDdcOptions(options); + if (_readWebHotReloadOption(options)) { + return DdcFrontendServerBuilder(); + } + return DevCompilerBuilder( useIncrementalCompiler: _readUseIncrementalCompilerOption(options), generateFullDill: _readGenerateFullDillOption(options), emitDebugSymbols: _readEmitDebugSymbolsOption(options), canaryFeatures: _readCanaryOption(options), + ddcModules: _readWebHotReloadOption(options), sdkKernelPath: sdkDdcKernelPath, trackUnusedInputs: _readTrackInputsCompilerOption(options), platform: ddcPlatform, @@ -42,6 +71,7 @@ final ddcKernelExtension = '.ddc.dill'; Builder ddcKernelBuilder(BuilderOptions options) { validateOptions(options.config, _supportedOptions, 'build_web_compilers:ddc'); + _ensureSameDdcHotReloadOptions(options); _ensureSameDdcOptions(options); return KernelBuilder( @@ -55,11 +85,16 @@ Builder ddcKernelBuilder(BuilderOptions options) { } Builder sdkJsCopyRequirejs(BuilderOptions _) => SdkJsCopyBuilder(); -Builder sdkJsCompile(BuilderOptions options) => SdkJsCompileBuilder( - sdkKernelPath: 'lib/_internal/ddc_platform.dill', - outputPath: 'lib/src/dev_compiler/dart_sdk.js', - canaryFeatures: _readCanaryOption(options), -); +Builder sdkJsCompile(BuilderOptions options) { + _ensureSameDdcHotReloadOptions(options); + return SdkJsCompileBuilder( + sdkKernelPath: 'lib/_internal/ddc_platform.dill', + outputPath: 'lib/src/dev_compiler/dart_sdk.js', + canaryFeatures: + _readWebHotReloadOption(options) || _readCanaryOption(options), + usesWebHotReload: _readWebHotReloadOption(options), + ); +} // Dart2js related builders Builder dart2jsMetaModuleBuilder(BuilderOptions options) => @@ -115,6 +150,26 @@ void _ensureSameDdcOptions(BuilderOptions options) { } } +void _ensureSameDdcHotReloadOptions(BuilderOptions options) { + final webHotReload = _readWebHotReloadOption(options); + if (_lastWebHotReloadValue != null) { + if (webHotReload != _lastWebHotReloadValue) { + throw ArgumentError( + '`web-hot-reload` must be configured the same across the following ' + 'builders: build_web_compilers:ddc, ' + 'build_web_compilers|sdk_js, ' + 'build_web_compilers|entrypoint, ' + 'build_web_compilers|entrypoint_marker, ' + 'and build_web_compilers|ddc_modules.' + '\n\nPlease use the `global_options` section in ' + '`build.yaml` or the `--define` flag to set global options.', + ); + } + } else { + _lastWebHotReloadValue = webHotReload; + } +} + bool _readUseIncrementalCompilerOption(BuilderOptions options) { return options.config[_useIncrementalCompilerOption] as bool? ?? true; } @@ -135,18 +190,24 @@ bool _readTrackInputsCompilerOption(BuilderOptions options) { return options.config[_trackUnusedInputsCompilerOption] as bool? ?? true; } +bool _readWebHotReloadOption(BuilderOptions options) { + return options.config[_webHotReloadOption] as bool? ?? false; +} + Map _readEnvironmentOption(BuilderOptions options) { final environment = options.config[_environmentOption] as Map? ?? const {}; return environment.map((key, value) => MapEntry('$key', '$value')); } Map? _previousDdcConfig; +bool? _lastWebHotReloadValue; const _useIncrementalCompilerOption = 'use-incremental-compiler'; const _generateFullDillOption = 'generate-full-dill'; const _emitDebugSymbolsOption = 'emit-debug-symbols'; const _canaryOption = 'canary'; const _trackUnusedInputsCompilerOption = 'track-unused-inputs'; const _environmentOption = 'environment'; +const _webHotReloadOption = 'web-hot-reload'; const _supportedOptions = [ _environmentOption, @@ -155,4 +216,5 @@ const _supportedOptions = [ _emitDebugSymbolsOption, _canaryOption, _trackUnusedInputsCompilerOption, + _webHotReloadOption, ]; diff --git a/build_web_compilers/lib/src/common.dart b/build_web_compilers/lib/src/common.dart index 2aaf3d9db9..2689809fd0 100644 --- a/build_web_compilers/lib/src/common.dart +++ b/build_web_compilers/lib/src/common.dart @@ -9,6 +9,14 @@ import 'package:build/build.dart'; import 'package:path/path.dart' as p; import 'package:scratch_space/scratch_space.dart'; +final multiRootScheme = 'org-dartlang-app'; +final jsModuleErrorsExtension = '.ddc.js.errors'; +final jsModuleExtension = '.ddc.js'; +final jsSourceMapExtension = '.ddc.js.map'; +final metadataExtension = '.ddc.js.metadata'; +final symbolsExtension = '.ddc.js.symbols'; +final fullKernelExtension = '.ddc.full.dill'; + final defaultAnalysisOptionsId = AssetId( 'build_modules', 'lib/src/analysis_options.default.yaml', @@ -50,6 +58,15 @@ void validateOptions( } } +/// The url to compile for a source. +/// +/// Use the package: path for files under lib and the full absolute path for +/// other files. +String sourceArg(AssetId id) { + final uri = canonicalUriFor(id); + return uri.startsWith('package:') ? uri : '$multiRootScheme:///${id.path}'; +} + /// If [id] exists, assume it is a source map and fix up the source uris from /// it so they make sense in a browser context, then write the modified version /// using [writer]. @@ -85,3 +102,69 @@ Future fixAndCopySourceMap( await writer.writeAsString(id, jsonEncode(json)); } } + +void fixMetadataSources(Map json, Uri scratchUri) { + String updatePath(String path) => + Uri.parse(path).path.replaceAll(scratchUri.path, ''); + + final sourceMapUri = json['sourceMapUri'] as String?; + if (sourceMapUri != null) { + json['sourceMapUri'] = updatePath(sourceMapUri); + } + + final moduleUri = json['moduleUri'] as String?; + if (moduleUri != null) { + json['moduleUri'] = updatePath(moduleUri); + } + + final fullDillUri = json['fullDillUri'] as String?; + if (fullDillUri != null) { + json['fullDillUri'] = updatePath(fullDillUri); + } + + final libraries = json['libraries'] as List?; + if (libraries != null) { + for (final lib in libraries) { + final libraryJson = lib as Map?; + if (libraryJson != null) { + final fileUri = libraryJson['fileUri'] as String?; + if (fileUri != null) { + libraryJson['fileUri'] = updatePath(fileUri); + } + } + } + } +} + +/// The module name of [jsId] corresponding to the actual key used by DDC in its +/// boostrapper (which may contain path prefixes). +/// +/// Corresponds to the library name for the Library Bundler module system. +String ddcModuleName(AssetId jsId) { + final jsPath = + jsId.path.startsWith('lib/') + ? jsId.path.replaceFirst('lib/', 'packages/${jsId.package}/') + : jsId.path; + return jsPath.substring(0, jsPath.length - jsModuleExtension.length); +} + +String ddcLibraryId(AssetId jsId) { + final jsPath = + jsId.path.startsWith('lib/') + ? jsId.path.replaceFirst('lib/', 'package:${jsId.package}/') + : '$multiRootScheme:///${jsId.path}'; + final prefix = jsPath.substring(0, jsPath.length - jsModuleExtension.length); + return '$prefix.dart'; +} + +AssetId changeAssetIdExtension( + AssetId inputId, + String inputExtension, + String outputExtension, +) { + assert(inputId.path.endsWith(inputExtension)); + final newPath = + inputId.path.substring(0, inputId.path.length - inputExtension.length) + + outputExtension; + return AssetId(inputId.package, newPath); +} diff --git a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart new file mode 100644 index 0000000000..ccceacf303 --- /dev/null +++ b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart @@ -0,0 +1,134 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; + +import 'package:build/build.dart'; +import 'package:build_modules/build_modules.dart'; + +import 'common.dart'; +import 'errors.dart'; +import 'platforms.dart'; + +/// A builder that compiles DDC modules with the Frontend Server. +class DdcFrontendServerBuilder implements Builder { + DdcFrontendServerBuilder(); + + @override + final Map> buildExtensions = { + moduleExtension(ddcPlatform): [ + jsModuleExtension, + jsModuleErrorsExtension, + jsSourceMapExtension, + metadataExtension, + ], + }; + + @override + Future build(BuildStep buildStep) async { + final moduleContents = await buildStep.readAsString(buildStep.inputId); + final module = Module.fromJson( + json.decode(moduleContents) as Map, + ); + final ddcEntrypointId = module.primarySource; + // Entrypoints always have a `.module` file for ease of looking them up, + // but they might not be the primary source. + if (ddcEntrypointId.changeExtension(moduleExtension(ddcPlatform)) != + buildStep.inputId) { + return; + } + + Future handleError(Object e) async { + await buildStep.writeAsString( + ddcEntrypointId.changeExtension(jsModuleErrorsExtension), + '$e', + ); + log.severe('Error encountered: $e'); + } + + try { + await _compile(module, buildStep); + } on FrontendServerCompilationException catch (e) { + await handleError(e); + } on MissingModulesException catch (e) { + await handleError(e); + } + } + + /// Compile [module] with Frontend Server. + Future _compile(Module module, BuildStep buildStep) async { + final transitiveAssets = await buildStep.trackStage( + 'CollectTransitiveDeps', + () => module.computeTransitiveAssets(buildStep), + ); + final frontendServerState = await buildStep.fetchResource( + frontendServerStateResource, + ); + final scratchSpace = await buildStep.fetchResource(scratchSpaceResource); + final webEntrypointAsset = frontendServerState.entrypointAssetId; + await buildStep.trackStage( + 'EnsureAssets', + () => scratchSpace.ensureAssets([ + webEntrypointAsset, + ...transitiveAssets, + ], buildStep), + ); + final changedAssetUris = [ + for (final asset in scratchSpace.changedFilesInBuild) asset.uri, + ]; + final ddcEntrypointId = module.primarySource; + final jsOutputId = ddcEntrypointId.changeExtension(jsModuleExtension); + final jsFESOutputId = ddcEntrypointId.changeExtension('.dart.lib.js'); + + final frontendServer = await buildStep.fetchResource( + persistentFrontendServerResource, + ); + final driver = await buildStep.fetchResource( + frontendServerProxyDriverResource, + ); + driver.init(frontendServer); + + // Request from the Frontend Server exactly the JS file requested by + // build_runner. Frontend Server's recompilation logic will avoid + // extraneous recompilation. + final compilerOutput = await driver.recompileAndRecord( + sourceArg(webEntrypointAsset), + changedAssetUris, + [sourceArg(jsFESOutputId)], + ); + if (compilerOutput == null) { + throw FrontendServerCompilationException( + webEntrypointAsset, + 'Frontend Server produced no output.', + ); + } + if (compilerOutput.errorCount != 0 || compilerOutput.errorMessage != null) { + throw FrontendServerCompilationException( + webEntrypointAsset, + compilerOutput.errorMessage!, + ); + } + final outputFile = scratchSpace.fileFor(jsOutputId); + // Write an empty file if this output was deemed extraneous by FES. + if (!!await outputFile.exists()) { + await outputFile.create(recursive: true); + } + await scratchSpace.copyOutput(jsOutputId, buildStep); + await fixAndCopySourceMap( + ddcEntrypointId.changeExtension(jsSourceMapExtension), + scratchSpace, + buildStep, + ); + + // Copy the metadata output, modifying its contents to remove the temp + // directory from paths + final metadataId = ddcEntrypointId.changeExtension(metadataExtension); + final file = scratchSpace.fileFor(metadataId); + final content = await file.readAsString(); + final json = jsonDecode(content) as Map; + fixMetadataSources(json, scratchSpace.tempDir.uri); + await buildStep.writeAsString(metadataId, jsonEncode(json)); + } +} diff --git a/build_web_compilers/lib/src/dev_compiler_bootstrap.dart b/build_web_compilers/lib/src/dev_compiler_bootstrap.dart index 65c73fe1a2..32146bf6ea 100644 --- a/build_web_compilers/lib/src/dev_compiler_bootstrap.dart +++ b/build_web_compilers/lib/src/dev_compiler_bootstrap.dart @@ -5,6 +5,7 @@ import 'dart:async'; import 'dart:collection'; import 'dart:convert'; +import 'dart:io'; import 'package:build/build.dart'; import 'package:build_modules/build_modules.dart'; @@ -12,8 +13,7 @@ import 'package:build_modules/build_modules.dart'; import 'package:path/path.dart' as _p; import 'package:pool/pool.dart'; -import 'ddc_names.dart'; -import 'dev_compiler_builder.dart'; +import 'common.dart'; import 'platforms.dart'; import 'web_entrypoint_builder.dart'; @@ -22,6 +22,10 @@ _p.Context get _context => _p.url; final _modulePartialExtension = _context.withoutExtension(jsModuleExtension); +final stackTraceMapperPath = + 'packages/build_web_compilers/src/' + 'dev_compiler_stack_trace/stack_trace_mapper.dart.js'; + /// Bootstraps a ddc application, creating the main entrypoint as well as the /// bootstrap and digest entrypoints. /// @@ -33,6 +37,7 @@ Future bootstrapDdc( Iterable requiredAssets = const [], String entrypointExtension = jsEntrypointExtension, required bool? nativeNullAssertions, + bool usesWebHotReload = false, }) async { platform = ddcPlatform; // Ensures that the sdk resources are built and available. @@ -47,7 +52,11 @@ Future bootstrapDdc( // First, ensure all transitive modules are built. List transitiveJsModules; try { - transitiveJsModules = await _ensureTransitiveJsModules(module, buildStep); + transitiveJsModules = await _ensureTransitiveJsModules( + module, + buildStep, + computeStronglyConnectedComponents: !usesWebHotReload, + ); } on UnsupportedModules catch (e) { final librariesString = (await e.exactLibraries(buildStep).toList()) .map( @@ -93,69 +102,124 @@ $librariesString _context.withoutExtension(buildStep.inputId.path), ); - // Map from module name to module path for custom modules. - final modulePaths = SplayTreeMap.of({ - 'dart_sdk': r'packages/build_web_compilers/src/dev_compiler/dart_sdk', - }); - for (final jsId in transitiveJsModules) { - // Strip out the top level dir from the path for any module, and set it to - // `packages/` for lib modules. We set baseUrl to `/` to simplify things, - // and we only allow you to serve top level directories. - final moduleName = ddcModuleName(jsId); - modulePaths[moduleName] = _context.withoutExtension( - jsId.path.startsWith('lib') - ? '$moduleName$jsModuleExtension' - : _context.joinAll(_context.split(jsId.path).skip(1)), - ); - } - final bootstrapId = dartEntrypointId.changeExtension(ddcBootstrapExtension); - final bootstrapModuleName = _context.withoutExtension( - _context.relative( - bootstrapId.path, - from: _context.dirname(dartEntrypointId.path), - ), + final bootstrapEndId = dartEntrypointId.changeExtension( + ddcBootstrapEndExtension, ); final dartEntrypointParts = _context.split(dartEntrypointId.path); - final entrypointLibraryName = _context.joinAll([ - // Convert to a package: uri for files under lib. - if (dartEntrypointParts.first == 'lib') - 'package:${module.primarySource.package}', - // Strip top-level directory from the path. - ...dartEntrypointParts.skip(1), - ]); - - final bootstrapContent = - StringBuffer('$_entrypointExtensionMarker\n(function() {\n') - ..write( - _dartLoaderSetup( - modulePaths, - _p.url.relative( - appDigestsOutput.path, - from: _p.url.dirname(bootstrapId.path), - ), - ), - ) - ..write(_requireJsConfig) - ..write( - _appBootstrap( - bootstrapModuleName: bootstrapModuleName, - entrypointLibraryName: entrypointLibraryName, - moduleName: appModuleName, - moduleScope: appModuleScope, - nativeNullAssertions: nativeNullAssertions, - oldModuleScope: oldAppModuleScope, - ), - ); + final packageName = module.primarySource.package; + final entrypointLibraryName = + usesWebHotReload + ? _context.joinAll([ + // Convert to a package: uri for files under lib. + if (dartEntrypointParts.first == 'lib') 'package:$packageName', + ...dartEntrypointParts, + ]) + : _context.joinAll([ + // Convert to a package: uri for files under lib. + if (dartEntrypointParts.first == 'lib') 'package:$packageName', + // Strip top-level directory from the path. + ...dartEntrypointParts.skip(1), + ]); + + final entrypointJsId = dartEntrypointId.changeExtension(entrypointExtension); - await buildStep.writeAsString(bootstrapId, bootstrapContent.toString()); + // Map from module name to module path for custom modules. + final modulePaths = SplayTreeMap(); + String entrypointJsContent; + String bootstrapContent; + String bootstrapEndContent; + if (usesWebHotReload) { + final ddcSdkUrl = + r'packages/build_web_compilers/src/dev_compiler/dart_sdk.js'; + modulePaths['dart_sdk'] = ddcSdkUrl; + for (final jsId in transitiveJsModules) { + // Strip out the top level dir from the path for any module, and set it to + // `packages/` for lib modules. We set baseUrl to `/` to simplify things, + // and we only allow you to serve top level directories. + final moduleName = ddcModuleName(jsId); + final libraryId = ddcLibraryId(jsId); + modulePaths[libraryId] = + jsId.path.startsWith('lib') + ? '$moduleName$jsModuleExtension' + : _context.joinAll(_context.split(jsId.path).skip(1)); + } + final bootstrapEndModuleName = _context.relative( + bootstrapId.path, + from: _context.dirname(bootstrapEndId.path), + ); + bootstrapContent = generateDDCLibraryBundleMainModule( + entrypoint: entrypointLibraryName, + nativeNullAssertions: nativeNullAssertions ?? false, + onLoadEndBootstrap: bootstrapEndModuleName, + ); + final bootstrapModuleName = _context.relative( + bootstrapId.path, + from: _context.dirname(dartEntrypointId.path), + ); + entrypointJsContent = generateDDCLibraryBundleBootstrapScript( + entrypoint: entrypointLibraryName, + ddcSdkUrl: ddcSdkUrl, + ddcModuleLoaderUrl: + 'packages/build_web_compilers/src/dev_compiler/ddc_module_loader.js', + mainBoostrapperUrl: bootstrapModuleName, + mapperUrl: stackTraceMapperPath, + isWindows: Platform.isWindows, + scriptIdsToPath: modulePaths, + ); + bootstrapEndContent = generateDDCLibraryBundleOnLoadEndBootstrap(); + } else { + modulePaths['dart_sdk'] = + r'packages/build_web_compilers/src/dev_compiler/dart_sdk'; + for (final jsId in transitiveJsModules) { + // Strip out the top level dir from the path for any module, and set it to + // `packages/` for lib modules. We set baseUrl to `/` to simplify things, + // and we only allow you to serve top level directories. + final moduleName = ddcModuleName(jsId); + modulePaths[moduleName] = _context.withoutExtension( + jsId.path.startsWith('lib') + ? '$moduleName$jsModuleExtension' + : _context.joinAll(_context.split(jsId.path).skip(1)), + ); + } + final bootstrapModuleName = _context.withoutExtension( + _context.relative( + bootstrapId.path, + from: _context.dirname(dartEntrypointId.path), + ), + ); + entrypointJsContent = _entryPointJs(bootstrapModuleName); + bootstrapContent = + (StringBuffer('$_entrypointExtensionMarker\n(function() {\n') + ..write( + _dartLoaderSetup( + modulePaths, + _p.url.relative( + appDigestsOutput.path, + from: _p.url.dirname(bootstrapId.path), + ), + ), + ) + ..write(_requireJsConfig) + ..write( + _appBootstrap( + bootstrapModuleName: bootstrapModuleName, + entrypointLibraryName: entrypointLibraryName, + moduleName: appModuleName, + moduleScope: appModuleScope, + nativeNullAssertions: nativeNullAssertions, + oldModuleScope: oldAppModuleScope, + ), + )) + .toString(); + // Unused for the AMD module system. + bootstrapEndContent = ''; + } - final entrypointJsContent = _entryPointJs(bootstrapModuleName); - await buildStep.writeAsString( - dartEntrypointId.changeExtension(entrypointExtension), - entrypointJsContent, - ); + await buildStep.writeAsString(entrypointJsId, entrypointJsContent); + await buildStep.writeAsString(bootstrapId, bootstrapContent); + await buildStep.writeAsString(bootstrapEndId, bootstrapEndContent); // Output the digests and merged_metadata for transitive modules. // These can be consumed for hot reloads and debugging. @@ -185,12 +249,14 @@ final _lazyBuildPool = Pool(16); /// unsupported modules. Future> _ensureTransitiveJsModules( Module module, - BuildStep buildStep, -) async { + BuildStep buildStep, { + bool computeStronglyConnectedComponents = true, +}) async { // Collect all the modules this module depends on, plus this module. final transitiveDeps = await module.computeTransitiveDependencies( buildStep, throwIfUnsupported: true, + computeStronglyConnectedComponents: computeStronglyConnectedComponents, ); final jsModules = [ @@ -277,8 +343,7 @@ String _entryPointJs(String bootstrapModuleName) => ''' $_currentDirectoryScript $_baseUrlScript - var mapperUri = baseUrl + "packages/build_web_compilers/src/" + - "dev_compiler_stack_trace/stack_trace_mapper.dart.js"; + var mapperUri = baseUrl + "$stackTraceMapperPath"; var requireUri = baseUrl + "packages/build_web_compilers/src/dev_compiler/require.js"; var mainUri = _currentDirectory + "$bootstrapModuleName"; @@ -570,3 +635,260 @@ Future _ensureResources( } } } + +const _simpleLoaderScript = r''' +window.$dartCreateScript = (function() { + // Find the nonce value. (Note, this is only computed once.) + var scripts = Array.from(document.getElementsByTagName("script")); + var nonce; + scripts.some( + script => (nonce = script.nonce || script.getAttribute("nonce"))); + // If present, return a closure that automatically appends the nonce. + if (nonce) { + return function() { + var script = document.createElement("script"); + script.nonce = nonce; + return script; + }; + } else { + return function() { + return document.createElement("script"); + }; + } +})(); + +// Loads a module [relativeUrl] relative to [root]. +// +// If not specified, [root] defaults to the directory serving the main app. +var forceLoadModule = function (relativeUrl, root) { + var actualRoot = root ?? _currentDirectory; + return new Promise(function(resolve, reject) { + var script = self.$dartCreateScript(); + let policy = { + createScriptURL: function(src) {return src;} + }; + if (self.trustedTypes && self.trustedTypes.createPolicy) { + policy = self.trustedTypes.createPolicy('dartDdcModuleUrl', policy); + } + script.onload = resolve; + script.onerror = reject; + script.src = policy.createScriptURL(actualRoot + relativeUrl); + document.head.appendChild(script); + }); +}; +'''; + +String generateDDCLibraryBundleBootstrapScript({ + required String entrypoint, + required String ddcSdkUrl, + required String ddcModuleLoaderUrl, + required String mainBoostrapperUrl, + required String mapperUrl, + required bool isWindows, + required Map scriptIdsToPath, +}) { + final scriptsJs = StringBuffer(); + scriptIdsToPath.forEach((id, path) { + scriptsJs.write('{"src": "$path", "id": "$id"},\n'); + }); + // Write the "true" main boostrapper last as part of the loader's convention. + scriptsJs.write('{"src": "$mainBoostrapperUrl", "id": "data-main"}\n'); + final boostrapScript = ''' +// Save the current directory so we can access it in a closure. + let _currentDirectory = (function () { + let _url = document.currentScript.src; + let lastSlash = _url.lastIndexOf('/'); + if (lastSlash == -1) return _url; + let currentDirectory = _url.substring(0, lastSlash + 1); + return currentDirectory; + })(); + + let trimmedDirectory = _currentDirectory.endsWith("/") ? + _currentDirectory.substring(0, _currentDirectory.length - 1) + : _currentDirectory; + +$_simpleLoaderScript + +(function() { + let appName = "$multiRootScheme:///$entrypoint"; + + // Load pre-requisite DDC scripts. We intentionally use invalid names to avoid + // namespace clashes. + let prerequisiteScripts = [ + { + "src": "$ddcModuleLoaderUrl", + "id": "ddc_module_loader \x00" + }, + { + "src": "$mapperUrl", + "id": "dart_stack_trace_mapper \x00" + } + ]; + + // Load ddc_module_loader.js to access DDC's module loader API. + let prerequisiteLoads = []; + for (let i = 0; i < prerequisiteScripts.length; i++) { + prerequisiteLoads.push(forceLoadModule(prerequisiteScripts[i].src)); + } + Promise.all(prerequisiteLoads).then((_) => afterPrerequisiteLogic()); + + // Save the current script so we can access it in a closure. + var _currentScript = document.currentScript; + + // Create a policy if needed to load the files during a hot restart. + let policy = { + createScriptURL: function(src) {return src;} + }; + if (self.trustedTypes && self.trustedTypes.createPolicy) { + policy = self.trustedTypes.createPolicy('dartDdcModuleUrl', policy); + } + + var afterPrerequisiteLogic = function() { + window.\$dartLoader.rootDirectories.push(_currentDirectory); + let scripts = [${scriptsJs.toString()}]; + + let loadConfig = new window.\$dartLoader.LoadConfiguration(); + // TODO(srujzs): Verify this is sufficient for Windows. + loadConfig.isWindows = $isWindows; + loadConfig.root = trimmedDirectory; + loadConfig.bootstrapScript = scripts[scripts.length - 1]; + + loadConfig.loadScriptFn = function(loader) { + loader.addScriptsToQueue(scripts, null); + loader.loadEnqueuedModules(); + } + loadConfig.ddcEventForLoadStart = /* LOAD_ALL_MODULES_START */ 1; + loadConfig.ddcEventForLoadedOk = /* LOAD_ALL_MODULES_END_OK */ 2; + loadConfig.ddcEventForLoadedError = /* LOAD_ALL_MODULES_END_ERROR */ 3; + + let loader = new window.\$dartLoader.DDCLoader(loadConfig); + + // Record prerequisite scripts' fully resolved URLs. + prerequisiteScripts.forEach(script => loader.registerScript(script)); + + // Note: these variables should only be used in non-multi-app scenarios + // since they can be arbitrarily overridden based on multi-app load order. + window.\$dartLoader.loadConfig = loadConfig; + window.\$dartLoader.loader = loader; + + // Begin loading libraries + loader.nextAttempt(); + + // Set up stack trace mapper. + if (window.\$dartStackTraceUtility && + !window.\$dartStackTraceUtility.ready) { + window.\$dartStackTraceUtility.ready = true; + window.\$dartStackTraceUtility.setSourceMapProvider(function(url) { + var baseUrl = window.location.protocol + '//' + window.location.host; + url = url.replace(baseUrl + '/', ''); + if (url == 'dart_sdk.js') { + return dartDevEmbedder.debugger.getSourceMap('dart_sdk'); + } + url = url.replace(".lib.js", ""); + return dartDevEmbedder.debugger.getSourceMap(url); + }); + } + + let currentUri = _currentScript.src; + // We should have written a file containing all the scripts that need to be + // reloaded into the page. This is then read when a hot restart is triggered + // in DDC via the `\$dartReloadModifiedModules` callback. + let restartScripts = _currentDirectory + 'restart_scripts.json'; + + if (!window.\$dartReloadModifiedModules) { + window.\$dartReloadModifiedModules = (function(appName, callback) { + var xhttp = new XMLHttpRequest(); + xhttp.withCredentials = true; + xhttp.onreadystatechange = function() { + // https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/readyState + if (this.readyState == 4 && this.status == 200 || this.status == 304) { + var scripts = JSON.parse(this.responseText); + var numToLoad = 0; + var numLoaded = 0; + for (var i = 0; i < scripts.length; i++) { + var script = scripts[i]; + if (script.id == null) continue; + var src = script.src.toString(); + var oldSrc = window.\$dartLoader.moduleIdToUrl.get(script.id); + + // We might actually load from a different uri, delete the old one + // just to be sure. + window.\$dartLoader.urlToModuleId.delete(oldSrc); + + window.\$dartLoader.moduleIdToUrl.set(script.id, src); + window.\$dartLoader.urlToModuleId.set(src, script.id); + + numToLoad++; + + var el = document.getElementById(script.id); + if (el) el.remove(); + el = window.\$dartCreateScript(); + el.src = policy.createScriptURL(src); + el.async = false; + el.defer = true; + el.id = script.id; + el.onload = function() { + numLoaded++; + if (numToLoad == numLoaded) callback(); + }; + document.head.appendChild(el); + } + // Call `callback` right away if we found no updated scripts. + if (numToLoad == 0) callback(); + } + }; + xhttp.open("GET", restartScripts, true); + xhttp.send(); + }); + } + }; +})(); +'''; + return boostrapScript; +} + +String generateDDCLibraryBundleMainModule({ + required String entrypoint, + required bool nativeNullAssertions, + required String onLoadEndBootstrap, +}) { + // The typo below in "EXTENTION" is load-bearing, package:build depends on it. + return ''' +/* ENTRYPOINT_EXTENTION_MARKER */ + +(function() { + let appName = "$multiRootScheme:///$entrypoint"; + dartDevEmbedder.debugger.registerDevtoolsFormatter(); + + // Set up a final script that lets us know when all scripts have been loaded. + // Only then can we call the main method. + let onLoadEndSrc = '$onLoadEndBootstrap'; + window.\$dartLoader.loadConfig.bootstrapScript = { + src: onLoadEndSrc, + id: onLoadEndSrc, + }; + window.\$dartLoader.loadConfig.tryLoadBootstrapScript = true; + // Should be called by $onLoadEndBootstrap once all the scripts have been + // loaded. + window.$_onLoadEndCallback = function() { + let child = {}; + child.main = function() { + let sdkOptions = { + nativeNonNullAsserts: $nativeNullAssertions, + }; + dartDevEmbedder.runMain(appName, sdkOptions); + } + /* MAIN_EXTENSION_MARKER */ + child.main(); + } + // Call this immediately in build_web_compilers. + window.$_onLoadEndCallback(); +})(); +'''; +} + +String generateDDCLibraryBundleOnLoadEndBootstrap() { + return '''window.$_onLoadEndCallback();'''; +} + +const _onLoadEndCallback = r'$onLoadEndCallback'; diff --git a/build_web_compilers/lib/src/dev_compiler_builder.dart b/build_web_compilers/lib/src/dev_compiler_builder.dart index c4c5555f0f..66cbe235cd 100644 --- a/build_web_compilers/lib/src/dev_compiler_builder.dart +++ b/build_web_compilers/lib/src/dev_compiler_builder.dart @@ -11,19 +11,11 @@ import 'package:build/build.dart'; import 'package:build/experiments.dart'; import 'package:build_modules/build_modules.dart'; import 'package:path/path.dart' as p; -import 'package:scratch_space/scratch_space.dart'; import '../builders.dart'; import 'common.dart'; import 'errors.dart'; -final jsModuleErrorsExtension = '.ddc.js.errors'; -final jsModuleExtension = '.ddc.js'; -final jsSourceMapExtension = '.ddc.js.map'; -final metadataExtension = '.ddc.js.metadata'; -final symbolsExtension = '.ddc.js.symbols'; -final fullKernelExtension = '.ddc.full.dill'; - /// A builder which can output ddc modules! class DevCompilerBuilder implements Builder { final bool useIncrementalCompiler; @@ -50,6 +42,9 @@ class DevCompilerBuilder implements Builder { /// Enables canary features in DDC. final bool canaryFeatures; + /// Emits code with the DDC module system. + final bool ddcModules; + final bool trackUnusedInputs; final DartPlatform platform; @@ -79,6 +74,7 @@ class DevCompilerBuilder implements Builder { this.generateFullDill = false, this.emitDebugSymbols = false, this.canaryFeatures = false, + this.ddcModules = false, this.trackUnusedInputs = false, required this.platform, String? sdkKernelPath, @@ -133,6 +129,7 @@ class DevCompilerBuilder implements Builder { generateFullDill, emitDebugSymbols, canaryFeatures, + ddcModules, trackUnusedInputs, platformSdk, sdkKernelPath, @@ -155,6 +152,7 @@ Future _createDevCompilerModule( bool generateFullDill, bool emitDebugSymbols, bool canaryFeatures, + bool ddcModules, bool trackUnusedInputs, String dartSdk, String sdkKernelPath, @@ -204,7 +202,7 @@ Future _createDevCompilerModule( WorkRequest() ..arguments.addAll([ '--dart-sdk-summary=$sdkSummary', - '--modules=amd', + '--modules=${ddcModules ? 'ddc' : 'amd'}', '--no-summarize', if (generateFullDill) '--experimental-output-compiled-kernel', if (emitDebugSymbols) '--emit-debug-symbols', @@ -227,7 +225,7 @@ Future _createDevCompilerModule( ], if (usedInputsFile != null) '--used-inputs-file=${usedInputsFile.uri.toFilePath()}', - for (final source in module.sources) _sourceArg(source), + for (final source in module.sources) sourceArg(source), for (final define in environment.entries) '-D${define.key}=${define.value}', for (final experiment in enabledExperiments) @@ -303,7 +301,7 @@ Future _createDevCompilerModule( final file = scratchSpace.fileFor(metadataId); final content = await file.readAsString(); final json = jsonDecode(content) as Map; - _fixMetadataSources(json, scratchSpace.tempDir.uri); + fixMetadataSources(json, scratchSpace.tempDir.uri); await buildStep.writeAsString(metadataId, jsonEncode(json)); // Copy the symbols output, modifying its contents to remove the temp @@ -339,55 +337,3 @@ String _summaryArg(Module module) { ); return '--summary=${scratchSpace.fileFor(kernelAsset).path}=$moduleName'; } - -/// The url to compile for a source. -/// -/// Use the package: path for files under lib and the full absolute path for -/// other files. -String _sourceArg(AssetId id) { - final uri = canonicalUriFor(id); - return uri.startsWith('package:') ? uri : '$multiRootScheme:///${id.path}'; -} - -/// The module name according to ddc for [jsId] which represents the real js -/// module file. -String ddcModuleName(AssetId jsId) { - final jsPath = - jsId.path.startsWith('lib/') - ? jsId.path.replaceFirst('lib/', 'packages/${jsId.package}/') - : jsId.path; - return jsPath.substring(0, jsPath.length - jsModuleExtension.length); -} - -void _fixMetadataSources(Map json, Uri scratchUri) { - String updatePath(String path) => - Uri.parse(path).path.replaceAll(scratchUri.path, ''); - - final sourceMapUri = json['sourceMapUri'] as String?; - if (sourceMapUri != null) { - json['sourceMapUri'] = updatePath(sourceMapUri); - } - - final moduleUri = json['moduleUri'] as String?; - if (moduleUri != null) { - json['moduleUri'] = updatePath(moduleUri); - } - - final fullDillUri = json['fullDillUri'] as String?; - if (fullDillUri != null) { - json['fullDillUri'] = updatePath(fullDillUri); - } - - final libraries = json['libraries'] as List?; - if (libraries != null) { - for (final lib in libraries) { - final libraryJson = lib as Map?; - if (libraryJson != null) { - final fileUri = libraryJson['fileUri'] as String?; - if (fileUri != null) { - libraryJson['fileUri'] = updatePath(fileUri); - } - } - } - } -} diff --git a/build_web_compilers/lib/src/errors.dart b/build_web_compilers/lib/src/errors.dart index 38ea3a0c5b..c73d82a513 100644 --- a/build_web_compilers/lib/src/errors.dart +++ b/build_web_compilers/lib/src/errors.dart @@ -26,3 +26,11 @@ class DartDevcCompilationException extends _WorkerException { DartDevcCompilationException(super.jsId, super.error); } + +/// An [Exception] that is thrown when DDC + Frontend Server compilation fails. +class FrontendServerCompilationException extends _WorkerException { + @override + final String message = 'Error compiling DDC with Frontend Server'; + + FrontendServerCompilationException(super.jsId, super.error); +} diff --git a/build_web_compilers/lib/src/sdk_js_compile_builder.dart b/build_web_compilers/lib/src/sdk_js_compile_builder.dart index 77ae41a376..824fc89836 100644 --- a/build_web_compilers/lib/src/sdk_js_compile_builder.dart +++ b/build_web_compilers/lib/src/sdk_js_compile_builder.dart @@ -42,12 +42,17 @@ class SdkJsCompileBuilder implements Builder { /// Enables canary features in DDC. final bool canaryFeatures; + /// Emits DDC code with the Library Bundle module system, which supports hot + /// reload. + final bool usesWebHotReload; + SdkJsCompileBuilder({ required this.sdkKernelPath, required String outputPath, String? librariesPath, String? platformSdk, required this.canaryFeatures, + required this.usesWebHotReload, }) : platformSdk = platformSdk ?? sdkDir, librariesPath = librariesPath ?? @@ -72,6 +77,7 @@ class SdkJsCompileBuilder implements Builder { librariesPath, jsOutputId, canaryFeatures, + usesWebHotReload, ); } } @@ -84,6 +90,7 @@ Future _createDevCompilerModule( String librariesPath, AssetId jsOutputId, bool canaryFeatures, + bool usesWebHotReload, ) async { final scratchSpace = await buildStep.fetchResource(scratchSpaceResource); final jsOutputFile = scratchSpace.fileFor(jsOutputId); @@ -112,8 +119,8 @@ Future _createDevCompilerModule( result = await Process.run(dartPath, [ snapshotPath, '--multi-root-scheme=org-dartlang-sdk', - '--modules=amd', - if (canaryFeatures) '--canary', + '--modules=${usesWebHotReload ? 'ddc' : 'amd'}', + if (canaryFeatures || usesWebHotReload) '--canary', '--module-name=dart_sdk', '-o', jsOutputFile.path, diff --git a/build_web_compilers/lib/src/sdk_js_copy_builder.dart b/build_web_compilers/lib/src/sdk_js_copy_builder.dart index e6432ad1b4..631cca43fb 100644 --- a/build_web_compilers/lib/src/sdk_js_copy_builder.dart +++ b/build_web_compilers/lib/src/sdk_js_copy_builder.dart @@ -10,12 +10,15 @@ import 'package:path/path.dart' as p; import 'common.dart'; -/// Copies the require.js file from the sdk itself, into the -/// build_web_compilers package at `lib/require.js`. +/// Copies the `require.js` and `ddc_module_loader.js` files from the SDK +/// into the `build_web_compilers` package under `lib/`. class SdkJsCopyBuilder implements Builder { @override final buildExtensions = { - r'$package$': ['lib/src/dev_compiler/require.js'], + r'$package$': [ + 'lib/src/dev_compiler/require.js', + 'lib/src/dev_compiler/ddc_module_loader.js', + ], }; /// Path to the require.js file that should be used for all ddc web apps. @@ -27,6 +30,16 @@ class SdkJsCopyBuilder implements Builder { 'require.js', ); + /// Path to the ddc_module_loader.js file that should be used for all ddc web + /// apps running with the library bundle module system. + final _sdkModuleLoaderJsLocation = p.join( + sdkDir, + 'lib', + 'dev_compiler', + 'ddc', + 'ddc_module_loader.js', + ); + @override FutureOr build(BuildStep buildStep) async { if (buildStep.inputId.package != 'build_web_compilers') { @@ -39,5 +52,12 @@ class SdkJsCopyBuilder implements Builder { AssetId('build_web_compilers', 'lib/src/dev_compiler/require.js'), await File(_sdkRequireJsLocation).readAsBytes(), ); + await buildStep.writeAsBytes( + AssetId( + 'build_web_compilers', + 'lib/src/dev_compiler/ddc_module_loader.js', + ), + await File(_sdkModuleLoaderJsLocation).readAsBytes(), + ); } } diff --git a/build_web_compilers/lib/src/web_entrypoint_builder.dart b/build_web_compilers/lib/src/web_entrypoint_builder.dart index 82c001a91c..8452e7590d 100644 --- a/build_web_compilers/lib/src/web_entrypoint_builder.dart +++ b/build_web_compilers/lib/src/web_entrypoint_builder.dart @@ -17,6 +17,7 @@ import 'dart2wasm_bootstrap.dart'; import 'dev_compiler_bootstrap.dart'; const ddcBootstrapExtension = '.dart.bootstrap.js'; +const ddcBootstrapEndExtension = '.dart.bootstrap.end.js'; const jsEntrypointExtension = '.dart.js'; const wasmExtension = '.wasm'; const wasmSourceMapExtension = '.wasm.map'; @@ -123,10 +124,16 @@ final class EntrypointBuilderOptions { /// necessary. final String? loaderExtension; + /// Whether or not to emit DDC entrypoints that support web hot reload. + /// + /// Web hot reload is only supported for DDC's Library Bundle module system. + final bool usesWebHotReload; + EntrypointBuilderOptions({ required this.compilers, this.nativeNullAssertions, this.loaderExtension, + this.usesWebHotReload = false, }); factory EntrypointBuilderOptions.fromOptions(BuilderOptions options) { @@ -138,6 +145,7 @@ final class EntrypointBuilderOptions { const dart2wasmArgsOption = 'dart2wasm_args'; const nativeNullAssertionsOption = 'native_null_assertions'; const loaderOption = 'loader'; + const webHotReloadOption = 'web-hot-reload'; String? defaultLoaderOption; const supportedOptions = [ @@ -147,11 +155,13 @@ final class EntrypointBuilderOptions { nativeNullAssertionsOption, dart2wasmArgsOption, loaderOption, + webHotReloadOption, ]; final config = options.config; final nativeNullAssertions = options.config[nativeNullAssertionsOption] as bool?; + final usesWebHotReload = options.config[webHotReloadOption] as bool?; final compilers = []; validateOptions( @@ -238,6 +248,7 @@ final class EntrypointBuilderOptions { config.containsKey(loaderOption) ? config[loaderOption] as String? : defaultLoaderOption, + usesWebHotReload: usesWebHotReload ?? false, ); } @@ -250,6 +261,7 @@ final class EntrypointBuilderOptions { '.dart': [ if (optionsFor(WebCompiler.DartDevc) case final ddc?) ...[ ddcBootstrapExtension, + ddcBootstrapEndExtension, mergedMetadataExtension, digestsEntrypointExtension, ddc.extension, @@ -317,10 +329,15 @@ class WebEntrypointBuilder implements Builder { compilationSteps.add( Future(() async { try { + final usesWebHotReload = options.usesWebHotReload; await bootstrapDdc( buildStep, nativeNullAssertions: options.nativeNullAssertions, - requiredAssets: _ddcSdkResources, + requiredAssets: + usesWebHotReload + ? _ddcLibraryBundleSdkResources + : _ddcSdkResources, + usesWebHotReload: usesWebHotReload, ); } on MissingModulesException catch (e) { log.severe('$e'); @@ -451,8 +468,16 @@ Future _isAppEntryPoint(AssetId dartId, AssetReader reader) async { } /// Files copied from the SDK that are required at runtime to run a DDC -/// application. +/// application with the AMD module system. final _ddcSdkResources = [ AssetId('build_web_compilers', 'lib/src/dev_compiler/dart_sdk.js'), AssetId('build_web_compilers', 'lib/src/dev_compiler/require.js'), ]; + +/// Files copied from the SDK that are required at runtime to run a DDC +/// application with the Library Bundle module system (which supports hot +/// reload). +final _ddcLibraryBundleSdkResources = [ + AssetId('build_web_compilers', 'lib/src/dev_compiler/dart_sdk.js'), + AssetId('build_web_compilers', 'lib/src/dev_compiler/ddc_module_loader.js'), +]; diff --git a/build_web_compilers/lib/src/web_entrypoint_marker_builder.dart b/build_web_compilers/lib/src/web_entrypoint_marker_builder.dart new file mode 100644 index 0000000000..0d56da9e6f --- /dev/null +++ b/build_web_compilers/lib/src/web_entrypoint_marker_builder.dart @@ -0,0 +1,57 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; + +import 'package:build/build.dart'; +import 'package:build_modules/build_modules.dart'; +import 'package:glob/glob.dart'; + +/// A builder that gathers information about a web target's 'main' entrypoint. +class WebEntrypointMarkerBuilder implements Builder { + /// Records state (such as the web entrypoint) required when compiling DDC + /// with the Library Bundle module system. + /// + /// A no-op if [usesWebHotReload] is not set. + final bool usesWebHotReload; + + WebEntrypointMarkerBuilder({this.usesWebHotReload = false}); + + @override + final buildExtensions = const { + r'$web$': ['.web.entrypoint.json'], + }; + + @override + Future build(BuildStep buildStep) async { + if (!usesWebHotReload) return; + + final frontendServerState = await buildStep.fetchResource( + frontendServerStateResource, + ); + final webAssets = await buildStep.findAssets(Glob('web/**')).toList(); + final webEntrypointJson = {}; + + for (final asset in webAssets) { + if (asset.extension == '.dart') { + final moduleLibrary = ModuleLibrary.fromSource( + asset, + await buildStep.readAsString(asset), + ); + if (moduleLibrary.hasMain && moduleLibrary.isEntryPoint) { + // We must save the main entrypoint as the recompilation target for + // the Frontend Server before any JS files are emitted. + frontendServerState.entrypointAssetId = asset; + webEntrypointJson['entrypoint'] = asset.uri.toString(); + break; + } + } + } + + await buildStep.writeAsString( + AssetId(buildStep.inputId.package, 'web/.web.entrypoint.json'), + jsonEncode(webEntrypointJson), + ); + } +} diff --git a/build_web_compilers/pubspec.yaml b/build_web_compilers/pubspec.yaml index 0f37d62fa1..a6a59511f4 100644 --- a/build_web_compilers/pubspec.yaml +++ b/build_web_compilers/pubspec.yaml @@ -1,5 +1,5 @@ name: build_web_compilers -version: 4.3.2 +version: 4.4.0 description: Builder implementations wrapping the dart2js and DDC compilers. repository: https://github.com/dart-lang/build/tree/master/build_web_compilers resolution: workspace @@ -12,13 +12,14 @@ dependencies: archive: '>=3.0.0 <5.0.0' bazel_worker: ^1.0.0 build: '>=2.0.0 <5.0.0' - build_modules: ^5.0.0 + build_modules: ^5.1.0 + build_runner: ^2.0.0 collection: ^1.15.0 glob: ^2.0.0 logging: ^1.0.0 path: ^1.8.0 pool: ^1.5.0 - scratch_space: ^1.0.0 + scratch_space: ^1.2.0 source_maps: ^0.10.10 source_span: ^1.8.0 stack_trace: ^1.10.0 @@ -28,7 +29,6 @@ dev_dependencies: path: ../build_modules/test/fixtures/a b: path: ../build_modules/test/fixtures/b - build_runner: ^2.0.0 build_test: ^3.0.0 c: path: test/fixtures/c diff --git a/build_web_compilers/test/dev_compiler_bootstrap_test.dart b/build_web_compilers/test/dev_compiler_bootstrap_test.dart index e39fe16abe..0bda980b02 100644 --- a/build_web_compilers/test/dev_compiler_bootstrap_test.dart +++ b/build_web_compilers/test/dev_compiler_bootstrap_test.dart @@ -65,6 +65,8 @@ void main() { 'build_web_compilers|lib/.ddc.meta_module.raw': isNotNull, 'build_web_compilers|lib/src/dev_compiler/dart_sdk.js.map': isNotNull, 'build_web_compilers|lib/src/dev_compiler/dart_sdk.js': isNotNull, + 'build_web_compilers|lib/src/dev_compiler/ddc_module_loader.js': + isNotNull, 'build_web_compilers|lib/src/dev_compiler/require.js': isNotNull, }; @@ -102,6 +104,7 @@ void main() { isNot(contains('lib/a')), ]), ), + 'a|web/index.dart.bootstrap.end.js': isEmpty, 'a|web/index.dart.ddc_merged_metadata': isNotEmpty, 'a|web/index.dart.js': decodedMatches(contains('index.dart.bootstrap')), 'a|web/index.digests': decodedMatches(contains('packages/')), @@ -157,6 +160,7 @@ void main() { contains('if (childName === "b.dart")'), ]), ), + 'a|web/b.dart.bootstrap.end.js': isEmpty, 'a|web/b.dart.ddc_merged_metadata': isNotNull, 'a|web/b.dart.js': isNotNull, 'a|web/b.ddc.module': isNotNull, @@ -166,6 +170,8 @@ void main() { 'build_web_compilers|lib/.ddc.meta_module.raw': isNotNull, 'build_web_compilers|lib/src/dev_compiler/dart_sdk.js.map': isNotNull, 'build_web_compilers|lib/src/dev_compiler/dart_sdk.js': isNotNull, + 'build_web_compilers|lib/src/dev_compiler/ddc_module_loader.js': + isNotNull, 'build_web_compilers|lib/src/dev_compiler/require.js': isNotNull, }; @@ -198,6 +204,7 @@ void main() { contains('if (childName === "package:a/app.dart")'), ]), ), + 'a|lib/app.dart.bootstrap.end.js': isEmpty, 'a|lib/app.dart.ddc_merged_metadata': isNotEmpty, 'a|lib/app.dart.js': isNotEmpty, 'a|lib/app.ddc.dill': isNotNull, @@ -211,6 +218,8 @@ void main() { 'build_web_compilers|lib/.ddc.meta_module.raw': isNotNull, 'build_web_compilers|lib/src/dev_compiler/dart_sdk.js.map': isNotNull, 'build_web_compilers|lib/src/dev_compiler/dart_sdk.js': isNotNull, + 'build_web_compilers|lib/src/dev_compiler/ddc_module_loader.js': + isNotNull, 'build_web_compilers|lib/src/dev_compiler/require.js': isNotNull, }; diff --git a/scratch_space/CHANGELOG.md b/scratch_space/CHANGELOG.md index 8ae4cd83af..652981366a 100644 --- a/scratch_space/CHANGELOG.md +++ b/scratch_space/CHANGELOG.md @@ -1,3 +1,6 @@ +## 1.2.0 +- Adding `changedFilesInBuild`, which contains all modified files encountered since the last `ensureAssets` call, and `dispose`. + ## 1.1.1 - Allow `build` 4.0.0. diff --git a/scratch_space/lib/src/scratch_space.dart b/scratch_space/lib/src/scratch_space.dart index cf76e4bab6..b065d04fc5 100644 --- a/scratch_space/lib/src/scratch_space.dart +++ b/scratch_space/lib/src/scratch_space.dart @@ -3,6 +3,7 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:async'; +import 'dart:collection'; import 'dart:io'; import 'package:build/build.dart'; @@ -27,6 +28,11 @@ class ScratchSpace { /// The temp directory at the root of this [ScratchSpace]. final Directory tempDir; + /// Contains assets that changed between calls to [ensureAssets]. + /// + /// Cleared at the end of every build. + final _changedFilesInBuild = {}; + // Assets which have a file created but are still being written to. final _pendingWrites = >{}; @@ -44,6 +50,9 @@ class ScratchSpace { ), ); + Iterable get changedFilesInBuild => + UnmodifiableSetView(_changedFilesInBuild); + /// Copies [id] from the tmp dir and writes it back using the [writer]. /// /// Note that [BuildStep] implements [AssetWriter] and that is typically @@ -92,6 +101,9 @@ class ScratchSpace { /// Copies [assetIds] to [tempDir] if they don't exist, using [reader] to /// read assets and mark dependencies. /// + /// Assets that have changed since the last time they were seen by + /// [ensureAssets] are added to [_changedFilesInBuild]. + /// /// Note that [BuildStep] implements [AssetReader] and that is typically /// what you will want to pass in. /// @@ -102,7 +114,6 @@ class ScratchSpace { if (!exists) { throw StateError('Tried to use a deleted ScratchSpace!'); } - final futures = assetIds.map((id) async { final digest = await reader.digest(id); @@ -111,6 +122,9 @@ class ScratchSpace { await _pendingWrites[id]; return; } + if (existing != null) { + _changedFilesInBuild.add(id); + } _digests[id] = digest; try { @@ -140,6 +154,11 @@ class ScratchSpace { /// with [id] to make sure it is actually present. File fileFor(AssetId id) => File(p.join(tempDir.path, p.normalize(_relativePathFor(id)))); + + /// Performs cleanup required across builds. + void dispose() { + _changedFilesInBuild.clear(); + } } /// Returns a canonical uri for [id]. diff --git a/scratch_space/pubspec.yaml b/scratch_space/pubspec.yaml index 9d0a876ca6..160fc15e46 100644 --- a/scratch_space/pubspec.yaml +++ b/scratch_space/pubspec.yaml @@ -1,5 +1,5 @@ name: scratch_space -version: 1.1.1 +version: 1.2.0 description: A tool to manage running external executables within package:build. repository: https://github.com/dart-lang/build/tree/master/scratch_space resolution: workspace