Skip to content

Commit f7c5cf7

Browse files
sbc100sbc100
authored
Remove ES6 constructs from file_packager output (#25476)
This allows the generated code to run on older engines. Unlike the majority of the emscripten-generated code the file packager output does not always get run through babel when targeting older engines. Specifically tests the standalone file packages code on the oldest support node version. This also required removing the use of the `fs/Promises` packages which is not present in older versions of node. Split out from #25459
1 parent 8f6e552 commit f7c5cf7

File tree

5 files changed

+44
-39
lines changed

5 files changed

+44
-39
lines changed

‎.circleci/config.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -936,6 +936,7 @@ jobs:
936936
other.test_native_call_before_init
937937
other.test_js_optimizer_verbose
938938
other.test_node_unhandled_rejection
939+
other.test_file_packager_separate_metadata
939940
other.test_full_js_library*
940941
core2.test_hello_world
941942
core2.test_fcntl_open_nodefs

‎test/codesize/test_codesize_file_preload.expected.js‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ var ENVIRONMENT_IS_NODE = globalThis.process?.versions?.node && globalThis.proce
3232
// --pre-jses are emitted after the Module integration code, so that they can
3333
// refer to Module (if they choose; they can also define Module)
3434
// include: <FILENAME REPLACED>
35-
Module["expectedDataFileDownloads"] ??= 0;
35+
if (!Module["expectedDataFileDownloads"]) Module["expectedDataFileDownloads"] = 0;
3636

3737
Module["expectedDataFileDownloads"]++;
3838

@@ -41,7 +41,7 @@ Module["expectedDataFileDownloads"]++;
4141
var isPthread = typeof ENVIRONMENT_IS_PTHREAD != "undefined" && ENVIRONMENT_IS_PTHREAD;
4242
var isWasmWorker = typeof ENVIRONMENT_IS_WASM_WORKER != "undefined" && ENVIRONMENT_IS_WASM_WORKER;
4343
if (isPthread || isWasmWorker) return;
44-
var isNode = globalThis.process?.versions?.node && globalThis.process?.type != "renderer";
44+
var isNode = globalThis.process && globalThis.process.versions && globalThis.process.versions.node && globalThis.process.type != "renderer";
4545
async function loadPackage(metadata) {
4646
var PACKAGE_PATH = "";
4747
if (typeof window === "object") {
@@ -52,15 +52,14 @@ Module["expectedDataFileDownloads"]++;
5252
}
5353
var PACKAGE_NAME = "a.out.data";
5454
var REMOTE_PACKAGE_BASE = "a.out.data";
55-
var REMOTE_PACKAGE_NAME = Module["locateFile"]?.(REMOTE_PACKAGE_BASE, "") ?? REMOTE_PACKAGE_BASE;
55+
var REMOTE_PACKAGE_NAME = Module["locateFile"] ? Module["locateFile"](REMOTE_PACKAGE_BASE, "") : REMOTE_PACKAGE_BASE;
5656
var REMOTE_PACKAGE_SIZE = metadata["remote_package_size"];
5757
async function fetchRemotePackage(packageName, packageSize) {
5858
if (isNode) {
59-
var fsPromises = require("fs/promises");
60-
var contents = await fsPromises.readFile(packageName);
61-
return contents.buffer;
59+
var contents = require("fs").readFileSync(packageName);
60+
return new Uint8Array(contents).buffer;
6261
}
63-
Module["dataFileDownloads"] ??= {};
62+
if (!Module["dataFileDownloads"]) Module["dataFileDownloads"] = {};
6463
try {
6564
var response = await fetch(packageName);
6665
} catch (e) {
@@ -73,9 +72,9 @@ Module["expectedDataFileDownloads"]++;
7372
}
7473
const chunks = [];
7574
const headers = response.headers;
76-
const total = Number(headers.get("Content-Length") ?? packageSize);
75+
const total = Number(headers.get("Content-Length") || packageSize);
7776
let loaded = 0;
78-
Module["setStatus"]?.("Downloading data...");
77+
Module["setStatus"] && Module["setStatus"]("Downloading data...");
7978
const reader = response.body.getReader();
8079
while (1) {
8180
var {done, value} = await reader.read();
@@ -92,7 +91,7 @@ Module["expectedDataFileDownloads"]++;
9291
totalLoaded += download.loaded;
9392
totalSize += download.total;
9493
}
95-
Module["setStatus"]?.(`Downloading data... (${totalLoaded}/${totalSize})`);
94+
Module["setStatus"] && Module["setStatus"](`Downloading data... (${totalLoaded}/${totalSize})`);
9695
}
9796
const packageData = new Uint8Array(chunks.map(c => c.length).reduce((a, b) => a + b, 0));
9897
let offset = 0;
@@ -119,7 +118,7 @@ Module["expectedDataFileDownloads"]++;
119118
}
120119
async function processPackageData(arrayBuffer) {
121120
assert(arrayBuffer, "Loading data file failed.");
122-
assert(arrayBuffer.constructor.name === ArrayBuffer.name, "bad input to processPackageData");
121+
assert(arrayBuffer.constructor.name === ArrayBuffer.name, "bad input to processPackageData " + arrayBuffer.constructor.name);
123122
var byteArray = new Uint8Array(arrayBuffer);
124123
// Reuse the bytearray from the XHR as the source for file reads.
125124
for (var file of metadata["files"]) {
@@ -132,7 +131,7 @@ Module["expectedDataFileDownloads"]++;
132131
Module["removeRunDependency"]("datafile_a.out.data");
133132
}
134133
Module["addRunDependency"]("datafile_a.out.data");
135-
Module["preloadResults"] ??= {};
134+
if (!Module["preloadResults"]) Module["preloadResults"] = {};
136135
Module["preloadResults"][PACKAGE_NAME] = {
137136
fromCache: false
138137
};
@@ -144,7 +143,8 @@ Module["expectedDataFileDownloads"]++;
144143
if (Module["calledRun"]) {
145144
runWithFS(Module);
146145
} else {
147-
(Module["preRun"] ??= []).push(runWithFS);
146+
if (!Module["preRun"]) Module["preRun"] = [];
147+
Module["preRun"].push(runWithFS);
148148
}
149149
}
150150
loadPackage({

‎test/codesize/test_codesize_file_preload.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 22473,
3-
"a.out.js.gz": 9349,
2+
"a.out.js": 22552,
3+
"a.out.js.gz": 9344,
44
"a.out.nodebug.wasm": 1681,
55
"a.out.nodebug.wasm.gz": 960,
6-
"total": 24154,
7-
"total_gz": 10309,
6+
"total": 24233,
7+
"total_gz": 10304,
88
"sent": [
99
"a (fd_write)"
1010
],

‎test/test_other.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3849,7 +3849,7 @@ def test_file_packager_separate_metadata(self):
38493849
return 0;
38503850
}
38513851
''')
3852-
self.do_runf('src.c', cflags=['--pre-js=immutable.js', '-sFORCE_FILESYSTEM'])
3852+
self.do_runf('src.c', 'done\n', cflags=['--extern-pre-js=immutable.js', '-sFORCE_FILESYSTEM'])
38533853

38543854
def test_file_packager_unicode(self):
38553855
unicode_name = 'unicode…☃'

‎tools/file_packager.py‎

Lines changed: 25 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -612,7 +612,7 @@ def generate_js(data_target, data_files, metadata):
612612
var Module = typeof %(EXPORT_NAME)s != 'undefined' ? %(EXPORT_NAME)s : {};\n''' % {"EXPORT_NAME": options.export_name}
613613

614614
ret += '''
615-
Module['expectedDataFileDownloads'] ??= 0;
615+
if (!Module['expectedDataFileDownloads']) Module['expectedDataFileDownloads'] = 0;
616616
Module['expectedDataFileDownloads']++;'''
617617

618618
if not options.export_es6:
@@ -626,7 +626,7 @@ def generate_js(data_target, data_files, metadata):
626626
if (isPthread || isWasmWorker) return;\n'''
627627

628628
if options.support_node:
629-
ret += " var isNode = globalThis.process?.versions?.node && globalThis.process?.type != 'renderer';\n"
629+
ret += " var isNode = globalThis.process && globalThis.process.versions && globalThis.process.versions.node && globalThis.process.type != 'renderer';\n"
630630

631631
if options.support_node and options.export_es6:
632632
ret += '''if (isNode) {
@@ -682,7 +682,7 @@ def generate_js(data_target, data_files, metadata):
682682
await Module['FS_preloadFile'](name, null, data, true, true, false, true);
683683
Module['removeRunDependency'](`fp ${name}`);
684684
} catch (e) {
685-
err(`Preloading file ${name} failed`);
685+
err(`Preloading file ${name} failed`, e);
686686
}\n'''
687687
create_data = '''// canOwn this data in the filesystem, it is a slice into the heap that will never change
688688
Module['FS_createDataFile'](name, null, data, true, true, true);
@@ -771,7 +771,7 @@ def generate_js(data_target, data_files, metadata):
771771
}
772772
var PACKAGE_NAME = '%s';
773773
var REMOTE_PACKAGE_BASE = '%s';
774-
var REMOTE_PACKAGE_NAME = Module['locateFile']?.(REMOTE_PACKAGE_BASE, '') ?? REMOTE_PACKAGE_BASE;\n''' % (js_manipulation.escape_for_js_string(data_target), js_manipulation.escape_for_js_string(remote_package_name))
774+
var REMOTE_PACKAGE_NAME = Module['locateFile'] ? Module['locateFile'](REMOTE_PACKAGE_BASE, '') : REMOTE_PACKAGE_BASE;\n''' % (js_manipulation.escape_for_js_string(data_target), js_manipulation.escape_for_js_string(remote_package_name))
775775
metadata['remote_package_size'] = remote_package_size
776776
ret += " var REMOTE_PACKAGE_SIZE = metadata['remote_package_size'];\n"
777777

@@ -938,15 +938,14 @@ def generate_js(data_target, data_files, metadata):
938938
if options.support_node:
939939
node_support_code = '''
940940
if (isNode) {
941-
var fsPromises = require('fs/promises');
942-
var contents = await fsPromises.readFile(packageName);
943-
return contents.buffer;
941+
var contents = require('fs').readFileSync(packageName);
942+
return new Uint8Array(contents).buffer;
944943
}'''.strip()
945944

946945
ret += '''
947946
async function fetchRemotePackage(packageName, packageSize) {
948947
%(node_support_code)s
949-
Module['dataFileDownloads'] ??= {};
948+
if (!Module['dataFileDownloads']) Module['dataFileDownloads'] = {};
950949
try {
951950
var response = await fetch(packageName);
952951
} catch (e) {
@@ -958,10 +957,10 @@ def generate_js(data_target, data_files, metadata):
958957
959958
const chunks = [];
960959
const headers = response.headers;
961-
const total = Number(headers.get('Content-Length') ?? packageSize);
960+
const total = Number(headers.get('Content-Length') || packageSize);
962961
let loaded = 0;
963962
964-
Module['setStatus']?.('Downloading data...');
963+
Module['setStatus'] && Module['setStatus']('Downloading data...');
965964
const reader = response.body.getReader();
966965
967966
while (1) {
@@ -979,7 +978,7 @@ def generate_js(data_target, data_files, metadata):
979978
totalSize += download.total;
980979
}
981980
982-
Module['setStatus']?.(`Downloading data... (${totalLoaded}/${totalSize})`);
981+
Module['setStatus'] && Module['setStatus'](`Downloading data... (${totalLoaded}/${totalSize})`);
983982
}
984983
985984
const packageData = new Uint8Array(chunks.map((c) => c.length).reduce((a, b) => a + b, 0));
@@ -994,7 +993,7 @@ def generate_js(data_target, data_files, metadata):
994993
code += '''
995994
async function processPackageData(arrayBuffer) {
996995
assert(arrayBuffer, 'Loading data file failed.');
997-
assert(arrayBuffer.constructor.name === ArrayBuffer.name, 'bad input to processPackageData');
996+
assert(arrayBuffer.constructor.name === ArrayBuffer.name, 'bad input to processPackageData ' + arrayBuffer.constructor.name);
998997
var byteArray = new Uint8Array(arrayBuffer);
999998
var curr;
1000999
%s
@@ -1004,7 +1003,7 @@ def generate_js(data_target, data_files, metadata):
10041003
# we need to find the datafile in the same dir as the html file
10051004

10061005
code += '''
1007-
Module['preloadResults'] ??= {};\n'''
1006+
if (!Module['preloadResults']) Module['preloadResults'] = {};\n'''
10081007

10091008
if options.use_preload_cache:
10101009
code += '''
@@ -1034,7 +1033,7 @@ def generate_js(data_target, data_files, metadata):
10341033
await preloadFallback(e)%s;
10351034
}
10361035
1037-
Module['setStatus']?.('Downloading...');\n''' % catch_handler
1036+
Module['setStatus'] && Module['setStatus']('Downloading...');\n''' % catch_handler
10381037
else:
10391038
# Not using preload cache, so we might as well start the xhr ASAP,
10401039
# potentially before JS parsing of the main codebase if it's after us.
@@ -1065,17 +1064,21 @@ def generate_js(data_target, data_files, metadata):
10651064
if (Module['calledRun']) {
10661065
runWithFS(Module)%s;
10671066
} else {
1068-
(Module['preRun'] ??= []).push(runWithFS); // FS is not initialized yet, wait for it
1067+
if (!Module['preRun']) Module['preRun'] = [];
1068+
Module['preRun'].push(runWithFS); // FS is not initialized yet, wait for it
10691069
}\n''' % catch_handler
10701070

10711071
if options.separate_metadata:
10721072
node_support_code = ''
10731073
if options.support_node:
10741074
node_support_code = '''
10751075
if (isNode) {
1076-
var fsPromises = require('fs/promises');
1077-
var contents = await fsPromises.readFile(metadataUrl, 'utf8');
1078-
return loadPackage(JSON.parse(contents));
1076+
var contents = require('fs').readFileSync(metadataUrl, 'utf8');
1077+
// The await here is needed, even though JSON.parse is a sync API. It works
1078+
// around a issue with `removeRunDependency` otherwise being called to early
1079+
// on the metadata object.
1080+
var json = await JSON.parse(contents);
1081+
return loadPackage(json);
10791082
}'''.strip()
10801083

10811084
ret += '''
@@ -1084,20 +1087,21 @@ def generate_js(data_target, data_files, metadata):
10841087
10851088
async function runMetaWithFS() {
10861089
Module['addRunDependency']('%(metadata_file)s');
1087-
var metadataUrl = Module['locateFile']?.('%(metadata_file)s', '') ?? '%(metadata_file)s';
1090+
var metadataUrl = Module['locateFile'] ? Module['locateFile']('%(metadata_file)s', '') : '%(metadata_file)s';
10881091
%(node_support_code)s
10891092
var response = await fetch(metadataUrl);
10901093
if (!response.ok) {
10911094
throw new Error(`${response.status}: ${response.url}`);
10921095
}
10931096
var json = await response.json();
1094-
return loadPackage(json);
1097+
await loadPackage(json);
10951098
}
10961099
10971100
if (Module['calledRun']) {
10981101
runMetaWithFS();
10991102
} else {
1100-
(Module['preRun'] ??= []).push(runMetaWithFS);
1103+
if (!Module['preRun']) Module['preRun'] = [];
1104+
Module['preRun'].push(runMetaWithFS);
11011105
}\n''' % {'node_support_code': node_support_code, 'metadata_file': os.path.basename(options.jsoutput + '.metadata')}
11021106
else:
11031107
ret += '''

0 commit comments

Comments
 (0)