Merge remote-tracking branch 'upstream/v2.1' into v2.1

Author: xrSimpodin

doc/extensions.html

@@ -160,13 +160,33 @@ <h3 id="xpcall"><tt>xpcall(f, err [,args...])</tt> passes arguments</h3>
 which is called in a protected context.
 </p>
 
-<h3 id="load"><tt>loadfile()</tt> etc. handle UTF-8 source code</h3>
+<h3 id="load"><tt>load*()</tt> handle UTF-8 source code</h3>
 <p>
 Non-ASCII characters are handled transparently by the Lua source code parser.
 This allows the use of UTF-8 characters in identifiers and strings.
 A UTF-8 BOM is skipped at the start of the source code.
 </p>
 
+<h3 id="load_mode"><tt>load*()</tt> add a mode parameter</h3>
+<p>
+As an extension from Lua 5.2, the functions <tt>loadstring()</tt>,
+<tt>loadfile()</tt> and (new) <tt>load()</tt> add an optional
+<tt>mode</tt> parameter.
+</p>
+<p>
+The default mode string is <tt>"bt"</tt>, which allows loading of both
+source code and bytecode. Use <tt>"t"</tt> to allow only source code
+or <tt>"b"</tt> to allow only bytecode to be loaded.
+</p>
+<p>
+By default, the <tt>load*</tt> functions generate the native bytecode format.
+For cross-compilation purposes, add <tt>W</tt> to the mode string to
+force the 32 bit format and <tt>X</tt> to force the 64 bit format.
+Add both to force the opposite format. Note that non-native bytecode
+generated by <tt>load*</tt> cannot be run, but can still be passed
+to <tt>string.dump</tt>.
+</p>
+
 <h3 id="tostring"><tt>tostring()</tt> etc. canonicalize NaN and &plusmn;Inf</h3>
 <p>
 All number-to-string conversions consistently convert non-finite numbers
@@ -186,26 +206,33 @@ <h3 id="tonumber"><tt>tonumber()</tt> etc. use builtin string to number conversi
 numbers (e.g. <tt>0x1.5p-3</tt>).
 </p>
 
-<h3 id="string_dump"><tt>string.dump(f [,strip])</tt> generates portable bytecode</h3>
+<h3 id="string_dump"><tt>string.dump(f [,mode])</tt> generates portable bytecode</h3>
 <p>
 An extra argument has been added to <tt>string.dump()</tt>. If set to
-<tt>true</tt>, 'stripped' bytecode without debug information is
-generated. This speeds up later bytecode loading and reduces memory
-usage. See also the
+<tt>true</tt> or to a string which contains the character <tt>s</tt>,
+'stripped' bytecode without debug information is generated. This speeds
+up later bytecode loading and reduces memory usage. See also the
 <a href="running.html#opt_b"><tt>-b</tt> command line option</a>.
 </p>
 <p>
 The generated bytecode is portable and can be loaded on any architecture
-that LuaJIT supports, independent of word size or endianess. However, the
-bytecode compatibility versions must match. Bytecode stays compatible
-for dot releases (x.y.0 &rarr; x.y.1), but may change with major or
-minor releases (2.0 &rarr; 2.1) or between any beta release. Foreign
-bytecode (e.g. from Lua 5.1) is incompatible and cannot be loaded.
+that LuaJIT supports. However, the bytecode compatibility versions must
+match. Bytecode only stays compatible within a major+minor version
+(x.y.aaa &rarr; x.y.bbb), except for development branches. Foreign bytecode
+(e.g. from Lua 5.1) is incompatible and cannot be loaded.
 </p>
 <p>
 Note: <tt>LJ_GC64</tt> mode requires a different frame layout, which implies
-a different, incompatible bytecode format for all 64 bit ports. This may be
-rectified in the future.
+a different, incompatible bytecode format between 32 bit and 64 bit ports.
+This may be rectified in the future. In the meantime, use the <tt>W</tt>
+and </tt>X</tt> <a href="#load_mode">modes of the <tt>load*</tt> functions</a>
+for cross-compilation purposes.
+</p>
+<p>
+Due to VM hardening, bytecode is not deterministic. Add <tt>d</tt> to the
+mode string to dump it in a deterministic manner: identical source code
+always gives a byte-for-byte identical bytecode dump. This feature is
+mainly useful for reproducible builds.
 </p>
 
 <h3 id="table_new"><tt>table.new(narray, nhash)</tt> allocates a pre-sized table</h3>
@@ -286,7 +313,7 @@ <h2 id="lua52">Extensions from Lua 5.2</h2>
 </p>
 <ul>
 <li><tt>goto</tt> and <tt>::labels::</tt>.</li>
-<li>Hex escapes <tt>'\x3F'</tt> and <tt>'\*'</tt> escape in strings.</li>
+<li>Hex escapes <tt>'\x3F'</tt> and <tt>'\z'</tt> escape in strings.</li>
 <li><tt>load(string|reader [, chunkname [,mode [,env]]])</tt>.</li>
 <li><tt>loadstring()</tt> is an alias for <tt>load()</tt>.</li>
 <li><tt>loadfile(filename [,mode [,env]])</tt>.</li>

doc/running.html

@@ -106,6 +106,9 @@ <h3 id="opt_b"><tt>-b[options] input output</tt></h3>
 <li><tt>-l</tt> &mdash; Only list bytecode.</li>
 <li><tt>-s</tt> &mdash; Strip debug info (this is the default).</li>
 <li><tt>-g</tt> &mdash; Keep debug info.</li>
+<li><tt>-W</tt> &mdash; Generate 32 bit (non-GC64) bytecode.</li>
+<li><tt>-X</tt> &mdash; Generate 64 bit (GC64) bytecode.</li>
+<li><tt>-d</tt> &mdash; Generate bytecode in deterministic manner.</li>
 <li><tt>-n name</tt> &mdash; Set module name (default: auto-detect from input name)</li>
 <li><tt>-t type</tt> &mdash; Set output file type (default: auto-detect from output name).</li>
 <li><tt>-a arch</tt> &mdash; Override architecture for object files (default: native).</li>

dynasm/dasm_x86.lua

@@ -1151,6 +1151,8 @@ local map_op = {
   rep_0 =	"F3",
   repe_0 =	"F3",
   repz_0 =	"F3",
+  endbr32_0 =	"F30F1EFB",
+  endbr64_0 =	"F30F1EFA",
   -- F4: *hlt
   cmc_0 =	"F5",
   -- F6: test... mb,i; div... mb

dynasm/dynasm.lua

@@ -75,7 +75,7 @@ local function wline(line, needindent)
   g_synclineno = g_synclineno + 1
 end
 
--- Write assembler line as a comment, if requestd.
+-- Write assembler line as a comment, if requested.
 local function wcomment(aline)
   if g_opt.comment then
     wline(g_opt.comment..aline..g_opt.endcomment, true)

src/host/genlibbc.lua

@@ -138,65 +138,73 @@ local function fixup_dump(dump, fixup)
   return { dump = ndump, startbc = startbc, sizebc = sizebc }
 end
 
-local function find_defs(src)
+local function find_defs(src, mode)
   local defs = {}
   for name, code in string.gmatch(src, "LJLIB_LUA%(([^)]*)%)%s*/%*(.-)%*/") do
-    local env = {}
     local tcode, fixup = transform_lua(code)
-    local func = assert(load(tcode, "", nil, env))()
-    defs[name] = fixup_dump(string.dump(func, true), fixup)
+    local func = assert(load(tcode, "", mode))
+    defs[name] = fixup_dump(string.dump(func, mode), fixup)
     defs[#defs+1] = name
   end
   return defs
 end
 
-local function gen_header(defs)
+local function gen_header(defs32, defs64)
   local t = {}
   local function w(x) t[#t+1] = x end
   w("/* This is a generated file. DO NOT EDIT! */\n\n")
   w("static const int libbc_endian = ") w(isbe and 1 or 0) w(";\n\n")
-  local s, sb = "", ""
-  for i,name in ipairs(defs) do
-    local d = defs[name]
-    s = s .. d.dump
-    sb = sb .. string.char(i) .. ("\0"):rep(d.startbc - 1)
-	    .. (isbe and "\0\0\0\255" or "\255\0\0\0"):rep(d.sizebc)
-	    .. ("\0"):rep(#d.dump - d.startbc - d.sizebc*4)
-  end
-  w("static const uint8_t libbc_code[] = {\n")
-  local n = 0
-  for i=1,#s do
-    local x = string.byte(s, i)
-    local xb = string.byte(sb, i)
-    if xb == 255 then
-      local name = BCN[x]
-      local m = #name + 4
-      if n + m > 78 then n = 0; w("\n") end
-      n = n + m
-      w("BC_"); w(name)
+  for j,defs in ipairs{defs64, defs32} do
+    local s, sb = "", ""
+    for i,name in ipairs(defs) do
+      local d = defs[name]
+      s = s .. d.dump
+      sb = sb .. string.char(i) .. ("\0"):rep(d.startbc - 1)
+	      .. (isbe and "\0\0\0\255" or "\255\0\0\0"):rep(d.sizebc)
+	      .. ("\0"):rep(#d.dump - d.startbc - d.sizebc*4)
+    end
+    if j == 1 then
+      w("static const uint8_t libbc_code[] = {\n#if LJ_FR2\n")
     else
-      local m = x < 10 and 2 or (x < 100 and 3 or 4)
-      if xb == 0 then
+      w("\n#else\n")
+    end
+    local n = 0
+    for i=1,#s do
+      local x = string.byte(s, i)
+      local xb = string.byte(sb, i)
+      if xb == 255 then
+	local name = BCN[x]
+	local m = #name + 4
 	if n + m > 78 then n = 0; w("\n") end
+	n = n + m
+	w("BC_"); w(name)
       else
-	local name = defs[xb]:gsub("_", ".")
-	if n ~= 0 then w("\n") end
-	w("/* "); w(name); w(" */ ")
-	n = #name + 7
+	local m = x < 10 and 2 or (x < 100 and 3 or 4)
+	if xb == 0 then
+	  if n + m > 78 then n = 0; w("\n") end
+	else
+	  local name = defs[xb]:gsub("_", ".")
+	  if n ~= 0 then w("\n") end
+	  w("/* "); w(name); w(" */ ")
+	  n = #name + 7
+	end
+	n = n + m
+	w(x)
       end
-      n = n + m
-      w(x)
+      w(",")
     end
-    w(",")
   end
-  w("\n0\n};\n\n")
+  w("\n#endif\n0\n};\n\n")
   w("static const struct { const char *name; int ofs; } libbc_map[] = {\n")
-  local m = 0
-  for _,name in ipairs(defs) do
-    w('{"'); w(name); w('",'); w(m) w('},\n')
-    m = m + #defs[name].dump
+  local m32, m64 = 0, 0
+  for i,name in ipairs(defs32) do
+    assert(name == defs64[i])
+    w('{"'); w(name); w('",'); w(m32) w('},\n')
+    m32 = m32 + #defs32[name].dump
+    m64 = m64 + #defs64[name].dump
+    assert(m32 == m64)
   end
-  w("{NULL,"); w(m); w("}\n};\n\n")
+  w("{NULL,"); w(m32); w("}\n};\n\n")
   return table.concat(t)
 end
 
@@ -219,7 +227,8 @@ end
 
 local outfile = parse_arg(arg)
 local src = read_files(arg)
-local defs = find_defs(src)
-local hdr = gen_header(defs)
+local defs32 = find_defs(src, "Wdts")
+local defs64 = find_defs(src, "Xdts")
+local hdr = gen_header(defs32, defs64)
 write_file(outfile, hdr)
 

src/host/genversion.lua

@@ -29,7 +29,7 @@ local function file_write_mod(file, data)
   assert(fp:close())
 end
 
-local text = file_read(FILE_ROLLING_H)
+local text = file_read(FILE_ROLLING_H):gsub("#error.-\n", "")
 local relver = file_read(FILE_RELVER_TXT):match("(%d+)")
 
 if relver then

src/jit/bcsave.lua

@@ -29,6 +29,9 @@ Save LuaJIT bytecode: luajit -b[options] input output
   -l        Only list bytecode.
   -s        Strip debug info (default).
   -g        Keep debug info.
+  -W        Generate 32 bit (non-GC64) bytecode.
+  -X        Generate 64 bit (GC64) bytecode.
+  -d        Generate bytecode in deterministic manner.
   -n name   Set module name (default: auto-detect from input name).
   -t type   Set output file type (default: auto-detect from output name).
   -a arch   Override architecture for object files (default: native).
@@ -51,8 +54,9 @@ local function check(ok, ...)
 end
 
 local function readfile(ctx, input)
-  if type(input) == "function" then return input end
-  if ctx.filename then
+  if ctx.string then
+    return check(loadstring(input, nil, ctx.mode))
+  elseif ctx.filename then
     local data
     if input == "-" then
       data = io.stdin:read("*a")
@@ -61,10 +65,10 @@ local function readfile(ctx, input)
       data = assert(fp:read("*a"))
       assert(fp:close())
     end
-    return check(load(data, ctx.filename))
+    return check(load(data, ctx.filename, ctx.mode))
   else
     if input == "-" then input = nil end
-    return check(loadfile(input))
+    return check(loadfile(input, ctx.mode))
   end
 end
 
@@ -624,7 +628,7 @@ end
 
 local function bcsave(ctx, input, output)
   local f = readfile(ctx, input)
-  local s = string.dump(f, ctx.strip)
+  local s = string.dump(f, ctx.mode)
   local t = ctx.type
   if not t then
     t = detecttype(output)
@@ -647,9 +651,11 @@ local function docmd(...)
   local n = 1
   local list = false
   local ctx = {
-    strip = true, arch = jit.arch, os = jit.os:lower(),
-    type = false, modname = false,
+    mode = "bt", arch = jit.arch, os = jit.os:lower(),
+    type = false, modname = false, string = false,
   }
+  local strip = "s"
+  local gc64 = ""
   while n <= #arg do
     local a = arg[n]
     if type(a) == "string" and a:sub(1, 1) == "-" and a ~= "-" then
@@ -660,14 +666,18 @@ local function docmd(...)
 	if opt == "l" then
 	  list = true
 	elseif opt == "s" then
-	  ctx.strip = true
+	  strip = "s"
 	elseif opt == "g" then
-	  ctx.strip = false
+	  strip = ""
+	elseif opt == "W" or opt == "X" then
+	  gc64 = opt
+	elseif opt == "d" then
+	  ctx.mode = ctx.mode .. opt
 	else
 	  if arg[n] == nil or m ~= #a then usage() end
 	  if opt == "e" then
 	    if n ~= 1 then usage() end
-	    arg[1] = check(loadstring(arg[1]))
+	    ctx.string = true
 	  elseif opt == "n" then
 	    ctx.modname = checkmodname(tremove(arg, n))
 	  elseif opt == "t" then
@@ -687,6 +697,7 @@ local function docmd(...)
       n = n + 1
     end
   end
+  ctx.mode = ctx.mode .. strip .. gc64
   if list then
     if #arg == 0 or #arg > 2 then usage() end
     bclist(ctx, arg[1], arg[2] or "-")

src/lib_base.c

@@ -360,7 +360,11 @@ LJLIB_ASM_(xpcall)		LJLIB_REC(.)
 static int load_aux(lua_State *L, int status, int envarg)
 {
   if (status == LUA_OK) {
-    if (tvistab(L->base+envarg-1)) {
+    /*
+    ** Set environment table for top-level function.
+    ** Don't do this for non-native bytecode, which returns a prototype.
+    */
+    if (tvistab(L->base+envarg-1) && tvisfunc(L->top-1)) {
       GCfunc *fn = funcV(L->top-1);
       GCtab *t = tabV(L->base+envarg-1);
       setgcref(fn->c.env, obj2gco(t));