Support for executing Javascript

docs/examples/jsfsx/bomb.js

@@ -0,0 +1 @@
+x_out = butts;

docs/examples/jsfsx/hello.js

@@ -0,0 +1 @@
+x_out = "Hack the Planet!";

docs/examples/jsfsx/input.js

@@ -0,0 +1 @@
+x_out = "The request method is: " + x_in.method;

docs/examples/jsfsx/logit.js

@@ -0,0 +1,2 @@
+x_err = x_err + "Something to log";
+x_out = x_err;

docs/examples/jsfsx/stream.js

@@ -0,0 +1,5 @@
+var loop_count = 5;
+
+for(var i=0;i<loop_count;i++){
+  x_push("drip\n");
+}

docs/examples/jsfsx/viewsource.js

@@ -0,0 +1,2 @@
+// If you're seeing this, the execute override worked!
+x_out = "Call me with an access-key to view my sourcecode!";

docs/jsfsx.md

@@ -0,0 +1,229 @@
+# jsfsx
+
+"The simplest thing that could possibly work."
+
+## TODO (in no strict order)
+
+### poc
+- [x] Add executable flag
+- [x] Execute executable files on GET
+- [x] Fix duplicated output error
+- [x] Standardize i/o interface (`x_in`, `x_out`, etc.)
+  - [x] Expose some/all request input to the executing code
+- [x] Refactor (code structure, logging, error handling, etc.)
+- [x] Come up with a way to fetch the source of an executable w/o running it
+  - [x] Don't execute if an access-key/token is presented
+- [x] Consider the impact of executables that emit large amounts of data (or continuous streams)
+- [x] Handle errors generated by uploaded code so they don't crash the whole server
+- [ ] Finalize X interface (`x_in`, `x_out`, etc.)
+- [ ] Execute executable files on POST
+
+### post-poc
+- [ ] Preserve `executable` bit through `PUT`s (note: this might be an existing bug, other properties appear to behave the same way...)
+- [ ] Consider adding "internal primatives" that let X code access JSFS data w/o HTTP overhead
+- [ ] Consider finding a way for a client to access `x_err` data (maybe a `debug` flag that dumps the entire context to `response`?)
+- [ ] Figure out how to set the `content-length`, `content-type` headers when executing code
+  - [ ] Allow the executable code to set these values somehow?
+- [ ] Experiment with `vm` settings to maximize stability, performance, security
+  - [ ] Caching/compiling scripts for faster startup time
+  - [ ] Other languages?
+  - [ ] How to handle modules/dependencies?
+  - [ ] Threading/eventloop impact/tuning
+  - [ ] Related: other workloads (external binaries, shell scripts, etc.)
+- [ ] Fix bug that allows invalid `access-key` to view source
+- [ ] Review types and make sure we're using the right ones to pass things around
+- [ ] Find a way to tell when all X source chunks are ready
+
+
+## curl to store an executable file
+```bash
+curl -X POST -H "content-type: text/javascript" -H "x-access-key: jjg" -H "x-executable: true" --data-binary @hello.js "http://localhost:7302/bin/hello.js"
+```
+
+### result
+```json
+{
+  "url": "/localhost/bin/hello.js",
+  "created": 1712694738665,
+  "version": 0,
+  "private": false,
+  "encrypted": false,
+  "fingerprint": "438754d26cf1daaf69f9a0e6421b3053e4c00f75",
+  "access_key": "jjg",
+  "content_type": "text/javascript",
+  "file_size": 33,
+  "block_size": 1048576,
+  "blocks_replicated": 0,
+  "inode_replicated": 0,
+  "blocks": [
+    {
+      "block_hash": "a3b622a18ce02eb4d6e609f842964f430325e3d4",
+      "last_seen": "./blocks/"
+    }
+  ],
+  "executable": true,
+  "media_type": "unknown"
+}
+```
+
+## flow
+```
+case "GET"
+send_blocks()
+load_from_last_seen(true)
+read_file()
+read_stream = operations.stream_read(path)
+read_stream.pipe(unzipper).pipe(decryptor).pipe(res)
+on_end()
+read_stream shutdown
+send_blocks() (until all blocks are sent)
+- or -
+search_for_block(0)
+read_file()
+```
+
+## it works!
+
+### source file
+`hello.js`
+```javascript
+x_out = "Hack the Planet!";
+```
+
+### upload
+```bash
+curl -X POST -H "content-type: text/javascript" -H "x-access-key: jjg" -H "x-executable: true" --data-binary @hello.js "http://localhost:7302/bin/hello.js"
+```
+
+### execute
+```bash
+curl "http://localhost:7302/bin/hello.js"
+```
+
+### output
+```bash
+Hack the Planet!Hack the Pla
+```
+
+There's clearly bugs, but I think it proves the concept.
+
+```bash
+curl -v "http://localhost:7302/bin/hello.js"
+*   Trying 127.0.0.1:7302...
+* Connected to localhost (127.0.0.1) port 7302 (#0)
+> GET /bin/hello.js HTTP/1.1
+> Host: localhost:7302
+> User-Agent: curl/7.81.0
+> Accept: */*
+> 
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< Access-Control-Allow-Methods: GET,POST,PUT,DELETE,OPTIONS
+< Access-Control-Allow-Headers: Accept,Accept-Version,Api-Version,Content-Type,Origin,Range,X_FILENAME,X-Access-Key,X-Access-Token,X-Append,X-Encrypted,X-Private,X-Replacement-Access-Key,X-Requested-With,X-Executable
+< Access-Control-Allow-Origin: *
+< Access-Control-Expose-Headers: X-Media-Bitrate,X-Media-Channels,X-Media-Duration,X-Media-Resolution,X-Media-Size,X-Media-Type
+< Content-Type: text/javascript
+< Content-Length: 28
+< Date: Thu, 11 Apr 2024 15:50:10 GMT
+< Connection: keep-alive
+< Keep-Alive: timeout=5
+< 
+* Excess found in a read: excess = 4, size = 28, maxdownload = 28, bytecount = 0
+* Closing connection 0
+Hack the Planet!Hack the Pla
+```
+
+## Refactoring and bugs
+
+For whatever reason refactoring the code a bit made the duplicate output bug go away ("IT'S MAGIC!").  Now executable `GET` requests work as expected, so it's probably time to think more about the interface between requests, reponses and the executable code.
+
+It's tempting to move on to `POST` but probably better to figure out the I/O first..
+
+
+## input
+
+Let's start by passing the whole `request` into the executor, what's the worst that can happen?
+
+Then this uploaded code:
+
+```javascript
+x_out = "The request method is: " + x_in.method;
+```
+
+...yeilds this result:
+```bash
+curl "http://localhost:7302/bin/input.js"
+The request method is: GET
+```
+
+## X runtime environment/interface
+
+There will be a lot more to explore here in the future, but in the spirit of MVP or whatever here's what we're going to do for now.
+
+Files marked executable are run as Javascript in a [Node.js VM](https://nodejs.org/api/vm.html#vm-executing-javascript).  At startup three variables will be initialized: `x_in`, `x_out` and `x_err`.  These map loosely to the `stdin`, `stdout` and `stderr` unix convention.
+* `x_in` is the entire `request` object sent by the user agent (for now)
+* `x_out` is returned to the user agent in the `response` object
+* `x_err` is written to the JSFS log
+
+It would be useful if `x_err` was more accessible by the user agent, and I have some ideas for this (maybe a `x-jsfs-debug` header that dumps the entire `context` to `response`?), but for now I'm just going to let it write to the log (if it's needed for debugging you can always do that using a local JSFS instance right?).
+
+## view source
+Now you can retrieve the data (as opposed to executing the code) in a stored file that is marked as executable:
+
+```bash
+curl "http://localhost:7302/bin/viewsource2.js"
+Call me with an access-key to view my sourcecode!
+
+curl -H "x-access-key: jjg" "http://localhost:7302/bin/viewsource2.js"
+// If you're seeing this, the execute override worked!
+x_out = "Call me with an access-key to view my sourcecode!";
+```
+
+## streaming
+What if an X generates output for a long time on purpose (imagine an audio stream, just as a random example...)?
+
+This is tricky, and the more I think about it, the more I'm not sure it makes sense for code executing in this context to behave this way.  Let me try to explain...
+
+An HTTP request is made to an X file, the file is loaded, passed the request data and run.  When the run is complete, any output is passed back to the requestor.  This is pipelined using a Node.js stream but since the relationship between input and output is asymetrical the stream can't work in a "pipelined" fashion like passing a regular file back to the caller.  The `Transform` component used to execute the code can't "yield" output outside of fixed events which consist of either one per "chunk" passed to the component (not an option because the chunks are the sourcecode that can't be executed piecemeal) or at the end of receiving all the chunks (how we do it now).
+
+So this means that if an X needs all of it's code to run, it can only send data back to the caller in one shot.  That means that the output must be buffered in memory, unless there is some non-obvious way to "drain" the output downstream in the pipeline that I'm not finding in the docs...
+
+This might be overcome using a `Duplex` stream, however we also need a way to stream data out of the X running under `vm.runInContext()`.  Right now I don't see a way to do this as running the X in `vm` is a blocking operation, but maybe there's a way to get events out if it?  Maybe the `context` that gets passed-in could include a function that could somehow cross that barrier?
+
+OK, so if I pass a function via the `context` object, I can invoke that function from inside the `vm` that is running the X.  So this *could* be used to send a signal to the `ExecutableStream` that there's data to send along the pipeline.  I'm not sure yet if this is a *good* idea (it's starting to feel like entangling the X-world too much in the outer world) but let's just go with it for awhile and see if it actually works.
+
+So what's it going to take to test this end-to-end?  I think `ExecutableStream` needs to be re-written as a `Duplex` stream provider, so let's checkpoint all this in git before we break everything...
+
+So it looks like I can cram what I need into the `_write()` and `_read()` methods of the `Duplex` stream, but I need a way to know when all the X code has been `_write()`-end, and I also need to know how to tell callers of `_read()` that the X is done generating output.
+
+### it works, but it's buggy
+
+`stream.js`
+```javascript
+var loop_count = 5;
+
+for(var i=0;i<loop_count;i++){
+  x_push("drip\n");
+}
+```
+
+```bash
+curl "http://localhost:7302/bin/stream.js"
+drip
+drip
+drip
+drip
+drip
+```
+
+This change introduces a new piece to the X interface: `x_push()`.  When called, this will immediately stream the data passed to the function to the client.  Right now this is a string, and it should probably be something more generic.  I've also preserved the previous "batch-mode" `x_out` interface but off the top of my head I can't think of why everything can't just use the `x_push()` interface other than it sounds kind of weird.
+
+The "buggy" part is that right now there's no way to run more than one block's worth of X code because I can't find a way for the `Duplex` stream to know when it's got all the blocks from the previous step in the pipeline.  I assume there is a way, but I haven't figured it out yet.
+
+
+## References
+* https://nodejs.org/api/vm.html
+* https://nodejs.org/docs/latest/api/stream.html#stream_implementing_a_transform_stream
+* https://v8.dev/docs
+* https://v8.dev/features
+* 

lib/constants.js

@@ -20,7 +20,8 @@ module.exports.ALLOWED_HEADERS = ["Accept",
                                   "X-Encrypted",
                                   "X-Private",
                                   "X-Replacement-Access-Key",
-                                  "X-Requested-With"];
+                                  "X-Requested-With",
+                                  "X-Executable"];
 
 module.exports.EXPOSED_HEADERS = ["X-Media-Bitrate",
                                   "X-Media-Channels",
@@ -38,4 +39,5 @@ module.exports.ACCEPTED_PARAMS = [{"access_key": "x"},
                                   {"inode_only": "x"},
                                   {"private": "x"},
                                   {"replacement_access_key": "x"},
-                                  {"version": "x"}];
+                                  {"version": "x"},
+                                  {"executable":"x"}];

lib/inode.js

@@ -33,6 +33,9 @@ var Inode = {
 
     // use fingerprint as default key
     this.file_metadata.access_key = this.file_metadata.fingerprint;
+
+    // experimental executable support (jsfsx)
+    this.file_metadata.executable = false;
   },
   write: function(chunk, req, callback){
     this.input_buffer = new Buffer.concat([this.input_buffer, chunk]);

lib/xstream.js

@@ -0,0 +1,68 @@
+const vm = require('node:vm');
+const { Duplex } = require('node:stream');
+
+var log = require("../jlog.js");
+
+class ExecutableStream extends Duplex {
+  constructor(x_in) {
+    super();
+    this.x_in = x_in;
+    this.code = "";
+    this.x_done = false;
+  }
+  _write(chunk, encoding, callback) {
+
+    // NOTE: This nonsense is needed to give the X access to the Duplex `push()` method
+    function make_x_push(t) {
+      return function (s) {
+        t.push(s);
+      };
+    }
+
+    // TODO: Figure out a way to buffer chunks until we have the whole program.
+    this.code = this.code + chunk.toString();
+
+    log.message(log.INFO, "Xing " + this.code.length + " bytes of Javascript...");
+
+    // This provides a basic unix-like in/out/error interface to executable code.
+    // * x_in is currently the entire `request` object sent by the client
+    // * x_out is written back to the client via the `response` object after X is done
+    // * x_push() streams output to the client via the `response` object while X is still running
+    // * x_err is written to JSFS logs
+    const context = {
+      x_in: this.x_in,
+      x_out:"",
+      x_err:"",
+      x_push: make_x_push(this)
+    };
+
+    vm.createContext(context);
+    log.message(log.INFO, "X context created.");
+
+    // TODO: There could be a lot more done here when an X blows-up, but
+    // for now let's just not crash the whole damn server...
+    try {
+      vm.runInContext(this.code, context)
+      log.message(log.INFO, "X complete!");
+    } catch(err) {
+      log.message(log.ERROR, "X exception: " + err);
+    }
+
+    if(context.x_err.length > 0) {
+      log.message(log.INFO, "X error logs: " + context.x_err);
+    }
+
+    // If the X used the x_out interface, write it out now
+    this.push(context.x_out);
+    this.x_done = true;
+
+    callback();
+  }
+  _read(size){
+    if(this.x_done){
+      this.push(null);
+    }
+  }
+}
+
+module.exports = ExecutableStream;