add webrtc wrapper around gstreamer, web client

FolkComputer · Jun 21, 2024 · ade7dcd · ade7dcd
1 parent 9d40814
commit ade7dcd
Show file tree

Hide file tree

Showing 4 changed files with 234 additions and 0 deletions.
diff --git a/test/webrtc.tcl b/test/webrtc.tcl
@@ -0,0 +1,19 @@
+loadVirtualPrograms [list "virtual-programs/web/webrtc.folk" "virtual-programs/web/web-keyboards.folk" "virtual-programs/keyboard.folk" "virtual-programs/gstreamer.folk" "virtual-programs/images.folk" "virtual-programs/web/new-program-web-editor.folk"]
+Step
+
+# Assert <unknown> wishes the-moon receives webrtc video earth
+
+When the-moon has webrtc video earth frame /image/ at /ts/ {
+  Wish the web server handles route "/rtc-image/$" with handler [list apply {{im} {
+   set filename "/tmp/web-image-frame.png"
+   image saveAsPng $im $filename
+   set fsize [file size $filename]
+   set fd [open $filename r]
+   fconfigure $fd -encoding binary -translation binary
+   set body [read $fd $fsize]
+   close $fd
+   dict create statusAndHeaders "HTTP/1.1 200 OK\nConnection: close\nContent-Type: image/png\nContent-Length: $fsize\n\n" body $body
+  }} $image]
+}
+
+forever { Step }
diff --git a/vendor/gstwebrtc/gstwebrtc-api-2.0.0.min.js b/vendor/gstwebrtc/gstwebrtc-api-2.0.0.min.js
diff --git a/virtual-programs/web/webrtc.folk b/virtual-programs/web/webrtc.folk
@@ -0,0 +1,202 @@
+try {
+  exec gst-webrtc-signalling-server &
+} on error {e} {
+  error "gst-webrtc-signalling-server not found, you probably need to install gst-plugins-rs"
+}
+
+try {
+  exec gst-inspect-1.0 rswebrtc
+} on error {e} {
+  error "gstreamer plugin 'rswebrtc' not found, you probably need to install gst-plugins-rs"
+}
+
+try {
+  exec gst-inspect-1.0 webrtc
+} on error {e} {
+  error "gstreamer plugin 'webrtc' not found, you probably need to install gst-plugins-bad"
+}
+
+When /someone/ wishes /p/ receives webrtc video /feed/ &\
+     /someone/ claims webrtc video /feed/ streams from peer /peer-id/ {
+  set uri gstwebrtc://127.0.0.1:8443?peer-id=${peer-id}
+  When the gstreamer pipeline "uridecodebin uri=$uri" frame is /image/ at /ts/ {
+    Claim $p has webrtc video $feed frame $image at $ts
+  }
+}
+
+When when /p/ has webrtc video /feed/ frame /image/ at /ts/ /lambda/ with environment /e/ {
+  Wish $p receives webrtc video $feed
+}
+
+Wish the web server handles route "/webrtc$" with handler {
+  html {
+    <!DOCTYPE html>
+    <html>
+    <body>
+      <span id="status">Status</span>
+      <h1>active feeds</h1>
+      <div id="feeds">
+      </div>
+
+      <script src="/lib/folk.js"></script>
+      <script>
+        async function init() {
+          ws = new FolkWS(document.getElementById('status'));
+          for await (const {action, match} of ws.watch('/someone/ wishes /someone/ receives webrtc video /feed/')) {
+            const [feed] = match;
+            if (action === "add") {
+              const link = document.createElement('a');
+              link.id = `feed-${feed}`;
+              link.style = "display: block";
+              link.innerText = `stream to ${feed}`;
+              link.href = `/webrtc/${encodeURIComponent(feed)}/send`;
+              feeds.append(link);
+            } else if (action === "del") {
+              document.getElementById(`feed-${feed}`).remove();
+            }
+          }
+        }
+        init();
+      </script>
+    </body>
+    </html>
+  }
+}
+
+Wish the web server handles route {/webrtc/(.*)/send$} with handler {
+  regexp {/webrtc/(.*)/send$} $path -> feed
+  html [format {
+    <!DOCTYPE html>
+    <html style="display: flex; min-height: 100vh">
+    <head>
+    </head>
+    <body style="display: flex; flex-direction: column; flex: 1;">
+      <span id="status">Status</span>
+      <h1>share video to feed %s</h1>
+      <div>
+        <button id="camera-button">start camera</button>
+        <button id="screen-button">start screenshare</button>
+        <button id="stop-button">stop</button>
+      </div>
+      <video style="flex: 1; background: #000; margin: auto; max-width: 100%;" id="preview"></video>
+
+      <script src="/lib/folk.js"></script>
+      <script src="/vendor/gstwebrtc/gstwebrtc-api-2.0.0.min.js"></script>
+      <script>
+        const protocol = window.location.protocol.replace("http", "ws");
+        const api = new GstWebRTCAPI({
+          meta: { name: `WebClient-${Date.now()}` },
+          signalingServerUrl: `${protocol}//${window.location.hostname}:8443/webrtc`,
+        });
+        const ws = new FolkWS(document.getElementById("status"));
+
+        const feed = '%s';
+        let peerId = null;
+        let session;
+
+        const videoElement = document.getElementById("preview");
+
+        api.registerConnectionListener({
+          connected: (clientId) => { peerId = clientId; },
+          disconnected: () => { peerId = null; }
+        });
+
+        const handleStream = (stream) => {
+          const sess = api.createProducerSession(stream);
+
+          if (!sess) {
+            for (const track of stream.getTracks()) {
+              track.stop();
+            }
+            // captureSection.classList.remove("starting");
+            return;
+          }
+
+          sess.addEventListener("error", (event) => console.error(event.message, event.error));
+
+          sess.addEventListener("closed", () => {
+            videoElement.pause();
+            videoElement.srcObject = null;
+            ws.send(tcl`Commit {}`);
+            session = null;
+            // captureSection.classList.remove("has-sess", "starting");
+          });
+
+          sess.addEventListener("stateChanged", (event) => {
+            if ((event.target.state === GstWebRTCAPI.SessionState.streaming)) {
+              videoElement.srcObject = stream;
+              videoElement.play().catch(() => {});
+              // captureSection.classList.remove("starting");
+
+              // sess ready, announce to folk consumers!
+              ws.send(tcl`Commit {
+                Claim webrtc video ${feed} streams from peer ${peerId}
+              }`);
+            }
+          });
+
+          /*
+          sess.addEventListener("clientConsumerAdded", (event) => {
+            if (captureSection._producerSession === sess) {
+              console.info(`client consumer added: ${event.detail.peerId}`);
+            }
+          });
+
+          sess.addEventListener("clientConsumerRemoved", (event) => {
+            if (captureSection._producerSession === sess) {
+              console.info(`client consumer removed: ${event.detail.peerId}`);
+            }
+          });
+          */
+
+          // captureSection.classList.add("has-sess");
+          sess.start();
+          return sess;
+        };
+
+        document.getElementById("stop-button").onclick = (event) => {
+          event.preventDefault();
+          if (session) {
+            session.close();
+          }
+        };
+
+        document.getElementById("camera-button").onclick = (event) => {
+          event.preventDefault();
+          if (session) return;
+
+          session = true;
+
+          const constraints = { video: true, audio: false };
+          navigator.mediaDevices.getUserMedia(constraints)
+            .then(handleStream)
+            .then(
+              (s) => { session = s; },
+              (error) => {
+                console.error("cannot have access to webcam and microphone", error);
+                session = null;
+              }
+            );
+        };
+
+        document.getElementById("screen-button").onclick = (event) => {
+          event.preventDefault();
+          if (session) return;
+
+          session = true;
+          const constraints = { video: true, audio: false };
+          navigator.mediaDevices.getDisplayMedia(constraints)
+            .then(handleStream)
+            .then(
+              (s) => { session = s; },
+              (error) => {
+                console.error("cannot have access to screen", error);
+                session = null;
+              }
+            );
+        };
+      </script>
+    </body>
+    </html>
+  } $feed $feed]
+}
diff --git a/web.tcl b/web.tcl
@@ -128,6 +128,14 @@ proc handlePage {path httpStatusVar contentTypeVar} {
         "/statements.pdf" {
             getDotAsPdf [Statements::dot] contentType
         }
+        "/lib/folk.js" {
+            set contentType "text/javascript"
+            readFile "lib/folk.js" contentType
+        }
+        "/vendor/gstwebrtc/gstwebrtc-api-2.0.0.min.js" {
+            set contentType "text/javascript"
+            readFile "vendor/gstwebrtc/gstwebrtc-api-2.0.0.min.js" contentType
+        }
         default {
             upvar $httpStatusVar httpStatus
             set httpStatus "HTTP/1.1 404 Not Found"