add webrtc wrapper around gstreamer, web client

FolkComputer · Jun 21, 2024 · e5d7eba · e5d7eba
1 parent 9d40814
commit e5d7eba
Show file tree

Hide file tree

Showing 4 changed files with 277 additions and 0 deletions.
diff --git a/test/webrtc.tcl b/test/webrtc.tcl
@@ -0,0 +1,20 @@
+loadVirtualPrograms [list "virtual-programs/web/webrtc.folk" "virtual-programs/web/web-keyboards.folk" "virtual-programs/keyboard.folk" "virtual-programs/gstreamer.folk" "virtual-programs/images.folk" "virtual-programs/web/new-program-web-editor.folk"]
+Step
+
+Assert <unknown> wishes the-moon receives webrtc video earth
+
+When the-moon has webrtc video earth frame /image/ at /ts/ {
+  puts "have feed"
+  Wish the web server handles route "/rtc-image/$" with handler [list apply {{im} {
+   set filename "/tmp/web-image-frame.png"
+   image saveAsPng $im $filename
+   set fsize [file size $filename]
+   set fd [open $filename r]
+   fconfigure $fd -encoding binary -translation binary
+   set body [read $fd $fsize]
+   close $fd
+   dict create statusAndHeaders "HTTP/1.1 200 OK\nConnection: close\nContent-Type: image/png\nContent-Length: $fsize\n\n" body $body
+  }} $image]
+}
+
+forever { Step }
diff --git a/vendor/gstwebrtc/gstwebrtc-api-2.0.0.min.js b/vendor/gstwebrtc/gstwebrtc-api-2.0.0.min.js
diff --git a/virtual-programs/web/webrtc.folk b/virtual-programs/web/webrtc.folk
@@ -0,0 +1,244 @@
+try {
+  exec gst-webrtc-signalling-server &
+} on error {e} {
+  error "gst-webrtc-signalling-server not found, you probably need to install gst-plugins-rs"
+}
+
+try {
+  exec gst-inspect-1.0 rswebrtc
+} on error {e} {
+  error "gstreamer plugin 'rswebrtc' not found, you probably need to install gst-plugins-rs"
+}
+
+try {
+  exec gst-inspect-1.0 webrtc
+} on error {e} {
+  error "gstreamer plugin 'webrtc' not found, you probably need to install gst-plugins-bad"
+}
+
+When /someone/ wishes /p/ receives webrtc video /feed/ &\
+     /someone/ claims webrtc video /feed/ streams from peer /peer-id/ {
+  set uri gstwebrtc://127.0.0.1:8443?peer-id=${peer-id}
+  When the gstreamer pipeline "uridecodebin uri=$uri" frame is /image/ at /ts/ {
+    Claim $p has webrtc video $feed frame $image at $ts
+  }
+}
+
+When when /p/ has webrtc video /feed/ frame /image/ at /ts/ /lambda/ with environment /e/ {
+  Wish $p receives webrtc video $feed
+}
+
+Wish the web server handles route "/webrtc$" with handler {
+  html {
+    <!DOCTYPE html>
+    <html>
+    <body>
+      <span id="status">Status</span>
+      <h1>active feeds</h1>
+      <div id="feeds">
+      </div>
+
+      <script src="/lib/folk.js"></script>
+      <script>
+      function wsConnect() {
+        ws = new WebSocket(window.location.origin.replace("http", "ws") + "/ws");
+        interval = null;
+
+        ws.onopen = () => {
+          document.getElementById('status').innerHTML = "<span style=background-color:seagreen;color:white;>Connnected</span>";
+          ws.send(`Commit {
+            When /someone/ wishes /someone/ receives webrtc video /feed/ {
+              ::websocket::send $this text "add $feed"
+              On unmatch { ::websocket::send $this text "del $feed" }
+            }
+          }`);
+        };
+        ws.onclose = window.onbeforeunload = () => {
+          document.getElementById('status').innerHTML = "<span style=background-color:red;color:white;>Disconnnected</span>";
+          feeds.innerHTML = '';
+    	    setTimeout(() => { wsConnect(); }, 1000);
+        };
+        ws.onerror = (err) => {
+          document.getElementById('status').innerText = "Error";
+          console.error('Socket encountered error: ', err.message, 'Closing socket');
+          ws.close();
+        }
+        ws.onmessage = (e) => {
+          const [action, feed] = loadList(e.data);
+          if (action === "add") {
+            const link = document.createElement('a');
+            link.id = `feed-${feed}`;
+            link.style = "display: block";
+            link.innerText = `stream to ${feed}`;
+            link.href = `/webrtc/${encodeURIComponent(feed)}/send`;
+            feeds.append(link);
+          } else if (action === "del") {
+            document.getElementById(`feed-${feed}`).remove();
+          }
+        }
+      };
+      wsConnect();
+    </script>
+    </body>
+    </html>
+  }
+}
+
+Wish the web server handles route {/webrtc/(.*)/send$} with handler {
+  regexp {/webrtc/(.*)/send$} $path -> feed
+  html [format {
+    <!DOCTYPE html>
+    <html style="display: flex; min-height: 100vh">
+    <head>
+    </head>
+    <body style="display: flex; flex-direction: column; flex: 1;">
+      <span id="status">Status</span>
+      <h1>share video to feed %s</h1>
+      <div>
+        <button id="camera-button">start camera</button>
+        <button id="screen-button">start screenshare</button>
+        <button id="stop-button">stop</button>
+      </div>
+      <video style="flex: 1; background: #000; margin: auto; max-width: 100%;" id="preview"></video>
+
+      <script src="/lib/folk.js"></script>
+      <script src="/vendor/gstwebrtc/gstwebrtc-api-2.0.0.min.js"></script>
+      <script>
+        const protocol = window.location.protocol.replace("http", "ws");
+        const api = new GstWebRTCAPI({
+          meta: { name: `WebClient-${Date.now()}` },
+          signalingServerUrl: `${protocol}//${window.location.hostname}:8443/webrtc`,
+        });
+
+        const feed = '%s';
+        let peerId = null;
+        let session;
+
+        const videoElement = document.getElementById("preview");
+
+        api.registerConnectionListener({
+          connected: (clientId) => { peerId = clientId; },
+          disconnected: () => { peerId = null; }
+        });
+
+        const handleStream = (stream) => {
+          const sess = api.createProducerSession(stream);
+
+          if (!sess) {
+            for (const track of stream.getTracks()) {
+              track.stop();
+            }
+            // captureSection.classList.remove("starting");
+            return;
+          }
+
+          sess.addEventListener("error", (event) => console.error(event.message, event.error));
+
+          sess.addEventListener("closed", () => {
+            videoElement.pause();
+            videoElement.srcObject = null;
+            ws.send(`Commit {}`);
+            session = null;
+            // captureSection.classList.remove("has-sess", "starting");
+          });
+
+          sess.addEventListener("stateChanged", (event) => {
+            if ((event.target.state === GstWebRTCAPI.SessionState.streaming)) {
+              videoElement.srcObject = stream;
+              videoElement.play().catch(() => {});
+              // captureSection.classList.remove("starting");
+
+              // sess ready, announce to folk consumers!
+              ws.send(`Commit {
+                Claim webrtc video ${feed} streams from peer ${peerId}
+              }`);
+            }
+          });
+
+          /*
+          sess.addEventListener("clientConsumerAdded", (event) => {
+            if (captureSection._producerSession === sess) {
+              console.info(`client consumer added: ${event.detail.peerId}`);
+            }
+          });
+
+          sess.addEventListener("clientConsumerRemoved", (event) => {
+            if (captureSection._producerSession === sess) {
+              console.info(`client consumer removed: ${event.detail.peerId}`);
+            }
+          });
+          */
+
+          // captureSection.classList.add("has-sess");
+          sess.start();
+          return sess;
+        };
+
+        document.getElementById("stop-button").onclick = (event) => {
+          event.preventDefault();
+          if (session) {
+            session.close();
+          }
+        };
+
+        document.getElementById("camera-button").onclick = (event) => {
+          event.preventDefault();
+          if (session) return;
+
+          session = true;
+
+          const constraints = { video: true, audio: false };
+          navigator.mediaDevices.getUserMedia(constraints)
+            .then(handleStream)
+            .then(
+              (s) => { session = s; },
+              (error) => {
+                console.error("cannot have access to webcam and microphone", error);
+                session = null;
+              }
+            );
+        };
+
+        document.getElementById("screen-button").onclick = (event) => {
+          event.preventDefault();
+          if (session) return;
+
+          session = true;
+          const constraints = { video: true, audio: false };
+          navigator.mediaDevices.getDisplayMedia(constraints)
+            .then(handleStream)
+            .then(
+              (s) => { session = s; },
+              (error) => {
+                console.error("cannot have access to screen", error);
+                session = null;
+              }
+            );
+        };
+      </script>
+      <script>
+      function wsConnect() {
+        ws = new WebSocket(window.location.origin.replace("http", "ws") + "/ws");
+        interval = null;
+
+        ws.onopen = () => {
+          document.getElementById('status').innerHTML = "<span style=background-color:seagreen;color:white;>Connnected</span>";
+        };
+        ws.onclose = window.onbeforeunload = () => {
+          document.getElementById('status').innerHTML = "<span style=background-color:red;color:white;>Disconnnected</span>";
+    	    setTimeout(() => { wsConnect(); }, 1000);
+        };
+        ws.onerror = (err) => {
+          document.getElementById('status').innerText = "Error";
+          console.error('Socket encountered error: ', err.message, 'Closing socket');
+          ws.close();
+        }
+        ws.onmessage = (e) => {
+        }
+      };
+      wsConnect();
+    </script>
+    </body>
+    </html>
+  } $feed $feed]
+}
diff --git a/web.tcl b/web.tcl
@@ -128,6 +128,14 @@ proc handlePage {path httpStatusVar contentTypeVar} {
         "/statements.pdf" {
             getDotAsPdf [Statements::dot] contentType
         }
+        "/lib/folk.js" {
+            set contentType "text/javascript"
+            readFile "lib/folk.js" contentType
+        }
+        "/vendor/gstwebrtc/gstwebrtc-api-2.0.0.min.js" {
+            set contentType "text/javascript"
+            readFile "vendor/gstwebrtc/gstwebrtc-api-2.0.0.min.js" contentType
+        }
         default {
             upvar $httpStatusVar httpStatus
             set httpStatus "HTTP/1.1 404 Not Found"