From 272b16ee1c0540cb52f08182dbde8b098527cc91 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Tue, 13 Jun 2023 20:34:45 +0200
Subject: [PATCH] webrtc: accept `iceServers: []` provided in `POST /webrtc`,
 support trickle ICE

Thanks https://github.com/ayufan/camera-streamer/pull/65
---
 RELEASE.md                 |   1 +
 cmd/camera-streamer/opts.c |   1 +
 html/control.html          |  43 +++++-----
 html/webrtc.html           |  98 ++++++++++-----------
 output/webrtc/webrtc.cc    | 172 ++++++++++++++++++++++++++++++-------
 output/webrtc/webrtc.h     |   1 +
 util/http/json.hh          |  27 ++++++
 7 files changed, 239 insertions(+), 104 deletions(-)
 create mode 100644 util/http/json.hh

diff --git a/RELEASE.md b/RELEASE.md
index a2b56fb..b8b20d2 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -9,6 +9,7 @@
 - http: change `/option` to accept `device=`, `key=`, and `value=`
 - device: show stddev estimates to measure frame pacing
 - webrtc: allow to specify `--webrtc-ice_servers=` on command line
+- webrtc: accept `iceServers:[{urls:[],username:,password:}]` provided in `POST /webrtc`
 
 ## Variants
 
diff --git a/cmd/camera-streamer/opts.c b/cmd/camera-streamer/opts.c
index b0c48c7..756dd29 100644
--- a/cmd/camera-streamer/opts.c
+++ b/cmd/camera-streamer/opts.c
@@ -132,6 +132,7 @@ option_t all_options[] = {
   DEFINE_OPTION_DEFAULT(rtsp, port, uint, "8554", "Set the RTSP server port (default: 8854)."),
 
   DEFINE_OPTION_PTR(webrtc, ice_servers, list, "Specify ICE servers: [(stun|turn|turns)(:|://)][username:password@]hostname[:port][?transport=udp|tcp|tls)]."),
+  DEFINE_OPTION_DEFAULT(webrtc, disable_client_ice, bool, "1", "Ignore ICE servers provided in '/webrtc' request."),
 
   DEFINE_OPTION_DEFAULT(log, debug, bool, "1", "Enable debug logging."),
   DEFINE_OPTION_DEFAULT(log, verbose, bool, "1", "Enable verbose logging."),
diff --git a/html/control.html b/html/control.html
index 98705e9..18f4fc0 100644
--- a/html/control.html
+++ b/html/control.html
@@ -759,8 +759,11 @@
         method: 'POST'
       }).then(function(response) {
         return response.json();
-      }).then(function(answer) {
-        rtcPeerConnection = new RTCPeerConnection(rtcPeerConfig);
+      }).then(function(request) {
+        rtcPeerConnection = new RTCPeerConnection({
+          sdpSemantics: 'unified-plan',
+          iceServers: request.iceServers
+        });
         rtcPeerConnection.addTransceiver('video', { direction: 'recvonly' });
         //pc.addTransceiver('audio', {direction: 'recvonly'});
         rtcPeerConnection.addEventListener('track', function(evt) {
@@ -771,27 +774,27 @@
             view.scrollIntoView(false);
           }
         });
-        rtcPeerConnection.remote_pc_id = answer.id;
-        return rtcPeerConnection.setRemoteDescription(answer);
+        rtcPeerConnection.addEventListener("icecandidate", function(e) {
+          if (e.candidate) {
+            return fetch(webrtcURL, {
+              body: JSON.stringify({
+                type: 'remote_candidate',
+                id: rtcPeerConnection.remote_pc_id,
+                candidates: [e.candidate]
+              }),
+              headers: { 'Content-Type': 'application/json' },
+              method: 'POST'
+            }).catch(function(e) {
+              console.log("Failed to send ICE WebRTC: "+e);
+            });
+          }
+        });
+        rtcPeerConnection.remote_pc_id = request.id;
+        return rtcPeerConnection.setRemoteDescription(request);
       }).then(function() {
         return rtcPeerConnection.createAnswer();
       }).then(function(answer) {
         return rtcPeerConnection.setLocalDescription(answer);
-      }).then(function() {
-        // wait for ICE gathering to complete
-        return new Promise(function(resolve) {
-          if (rtcPeerConnection.iceGatheringState === 'complete') {
-            resolve();
-          } else {
-            function checkState() {
-              if (rtcPeerConnection.iceGatheringState === 'complete') {
-                rtcPeerConnection.removeEventListener('icegatheringstatechange', checkState);
-                resolve();
-              }
-            }
-            rtcPeerConnection.addEventListener('icegatheringstatechange', checkState);
-          }
-        });
       }).then(function(answer) {
         var offer = rtcPeerConnection.localDescription;
 
@@ -807,7 +810,7 @@
       }).then(function(response) {
         return response.json();
       }).catch(function(e) {
-        alert(e);
+        console.log(e);
       });
     }
   })
diff --git a/html/webrtc.html b/html/webrtc.html
index 226a2f1..d643a3b 100644
--- a/html/webrtc.html
+++ b/html/webrtc.html
@@ -47,28 +47,13 @@
 
   <script>
     function startWebRTC() {
-        var config = {
-            sdpSemantics: 'unified-plan'
-        };
+        // syntax: https://github.com/paullouisageneau/libdatachannel/blob/master/DOC.md#rtcwebrtc_peer_connection
+        const ice_servers = [
+          // 'stun:stun.l.google.com:19302',
+          //'turn://7295a68e2381585126c6a19e:zDd3R415oPPXJKQT@a.relay.metered.ca:80'
+        ];
 
-        if (document.getElementById('use-stun') && document.getElementById('use-stun').checked) {
-          config.iceServers = [{urls: ['stun:stun.l.google.com:19302']}];
-        }
-
-        pc = new RTCPeerConnection(config);
-        pc.addTransceiver('video', {direction: 'recvonly'});
-        //pc.addTransceiver('audio', {direction: 'recvonly'});
-        pc.addEventListener('track', function(evt) {
-          console.log("track event " + evt.track.kind);
-            if (evt.track.kind == 'video') {
-                if (document.getElementById('stream')) {
-                  document.getElementById('stream').srcObject = evt.streams[0];
-                }
-            } else {
-                if (document.getElementById('audio'))
-                  document.getElementById('audio').srcObject = evt.streams[0];
-            }
-        });
+        var pc = null;
 
         const urlSearchParams = new URLSearchParams(window.location.search);
         const params = Object.fromEntries(urlSearchParams.entries());
@@ -76,7 +61,8 @@
         fetch(window.location.href, {
           body: JSON.stringify({
               type: 'request',
-              res: params.res
+              res: params.res,
+              ice_servers: ice_servers
           }),
           headers: {
               'Content-Type': 'application/json'
@@ -84,54 +70,62 @@
           method: 'POST'
         }).then(function(response) {
           return response.json();
-        }).then(function(answer) {
-          pc.remote_pc_id = answer.id;
-          return pc.setRemoteDescription(answer);
+        }).then(function(request) {
+          pc = new RTCPeerConnection({
+            sdpSemantics: 'unified-plan',
+            iceServers: request.iceServers
+          });
+          pc.remote_pc_id = request.id;
+          pc.addTransceiver('video', { direction: 'recvonly' });
+          pc.addEventListener('track', function(evt) {
+            if (document.getElementById('stream'))
+              document.getElementById('stream').srcObject = evt.streams[0];
+          });
+          pc.addEventListener("icecandidate", function(e) {
+            if (e.candidate) {
+              return fetch(window.location.href, {
+                body: JSON.stringify({
+                  type: 'remote_candidate',
+                  id: pc.remote_pc_id,
+                  candidates: [e.candidate]
+                }),
+                headers: { 'Content-Type': 'application/json' },
+                method: 'POST'
+              }).catch(function(e) {
+                console.log("Failed to send ICE WebRTC: "+e);
+              });
+            }
+          });
+
+          return pc.setRemoteDescription(request);
         }).then(function() {
           return pc.createAnswer();
         }).then(function(answer) {
           return pc.setLocalDescription(answer);
         }).then(function() {
-            // wait for ICE gathering to complete
-            return new Promise(function(resolve) {
-                if (pc.iceGatheringState === 'complete') {
-                    resolve();
-                } else {
-                    function checkState() {
-                        if (pc.iceGatheringState === 'complete') {
-                            pc.removeEventListener('icegatheringstatechange', checkState);
-                            resolve();
-                        }
-                    }
-                    pc.addEventListener('icegatheringstatechange', checkState);
-                }
-            });
-        }).then(function(answer) {
           var offer = pc.localDescription;
 
           return fetch(window.location.href, {
             body: JSON.stringify({
-                type: offer.type,
-                id: pc.remote_pc_id,
-                sdp: offer.sdp,
+              type: offer.type,
+              id: pc.remote_pc_id,
+              sdp: offer.sdp,
             }),
-            headers: {
-                'Content-Type': 'application/json'
-            },
+            headers: { 'Content-Type': 'application/json' },
             method: 'POST'
           })
         }).then(function(response) {
-            return response.json();
+          return response.json();
         }).catch(function(e) {
-            alert(e);
+          console.log(e);
         });
     }
 
     function stopWebRTC() {
-        setTimeout(function() {
-            pc.close();
-        }, 500);
-      }
+      setTimeout(function() {
+        pc.close();
+      }, 500);
+    }
   </script>
 
   <script>
diff --git a/output/webrtc/webrtc.cc b/output/webrtc/webrtc.cc
index 4d53c5c..b7d1731 100644
--- a/output/webrtc/webrtc.cc
+++ b/output/webrtc/webrtc.cc
@@ -14,6 +14,7 @@ extern "C" {
 };
 
 #include "util/opts/helpers.hh"
+#include "util/http/json.hh"
 
 #ifdef USE_LIBDATACHANNEL
 
@@ -24,17 +25,19 @@ extern "C" {
 #include <atomic>
 #include <chrono>
 #include <set>
-#include <nlohmann/json.hpp>
 #include <rtc/peerconnection.hpp>
 #include <rtc/rtcpsrreporter.hpp>
 #include <rtc/h264rtppacketizer.hpp>
 #include <rtc/h264packetizationhandler.hpp>
 #include <rtc/rtcpnackresponder.hpp>
 
+#include "third_party/magic_enum/include/magic_enum.hpp"
+
 using namespace std::chrono_literals;
 
 class Client;
 
+static webrtc_options_t *webrtc_options;
 static std::set<std::shared_ptr<Client> > webrtc_clients;
 static std::mutex webrtc_clients_lock;
 static const auto webrtc_client_lock_timeout = 3 * 1000ms;
@@ -131,6 +134,32 @@ public:
     video->track->send(data);
   }
 
+  void describePeerConnection(nlohmann::json &message)
+  {
+    nlohmann::json ice_servers = nlohmann::json::array();
+
+    for (const auto &ice_server : pc->config()->iceServers) {
+      nlohmann::json json;
+
+      std::string url;
+
+      if (ice_server.type == rtc::IceServer::Type::Turn) {
+        json["username"] = ice_server.username;
+        json["credential"] = ice_server.password;
+        url = ice_server.relayType == rtc::IceServer::RelayType::TurnTls ? "turns:" : "turn:";
+      } else {
+        url = "stun:";
+      }
+
+      url += ice_server.hostname + ":" + std::to_string(ice_server.port);
+      json["urls"] = url;
+
+      ice_servers.push_back(json);
+    }
+
+    message["iceServers"] = ice_servers;
+  }
+
 public:
   char *name = NULL;
   std::string id;
@@ -138,11 +167,13 @@ public:
   std::shared_ptr<ClientTrackData> video;
   std::mutex lock;
   std::condition_variable wait_for_complete;
+  std::vector<rtc::Candidate> pending_remote_candidates;
+  bool has_set_sdp_answer = false;
   bool had_key_frame = false;
   bool requested_key_frame = false;
 };
 
-std::shared_ptr<Client> findClient(std::string id)
+std::shared_ptr<Client> webrtc_find_client(std::string id)
 {
   std::unique_lock lk(webrtc_clients_lock);
   for (auto client : webrtc_clients) {
@@ -154,14 +185,14 @@ std::shared_ptr<Client> findClient(std::string id)
   return std::shared_ptr<Client>();
 }
 
-void removeClient(const std::shared_ptr<Client> &client, const char *reason)
+static void webrtc_remove_client(const std::shared_ptr<Client> &client, const char *reason)
 {
   std::unique_lock lk(webrtc_clients_lock);
   webrtc_clients.erase(client);
   LOG_INFO(client.get(), "Client removed: %s.", reason);
 }
 
-std::shared_ptr<ClientTrackData> addVideo(const std::shared_ptr<rtc::PeerConnection> pc, const uint8_t payloadType, const uint32_t ssrc, const std::string cname, const std::string msid)
+static std::shared_ptr<ClientTrackData> webrtc_add_video(const std::shared_ptr<rtc::PeerConnection> pc, const uint8_t payloadType, const uint32_t ssrc, const std::string cname, const std::string msid)
 {
   auto video = rtc::Description::Video(cname, rtc::Description::Direction::SendOnly);
   video.addH264Codec(payloadType);
@@ -179,8 +210,52 @@ std::shared_ptr<ClientTrackData> addVideo(const std::shared_ptr<rtc::PeerConnect
   return std::shared_ptr<ClientTrackData>(new ClientTrackData{track, srReporter});
 }
 
-std::shared_ptr<Client> createPeerConnection(const rtc::Configuration &config)
+static void webrtc_parse_ice_servers(rtc::Configuration &config, const nlohmann::json &message)
 {
+  auto ice_servers = message.find("ice_servers");
+  if (ice_servers == message.end() || !ice_servers->is_array())
+    return;
+
+  if (webrtc_options->disable_client_ice) {
+    LOG_VERBOSE(NULL, "ICE server from SDP request ignored due to `disable_client_ice`: %s",
+      ice_servers->dump().c_str());
+    return;
+  }
+
+  for (const auto& ice_server : *ice_servers) {
+    try {
+      auto urls = ice_server["urls"];
+
+      // convert non array to array
+      if (!urls.is_array()) {
+        urls = nlohmann::json::array();
+        urls.push_back(ice_server["urls"]);
+      }
+
+      for (const auto& url : urls) {
+        auto iceServer = rtc::IceServer(url.get<std::string>());
+        if (iceServer.type == rtc::IceServer::Type::Turn) {
+          if (ice_server.contains("username"))
+            iceServer.username = ice_server["username"].get<std::string>();
+          if (ice_server.contains("credential"))
+            iceServer.password = ice_server["credential"].get<std::string>();
+        }
+
+        config.iceServers.push_back(iceServer);
+        LOG_VERBOSE(NULL, "Added ICE server from SDP request json: %s", url.dump().c_str());
+      }
+
+    } catch (nlohmann::detail::exception &e) {
+      LOG_VERBOSE(NULL, "Failed to parse ICE server: %s: %s",
+        ice_server.dump().c_str(), e.what());
+    }
+  }
+}
+
+static std::shared_ptr<Client> webrtc_peer_connection(rtc::Configuration config, const nlohmann::json &message)
+{
+  webrtc_parse_ice_servers(config, message);
+
   auto pc = std::make_shared<rtc::PeerConnection>(config);
   auto client = std::make_shared<Client>(pc);
   auto wclient = std::weak_ptr(client);
@@ -207,11 +282,15 @@ std::shared_ptr<Client> createPeerConnection(const rtc::Configuration &config)
     if(auto client = wclient.lock()) {
       LOG_DEBUG(client.get(), "onStateChange: %d", (int)state);
 
-      if (state == rtc::PeerConnection::State::Disconnected ||
+      if(state == rtc::PeerConnection::State::Connected) {
+        // Start streaming once the client is connected, to ensure a keyframe is sent to start the stream.
+        std::unique_lock lock(client->lock);
+        client->video->startStreaming();
+      } else if (state == rtc::PeerConnection::State::Disconnected ||
         state == rtc::PeerConnection::State::Failed ||
         state == rtc::PeerConnection::State::Closed)
       {
-        removeClient(client, "stream closed");
+        webrtc_remove_client(client, "stream closed");
       }
     }
   });
@@ -253,10 +332,10 @@ static void webrtc_h264_capture(buffer_lock_t *buf_lock, buffer_t *buf)
 
 static void http_webrtc_request(http_worker_t *worker, FILE *stream, const nlohmann::json &message)
 {
-  auto client = createPeerConnection(webrtc_configuration);
+  auto client = webrtc_peer_connection(webrtc_configuration, message);
   LOG_INFO(client.get(), "Stream requested.");
 
-  client->video = addVideo(client->pc, webrtc_client_video_payload_type, rand(), "video", "");
+  client->video = webrtc_add_video(client->pc, webrtc_client_video_payload_type, rand(), "video", "");
 
   try {
     {
@@ -271,6 +350,7 @@ static void http_webrtc_request(http_worker_t *worker, FILE *stream, const nlohm
       message["id"] = client->id;
       message["type"] = description->typeString();
       message["sdp"] = std::string(description.value());
+      client->describePeerConnection(message);
       http_write_response(stream, "200 OK", "application/json", message.dump().c_str(), 0);
       LOG_VERBOSE(client.get(), "Local SDP Offer: %s", std::string(message["sdp"]).c_str());
     } else {
@@ -278,7 +358,7 @@ static void http_webrtc_request(http_worker_t *worker, FILE *stream, const nlohm
     }
   } catch(const std::exception &e) {
     http_500(stream, e.what());
-    removeClient(client, e.what());
+    webrtc_remove_client(client, e.what());
   }
 }
 
@@ -289,18 +369,24 @@ static void http_webrtc_answer(http_worker_t *worker, FILE *stream, const nlohma
     return;
   }
 
-  if (auto client = findClient(message["id"])) {
+  if (auto client = webrtc_find_client(message["id"])) {
     LOG_INFO(client.get(), "Answer received.");
     LOG_VERBOSE(client.get(), "Remote SDP Answer: %s", std::string(message["sdp"]).c_str());
 
     try {
       auto answer = rtc::Description(std::string(message["sdp"]), std::string(message["type"]));
       client->pc->setRemoteDescription(answer);
-      client->video->startStreaming();
+      client->has_set_sdp_answer = true;
+
+      // If there are any pending candidates that make it in before the answer request, add them now.
+      for(auto const &candidate : client->pending_remote_candidates) {
+        client->pc->addRemoteCandidate(candidate);
+      }
+      client->pending_remote_candidates.clear();
       http_write_response(stream, "200 OK", "application/json", "{}", 0);
     } catch(const std::exception &e) {
       http_500(stream, e.what());
-      removeClient(client, e.what());
+      webrtc_remove_client(client, e.what());
     }
   } else {
     http_404(stream, "No client found");
@@ -315,18 +401,18 @@ static void http_webrtc_offer(http_worker_t *worker, FILE *stream, const nlohman
   }
 
   auto offer = rtc::Description(std::string(message["sdp"]), std::string(message["type"]));
-  auto client = createPeerConnection(webrtc_configuration);
+  auto client = webrtc_peer_connection(webrtc_configuration, message);
 
   LOG_INFO(client.get(), "Offer received.");
   LOG_VERBOSE(client.get(), "Remote SDP Offer: %s", std::string(message["sdp"]).c_str());
 
   try {
-    client->video = addVideo(client->pc, webrtc_client_video_payload_type, rand(), "video", "");
-    client->video->startStreaming();
+    client->video = webrtc_add_video(client->pc, webrtc_client_video_payload_type, rand(), "video", "");
 
     {
       std::unique_lock lock(client->lock);
       client->pc->setRemoteDescription(offer);
+      client->has_set_sdp_answer = true;
       client->pc->setLocalDescription();
       client->wait_for_complete.wait_for(lock, webrtc_client_lock_timeout);
     }
@@ -336,6 +422,7 @@ static void http_webrtc_offer(http_worker_t *worker, FILE *stream, const nlohman
       nlohmann::json message;
       message["type"] = description->typeString();
       message["sdp"] = std::string(description.value());
+      client->describePeerConnection(message);
       http_write_response(stream, "200 OK", "application/json", message.dump().c_str(), 0);
 
       LOG_VERBOSE(client.get(), "Local SDP Answer: %s", std::string(message["sdp"]).c_str());
@@ -344,32 +431,49 @@ static void http_webrtc_offer(http_worker_t *worker, FILE *stream, const nlohman
     }
   } catch(const std::exception &e) {
     http_500(stream, e.what());
-    removeClient(client, e.what());
+    webrtc_remove_client(client, e.what());
   }
 }
 
-nlohmann::json http_parse_json_body(http_worker_t *worker, FILE *stream)
+static void http_webrtc_remote_candidate(http_worker_t *worker, FILE *stream, const nlohmann::json &message)
 {
-  std::string text;
-
-  size_t i = 0;
-  size_t n = (size_t)worker->content_length;
-  if (n < 0 || n > webrtc_client_max_json_body)
-    n = webrtc_client_max_json_body;
-
-  text.resize(n);
-
-  while (i < n && !feof(stream)) {
-    i += fread(&text[i], 1, n-i, stream);
+  if (!message.contains("candidates") || !message.contains("id") || !message["candidates"].is_array()) {
+    http_400(stream, "candidates is not array or no id");
+    return;
   }
-  text.resize(i);
 
-  return nlohmann::json::parse(text);
+  auto client = webrtc_find_client(message["id"]);
+  if (!client) {
+    http_404(stream, "No client found");
+    return;
+  }
+
+  for (auto const & entry : message["candidates"]) {
+    try {
+      auto remoteCandidate = rtc::Candidate(
+        entry["candidate"].get<std::string>(),
+        entry["sdpMid"].get<std::string>());
+  
+      std::unique_lock lock(client->lock);
+      // The ICE candidate http requests can race the sdp answer http request and win. But it's invalid to set the ICE
+      // candidates before the SDP answer is set.
+      if (client->has_set_sdp_answer) {
+        client->pc->addRemoteCandidate(remoteCandidate);
+      } else {
+        client->pending_remote_candidates.push_back(remoteCandidate);
+      }
+    } catch (nlohmann::detail::exception &e) {
+      http_400(stream, e.what());
+      return;
+    }
+  }
+
+  http_write_response(stream, "200 OK", "application/json", "{}", 0);
 }
 
 extern "C" void http_webrtc_offer(http_worker_t *worker, FILE *stream)
 {
-  auto message = http_parse_json_body(worker, stream);
+  auto message = http_parse_json_body(worker, stream, webrtc_client_max_json_body);
 
   if (!message.contains("type")) {
     http_400(stream, "missing 'type'");
@@ -386,6 +490,8 @@ extern "C" void http_webrtc_offer(http_worker_t *worker, FILE *stream)
     http_webrtc_answer(worker, stream, message);
   } else if (type == "offer") {
     http_webrtc_offer(worker, stream, message);
+  } else if (type == "remote_candidate") {
+    http_webrtc_remote_candidate(worker, stream, message);
   } else {
     http_400(stream, (std::string("Not expected: " + type)).c_str());
   }
@@ -393,6 +499,8 @@ extern "C" void http_webrtc_offer(http_worker_t *worker, FILE *stream)
 
 extern "C" int webrtc_server(webrtc_options_t *options)
 {
+  webrtc_options = options;
+
   for (const auto &ice_server : str_split(options->ice_servers, OPTION_VALUE_LIST_SEP_CHAR)) {
     webrtc_configuration.iceServers.push_back(rtc::IceServer(ice_server));
   }
diff --git a/output/webrtc/webrtc.h b/output/webrtc/webrtc.h
index 8fcb729..ebb8519 100644
--- a/output/webrtc/webrtc.h
+++ b/output/webrtc/webrtc.h
@@ -10,6 +10,7 @@ typedef struct webrtc_options_s {
   bool running;
   bool disabled;
   char ice_servers[WEBRTC_OPTIONS_LENGTH];
+  bool disable_client_ice;
 } webrtc_options_t;
 
 // WebRTC
diff --git a/util/http/json.hh b/util/http/json.hh
new file mode 100644
index 0000000..f2a44d8
--- /dev/null
+++ b/util/http/json.hh
@@ -0,0 +1,27 @@
+#pragma once
+
+extern "C" {
+#include "http.h"
+}
+
+#include <stdio.h>
+#include <nlohmann/json.hpp>
+
+inline nlohmann::json http_parse_json_body(http_worker_t *worker, FILE *stream, unsigned max_body_size)
+{
+  std::string text;
+
+  size_t i = 0;
+  size_t n = (size_t)worker->content_length;
+  if (n < 0 || n > max_body_size)
+    n = max_body_size;
+
+  text.resize(n);
+
+  while (i < n && !feof(stream)) {
+    i += fread(&text[i], 1, n-i, stream);
+  }
+  text.resize(i);
+
+  return nlohmann::json::parse(text);
+}