From 5ee0bee59f0054e1eae78b0f98aff436d63a9ee2 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Wed, 31 Aug 2022 19:04:54 +0200 Subject: [PATCH] Add WebRTC support using `libdatachannel` The WebRTC is exposed via `/video.html` endpoint and enabled by default as long as h264 stream is available. --- .gitignore | 3 +- .gitmodules | 4 + .vscode/c_cpp_properties.json | 5 +- Dockerfile | 2 +- Makefile | 26 ++- README.md | 13 +- cmd/camera-streamer/http.c | 5 +- cmd/camera-streamer/main.c | 4 +- cmd/camera-streamer/opts.c | 2 +- html/index.html | 2 +- html/jmuxer.min.js | 1 - html/video.html | 197 ++++++++++------ output/http_h264.c | 10 +- output/webrtc/webrtc.cc | 426 ++++++++++++++++++++++++++++++++++ output/webrtc/webrtc.h | 11 + third_party/libdatachannel | 1 + util/http/http.c | 9 +- util/http/http.h | 3 + util/http/http_methods.c | 7 +- 19 files changed, 644 insertions(+), 87 deletions(-) create mode 100644 .gitmodules delete mode 100644 html/jmuxer.min.js create mode 100644 output/webrtc/webrtc.cc create mode 100644 output/webrtc/webrtc.h create mode 160000 third_party/libdatachannel diff --git a/.gitignore b/.gitignore index dca0abd..c6f4b4c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ tmp/ *.o *.d -html/*.c +*.html.c +*.js.c /camera-streamer /test_* .vscode/settings.json diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..d0b3f49 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "third_party/libdatachannel"] + path = third_party/libdatachannel + url = https://github.com/paullouisageneau/libdatachannel.git + ignore = dirty diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 2ad184e..293606c 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -4,6 +4,8 @@ "name": "Linux", "includePath": [ "${workspaceFolder}/**", + "${workspaceFolder}/third_party/libdatachannel/include", + "${workspaceFolder}/third_party/libdatachannel/deps/json/include", "/usr/include/libcamera", "/usr/include/liveMedia", "/usr/include/groupsock", @@ -13,7 +15,8 @@ "defines": [ "USE_LIBCAMERA=1", "USE_FFMPEG=1", - "USE_RTSP=1" + "USE_RTSP=1", + "USE_LIBDATACHANNEL=1" ], "compilerPath": "/usr/bin/gcc", "cStandard": "gnu17", diff --git a/Dockerfile b/Dockerfile index 5fa8d2f..4c56d50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,5 +24,5 @@ ADD / /src WORKDIR /src RUN git clean -ffdx RUN git submodule update --init --recursive --recommend-shallow -RUN git submodule foreach git clean -ffdx +RUN git submodule foreach --recursive git clean -ffdx RUN make -j$(nproc) diff --git a/Makefile b/Makefile index 3f8893c..40aec69 100644 --- a/Makefile +++ b/Makefile @@ -10,9 +10,12 @@ ifneq (x,x$(shell which ccache)) CCACHE ?= ccache endif +LIBDATACHANNEL_PATH ?= third_party/libdatachannel + USE_FFMPEG ?= $(shell pkg-config libavutil libavformat libavcodec && echo 1) USE_LIBCAMERA ?= $(shell pkg-config libcamera && echo 1) USE_RTSP ?= $(shell pkg-config live555 && echo 1) +USE_LIBDATACHANNEL ?= $(shell [ -e $(LIBDATACHANNEL_PATH)/CMakeLists.txt ] && echo 1) ifeq (1,$(DEBUG)) CFLAGS += -g @@ -25,12 +28,25 @@ endif ifeq (1,$(USE_LIBCAMERA)) CFLAGS += -DUSE_LIBCAMERA $(shell pkg-config --cflags libcamera) -LDLIBS += $(shell pkg-config --libs libcamera) +LDLIBS += $(shell pkg-config --libs libcamera) endif ifeq (1,$(USE_RTSP)) CFLAGS += -DUSE_RTSP $(shell pkg-config --cflags live555) -LDLIBS += $(shell pkg-config --libs live555) +LDLIBS += $(shell pkg-config --libs live555) +endif + +ifeq (1,$(USE_LIBDATACHANNEL)) +CFLAGS += -DUSE_LIBDATACHANNEL +CFLAGS += -I$(LIBDATACHANNEL_PATH)/include +CFLAGS += -I$(LIBDATACHANNEL_PATH)/deps/json/include +LDLIBS += -L$(LIBDATACHANNEL_PATH)/build -ldatachannel-static +LDLIBS += -L$(LIBDATACHANNEL_PATH)/build/deps/usrsctp/usrsctplib -lusrsctp +LDLIBS += -L$(LIBDATACHANNEL_PATH)/build/deps/libsrtp -lsrtp2 +LDLIBS += -L$(LIBDATACHANNEL_PATH)/build/deps/libjuice -ljuice-static +LDLIBS += -lcrypto -lssl + +camera-streamer: $(LIBDATACHANNEL_PATH)/build/libdatachannel-static.a endif HTML_SRC = $(addsuffix .c,$(HTML)) @@ -40,7 +56,7 @@ OBJS = $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(SRC) $(HTML_SRC))) all: $(TARGET) -%: cmd/% $(OBJS) +%: cmd/% $(filter-out third_party/%, $(OBJS)) $(CCACHE) $(CXX) $(CFLAGS) -o $@ $(filter-out cmd/%, $^) $(filter $ $@.tmp mv $@.tmp $@ + +$(LIBDATACHANNEL_PATH)/build/libdatachannel-static.a: $(LIBDATACHANNEL_PATH) + [ -e $:8080/video` by default and is available when there's H264 output generated. + +WebRTC support is implemented using awesome [libdatachannel](https://github.com/paullouisageneau/libdatachannel/) library. + +The support will be compiled by default when doing `make`. + ## License GNU General Public License v3.0 diff --git a/cmd/camera-streamer/http.c b/cmd/camera-streamer/http.c index 7bbb213..fb89911 100644 --- a/cmd/camera-streamer/http.c +++ b/cmd/camera-streamer/http.c @@ -2,6 +2,7 @@ #include "util/opts/opts.h" #include "util/opts/log.h" #include "util/opts/fourcc.h" +#include "output/webrtc/webrtc.h" #include "device/camera/camera.h" #include "output/output.h" #include "output/rtsp/rtsp.h" @@ -10,8 +11,6 @@ extern unsigned char html_index_html[]; extern unsigned int html_index_html_len; extern unsigned char html_video_html[]; extern unsigned int html_video_html_len; -extern unsigned char html_jmuxer_min_js[]; -extern unsigned int html_jmuxer_min_js_len; extern camera_t *camera; void *camera_http_set_option(http_worker_t *worker, FILE *stream, const char *key, const char *value, void *headersp) @@ -81,8 +80,8 @@ http_method_t http_methods[] = { { "GET /video.h264?", http_h264_video }, { "GET /video.mkv?", http_mkv_video }, { "GET /video.mp4?", http_mp4_video }, + { "POST /video?", http_webrtc_offer }, { "GET /option?", camera_http_option }, - { "GET /jmuxer.min.js?", http_content, "text/javascript", html_jmuxer_min_js, 0, &html_jmuxer_min_js_len }, { "GET /?", http_content, "text/html", html_index_html, 0, &html_index_html_len }, { } }; diff --git a/cmd/camera-streamer/main.c b/cmd/camera-streamer/main.c index be58179..75d806b 100644 --- a/cmd/camera-streamer/main.c +++ b/cmd/camera-streamer/main.c @@ -1,9 +1,9 @@ #include "util/http/http.h" #include "util/opts/opts.h" #include "util/opts/log.h" -#include "util/opts/fourcc.h" #include "device/camera/camera.h" #include "output/rtsp/rtsp.h" +#include "output/webrtc/webrtc.h" #include #include @@ -46,6 +46,8 @@ int main(int argc, char *argv[]) goto error; } + webrtc_server(); + while (true) { camera = camera_open(&camera_options); if (camera) { diff --git a/cmd/camera-streamer/opts.c b/cmd/camera-streamer/opts.c index 9cd0b2d..7e7e56f 100644 --- a/cmd/camera-streamer/opts.c +++ b/cmd/camera-streamer/opts.c @@ -107,4 +107,4 @@ option_t all_options[] = { DEFINE_OPTION_PTR(log, filter, list, "Enable debug logging from the given files. Ex.: `-log-filter=buffer.cc`"), {} -}; \ No newline at end of file +}; diff --git a/html/index.html b/html/index.html index 0f92e09..c099335 100644 --- a/html/index.html +++ b/html/index.html @@ -40,7 +40,7 @@
  • /video
    - Get a live video (H264) stream.
    + Get a live WebRTC (H264) stream.

    • /video.mp4
      get a live video stream in MP4 format (if FFMPEG enabled).
      • diff --git a/html/jmuxer.min.js b/html/jmuxer.min.js deleted file mode 100644 index 1eebe7e..0000000 --- a/html/jmuxer.min.js +++ /dev/null @@ -1 +0,0 @@ -!function(e,t){"object"==typeof exports&&"undefined"!=typeof module?module.exports=t(require("stream")):"function"==typeof define&&define.amd?define(["stream"],t):(e="undefined"!=typeof globalThis?globalThis:e||self).JMuxer=t(e.stream)}(this,(function(e){"use strict";function t(e){return t="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&"function"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e},t(e)}function n(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function r(e,t){for(var n=0;ne.length)&&(t=e.length);for(var n=0,r=new Array(t);n=e.length?{done:!0}:{done:!1,value:e[r++]}},e:function(e){throw e},f:i}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}var a,s=!0,o=!1;return{s:function(){n=n.call(e)},n:function(){var e=n.next();return s=e.done,e},e:function(e){o=!0,a=e},f:function(){try{s||null==n.return||n.return()}finally{if(o)throw a}}}}var v,m;function k(e){if(v){for(var t=arguments.length,n=new Array(t>1?t-1:0),r=1;r1?t-1:0),r=1;r>5,this.ntype=31&this.payload[0],this.isvcl=1==this.ntype||5==this.ntype,this.stype="",this.isfmb=!1}return i(e,[{key:"toString",value:function(){return"".concat(e.type(this),": NRI: ").concat(this.getNri())}},{key:"getNri",value:function(){return this.nri}},{key:"type",value:function(){return this.ntype}},{key:"isKeyframe",value:function(){return this.ntype===e.IDR}},{key:"getPayload",value:function(){return this.payload}},{key:"getPayloadSize",value:function(){return this.payload.byteLength}},{key:"getSize",value:function(){return 4+this.getPayloadSize()}},{key:"getData",value:function(){var e=new Uint8Array(this.getSize());return new DataView(e.buffer).setUint32(0,this.getSize()-4),e.set(this.getPayload(),4),e}}],[{key:"NDR",get:function(){return 1}},{key:"IDR",get:function(){return 5}},{key:"SEI",get:function(){return 6}},{key:"SPS",get:function(){return 7}},{key:"PPS",get:function(){return 8}},{key:"AUD",get:function(){return 9}},{key:"TYPES",get:function(){var t;return a(t={},e.IDR,"IDR"),a(t,e.SEI,"SEI"),a(t,e.SPS,"SPS"),a(t,e.PPS,"PPS"),a(t,e.NDR,"NDR"),a(t,e.AUD,"AUD"),t}},{key:"type",value:function(t){return t.ntype in e.TYPES?e.TYPES[t.ntype]:"UNKNOWN"}}]),e}();function S(e,t){var n=new Uint8Array((0|e.byteLength)+(0|t.byteLength));return n.set(e,0),n.set(t,0|e.byteLength),n}var w,x=function(){function e(t){n(this,e),this.data=t,this.index=0,this.bitLength=8*t.byteLength}return i(e,[{key:"setData",value:function(e){this.data=e,this.index=0,this.bitLength=8*e.byteLength}},{key:"bitsAvailable",get:function(){return this.bitLength-this.index}},{key:"skipBits",value:function(e){if(this.bitsAvailable1&&void 0!==arguments[1])||arguments[1],n=this.getBits(e,this.index,t);return n}},{key:"getBits",value:function(e,t){var n=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];if(this.bitsAvailable>>r,a=8-r;if(a>=e)return n&&(this.index+=e),i>>a-e;n&&(this.index+=a);var s=e-a;return i<>>1:-1*(e>>>1)}},{key:"readBoolean",value:function(){return 1===this.readBits(1)}},{key:"readUByte",value:function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:1;return this.readBits(8*e)}},{key:"readUShort",value:function(){return this.readBits(16)}},{key:"readUInt",value:function(){return this.readBits(32)}}]),e}(),A=function(){function e(t){n(this,e),this.remuxer=t,this.track=t.mp4track}return i(e,[{key:"parseSPS",value:function(t){var n=e.readSPS(new Uint8Array(t));this.track.fps=n.fps,this.track.width=n.width,this.track.height=n.height,this.track.sps=[new Uint8Array(t)],this.track.codec="avc1.";for(var r=new DataView(t.buffer,t.byteOffset+1,4),i=0;i<3;++i){var a=r.getUint8(i).toString(16);a.length<2&&(a="0"+a),this.track.codec+=a}}},{key:"parsePPS",value:function(e){this.track.pps=[new Uint8Array(e)]}},{key:"parseNAL",value:function(e){if(!e)return!1;var t=!1;switch(e.type()){case b.IDR:case b.NDR:t=!0;break;case b.PPS:this.track.pps||(this.parsePPS(e.getPayload()),!this.remuxer.readyToDecode&&this.track.pps&&this.track.sps&&(this.remuxer.readyToDecode=!0)),t=!0;break;case b.SPS:this.track.sps||(this.parseSPS(e.getPayload()),!this.remuxer.readyToDecode&&this.track.pps&&this.track.sps&&(this.remuxer.readyToDecode=!0)),t=!0;break;case b.AUD:k("AUD - ignoing");break;case b.SEI:k("SEI - ignoing")}return t}}],[{key:"extractNALu",value:function(e){for(var t,n,r=0,i=e.byteLength,a=0,s=[],o=0;r0&&w[1]>0&&(d=w[0]/w[1])}if(u.readBoolean()&&u.skipBits(1),u.readBoolean()&&(u.skipBits(4),u.readBoolean()&&u.skipBits(24)),u.readBoolean()&&(u.skipUEG(),u.skipUEG()),u.readBoolean()){var A=u.readUInt(),U=u.readUInt();u.readBoolean()&&(y=U/(2*A))}}return{fps:y>0?y:void 0,width:Math.ceil((16*(i+1)-2*c-2*f)*d),height:(2-s)*(a+1)*16-(s?2:4)*(l+h)}}},{key:"parseHeader",value:function(e){var t=new x(e.getPayload());t.readUByte(),e.isfmb=0===t.readUEG(),e.stype=t.readUEG()}}]),e}(),U=function(){function e(t){n(this,e),this.remuxer=t,this.track=t.mp4track}return i(e,[{key:"setAACConfig",value:function(){var t,n,r,i=new Uint8Array(2),a=e.getAACHeaderData;a&&(t=1+((192&a[2])>>>6),n=(60&a[2])>>>2,r=(1&a[2])<<2,r|=(192&a[3])>>>6,i[0]=t<<3,i[0]|=(14&n)>>1,i[1]|=(1&n)<<7,i[1]|=r<<3,this.track.codec="mp4a.40."+t,this.track.channelCount=r,this.track.config=i,this.remuxer.readyToDecode=!0)}}],[{key:"samplingRateMap",get:function(){return[96e3,88200,64e3,48e3,44100,32e3,24e3,22050,16e3,12e3,11025,8e3,7350]}},{key:"getAACHeaderData",get:function(){return w}},{key:"getHeaderLength",value:function(e){return 1&e[1]?7:9}},{key:"getFrameLength",value:function(e){return(3&e[3])<<11|e[4]<<3|(224&e[5])>>>5}},{key:"isAACPattern",value:function(e){return 255===e[0]&&240==(240&e[1])&&0==(6&e[1])}},{key:"extractAAC",value:function(t){var n,r,i=0,a=t.byteLength,s=[];if(!e.isAACPattern(t))return g("Invalid ADTS audio format"),s;for(n=e.getHeaderLength(t),w||(w=t.subarray(0,n));i-1&&this.listener[e].splice(n,1),!0}return!1}},{key:"offAll",value:function(){this.listener={}}},{key:"dispatch",value:function(e,t){return!!this.listener[e]&&(this.listener[e].map((function(e){e.apply(null,[t])})),!0)}}]),e}(),D=function(){function e(){n(this,e)}return i(e,null,[{key:"init",value:function(){var t;for(t in e.types={avc1:[],avcC:[],btrt:[],dinf:[],dref:[],esds:[],ftyp:[],hdlr:[],mdat:[],mdhd:[],mdia:[],mfhd:[],minf:[],moof:[],moov:[],mp4a:[],mvex:[],mvhd:[],sdtp:[],stbl:[],stco:[],stsc:[],stsd:[],stsz:[],stts:[],tfdt:[],tfhd:[],traf:[],trak:[],trun:[],trex:[],tkhd:[],vmhd:[],smhd:[]},e.types)e.types.hasOwnProperty(t)&&(e.types[t]=[t.charCodeAt(0),t.charCodeAt(1),t.charCodeAt(2),t.charCodeAt(3)]);var n=new Uint8Array([0,0,0,0,0,0,0,0,118,105,100,101,0,0,0,0,0,0,0,0,0,0,0,0,86,105,100,101,111,72,97,110,100,108,101,114,0]),r=new Uint8Array([0,0,0,0,0,0,0,0,115,111,117,110,0,0,0,0,0,0,0,0,0,0,0,0,83,111,117,110,100,72,97,110,100,108,101,114,0]);e.HDLR_TYPES={video:n,audio:r};var i=new Uint8Array([0,0,0,0,0,0,0,1,0,0,0,12,117,114,108,32,0,0,0,1]),a=new Uint8Array([0,0,0,0,0,0,0,0]);e.STTS=e.STSC=e.STCO=a,e.STSZ=new Uint8Array([0,0,0,0,0,0,0,0,0,0,0,0]),e.VMHD=new Uint8Array([0,0,0,1,0,0,0,0,0,0,0,0]),e.SMHD=new Uint8Array([0,0,0,0,0,0,0,0]),e.STSD=new Uint8Array([0,0,0,0,0,0,0,1]);var s=new Uint8Array([105,115,111,109]),o=new Uint8Array([97,118,99,49]),u=new Uint8Array([0,0,0,1]);e.FTYP=e.box(e.types.ftyp,s,u,s,o),e.DINF=e.box(e.types.dinf,e.box(e.types.dref,i))}},{key:"box",value:function(e){for(var t=arguments.length,n=new Array(t>1?t-1:0),r=1;r>24&255,i[1]=a>>16&255,i[2]=a>>8&255,i[3]=255&a,i.set(e,4),s=0,a=8;s>24&255,t>>16&255,t>>8&255,255&t,n>>24,n>>16&255,n>>8&255,255&n,85,196,0,0]))}},{key:"mdia",value:function(t){return e.box(e.types.mdia,e.mdhd(t.timescale,t.duration),e.hdlr(t.type),e.minf(t))}},{key:"mfhd",value:function(t){return e.box(e.types.mfhd,new Uint8Array([0,0,0,0,t>>24,t>>16&255,t>>8&255,255&t]))}},{key:"minf",value:function(t){return"audio"===t.type?e.box(e.types.minf,e.box(e.types.smhd,e.SMHD),e.DINF,e.stbl(t)):e.box(e.types.minf,e.box(e.types.vmhd,e.VMHD),e.DINF,e.stbl(t))}},{key:"moof",value:function(t,n,r){return e.box(e.types.moof,e.mfhd(t),e.traf(r,n))}},{key:"moov",value:function(t,n,r){for(var i=t.length,a=[];i--;)a[i]=e.trak(t[i]);return e.box.apply(null,[e.types.moov,e.mvhd(r,n)].concat(a).concat(e.mvex(t)))}},{key:"mvex",value:function(t){for(var n=t.length,r=[];n--;)r[n]=e.trex(t[n]);return e.box.apply(null,[e.types.mvex].concat(r))}},{key:"mvhd",value:function(t,n){var r=new Uint8Array([0,0,0,0,0,0,0,1,0,0,0,2,t>>24&255,t>>16&255,t>>8&255,255&t,n>>24&255,n>>16&255,n>>8&255,255&n,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,64,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255]);return e.box(e.types.mvhd,r)}},{key:"sdtp",value:function(t){var n,r,i=t.samples||[],a=new Uint8Array(4+i.length);for(r=0;r>>8&255),a.push(255&i),a=a.concat(Array.prototype.slice.call(r));for(n=0;n>>8&255),s.push(255&i),s=s.concat(Array.prototype.slice.call(r));var o=e.box(e.types.avcC,new Uint8Array([1,a[3],a[4],a[5],255,224|t.sps.length].concat(a).concat([t.pps.length]).concat(s))),u=t.width,c=t.height;return e.box(e.types.avc1,new Uint8Array([0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,u>>8&255,255&u,c>>8&255,255&c,0,72,0,0,0,72,0,0,0,0,0,0,0,1,18,98,105,110,101,108,112,114,111,46,114,117,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,24,17,17]),o,e.box(e.types.btrt,new Uint8Array([0,28,156,128,0,45,198,192,0,45,198,192])))}},{key:"esds",value:function(e){var t=e.config.byteLength,n=new Uint8Array(26+t+3);return n.set([0,0,0,0,3,23+t,0,1,0,4,15+t,64,21,0,0,0,0,0,0,0,0,0,0,0,5,t]),n.set(e.config,26),n.set([6,1,2],26+t),n}},{key:"mp4a",value:function(t){var n=t.audiosamplerate;return e.box(e.types.mp4a,new Uint8Array([0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,t.channelCount,0,16,0,0,0,0,n>>8&255,255&n,0,0]),e.box(e.types.esds,e.esds(t)))}},{key:"stsd",value:function(t){return"audio"===t.type?e.box(e.types.stsd,e.STSD,e.mp4a(t)):e.box(e.types.stsd,e.STSD,e.avc1(t))}},{key:"tkhd",value:function(t){var n=t.id,r=t.duration,i=t.width,a=t.height,s=t.volume;return e.box(e.types.tkhd,new Uint8Array([0,0,0,7,0,0,0,0,0,0,0,0,n>>24&255,n>>16&255,n>>8&255,255&n,0,0,0,0,r>>24,r>>16&255,r>>8&255,255&r,0,0,0,0,0,0,0,0,0,0,0,0,s>>0&255,s%1*10>>0&255,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,64,0,0,0,i>>8&255,255&i,0,0,a>>8&255,255&a,0,0]))}},{key:"traf",value:function(t,n){var r=e.sdtp(t),i=t.id;return e.box(e.types.traf,e.box(e.types.tfhd,new Uint8Array([0,0,0,0,i>>24,i>>16&255,i>>8&255,255&i])),e.box(e.types.tfdt,new Uint8Array([0,0,0,0,n>>24,n>>16&255,n>>8&255,255&n])),e.trun(t,r.length+16+16+8+16+8+8),r)}},{key:"trak",value:function(t){return t.duration=t.duration||4294967295,e.box(e.types.trak,e.tkhd(t),e.mdia(t))}},{key:"trex",value:function(t){var n=t.id;return e.box(e.types.trex,new Uint8Array([0,0,0,0,n>>24,n>>16&255,n>>8&255,255&n,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,1]))}},{key:"trun",value:function(t,n){var r,i,a,s,o,u,c=t.samples||[],f=c.length,l=12+16*f,h=new Uint8Array(l);for(n+=8+l,h.set([0,0,15,1,f>>>24&255,f>>>16&255,f>>>8&255,255&f,n>>>24&255,n>>>16&255,n>>>8&255,255&n],0),r=0;r>>24&255,a>>>16&255,a>>>8&255,255&a,s>>>24&255,s>>>16&255,s>>>8&255,255&s,o.isLeading<<2|o.dependsOn,o.isDependedOn<<6|o.hasRedundancy<<4|o.paddingValue<<1|o.isNonSync,61440&o.degradPrio,15&o.degradPrio,u>>>24&255,u>>>16&255,u>>>8&255,255&u],12+16*r);return e.box(e.types.trun,h)}},{key:"initSegment",value:function(t,n,r){e.types||e.init();var i,a=e.moov(t,n,r);return(i=new Uint8Array(e.FTYP.byteLength+a.byteLength)).set(e.FTYP),i.set(a,e.FTYP.byteLength),i}}]),e}(),C=1,E=function(){function e(){n(this,e)}return i(e,[{key:"flush",value:function(){this.mp4track.len=0,this.mp4track.samples=[]}},{key:"isReady",value:function(){return!(!this.readyToDecode||!this.samples.length)||null}}],[{key:"getTrackID",value:function(){return C++}}]),e}(),T=function(e){s(r,e);var t=l(r);function r(e){var i;return n(this,r),(i=t.call(this)).readyToDecode=!1,i.nextDts=0,i.dts=0,i.mp4track={id:E.getTrackID(),type:"audio",channelCount:0,len:0,fragmented:!0,timescale:e,duration:e,samples:[],config:"",codec:""},i.samples=[],i.aac=new U(c(i)),i}return i(r,[{key:"resetTrack",value:function(){this.readyToDecode=!1,this.mp4track.codec="",this.mp4track.channelCount="",this.mp4track.config="",this.mp4track.timescale=this.timescale,this.nextDts=0,this.dts=0}},{key:"remux",value:function(e){if(e.length>0)for(var t=0;t0&&this.readyToDecode&&(this.mp4track.len+=s,this.samples.push({units:a,size:s,keyFrame:i.keyFrame,duration:i.duration}))}}catch(e){n.e(e)}finally{n.f()}}},{key:"getPayload",value:function(){if(!this.isReady())return null;var e,t,n=new Uint8Array(this.mp4track.len),r=0,i=this.mp4track.samples;for(this.dts=this.nextDts;this.samples.length;){var a=this.samples.shift(),s=a.units;if((t=a.duration)<=0)k("remuxer: invalid sample duration at DTS: ".concat(this.nextDts," :").concat(t)),this.mp4track.len-=a.size;else{this.nextDts+=t,e={size:a.size,duration:t,cts:0,flags:{isLeading:0,isDependedOn:0,hasRedundancy:0,degradPrio:0,isNonSync:a.keyFrame?0:1,dependsOn:a.keyFrame?2:1}};var o,u=p(s);try{for(u.s();!(o=u.n()).done;){var c=o.value;n.set(c.getData(),r),r+=c.getSize()}}catch(e){u.e(e)}finally{u.f()}i.push(e)}}return i.length?new Uint8Array(n.buffer,0,this.mp4track.len):null}}]),r}(E),L=function(e){s(r,e);var t=l(r);function r(e){var i;return n(this,r),(i=t.call(this,"remuxer")).initialized=!1,i.trackTypes=[],i.tracks={},i.seq=1,i.env=e,i.timescale=1e3,i.mediaDuration=0,i}return i(r,[{key:"addTrack",value:function(e){"video"!==e&&"both"!==e||(this.tracks.video=new P(this.timescale),this.trackTypes.push("video")),"audio"!==e&&"both"!==e||(this.tracks.audio=new T(this.timescale),this.trackTypes.push("audio"))}},{key:"reset",value:function(){var e,t=p(this.trackTypes);try{for(t.s();!(e=t.n()).done;){var n=e.value;this.tracks[n].resetTrack()}}catch(e){t.e(e)}finally{t.f()}this.initialized=!1}},{key:"destroy",value:function(){this.tracks={},this.offAll()}},{key:"flush",value:function(){if(this.initialized){var e,t=p(this.trackTypes);try{for(t.s();!(e=t.n()).done;){var n=e.value,r=this.tracks[n],i=r.getPayload();if(i&&i.byteLength){var a={type:n,payload:S(D.moof(this.seq,r.dts,r.mp4track),D.mdat(i)),dts:r.dts};"video"===n&&(a.fps=r.mp4track.fps),this.dispatch("buffer",a);var s=(o=r.dts/this.timescale,u=void 0,c=void 0,f=void 0,l=void 0,l="",u=Math.floor(o),(c=parseInt(u/3600,10)%24)>0&&(l+=(c<10?"0"+c:c)+":"),l+=((f=parseInt(u/60,10)%60)<10?"0"+f:f)+":"+((u=u<0?0:u%60)<10?"0"+u:u));k("put segment (".concat(n,"): dts: ").concat(r.dts," frames: ").concat(r.mp4track.samples.length," second: ").concat(s)),r.flush(),this.seq++}}}catch(e){t.e(e)}finally{t.f()}}else this.isReady()&&(this.dispatch("ready"),this.initSegment(),this.initialized=!0,this.flush());var o,u,c,f,l}},{key:"initSegment",value:function(){var e,t=[],n=p(this.trackTypes);try{for(n.s();!(e=n.n()).done;){var r=e.value,i=this.tracks[r];if("browser"==this.env){var a={type:r,payload:D.initSegment([i.mp4track],this.mediaDuration,this.timescale)};this.dispatch("buffer",a)}else t.push(i.mp4track)}}catch(e){n.e(e)}finally{n.f()}if("node"==this.env){var s={type:"all",payload:D.initSegment(t,this.mediaDuration,this.timescale)};this.dispatch("buffer",s)}k("Initial segment generated.")}},{key:"isReady",value:function(){var e,t=p(this.trackTypes);try{for(t.s();!(e=t.n()).done;){var n=e.value;if(!this.tracks[n].readyToDecode||!this.tracks[n].samples.length)return!1}}catch(e){t.e(e)}finally{t.f()}return!0}},{key:"remux",value:function(e){var t,n=p(this.trackTypes);try{for(n.s();!(t=n.n()).done;){var r=t.value,i=e[r];"audio"===r&&this.tracks.video&&!this.tracks.video.readyToDecode||i.length>0&&this.tracks[r].remux(i)}}catch(e){n.e(e)}finally{n.f()}this.flush()}}]),r}(B),R=function(e){s(r,e);var t=l(r);function r(e,i){var a;return n(this,r),(a=t.call(this,"buffer")).type=i,a.queue=new Uint8Array,a.cleaning=!1,a.pendingCleaning=0,a.cleanOffset=30,a.cleanRanges=[],a.sourceBuffer=e,a.sourceBuffer.addEventListener("updateend",(function(){a.pendingCleaning>0&&(a.initCleanup(a.pendingCleaning),a.pendingCleaning=0),a.cleaning=!1,a.cleanRanges.length&&a.doCleanup()})),a.sourceBuffer.addEventListener("error",(function(){a.dispatch("error",{type:a.type,name:"buffer",error:"buffer error"})})),a}return i(r,[{key:"destroy",value:function(){this.queue=null,this.sourceBuffer=null,this.offAll()}},{key:"doCleanup",value:function(){if(this.cleanRanges.length){var e=this.cleanRanges.shift();k("".concat(this.type," remove range [").concat(e[0]," - ").concat(e[1],")")),this.cleaning=!0,this.sourceBuffer.remove(e[0],e[1])}else this.cleaning=!1}},{key:"initCleanup",value:function(e){try{if(this.sourceBuffer.updating)return void(this.pendingCleaning=e);if(this.sourceBuffer.buffered&&this.sourceBuffer.buffered.length&&!this.cleaning){for(var t=0;tthis.cleanOffset&&n<(r=e-this.cleanOffset)&&this.cleanRanges.push([n,r])}this.doCleanup()}}catch(e){g("Error occured while cleaning ".concat(this.type," buffer - ").concat(e.name,": ").concat(e.message))}}},{key:"doAppend",value:function(){if(this.queue.length&&this.sourceBuffer&&!this.sourceBuffer.updating)try{this.sourceBuffer.appendBuffer(this.queue),this.queue=new Uint8Array}catch(t){var e="unexpectedError";"QuotaExceededError"===t.name?(k("".concat(this.type," buffer quota full")),e="QuotaExceeded"):(g("Error occured while appending ".concat(this.type," buffer - ").concat(t.name,": ").concat(t.message)),e="InvalidStateError"),this.dispatch("error",{type:this.type,name:e,error:"buffer error"})}}},{key:"feed",value:function(e){this.queue=S(this.queue,e)}}]),r}(B);return function(r){s(o,r);var a=l(o);function o(e){var r;n(this,o),(r=a.call(this,"jmuxer")).isReset=!1;return r.options=Object.assign({},{node:"",mode:"both",flushingTime:500,maxDelay:500,clearBuffer:!0,fps:30,readFpsFromTrack:!1,debug:!1,onReady:function(){},onError:function(){}},e),r.env="object"===("undefined"==typeof process?"undefined":t(process))&&"undefined"==typeof window?"node":"browser",r.options.debug&&(v=console.log,m=console.error),r.options.fps||(r.options.fps=30),r.frameDuration=1e3/r.options.fps|0,r.remuxController=new L(r.env),r.remuxController.addTrack(r.options.mode),r.initData(),r.remuxController.on("buffer",r.onBuffer.bind(c(r))),"browser"==r.env&&(r.remuxController.on("ready",r.createBuffer.bind(c(r))),r.initBrowser()),r}return i(o,[{key:"initData",value:function(){this.lastCleaningTime=Date.now(),this.kfPosition=[],this.kfCounter=0,this.pendingUnits={},this.remainingData=new Uint8Array,this.startInterval()}},{key:"initBrowser",value:function(){"string"==typeof this.options.node&&""==this.options.node&&g("no video element were found to render, provide a valid video element"),this.node="string"==typeof this.options.node?document.getElementById(this.options.node):this.options.node,this.mseReady=!1,this.setupMSE()}},{key:"createStream",value:function(){var t=this.feed.bind(this),n=this.destroy.bind(this);return this.stream=new e.Duplex({writableObjectMode:!0,read:function(e){},write:function(e,n,r){t(e),r()},final:function(e){n(),e()}}),this.stream}},{key:"setupMSE",value:function(){if(window.MediaSource=window.MediaSource||window.WebKitMediaSource,!window.MediaSource)throw"Oops! Browser does not support media source extension.";this.isMSESupported=!!window.MediaSource,this.mediaSource=new MediaSource,this.url=URL.createObjectURL(this.mediaSource),this.node.src=this.url,this.mseEnded=!1,this.mediaSource.addEventListener("sourceopen",this.onMSEOpen.bind(this)),this.mediaSource.addEventListener("sourceclose",this.onMSEClose.bind(this)),this.mediaSource.addEventListener("webkitsourceopen",this.onMSEOpen.bind(this)),this.mediaSource.addEventListener("webkitsourceclose",this.onMSEClose.bind(this))}},{key:"endMSE",value:function(){if(!this.mseEnded)try{this.mseEnded=!0,this.mediaSource.endOfStream()}catch(e){g("mediasource is not available to end")}}},{key:"feed",value:function(e){var t,n,r,i=!1,a={video:[],audio:[]};if(e&&this.remuxController){if(r=e.duration?parseInt(e.duration):0,e.video){e.video=S(this.remainingData,e.video);var s=h(A.extractNALu(e.video),2);if(t=s[0],n=s[1],this.remainingData=n||new Uint8Array,!(t.length>0))return void g("Failed to extract any NAL units from video data:",n);a.video=this.getVideoFrames(t,r),i=!0}if(e.audio){if(!((t=U.extractAAC(e.audio)).length>0))return void g("Failed to extract audio data from:",e.audio);a.audio=this.getAudioFrames(t,r),i=!0}i?this.remuxController.remux(a):g("Input object must have video and/or audio property. Make sure it is a valid typed array")}}},{key:"getVideoFrames",value:function(e,t){var n,r=this,i=[],a=[],s=0,o=!1,u=!1;this.pendingUnits.units&&(i=this.pendingUnits.units,u=this.pendingUnits.vcl,o=this.pendingUnits.keyFrame,this.pendingUnits={});var c,f=p(e);try{for(f.s();!(c=f.n()).done;){var l=c.value,h=new b(l);h.type()!==b.IDR&&h.type()!==b.NDR||A.parseHeader(h),i.length&&u&&(h.isfmb||!h.isvcl)&&(a.push({units:i,keyFrame:o}),i=[],o=!1,u=!1),i.push(h),o=o||h.isKeyframe(),u=u||h.isvcl}}catch(e){f.e(e)}finally{f.f()}if(i.length)if(t)if(u)a.push({units:i,keyFrame:o});else{var d=a.length-1;d>=0&&(a[d].units=a[d].units.concat(i))}else this.pendingUnits={units:i,keyFrame:o,vcl:u};return n=t?t/a.length|0:this.frameDuration,s=t?t-n*a.length:0,a.map((function(e){e.duration=n,s>0&&(e.duration++,s--),r.kfCounter++,e.keyFrame&&r.options.clearBuffer&&r.kfPosition.push(r.kfCounter*n/1e3)})),k("jmuxer: No. of frames of the last chunk: ".concat(a.length)),a}},{key:"getAudioFrames",value:function(e,t){var n,r,i=[],a=0,s=p(e);try{for(s.s();!(r=s.n()).done;){var o=r.value;i.push({units:o})}}catch(e){s.e(e)}finally{s.f()}return n=t?t/i.length|0:this.frameDuration,a=t?t-n*i.length:0,i.map((function(e){e.duration=n,a>0&&(e.duration++,a--)})),i}},{key:"destroy",value:function(){if(this.stopInterval(),this.stream&&(this.remuxController.flush(),this.stream.push(null),this.stream=null),this.remuxController&&(this.remuxController.destroy(),this.remuxController=null),this.bufferControllers){for(var e in this.bufferControllers)this.bufferControllers[e].destroy();this.bufferControllers=null,this.endMSE()}this.node=!1,this.mseReady=!1,this.videoStarted=!1,this.mediaSource=null}},{key:"reset",value:function(){if(this.stopInterval(),this.isReset=!0,this.node.pause(),this.remuxController&&this.remuxController.reset(),this.bufferControllers){for(var e in this.bufferControllers)this.bufferControllers[e].destroy();this.bufferControllers=null,this.endMSE()}this.initData(),"browser"==this.env&&this.initBrowser(),k("JMuxer was reset")}},{key:"createBuffer",value:function(){if(this.mseReady&&this.remuxController&&this.remuxController.isReady()&&!this.bufferControllers)for(var e in this.bufferControllers={},this.remuxController.tracks){var t=this.remuxController.tracks[e];if(!o.isSupported("".concat(e,'/mp4; codecs="').concat(t.mp4track.codec,'"')))return g("Browser does not support codec"),!1;var n=this.mediaSource.addSourceBuffer("".concat(e,'/mp4; codecs="').concat(t.mp4track.codec,'"'));this.bufferControllers[e]=new R(n,e),this.bufferControllers[e].on("error",this.onBufferError.bind(this))}}},{key:"startInterval",value:function(){var e=this;this.interval=setInterval((function(){e.options.flushingTime?e.applyAndClearBuffer():e.bufferControllers&&e.cancelDelay()}),this.options.flushingTime||1e3)}},{key:"stopInterval",value:function(){this.interval&&clearInterval(this.interval)}},{key:"cancelDelay",value:function(){if(this.node.buffered&&this.node.buffered.length>0&&!this.node.seeking){var e=this.node.buffered.end(0);e-this.node.currentTime>this.options.maxDelay/1e3&&(console.log("delay"),this.node.currentTime=e-.001)}}},{key:"releaseBuffer",value:function(){for(var e in this.bufferControllers)this.bufferControllers[e].doAppend()}},{key:"applyAndClearBuffer",value:function(){this.bufferControllers&&(this.releaseBuffer(),this.clearBuffer())}},{key:"getSafeClearOffsetOfBuffer",value:function(e){for(var t,n="audio"===this.options.mode&&e||0,r=0;r=e);r++)t=this.kfPosition[r];return t&&(this.kfPosition=this.kfPosition.filter((function(e){return e=t}))),n}},{key:"clearBuffer",value:function(){if(this.options.clearBuffer&&Date.now()-this.lastCleaningTime>1e4){for(var e in this.bufferControllers){var t=this.getSafeClearOffsetOfBuffer(this.node.currentTime);this.bufferControllers[e].initCleanup(t)}this.lastCleaningTime=Date.now()}}},{key:"onBuffer",value:function(e){this.options.readFpsFromTrack&&void 0!==e.fps&&this.options.fps!=e.fps&&(this.options.fps=e.fps,this.frameDuration=Math.ceil(1e3/e.fps),k("JMuxer changed FPS to ".concat(e.fps," from track data"))),"browser"==this.env?this.bufferControllers&&this.bufferControllers[e.type]&&this.bufferControllers[e.type].feed(e.payload):this.stream&&this.stream.push(e.payload),0===this.options.flushingTime&&this.applyAndClearBuffer()}},{key:"onMSEOpen",value:function(){this.mseReady=!0,URL.revokeObjectURL(this.url),"function"==typeof this.options.onReady&&this.options.onReady.call(null,this.isReset)}},{key:"onMSEClose",value:function(){this.mseReady=!1,this.videoStarted=!1}},{key:"onBufferError",value:function(e){if("QuotaExceeded"==e.name)return k("JMuxer cleaning ".concat(e.type," buffer due to QuotaExceeded error")),void this.bufferControllers[e.type].initCleanup(this.node.currentTime);"InvalidStateError"==e.name?(k("JMuxer is reseting due to InvalidStateError"),this.reset()):this.endMSE(),"function"==typeof this.options.onError&&this.options.onError.call(null,e)}}],[{key:"isSupported",value:function(e){return window.MediaSource&&window.MediaSource.isTypeSupported(e)}}]),o}(B)})); diff --git a/html/video.html b/html/video.html index 8f48534..229e161 100644 --- a/html/video.html +++ b/html/video.html @@ -1,83 +1,142 @@ - - - - - - + #stream { + max-height: 100%; + max-width: 100%; + margin: auto; + position: absolute; + top: 0; left: 0; bottom: 0; right: 0; + } + -
      - -
      - - + const urlSearchParams = new URLSearchParams(window.location.search); + const params = Object.fromEntries(urlSearchParams.entries()); + + fetch('/video', { + body: JSON.stringify({ + type: 'request', + res: params.res + }), + headers: { + 'Content-Type': 'application/json' + }, + method: 'POST' + }).then(function(response) { + return response.json(); + }).then(function(answer) { + pc.remote_pc_id = answer.id; + return pc.setRemoteDescription(answer); + }).then(function() { + return pc.createAnswer(); + }).then(function(answer) { + return pc.setLocalDescription(answer); + }).then(function() { + // wait for ICE gathering to complete + return new Promise(function(resolve) { + if (pc.iceGatheringState === 'complete') { + resolve(); + } else { + function checkState() { + if (pc.iceGatheringState === 'complete') { + pc.removeEventListener('icegatheringstatechange', checkState); + resolve(); + } + } + pc.addEventListener('icegatheringstatechange', checkState); + } + }); + }).then(function(answer) { + var offer = pc.localDescription; + + return fetch('/video', { + body: JSON.stringify({ + type: offer.type, + id: pc.remote_pc_id, + sdp: offer.sdp, + }), + headers: { + 'Content-Type': 'application/json' + }, + method: 'POST' + }) + }).then(function(response) { + return response.json(); + }).catch(function(e) { + alert(e); + }); + } + + function stopWebRTC() { + setTimeout(function() { + pc.close(); + }, 500); + } + + + - \ No newline at end of file + diff --git a/output/http_h264.c b/output/http_h264.c index 529e47d..e9bb57e 100644 --- a/output/http_h264.c +++ b/output/http_h264.c @@ -35,11 +35,17 @@ bool h264_is_key_frame(buffer_t *buf) { unsigned char *data = buf->start; + static const int N = 8; + char buffer [3*N+1]; + buffer[sizeof(buffer)-1] = 0; + for(int j = 0; j < N; j++) + sprintf(&buffer[sizeof(buffer)/N*j], "%02X ", data[j]); + if (buf->flags.is_keyframe) { - LOG_DEBUG(buf, "Got key frame (from V4L2)!"); + LOG_DEBUG(buf, "Got key frame (from V4L2)!: %s", buffer); return true; } else if (buf->used >= 5 && (data[4] & 0x1F) == 0x07) { - LOG_DEBUG(buf, "Got key frame (from buffer)!"); + LOG_DEBUG(buf, "Got key frame (from buffer)!: %s", buffer); return true; } diff --git a/output/webrtc/webrtc.cc b/output/webrtc/webrtc.cc new file mode 100644 index 0000000..60c3fdd --- /dev/null +++ b/output/webrtc/webrtc.cc @@ -0,0 +1,426 @@ +extern "C" { +#include "webrtc.h" +#include "device/buffer.h" +#include "device/buffer_list.h" +#include "device/buffer_lock.h" +#include "device/device.h" +#include "output/output.h" +}; + +#ifdef USE_LIBDATACHANNEL + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std::chrono_literals; + +class Client; + +static std::set > webrtc_clients; +static std::mutex webrtc_clients_lock; +static const auto webrtc_client_lock_timeout = 3 * 1000ms; +static const auto webrtc_client_max_json_body = 10 * 1024; +static const auto webrtc_client_video_payload_type = 102; // H264 +static const rtc::Configuration webrtc_configuration = { + .iceServers = { rtc::IceServer("stun:stun.l.google.com:19302") }, + .disableAutoNegotiation = true +}; + +struct ClientTrackData +{ + std::shared_ptr track; + std::shared_ptr sender; + + void startStreaming() + { + double currentTime_s = get_monotonic_time_us(NULL, NULL)/(1000.0*1000.0); + sender->rtpConfig->setStartTime(currentTime_s, rtc::RtpPacketizationConfig::EpochStart::T1970); + sender->startRecording(); + } + + void sendTime() + { + double currentTime_s = get_monotonic_time_us(NULL, NULL)/(1000.0*1000.0); + + auto rtpConfig = sender->rtpConfig; + uint32_t elapsedTimestamp = rtpConfig->secondsToTimestamp(currentTime_s); + + sender->rtpConfig->timestamp = sender->rtpConfig->startTimestamp + elapsedTimestamp; + auto reportElapsedTimestamp = sender->rtpConfig->timestamp - sender->previousReportedTimestamp; + if (sender->rtpConfig->timestampToSeconds(reportElapsedTimestamp) > 1) { + sender->setNeedsToReport(); + } + } + + bool wantsFrame() const + { + if (!track) + return false; + + return track->isOpen(); + } +}; + +class Client +{ +public: + Client(std::shared_ptr pc_) + : pc(pc_), use_low_res(false) + { + id.resize(20); + for (auto & c : id) { + c = 'a' + (rand() % 26); + } + id = "rtc-" + id; + name = strdup(id.c_str()); + } + + ~Client() + { + free(name); + } + + bool wantsFrame() const + { + if (!pc || !video) + return false; + if (pc->state() != rtc::PeerConnection::State::Connected) + return false; + return video->wantsFrame(); + } + + void pushFrame(buffer_t *buf, bool low_res) + { + auto self = this; + + if (!video || !video->track) { + return; + } + + if (use_low_res != low_res) { + return; + } + + if (!had_key_frame) { + if (!h264_is_key_frame(buf)) { + device_video_force_key(buf->buf_list->dev); + LOG_VERBOSE(self, "Skipping as key frame was not yet sent."); + return; + } + had_key_frame = true; + } + + rtc::binary data((std::byte*)buf->start, (std::byte*)buf->start + buf->used); + video->sendTime(); + video->track->send(data); + } + +public: + char *name; + std::string id; + std::shared_ptr pc; + std::shared_ptr video; + std::mutex lock; + std::condition_variable wait_for_complete; + bool had_key_frame; + bool use_low_res; +}; + +std::shared_ptr findClient(std::string id) +{ + std::unique_lock lk(webrtc_clients_lock); + for (auto client : webrtc_clients) { + if (client && client->id == id) { + return client; + } + } + + return std::shared_ptr(); +} + +void removeClient(const std::shared_ptr &client, const char *reason) +{ + std::unique_lock lk(webrtc_clients_lock); + webrtc_clients.erase(client); + LOG_INFO(client.get(), "Client removed: %s.", reason); +} + +std::shared_ptr addVideo(const std::shared_ptr pc, const uint8_t payloadType, const uint32_t ssrc, const std::string cname, const std::string msid) +{ + auto video = rtc::Description::Video(cname, rtc::Description::Direction::SendOnly); + video.addH264Codec(payloadType); + video.setBitrate(1000); + video.addSSRC(ssrc, cname, msid, cname); + auto track = pc->addTrack(video); + auto rtpConfig = std::make_shared(ssrc, cname, payloadType, rtc::H264RtpPacketizer::defaultClockRate); + auto packetizer = std::make_shared(rtc::H264RtpPacketizer::Separator::LongStartSequence, rtpConfig); + auto h264Handler = std::make_shared(packetizer); + auto srReporter = std::make_shared(rtpConfig); + h264Handler->addToChain(srReporter); + auto nackResponder = std::make_shared(); + h264Handler->addToChain(nackResponder); + track->setMediaHandler(h264Handler); + return std::shared_ptr(new ClientTrackData{track, srReporter}); +} + +std::shared_ptr createPeerConnection(const rtc::Configuration &config) +{ + auto pc = std::make_shared(config); + auto client = std::make_shared(pc); + auto wclient = std::weak_ptr(client); + + pc->onTrack([wclient](std::shared_ptr track) { + if(auto client = wclient.lock()) { + LOG_DEBUG(client.get(), "onTrack: %s", track->mid().c_str()); + } + }); + + pc->onLocalDescription([wclient](rtc::Description description) { + if(auto client = wclient.lock()) { + LOG_DEBUG(client.get(), "onLocalDescription: %s", description.typeString().c_str()); + } + }); + + pc->onSignalingStateChange([wclient](rtc::PeerConnection::SignalingState state) { + if(auto client = wclient.lock()) { + LOG_DEBUG(client.get(), "onSignalingStateChange: %d", (int)state); + } + }); + + pc->onStateChange([wclient](rtc::PeerConnection::State state) { + if(auto client = wclient.lock()) { + LOG_DEBUG(client.get(), "onStateChange: %d", (int)state); + + if (state == rtc::PeerConnection::State::Disconnected || + state == rtc::PeerConnection::State::Failed || + state == rtc::PeerConnection::State::Closed) + { + removeClient(client, "stream closed"); + } + } + }); + + pc->onGatheringStateChange([wclient](rtc::PeerConnection::GatheringState state) { + if(auto client = wclient.lock()) { + LOG_DEBUG(client.get(), "onGatheringStateChange: %d", (int)state); + + if (state == rtc::PeerConnection::GatheringState::Complete) { + client->wait_for_complete.notify_all(); + } + } + }); + + std::unique_lock lk(webrtc_clients_lock); + webrtc_clients.insert(client); + return client; +} + +static bool webrtc_h264_needs_buffer(buffer_lock_t *buf_lock) +{ + std::unique_lock lk(webrtc_clients_lock); + for (auto client : webrtc_clients) { + if (client->wantsFrame()) + return true; + } + + return false; +} + +static void webrtc_h264_capture(buffer_lock_t *buf_lock, buffer_t *buf) +{ + std::unique_lock lk(webrtc_clients_lock); + for (auto client : webrtc_clients) { + if (client->wantsFrame()) { + client->pushFrame(buf, false); + + if (!http_h264_lowres.buf_list) { + client->pushFrame(buf, true); + } + } + } +} + +static void webrtc_h264_low_res_capture(buffer_lock_t *buf_lock, buffer_t *buf) +{ + std::unique_lock lk(webrtc_clients_lock); + for (auto client : webrtc_clients) { + if (client->wantsFrame()) { + client->pushFrame(buf, true); + } + } +} + +static void http_webrtc_request(http_worker_t *worker, FILE *stream, const nlohmann::json &message) +{ + auto client = createPeerConnection(webrtc_configuration); + LOG_INFO(client.get(), "Stream requested."); + + client->video = addVideo(client->pc, webrtc_client_video_payload_type, rand(), "video", ""); + if (message.contains("res")) { + client->use_low_res = (message["res"] == "low"); + } + + try { + { + std::unique_lock lock(client->lock); + client->pc->setLocalDescription(); + client->wait_for_complete.wait_for(lock, webrtc_client_lock_timeout); + } + + if (client->pc->gatheringState() == rtc::PeerConnection::GatheringState::Complete) { + auto description = client->pc->localDescription(); + nlohmann::json message; + message["id"] = client->id; + message["type"] = description->typeString(); + message["sdp"] = std::string(description.value()); + http_write_response(stream, "200 OK", "application/json", message.dump().c_str(), 0); + LOG_VERBOSE(client.get(), "Local SDP Offer: %s", std::string(message["sdp"]).c_str()); + } else { + http_500(stream, "Not complete"); + } + } catch(const std::exception &e) { + http_500(stream, e.what()); + removeClient(client, e.what()); + } +} + +static void http_webrtc_answer(http_worker_t *worker, FILE *stream, const nlohmann::json &message) +{ + if (!message.contains("id") || !message.contains("sdp")) { + http_400(stream, "no sdp or id"); + return; + } + + if (auto client = findClient(message["id"])) { + LOG_INFO(client.get(), "Answer received."); + LOG_VERBOSE(client.get(), "Remote SDP Answer: %s", std::string(message["sdp"]).c_str()); + + try { + auto answer = rtc::Description(std::string(message["sdp"]), std::string(message["type"])); + client->pc->setRemoteDescription(answer); + client->video->startStreaming(); + http_write_response(stream, "200 OK", "application/json", "{}", 0); + } catch(const std::exception &e) { + http_500(stream, e.what()); + removeClient(client, e.what()); + } + } else { + http_404(stream, "No client found"); + } +} + +static void http_webrtc_offer(http_worker_t *worker, FILE *stream, const nlohmann::json &message) +{ + if (!message.contains("sdp")) { + http_400(stream, "no sdp"); + return; + } + + auto offer = rtc::Description(std::string(message["sdp"]), std::string(message["type"])); + auto client = createPeerConnection(webrtc_configuration); + + LOG_INFO(client.get(), "Offer received."); + LOG_VERBOSE(client.get(), "Remote SDP Offer: %s", std::string(message["sdp"]).c_str()); + + try { + client->video = addVideo(client->pc, webrtc_client_video_payload_type, rand(), "video", ""); + client->video->startStreaming(); + + { + std::unique_lock lock(client->lock); + client->pc->setRemoteDescription(offer); + client->pc->setLocalDescription(); + client->wait_for_complete.wait_for(lock, webrtc_client_lock_timeout); + } + + if (client->pc->gatheringState() == rtc::PeerConnection::GatheringState::Complete) { + auto description = client->pc->localDescription(); + nlohmann::json message; + message["type"] = description->typeString(); + message["sdp"] = std::string(description.value()); + http_write_response(stream, "200 OK", "application/json", message.dump().c_str(), 0); + + LOG_VERBOSE(client.get(), "Local SDP Answer: %s", std::string(message["sdp"]).c_str()); + } else { + http_500(stream, "Not complete"); + } + } catch(const std::exception &e) { + http_500(stream, e.what()); + removeClient(client, e.what()); + } +} + +nlohmann::json http_parse_json_body(http_worker_t *worker, FILE *stream) +{ + std::string text; + + size_t i = 0; + size_t n = (size_t)worker->content_length; + if (n < 0 || n > webrtc_client_max_json_body) + n = webrtc_client_max_json_body; + + text.resize(n); + + while (i < n && !feof(stream)) { + i += fread(&text[i], 1, n-i, stream); + } + text.resize(i); + + return nlohmann::json::parse(text); +} + +extern "C" void http_webrtc_offer(http_worker_t *worker, FILE *stream) +{ + auto message = http_parse_json_body(worker, stream); + + if (!message.contains("type")) { + http_400(stream, "missing 'type'"); + return; + } + + std::string type = message["type"]; + + LOG_DEBUG(worker, "Recevied: '%s'", type.c_str()); + + if (type == "request") { + http_webrtc_request(worker, stream, message); + } else if (type == "answer") { + http_webrtc_answer(worker, stream, message); + } else if (type == "offer") { + http_webrtc_offer(worker, stream, message); + } else { + http_400(stream, (std::string("Not expected: " + type)).c_str()); + } +} + +extern "C" void webrtc_server() +{ + buffer_lock_register_check_streaming(&http_h264, webrtc_h264_needs_buffer); + buffer_lock_register_notify_buffer(&http_h264, webrtc_h264_capture); + buffer_lock_register_check_streaming(&http_h264_lowres, webrtc_h264_needs_buffer); + buffer_lock_register_notify_buffer(&http_h264_lowres, webrtc_h264_low_res_capture); +} + +#else // USE_LIBDATACHANNEL + +extern "C" void http_webrtc_offer(http_worker_t *worker, FILE *stream) +{ + http_404(stream, NULL); +} + +extern "C" void webrtc_server() +{ +} + +#endif // USE_LIBDATACHANNEL diff --git a/output/webrtc/webrtc.h b/output/webrtc/webrtc.h new file mode 100644 index 0000000..dd5ca86 --- /dev/null +++ b/output/webrtc/webrtc.h @@ -0,0 +1,11 @@ +#pragma once + +#include "util/http/http.h" +#include "util/opts/log.h" +#include "util/opts/fourcc.h" +#include "util/opts/control.h" +#include "device/buffer.h" + +// WebRTC +void http_webrtc_offer(http_worker_t *worker, FILE *stream); +void webrtc_server(); diff --git a/third_party/libdatachannel b/third_party/libdatachannel new file mode 160000 index 0000000..04cf473 --- /dev/null +++ b/third_party/libdatachannel @@ -0,0 +1 @@ +Subproject commit 04cf4738961f55ba3f0aa39b4a61342f66bb3781 diff --git a/util/http/http.c b/util/http/http.c index d495edd..7eb3bb6 100644 --- a/util/http/http.c +++ b/util/http/http.c @@ -14,6 +14,9 @@ #include "http.h" #include "util/opts/log.h" +#define HEADER_RANGE "Range:" +#define HEADER_CONTENT_LENGTH "Content-Length:" + static int http_listen(int port, int maxcons) { struct sockaddr_in server = {0}; @@ -92,6 +95,7 @@ static void http_process(http_worker_t *worker, FILE *stream) } worker->range_header[0] = 0; + worker->content_length = -1; // Consume headers for(int i = 0; i < 50; i++) { @@ -101,9 +105,12 @@ static void http_process(http_worker_t *worker, FILE *stream) if (line[0] == '\r' && line[1] == '\n') break; - if (strcasestr(line, "Range:") == line) { + if (strcasestr(line, HEADER_RANGE) == line) { strcpy(worker->range_header, line); } + if (strcasestr(line, HEADER_CONTENT_LENGTH) == line) { + worker->content_length = atoi(line + strlen(HEADER_CONTENT_LENGTH)); + } } worker->current_method = NULL; diff --git a/util/http/http.h b/util/http/http.h index 83b5ff6..ffc697d 100644 --- a/util/http/http.h +++ b/util/http/http.h @@ -31,6 +31,7 @@ typedef struct http_worker_s { pthread_t thread; int client_fd; + int content_length; struct sockaddr_in client_addr; char *client_host; char client_method[BUFSIZE]; @@ -46,7 +47,9 @@ typedef struct http_server_options_s { int http_server(http_server_options_t *options, http_method_t *methods); void http_content(http_worker_t *worker, FILE *stream); +void http_write_response(FILE *stream, const char *status, const char *content_type, const char *body, unsigned content_length); void http_200(FILE *stream, const char *data); +void http_400(FILE *stream, const char *data); void http_404(FILE *stream, const char *data); void http_500(FILE *stream, const char *data); void *http_enum_params(http_worker_t *worker, FILE *stream, http_param_fn fn, void *opaque); diff --git a/util/http/http_methods.c b/util/http/http_methods.c index 0e2b292..8c1cb68 100644 --- a/util/http/http_methods.c +++ b/util/http/http_methods.c @@ -3,7 +3,7 @@ #include "http.h" -static void http_write_response( +void http_write_response( FILE *stream, const char *status, const char *content_type, @@ -46,6 +46,11 @@ void http_200(FILE *stream, const char *data) http_write_response(stream, "200 OK", NULL, data ? data : "Nothing here.\n", 0); } +void http_400(FILE *stream, const char *data) +{ + http_write_response(stream, "400 Bad Request", NULL, data ? data : "Nothing here.\n", 0); +} + void http_404(FILE *stream, const char *data) { http_write_response(stream, "404 Not Found", NULL, data ? data : "Nothing here.\n", 0);