mirror of
https://github.com/simplex-chat/simplex-chat.git
synced 2025-03-14 09:45:42 +00:00
webrtc calls in webview to typescript (#592)
* mobile: webrtc calls in webview (typescript WIP) * typsecript compiles * fix error messages * TS works in chrome * include ICE candidates into offer/answer, report connection state changes to host, end call on disconnection * refactor, readme for .js file
This commit is contained in:
Evgeny Poberezkin
GitHub
parent
18d1a0605e
commit
f78ec3584f
12 changed files with 1107 additions and 276 deletions
1
apps/android/app/src/main/assets/www/README.md
Normal file
1
apps/android/app/src/main/assets/www/README.md
Normal file
|
|
@ -0,0 +1 @@
|
|||
Do NOT edit call.js here, it is compiled from call.ts in packages/simplex-chat-webrtc
|
||||
|
|
@ -4,8 +4,8 @@
|
|||
<link href="./style.css" rel="stylesheet" />
|
||||
</head>
|
||||
<body>
|
||||
<video id="incoming-video-stream" autoplay></video>
|
||||
<video id="outgoing-video-stream" muted autoplay></video>
|
||||
<video id="remote-video-stream" autoplay></video>
|
||||
<video id="local-video-stream" muted autoplay></video>
|
||||
</body>
|
||||
<footer>
|
||||
<script src="./call.js"></script>
|
||||
|
|
|
|||
|
|
@ -1,299 +1,455 @@
|
|||
"use strict";
|
||||
// Inspired by
|
||||
// https://github.com/webrtc/samples/blob/gh-pages/src/content/insertable-streams/endtoend-encryption
|
||||
|
||||
let incomingVideo = document.getElementById("incoming-video-stream")
|
||||
let outgoingVideo = document.getElementById("outgoing-video-stream")
|
||||
incomingVideo.style.opacity = 0
|
||||
outgoingVideo.style.opacity = 0
|
||||
incomingVideo.onplaying = () => {
|
||||
incomingVideo.style.opacity = 1
|
||||
}
|
||||
outgoingVideo.onplaying = () => {
|
||||
outgoingVideo.style.opacity = 1
|
||||
}
|
||||
|
||||
var CallMediaType;
|
||||
(function (CallMediaType) {
|
||||
CallMediaType["Audio"] = "audio";
|
||||
CallMediaType["Video"] = "video";
|
||||
})(CallMediaType || (CallMediaType = {}));
|
||||
// STUN servers
|
||||
const peerConnectionConfig = {
|
||||
iceServers: [{urls: ["stun:stun.l.google.com:19302"]}],
|
||||
iceCandidatePoolSize: 10,
|
||||
encodedInsertableStreams: true,
|
||||
}
|
||||
let keyGenConfig = {
|
||||
name: "AES-GCM",
|
||||
length: 256,
|
||||
tagLength: 128,
|
||||
}
|
||||
let keyUsages = ["encrypt", "decrypt"]
|
||||
|
||||
// Hardcode a key for development
|
||||
let keyData = {alg: "A256GCM", ext: true, k: "JCMDWkhxLmPDhua0BUdhgv6Ac6hOtB9frSxJlnkTAK8", key_ops: keyUsages, kty: "oct"}
|
||||
|
||||
let pc
|
||||
let key
|
||||
let IV_LENGTH = 12
|
||||
iceServers: [{ urls: ["stun:stun.l.google.com:19302"] }],
|
||||
iceCandidatePoolSize: 10,
|
||||
encodedInsertableStreams: true,
|
||||
};
|
||||
const keyAlgorithm = {
|
||||
name: "AES-GCM",
|
||||
length: 256
|
||||
};
|
||||
const keyUsages = ["encrypt", "decrypt"];
|
||||
let pc;
|
||||
const IV_LENGTH = 12;
|
||||
const initialPlainTextRequired = {
|
||||
key: 10,
|
||||
delta: 3,
|
||||
undefined: 1,
|
||||
}
|
||||
|
||||
// let encryptKeyRepresentation
|
||||
let candidates = []
|
||||
run()
|
||||
|
||||
async function run() {
|
||||
pc = new RTCPeerConnection(peerConnectionConfig)
|
||||
|
||||
pc.onicecandidate = (event) => {
|
||||
// add candidate to maintained list to be sent all at once
|
||||
if (event.candidate) {
|
||||
candidates.push(event.candidate)
|
||||
key: 10,
|
||||
delta: 3,
|
||||
undefined: 1,
|
||||
};
|
||||
const defaultCallConfig = {
|
||||
iceCandidates: {
|
||||
delay: 2000,
|
||||
extrasInterval: 2000,
|
||||
extrasTimeout: 8000
|
||||
}
|
||||
}
|
||||
pc.onicegatheringstatechange = (_) => {
|
||||
if (pc.iceGatheringState == "complete") {
|
||||
// Give command for other caller to use
|
||||
console.log(JSON.stringify({action: "processIceCandidates", content: candidates}))
|
||||
};
|
||||
async function initializeCall(config, mediaType, aesKey) {
|
||||
const conn = new RTCPeerConnection(peerConnectionConfig);
|
||||
const remoteStream = new MediaStream();
|
||||
const localStream = await navigator.mediaDevices.getUserMedia(callMediaContraints(mediaType));
|
||||
await setUpMediaStreams(conn, localStream, remoteStream, aesKey);
|
||||
conn.addEventListener("connectionstatechange", connectionStateChange);
|
||||
const iceCandidates = new Promise((resolve, _) => {
|
||||
let candidates = [];
|
||||
let resolved = false;
|
||||
let extrasInterval;
|
||||
let extrasTimeout;
|
||||
const delay = setTimeout(() => {
|
||||
if (!resolved) {
|
||||
resolveIceCandidates();
|
||||
extrasInterval = setInterval(() => {
|
||||
sendIceCandidates();
|
||||
}, config.iceCandidates.extrasInterval);
|
||||
extrasTimeout = setTimeout(() => {
|
||||
clearInterval(extrasInterval);
|
||||
sendIceCandidates();
|
||||
}, config.iceCandidates.extrasTimeout);
|
||||
}
|
||||
}, config.iceCandidates.delay);
|
||||
conn.onicecandidate = ({ candidate: c }) => c && candidates.push(c);
|
||||
conn.onicegatheringstatechange = () => {
|
||||
if (conn.iceGatheringState == "complete") {
|
||||
if (resolved) {
|
||||
if (extrasInterval)
|
||||
clearInterval(extrasInterval);
|
||||
if (extrasTimeout)
|
||||
clearTimeout(extrasTimeout);
|
||||
sendIceCandidates();
|
||||
}
|
||||
else {
|
||||
resolveIceCandidates();
|
||||
}
|
||||
}
|
||||
};
|
||||
function resolveIceCandidates() {
|
||||
if (delay)
|
||||
clearTimeout(delay);
|
||||
resolved = true;
|
||||
const iceCandidates = candidates.slice();
|
||||
candidates = [];
|
||||
resolve(iceCandidates);
|
||||
}
|
||||
function sendIceCandidates() {
|
||||
if (candidates.length === 0)
|
||||
return;
|
||||
const iceCandidates = candidates.slice();
|
||||
candidates = [];
|
||||
sendMessageToNative({ type: "ice", iceCandidates });
|
||||
}
|
||||
});
|
||||
return { connection: conn, iceCandidates };
|
||||
function connectionStateChange() {
|
||||
sendMessageToNative({
|
||||
type: "connection",
|
||||
state: {
|
||||
connectionState: conn.connectionState,
|
||||
iceConnectionState: conn.iceConnectionState,
|
||||
iceGatheringState: conn.iceGatheringState,
|
||||
signalingState: conn.signalingState,
|
||||
}
|
||||
});
|
||||
if (conn.connectionState == "disconnected" || conn.connectionState == "failed") {
|
||||
conn.removeEventListener("connectionstatechange", connectionStateChange);
|
||||
sendMessageToNative({ type: "ended" });
|
||||
conn.close();
|
||||
pc = undefined;
|
||||
resetVideoElements();
|
||||
}
|
||||
}
|
||||
}
|
||||
let remoteStream = new MediaStream()
|
||||
key = await crypto.subtle.importKey("jwk", keyData, keyGenConfig, true, keyUsages)
|
||||
let localStream = await getLocalVideoStream()
|
||||
setUpVideos(pc, localStream, remoteStream)
|
||||
}
|
||||
|
||||
async function processCommand(data) {
|
||||
switch (data.action) {
|
||||
case "initiateCall":
|
||||
console.log("initiating call")
|
||||
let result = await makeOffer(pc)
|
||||
// Give command for callee to use
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
action: "processAndAnswerOffer",
|
||||
content: result,
|
||||
})
|
||||
)
|
||||
return result
|
||||
case "processAndAnswerOffer":
|
||||
await processOffer(data.content)
|
||||
let answer = await answerOffer(pc)
|
||||
// Give command for callee to use
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
action: "processOffer",
|
||||
content: answer,
|
||||
})
|
||||
)
|
||||
return answer
|
||||
case "processOffer":
|
||||
await processOffer(data.content)
|
||||
break
|
||||
case "processIceCandidates":
|
||||
processIceCandidates(data.content)
|
||||
break
|
||||
default:
|
||||
console.log("JS: Unknown Command")
|
||||
}
|
||||
// TODO remove WCallCommand from parameter type
|
||||
function sendMessageToNative(msg) {
|
||||
console.log(JSON.stringify(msg));
|
||||
}
|
||||
|
||||
async function makeOffer(pc) {
|
||||
// For initiating a call. Send offer to callee
|
||||
let offerDescription = await pc.createOffer()
|
||||
await pc.setLocalDescription(offerDescription)
|
||||
let offer = {
|
||||
sdp: offerDescription.sdp,
|
||||
type: offerDescription.type,
|
||||
}
|
||||
return offer
|
||||
// TODO remove WCallCommand from result type
|
||||
async function processCommand(command) {
|
||||
let resp;
|
||||
switch (command.type) {
|
||||
case "capabilities":
|
||||
const encryption = supportsInsertableStreams();
|
||||
resp = { type: "capabilities", capabilities: { encryption } };
|
||||
break;
|
||||
case "start":
|
||||
console.log("starting call");
|
||||
if (pc) {
|
||||
resp = { type: "error", message: "start: call already started" };
|
||||
}
|
||||
else if (!supportsInsertableStreams() && command.aesKey) {
|
||||
resp = { type: "error", message: "start: encryption is not supported" };
|
||||
}
|
||||
else {
|
||||
try {
|
||||
const { media, aesKey } = command;
|
||||
const call = await initializeCall(defaultCallConfig, media, aesKey);
|
||||
const { connection, iceCandidates } = call;
|
||||
pc = connection;
|
||||
const offer = await pc.createOffer();
|
||||
await pc.setLocalDescription(offer);
|
||||
// for debugging, returning the command for callee to use
|
||||
resp = { type: "accept", offer, iceCandidates: await iceCandidates, media, aesKey };
|
||||
// resp = {type: "offer", offer, iceCandidates: await iceCandidates}
|
||||
}
|
||||
catch (e) {
|
||||
resp = { type: "error", message: e.message };
|
||||
}
|
||||
}
|
||||
break;
|
||||
case "accept":
|
||||
if (pc) {
|
||||
resp = { type: "error", message: "accept: call already started" };
|
||||
}
|
||||
else if (!supportsInsertableStreams() && command.aesKey) {
|
||||
resp = { type: "error", message: "accept: encryption is not supported" };
|
||||
}
|
||||
else {
|
||||
try {
|
||||
const call = await initializeCall(defaultCallConfig, command.media, command.aesKey);
|
||||
const { connection, iceCandidates } = call;
|
||||
pc = connection;
|
||||
await pc.setRemoteDescription(new RTCSessionDescription(command.offer));
|
||||
const answer = await pc.createAnswer();
|
||||
await pc.setLocalDescription(answer);
|
||||
addIceCandidates(pc, command.iceCandidates);
|
||||
// same as command for caller to use
|
||||
resp = { type: "answer", answer, iceCandidates: await iceCandidates };
|
||||
}
|
||||
catch (e) {
|
||||
resp = { type: "error", message: e.message };
|
||||
}
|
||||
}
|
||||
break;
|
||||
case "answer":
|
||||
if (!pc) {
|
||||
resp = { type: "error", message: "answer: call not started" };
|
||||
}
|
||||
else if (!pc.localDescription) {
|
||||
resp = { type: "error", message: "answer: local description is not set" };
|
||||
}
|
||||
else if (pc.currentRemoteDescription) {
|
||||
resp = { type: "error", message: "answer: remote description already set" };
|
||||
}
|
||||
else {
|
||||
await pc.setRemoteDescription(new RTCSessionDescription(command.answer));
|
||||
addIceCandidates(pc, command.iceCandidates);
|
||||
resp = { type: "ok" };
|
||||
}
|
||||
break;
|
||||
case "ice":
|
||||
if (pc) {
|
||||
addIceCandidates(pc, command.iceCandidates);
|
||||
resp = { type: "ok" };
|
||||
}
|
||||
else {
|
||||
resp = { type: "error", message: "ice: call not started" };
|
||||
}
|
||||
break;
|
||||
case "end":
|
||||
if (pc) {
|
||||
pc.close();
|
||||
pc = undefined;
|
||||
resetVideoElements();
|
||||
resp = { type: "ok" };
|
||||
}
|
||||
else {
|
||||
resp = { type: "error", message: "end: call not started" };
|
||||
}
|
||||
break;
|
||||
default:
|
||||
resp = { type: "error", message: "unknown command" };
|
||||
break;
|
||||
}
|
||||
sendMessageToNative(resp);
|
||||
return resp;
|
||||
}
|
||||
|
||||
async function answerOffer(pc) {
|
||||
let answerDescription = await pc.createAnswer()
|
||||
await pc.setLocalDescription(answerDescription)
|
||||
let answer = {
|
||||
sdp: answerDescription.sdp,
|
||||
type: answerDescription.type,
|
||||
}
|
||||
return answer
|
||||
function addIceCandidates(conn, iceCandidates) {
|
||||
for (const c of iceCandidates) {
|
||||
conn.addIceCandidate(new RTCIceCandidate(c));
|
||||
}
|
||||
}
|
||||
|
||||
function processIceCandidates(iceCandidates) {
|
||||
iceCandidates.forEach((candidate) => processIceCandidate(candidate))
|
||||
async function setUpMediaStreams(pc, localStream, remoteStream, aesKey) {
|
||||
var _a;
|
||||
const videos = getVideoElements();
|
||||
if (!videos)
|
||||
throw Error("no video elements");
|
||||
let key;
|
||||
if (aesKey) {
|
||||
const keyData = decodeBase64(encodeAscii(aesKey));
|
||||
if (keyData)
|
||||
key = await crypto.subtle.importKey("raw", keyData, keyAlgorithm, false, keyUsages);
|
||||
}
|
||||
for (const track of localStream.getTracks()) {
|
||||
pc.addTrack(track, localStream);
|
||||
}
|
||||
if (key) {
|
||||
console.log("set up encryption for sending");
|
||||
for (const sender of pc.getSenders()) {
|
||||
setupPeerTransform(sender, encodeFunction(key));
|
||||
}
|
||||
}
|
||||
// Pull tracks from remote stream as they arrive add them to remoteStream video
|
||||
pc.ontrack = (event) => {
|
||||
if (key) {
|
||||
console.log("set up decryption for receiving");
|
||||
setupPeerTransform(event.receiver, decodeFunction(key));
|
||||
}
|
||||
for (const track of event.streams[0].getTracks()) {
|
||||
remoteStream.addTrack(track);
|
||||
}
|
||||
};
|
||||
// We assume VP8 encoding in the decode/encode stages to get the initial
|
||||
// bytes to pass as plaintext so we enforce that here.
|
||||
// VP8 is supported by all supports of webrtc.
|
||||
// Use of VP8 by default may also reduce depacketisation issues.
|
||||
// We do not encrypt the first couple of bytes of the payload so that the
|
||||
// video elements can work by determining video keyframes and the opus mode
|
||||
// being used. This appears to be necessary for any video feed at all.
|
||||
// For VP8 this is the content described in
|
||||
// https://tools.ietf.org/html/rfc6386#section-9.1
|
||||
// which is 10 bytes for key frames and 3 bytes for delta frames.
|
||||
// For opus (where encodedFrame.type is not set) this is the TOC byte from
|
||||
// https://tools.ietf.org/html/rfc6716#section-3.1
|
||||
const capabilities = RTCRtpSender.getCapabilities("video");
|
||||
if (capabilities) {
|
||||
const { codecs } = capabilities;
|
||||
const selectedCodecIndex = codecs.findIndex((c) => c.mimeType === "video/VP8");
|
||||
const selectedCodec = codecs[selectedCodecIndex];
|
||||
codecs.splice(selectedCodecIndex, 1);
|
||||
codecs.unshift(selectedCodec);
|
||||
for (const t of pc.getTransceivers()) {
|
||||
if (((_a = t.sender.track) === null || _a === void 0 ? void 0 : _a.kind) === "video") {
|
||||
t.setCodecPreferences(codecs);
|
||||
}
|
||||
}
|
||||
}
|
||||
// setupVideoElement(videos.local)
|
||||
// setupVideoElement(videos.remote)
|
||||
videos.local.srcObject = localStream;
|
||||
videos.remote.srcObject = remoteStream;
|
||||
}
|
||||
|
||||
function processIceCandidate(iceCandidate) {
|
||||
let candidate = new RTCIceCandidate(iceCandidate)
|
||||
pc.addIceCandidate(candidate)
|
||||
function callMediaContraints(mediaType) {
|
||||
switch (mediaType) {
|
||||
case CallMediaType.Audio:
|
||||
return { audio: true, video: false };
|
||||
case CallMediaType.Video:
|
||||
return {
|
||||
audio: true,
|
||||
video: {
|
||||
frameRate: 24,
|
||||
width: {
|
||||
min: 480,
|
||||
ideal: 720,
|
||||
max: 1280,
|
||||
},
|
||||
aspectRatio: 1.33,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function processOffer(offer) {
|
||||
// Negotiating initial connection
|
||||
if (!pc.currentRemoteDescription) {
|
||||
let remoteSessionDescription = new RTCSessionDescription(offer)
|
||||
await pc.setRemoteDescription(remoteSessionDescription)
|
||||
}
|
||||
function supportsInsertableStreams() {
|
||||
return ("createEncodedStreams" in RTCRtpSender.prototype)
|
||||
&& ("createEncodedStreams" in RTCRtpReceiver.prototype);
|
||||
}
|
||||
|
||||
function setUpVideos(pc, localStream, remoteStream) {
|
||||
localStream.getTracks().forEach((track) => {
|
||||
pc.addTrack(track, localStream)
|
||||
})
|
||||
pc.getSenders().forEach(setupSenderTransform)
|
||||
// Pull tracks from remote stream as they arrive add them to remoteStream video
|
||||
pc.ontrack = (event) => {
|
||||
setupReceiverTransform(event.receiver)
|
||||
event.streams[0].getTracks().forEach((track) => {
|
||||
remoteStream.addTrack(track)
|
||||
})
|
||||
}
|
||||
// We assume VP8 encoding in the decode/encode stages to get the initial
|
||||
// bytes to pass as plaintext so we enforce that here.
|
||||
// VP8 is supported by all supports of webrtc.
|
||||
// Use of VP8 by default may also reduce depacketisation issues.
|
||||
// We do not encrypt the first couple of bytes of the payload so that the
|
||||
// video elements can work by determining video keyframes and the opus mode
|
||||
// being used. This appears to be necessary for any video feed at all.
|
||||
// For VP8 this is the content described in
|
||||
// https://tools.ietf.org/html/rfc6386#section-9.1
|
||||
// which is 10 bytes for key frames and 3 bytes for delta frames.
|
||||
// For opus (where encodedFrame.type is not set) this is the TOC byte from
|
||||
// https://tools.ietf.org/html/rfc6716#section-3.1
|
||||
|
||||
const {codecs} = RTCRtpSender.getCapabilities("video")
|
||||
const selectedCodecIndex = codecs.findIndex((c) => c.mimeType === "video/VP8")
|
||||
const selectedCodec = codecs[selectedCodecIndex]
|
||||
codecs.splice(selectedCodecIndex, 1)
|
||||
codecs.unshift(selectedCodec)
|
||||
const transceiver = pc.getTransceivers().find((t) => t.sender && t.sender.track.kind === "video")
|
||||
transceiver.setCodecPreferences(codecs)
|
||||
|
||||
outgoingVideo.srcObject = localStream
|
||||
incomingVideo.srcObject = remoteStream
|
||||
function resetVideoElements() {
|
||||
const videos = getVideoElements();
|
||||
if (!videos)
|
||||
return;
|
||||
videos.local.srcObject = null;
|
||||
videos.remote.srcObject = null;
|
||||
}
|
||||
|
||||
async function getLocalVideoStream() {
|
||||
return await navigator.mediaDevices.getUserMedia({
|
||||
audio: true,
|
||||
video: {
|
||||
frameRate: 24,
|
||||
width: {
|
||||
min: 480,
|
||||
ideal: 720,
|
||||
max: 1280,
|
||||
},
|
||||
aspectRatio: 1.33,
|
||||
},
|
||||
})
|
||||
function getVideoElements() {
|
||||
const local = document.getElementById("local-video-stream");
|
||||
const remote = document.getElementById("remote-video-stream");
|
||||
if (!(local && remote && local instanceof HTMLMediaElement && remote instanceof HTMLMediaElement))
|
||||
return;
|
||||
return { local, remote };
|
||||
}
|
||||
|
||||
function endCall() {
|
||||
pc.close()
|
||||
}
|
||||
|
||||
function toggleVideo(b) {
|
||||
if (b == "true") {
|
||||
localStream.getVideoTracks()[0].enabled = true
|
||||
} else {
|
||||
localStream.getVideoTracks()[0].enabled = false
|
||||
}
|
||||
}
|
||||
|
||||
// function setupVideoElement(video: HTMLElement) {
|
||||
// // TODO use display: none
|
||||
// video.style.opacity = "0"
|
||||
// video.onplaying = () => {
|
||||
// video.style.opacity = "1"
|
||||
// }
|
||||
// }
|
||||
// what does it do?
|
||||
// function toggleVideo(b) {
|
||||
// if (b == "true") {
|
||||
// localStream.getVideoTracks()[0].enabled = true
|
||||
// } else {
|
||||
// localStream.getVideoTracks()[0].enabled = false
|
||||
// }
|
||||
// }
|
||||
function f() {
|
||||
console.log("Debug Function")
|
||||
return "Debugging Return"
|
||||
console.log("Debug Function");
|
||||
return "Debugging Return";
|
||||
}
|
||||
|
||||
/* Stream Transforms */
|
||||
function setupSenderTransform(sender) {
|
||||
const senderStreams = sender.createEncodedStreams()
|
||||
const transformStream = new TransformStream({
|
||||
transform: encodeFunction,
|
||||
})
|
||||
senderStreams.readable.pipeThrough(transformStream).pipeTo(senderStreams.writable)
|
||||
function setupPeerTransform(peer, transform) {
|
||||
const streams = peer.createEncodedStreams();
|
||||
streams.readable.pipeThrough(new TransformStream({ transform })).pipeTo(streams.writable);
|
||||
}
|
||||
|
||||
function setupReceiverTransform(receiver) {
|
||||
const receiverStreams = receiver.createEncodedStreams()
|
||||
const transformStream = new TransformStream({
|
||||
transform: decodeFunction,
|
||||
})
|
||||
receiverStreams.readable.pipeThrough(transformStream).pipeTo(receiverStreams.writable)
|
||||
}
|
||||
|
||||
/* Cryptography */
|
||||
function encodeFunction(frame, controller) {
|
||||
// frame is an RTCEncodedAudioFrame
|
||||
// frame.data is ArrayBuffer
|
||||
let data = new Uint8Array(frame.data)
|
||||
let n = frame instanceof RTCEncodedVideoFrame ? initialPlainTextRequired[frame.type] : 0
|
||||
let iv = randomIV()
|
||||
let initial = data.subarray(0, n)
|
||||
let plaintext = data.subarray(n, data.byteLength)
|
||||
crypto.subtle
|
||||
.encrypt({name: "AES-GCM", iv: iv.buffer}, key, plaintext)
|
||||
.then((c) => {
|
||||
frame.data = concatN(initial, new Uint8Array(c), iv).buffer
|
||||
controller.enqueue(frame)
|
||||
})
|
||||
.catch((e) => {
|
||||
console.log("encrypt error")
|
||||
endCall()
|
||||
throw e
|
||||
})
|
||||
function encodeFunction(key) {
|
||||
return async (frame, controller) => {
|
||||
const data = new Uint8Array(frame.data);
|
||||
const n = frame instanceof RTCEncodedVideoFrame ? initialPlainTextRequired[frame.type] : 0;
|
||||
const iv = randomIV();
|
||||
const initial = data.subarray(0, n);
|
||||
const plaintext = data.subarray(n, data.byteLength);
|
||||
try {
|
||||
const ciphertext = await crypto.subtle.encrypt({ name: "AES-GCM", iv: iv.buffer }, key, plaintext);
|
||||
frame.data = concatN(initial, new Uint8Array(ciphertext), iv).buffer;
|
||||
controller.enqueue(frame);
|
||||
}
|
||||
catch (e) {
|
||||
console.log(`encryption error ${e}`);
|
||||
throw e;
|
||||
}
|
||||
};
|
||||
}
|
||||
function decodeFunction(frame, controller) {
|
||||
let data = new Uint8Array(frame.data)
|
||||
let n = frame instanceof RTCEncodedVideoFrame ? initialPlainTextRequired[frame.type] : 0
|
||||
let initial = data.subarray(0, n)
|
||||
let ciphertext = data.subarray(n, data.byteLength - IV_LENGTH)
|
||||
let iv = data.subarray(data.byteLength - IV_LENGTH, data.byteLength)
|
||||
crypto.subtle
|
||||
.decrypt({name: "AES-GCM", iv: iv}, key, ciphertext)
|
||||
.then((p) => {
|
||||
frame.data = concatN(initial, new Uint8Array(p)).buffer
|
||||
controller.enqueue(frame)
|
||||
})
|
||||
.catch((e) => {
|
||||
console.log("decrypt error")
|
||||
endCall()
|
||||
throw e
|
||||
})
|
||||
function decodeFunction(key) {
|
||||
return async (frame, controller) => {
|
||||
const data = new Uint8Array(frame.data);
|
||||
const n = frame instanceof RTCEncodedVideoFrame ? initialPlainTextRequired[frame.type] : 0;
|
||||
const initial = data.subarray(0, n);
|
||||
const ciphertext = data.subarray(n, data.byteLength - IV_LENGTH);
|
||||
const iv = data.subarray(data.byteLength - IV_LENGTH, data.byteLength);
|
||||
try {
|
||||
const plaintext = await crypto.subtle.decrypt({ name: "AES-GCM", iv }, key, ciphertext);
|
||||
frame.data = concatN(initial, new Uint8Array(plaintext)).buffer;
|
||||
controller.enqueue(frame);
|
||||
}
|
||||
catch (e) {
|
||||
console.log(`decryption error ${e}`);
|
||||
throw e;
|
||||
}
|
||||
};
|
||||
}
|
||||
class RTCEncodedVideoFrame {
|
||||
constructor(type, data) {
|
||||
this.type = type;
|
||||
this.data = data;
|
||||
}
|
||||
}
|
||||
|
||||
function randomIV() {
|
||||
return crypto.getRandomValues(new Uint8Array(IV_LENGTH))
|
||||
return crypto.getRandomValues(new Uint8Array(IV_LENGTH));
|
||||
}
|
||||
async function loadKey(keyData) {
|
||||
key = await crypto.subtle.importKey("jwk", keyData, keyGenConfig, false, keyUsages)
|
||||
}
|
||||
|
||||
const char_equal = "=".charCodeAt(0);
|
||||
function concatN(...bs) {
|
||||
const a = new Uint8Array(bs.reduce((size, b) => size + b.byteLength, 0))
|
||||
bs.reduce((offset, b) => {
|
||||
a.set(b, offset)
|
||||
return offset + b.byteLength
|
||||
}, 0)
|
||||
return a
|
||||
const a = new Uint8Array(bs.reduce((size, b) => size + b.byteLength, 0));
|
||||
bs.reduce((offset, b) => {
|
||||
a.set(b, offset);
|
||||
return offset + b.byteLength;
|
||||
}, 0);
|
||||
return a;
|
||||
}
|
||||
|
||||
async function generateKey() {
|
||||
crypto.subtle
|
||||
.generateKey(keyGenConfig, true, keyUsages)
|
||||
.then((k) => {
|
||||
encryptKey = k
|
||||
return crypto.subtle.exportKey("jwk", encryptKey)
|
||||
})
|
||||
.then((r) => {
|
||||
encryptKeyRepresentation = r
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
action: "processDecryptionKey",
|
||||
content: {
|
||||
key: encryptKeyRepresentation,
|
||||
iv: encryptIv,
|
||||
},
|
||||
})
|
||||
)
|
||||
})
|
||||
function encodeAscii(s) {
|
||||
const a = new Uint8Array(s.length);
|
||||
let i = s.length;
|
||||
while (i--)
|
||||
a[i] = s.charCodeAt(i);
|
||||
return a;
|
||||
}
|
||||
function decodeAscii(a) {
|
||||
let s = "";
|
||||
for (let i = 0; i < a.length; i++)
|
||||
s += String.fromCharCode(a[i]);
|
||||
return s;
|
||||
}
|
||||
const base64chars = new Uint8Array("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".split("").map((c) => c.charCodeAt(0)));
|
||||
const base64lookup = new Array(256);
|
||||
base64chars.forEach((c, i) => (base64lookup[c] = i));
|
||||
function encodeBase64(a) {
|
||||
const len = a.length;
|
||||
const b64len = Math.ceil(len / 3) * 4;
|
||||
const b64 = new Uint8Array(b64len);
|
||||
let j = 0;
|
||||
for (let i = 0; i < len; i += 3) {
|
||||
b64[j++] = base64chars[a[i] >> 2];
|
||||
b64[j++] = base64chars[((a[i] & 3) << 4) | (a[i + 1] >> 4)];
|
||||
b64[j++] = base64chars[((a[i + 1] & 15) << 2) | (a[i + 2] >> 6)];
|
||||
b64[j++] = base64chars[a[i + 2] & 63];
|
||||
}
|
||||
if (len % 3)
|
||||
b64[b64len - 1] = char_equal;
|
||||
if (len % 3 === 1)
|
||||
b64[b64len - 2] = char_equal;
|
||||
return b64;
|
||||
}
|
||||
function decodeBase64(b64) {
|
||||
let len = b64.length;
|
||||
if (len % 4)
|
||||
return;
|
||||
let bLen = (len * 3) / 4;
|
||||
if (b64[len - 1] === char_equal) {
|
||||
len--;
|
||||
bLen--;
|
||||
if (b64[len - 1] === char_equal) {
|
||||
len--;
|
||||
bLen--;
|
||||
}
|
||||
}
|
||||
const bytes = new Uint8Array(bLen);
|
||||
let i = 0;
|
||||
let pos = 0;
|
||||
while (i < len) {
|
||||
const enc1 = base64lookup[b64[i++]];
|
||||
const enc2 = i < len ? base64lookup[b64[i++]] : 0;
|
||||
const enc3 = i < len ? base64lookup[b64[i++]] : 0;
|
||||
const enc4 = i < len ? base64lookup[b64[i++]] : 0;
|
||||
if (enc1 === undefined || enc2 === undefined || enc3 === undefined || enc4 === undefined)
|
||||
return;
|
||||
bytes[pos++] = (enc1 << 2) | (enc2 >> 4);
|
||||
bytes[pos++] = ((enc2 & 15) << 4) | (enc3 >> 2);
|
||||
bytes[pos++] = ((enc3 & 3) << 6) | (enc4 & 63);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
//# sourceMappingURL=call.js.map
|
||||
|
|
@ -5,14 +5,14 @@ html, body {
|
|||
padding: 0;
|
||||
margin: 0;
|
||||
}
|
||||
#incoming-video-stream {
|
||||
#remote-video-stream {
|
||||
position: absolute;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
#outgoing-video-stream {
|
||||
#local-video-stream {
|
||||
position: absolute;
|
||||
width: 30%;
|
||||
max-width: 30%;
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ fun VideoCallView(close: () -> Unit) {
|
|||
}
|
||||
val localContext = LocalContext.current
|
||||
val iceCandidateCommand = remember { mutableStateOf("") }
|
||||
val commandToShow = remember { mutableStateOf("processCommand({action: \"initiateCall\"})") }
|
||||
val commandToShow = remember { mutableStateOf("processCommand({type: 'start', media: 'video', aesKey: 'FwW+t6UbnwHoapYOfN4mUBUuqR7UtvYWxW16iBqM29U='})") }
|
||||
val assetLoader = WebViewAssetLoader.Builder()
|
||||
.addPathHandler("/assets/www/", WebViewAssetLoader.AssetsPathHandler(localContext))
|
||||
.build()
|
||||
|
|
|
|||
3
packages/simplex-chat-webrtc/.gitignore
vendored
Normal file
3
packages/simplex-chat-webrtc/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
node_modules
|
||||
package-lock.json
|
||||
dist
|
||||
5
packages/simplex-chat-webrtc/.prettierrc.json
Normal file
5
packages/simplex-chat-webrtc/.prettierrc.json
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"bracketSpacing": false,
|
||||
"semi": false,
|
||||
"printWidth": 140
|
||||
}
|
||||
24
packages/simplex-chat-webrtc/package.json
Normal file
24
packages/simplex-chat-webrtc/package.json
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
{
|
||||
"name": "simplex-chat-webrtc",
|
||||
"version": "0.0.1",
|
||||
"description": "WebRTC call in browser and webview",
|
||||
"main": "dist/call.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [
|
||||
"SimpleX",
|
||||
"WebRTC"
|
||||
],
|
||||
"author": "",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"devDependencies": {
|
||||
"husky": "^7.0.4",
|
||||
"lint-staged": "^12.4.1",
|
||||
"prettier": "^2.6.2",
|
||||
"typescript": "^4.6.4"
|
||||
},
|
||||
"lint-staged": {
|
||||
"**/*": "prettier --write --ignore-unknown"
|
||||
}
|
||||
}
|
||||
13
packages/simplex-chat-webrtc/src/call.html
Normal file
13
packages/simplex-chat-webrtc/src/call.html
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<link href="./style.css" rel="stylesheet" />
|
||||
</head>
|
||||
<body>
|
||||
<video id="remote-video-stream" autoplay></video>
|
||||
<video id="local-video-stream" muted autoplay></video>
|
||||
</body>
|
||||
<footer>
|
||||
<script src="./call.js"></script>
|
||||
</footer>
|
||||
</html>
|
||||
584
packages/simplex-chat-webrtc/src/call.ts
Normal file
584
packages/simplex-chat-webrtc/src/call.ts
Normal file
|
|
@ -0,0 +1,584 @@
|
|||
// Inspired by
|
||||
// https://github.com/webrtc/samples/blob/gh-pages/src/content/insertable-streams/endtoend-encryption
|
||||
|
||||
// type WCallMessage = WCallCommand | WCallResponse
|
||||
|
||||
type WCallCommand = WCCapabilities | WCStartCall | WCAcceptOffer | WCEndCall | WCallCommandResponse
|
||||
|
||||
type WCallResponse = WRCapabilities | WRConnection | WRCallEnded | WROk | WRError | WCallCommandResponse
|
||||
|
||||
type WCallCommandResponse = WCallOffer | WCallAnswer | WCallIceCandidates
|
||||
|
||||
type WCallMessageTag = "capabilities" | "connection" | "start" | "offer" | "accept" | "answer" | "ice" | "end" | "ended" | "ok" | "error"
|
||||
|
||||
enum CallMediaType {
|
||||
Audio = "audio",
|
||||
Video = "video",
|
||||
}
|
||||
|
||||
interface IWebCallMessage {
|
||||
type: WCallMessageTag
|
||||
}
|
||||
|
||||
interface WCCapabilities extends IWebCallMessage {
|
||||
type: "capabilities"
|
||||
}
|
||||
|
||||
interface WRConnection extends IWebCallMessage {
|
||||
type: "connection",
|
||||
state: {
|
||||
connectionState: string
|
||||
iceConnectionState: string
|
||||
iceGatheringState: string
|
||||
signalingState: string
|
||||
}
|
||||
}
|
||||
|
||||
interface WCStartCall extends IWebCallMessage {
|
||||
type: "start"
|
||||
media: CallMediaType
|
||||
aesKey?: string
|
||||
}
|
||||
|
||||
interface WCEndCall extends IWebCallMessage {
|
||||
type: "end"
|
||||
}
|
||||
|
||||
interface WCAcceptOffer extends IWebCallMessage {
|
||||
type: "accept"
|
||||
offer: RTCSessionDescriptionInit
|
||||
iceCandidates: RTCIceCandidateInit[]
|
||||
media: CallMediaType
|
||||
aesKey?: string
|
||||
}
|
||||
|
||||
interface WCallOffer extends IWebCallMessage {
|
||||
type: "offer"
|
||||
offer: RTCSessionDescriptionInit
|
||||
iceCandidates: RTCIceCandidateInit[]
|
||||
}
|
||||
|
||||
interface WCallAnswer extends IWebCallMessage {
|
||||
type: "answer"
|
||||
answer: RTCSessionDescriptionInit
|
||||
iceCandidates: RTCIceCandidateInit[]
|
||||
}
|
||||
|
||||
interface WCallIceCandidates extends IWebCallMessage {
|
||||
type: "ice"
|
||||
iceCandidates: RTCIceCandidateInit[]
|
||||
}
|
||||
|
||||
interface WRCapabilities {
|
||||
type: "capabilities"
|
||||
capabilities: CallCapabilities
|
||||
}
|
||||
|
||||
interface CallCapabilities {
|
||||
encryption: boolean
|
||||
}
|
||||
|
||||
interface WRCallEnded extends IWebCallMessage {
|
||||
type: "ended"
|
||||
}
|
||||
|
||||
interface WROk extends IWebCallMessage {
|
||||
type: "ok"
|
||||
}
|
||||
|
||||
interface WRError extends IWebCallMessage {
|
||||
type: "error"
|
||||
message: string
|
||||
}
|
||||
|
||||
type RTCRtpSenderWithEncryption = RTCRtpSender & {
|
||||
createEncodedStreams: () => TransformStream
|
||||
}
|
||||
|
||||
type RTCRtpReceiverWithEncryption = RTCRtpReceiver & {
|
||||
createEncodedStreams: () => TransformStream
|
||||
}
|
||||
|
||||
type RTCConfigurationWithEncryption = RTCConfiguration & {
|
||||
encodedInsertableStreams: boolean
|
||||
}
|
||||
|
||||
// STUN servers
|
||||
const peerConnectionConfig: RTCConfigurationWithEncryption = {
|
||||
iceServers: [{urls: ["stun:stun.l.google.com:19302"]}],
|
||||
iceCandidatePoolSize: 10,
|
||||
encodedInsertableStreams: true,
|
||||
}
|
||||
|
||||
const keyAlgorithm: AesKeyAlgorithm = {
|
||||
name: "AES-GCM",
|
||||
length: 256
|
||||
}
|
||||
|
||||
const keyUsages: KeyUsage[] = ["encrypt", "decrypt"]
|
||||
|
||||
let pc: RTCPeerConnection | undefined
|
||||
|
||||
const IV_LENGTH = 12
|
||||
|
||||
const initialPlainTextRequired = {
|
||||
key: 10,
|
||||
delta: 3,
|
||||
undefined: 1,
|
||||
}
|
||||
|
||||
interface Call {
|
||||
connection: RTCPeerConnection
|
||||
iceCandidates: Promise<RTCIceCandidate[]>
|
||||
}
|
||||
|
||||
interface CallConfig {
|
||||
iceCandidates: {
|
||||
delay: number
|
||||
extrasInterval: number
|
||||
extrasTimeout: number
|
||||
}
|
||||
}
|
||||
|
||||
const defaultCallConfig: CallConfig = {
|
||||
iceCandidates: {
|
||||
delay: 2000,
|
||||
extrasInterval: 2000,
|
||||
extrasTimeout: 8000
|
||||
}
|
||||
}
|
||||
|
||||
async function initializeCall(config: CallConfig, mediaType: CallMediaType, aesKey?: string): Promise<Call> {
|
||||
const conn = new RTCPeerConnection(peerConnectionConfig)
|
||||
const remoteStream = new MediaStream()
|
||||
const localStream = await navigator.mediaDevices.getUserMedia(callMediaContraints(mediaType))
|
||||
await setUpMediaStreams(conn, localStream, remoteStream, aesKey)
|
||||
conn.addEventListener("connectionstatechange", connectionStateChange)
|
||||
const iceCandidates = new Promise<RTCIceCandidate[]>((resolve, _) => {
|
||||
let candidates: RTCIceCandidate[] = []
|
||||
let resolved = false
|
||||
let extrasInterval: number | undefined
|
||||
let extrasTimeout: number | undefined
|
||||
const delay = setTimeout(() => {
|
||||
if (!resolved) {
|
||||
resolveIceCandidates()
|
||||
extrasInterval = setInterval(() => {
|
||||
sendIceCandidates()
|
||||
}, config.iceCandidates.extrasInterval)
|
||||
extrasTimeout = setTimeout(() => {
|
||||
clearInterval(extrasInterval)
|
||||
sendIceCandidates()
|
||||
}, config.iceCandidates.extrasTimeout)
|
||||
}
|
||||
}, config.iceCandidates.delay)
|
||||
|
||||
conn.onicecandidate = ({candidate: c}) => c && candidates.push(c)
|
||||
conn.onicegatheringstatechange = () => {
|
||||
if (conn.iceGatheringState == "complete") {
|
||||
if (resolved) {
|
||||
if (extrasInterval) clearInterval(extrasInterval)
|
||||
if (extrasTimeout) clearTimeout(extrasTimeout)
|
||||
sendIceCandidates()
|
||||
} else {
|
||||
resolveIceCandidates()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function resolveIceCandidates() {
|
||||
if (delay) clearTimeout(delay)
|
||||
resolved = true
|
||||
const iceCandidates = candidates.slice()
|
||||
candidates = []
|
||||
resolve(iceCandidates)
|
||||
}
|
||||
|
||||
function sendIceCandidates() {
|
||||
if (candidates.length === 0) return
|
||||
const iceCandidates = candidates.slice()
|
||||
candidates = []
|
||||
sendMessageToNative({type: "ice", iceCandidates})
|
||||
}
|
||||
})
|
||||
|
||||
return {connection: conn, iceCandidates}
|
||||
|
||||
function connectionStateChange() {
|
||||
sendMessageToNative({
|
||||
type: "connection",
|
||||
state: {
|
||||
connectionState: conn.connectionState,
|
||||
iceConnectionState: conn.iceConnectionState,
|
||||
iceGatheringState: conn.iceGatheringState,
|
||||
signalingState: conn.signalingState,
|
||||
}
|
||||
})
|
||||
if (conn.connectionState == "disconnected" || conn.connectionState == "failed") {
|
||||
conn.removeEventListener("connectionstatechange", connectionStateChange)
|
||||
sendMessageToNative({type: "ended"})
|
||||
conn.close()
|
||||
pc = undefined
|
||||
resetVideoElements()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO remove WCallCommand from parameter type
|
||||
function sendMessageToNative(msg: WCallResponse | WCallCommand) {
|
||||
console.log(JSON.stringify(msg))
|
||||
}
|
||||
|
||||
// TODO remove WCallCommand from result type
|
||||
async function processCommand(command: WCallCommand): Promise<WCallResponse | WCallCommand> {
|
||||
let resp: WCallResponse | WCallCommand
|
||||
switch (command.type) {
|
||||
case "capabilities":
|
||||
const encryption = supportsInsertableStreams()
|
||||
resp = {type: "capabilities", capabilities: {encryption}}
|
||||
break
|
||||
case "start":
|
||||
console.log("starting call")
|
||||
if (pc) {
|
||||
resp = {type: "error", message: "start: call already started"}
|
||||
} else if (!supportsInsertableStreams() && command.aesKey) {
|
||||
resp = {type: "error", message: "start: encryption is not supported"}
|
||||
} else {
|
||||
try {
|
||||
const {media, aesKey} = command
|
||||
const call = await initializeCall(defaultCallConfig, media, aesKey)
|
||||
const {connection, iceCandidates} = call
|
||||
pc = connection
|
||||
const offer = await pc.createOffer()
|
||||
await pc.setLocalDescription(offer)
|
||||
// for debugging, returning the command for callee to use
|
||||
resp = {type: "accept", offer, iceCandidates: await iceCandidates, media, aesKey}
|
||||
// resp = {type: "offer", offer, iceCandidates: await iceCandidates}
|
||||
} catch (e) {
|
||||
resp = {type: "error", message: (e as Error).message}
|
||||
}
|
||||
}
|
||||
break
|
||||
case "accept":
|
||||
if (pc) {
|
||||
resp = {type: "error", message: "accept: call already started"}
|
||||
} else if (!supportsInsertableStreams() && command.aesKey) {
|
||||
resp = {type: "error", message: "accept: encryption is not supported"}
|
||||
} else {
|
||||
try {
|
||||
const call = await initializeCall(defaultCallConfig, command.media, command.aesKey)
|
||||
const {connection, iceCandidates} = call
|
||||
pc = connection
|
||||
await pc.setRemoteDescription(new RTCSessionDescription(command.offer))
|
||||
const answer = await pc.createAnswer()
|
||||
await pc.setLocalDescription(answer)
|
||||
addIceCandidates(pc, command.iceCandidates)
|
||||
// same as command for caller to use
|
||||
resp = {type: "answer", answer, iceCandidates: await iceCandidates}
|
||||
} catch (e) {
|
||||
resp = {type: "error", message: (e as Error).message}
|
||||
}
|
||||
}
|
||||
break
|
||||
case "answer":
|
||||
if (!pc) {
|
||||
resp = {type: "error", message: "answer: call not started"}
|
||||
} else if (!pc.localDescription) {
|
||||
resp = {type: "error", message: "answer: local description is not set"}
|
||||
} else if (pc.currentRemoteDescription) {
|
||||
resp = {type: "error", message: "answer: remote description already set"}
|
||||
} else {
|
||||
await pc.setRemoteDescription(new RTCSessionDescription(command.answer))
|
||||
addIceCandidates(pc, command.iceCandidates)
|
||||
resp = {type: "ok"}
|
||||
}
|
||||
break
|
||||
case "ice":
|
||||
if (pc) {
|
||||
addIceCandidates(pc, command.iceCandidates)
|
||||
resp = {type: "ok"}
|
||||
} else {
|
||||
resp = {type: "error", message: "ice: call not started"}
|
||||
}
|
||||
break
|
||||
case "end":
|
||||
if (pc) {
|
||||
pc.close()
|
||||
pc = undefined
|
||||
resetVideoElements()
|
||||
resp = {type: "ok"}
|
||||
} else {
|
||||
resp = {type: "error", message: "end: call not started"}
|
||||
}
|
||||
break
|
||||
default:
|
||||
resp = {type: "error", message: "unknown command"}
|
||||
break
|
||||
}
|
||||
sendMessageToNative(resp)
|
||||
return resp
|
||||
}
|
||||
|
||||
function addIceCandidates(conn: RTCPeerConnection, iceCandidates: RTCIceCandidateInit[]) {
|
||||
for (const c of iceCandidates) {
|
||||
conn.addIceCandidate(new RTCIceCandidate(c))
|
||||
}
|
||||
}
|
||||
|
||||
async function setUpMediaStreams(pc: RTCPeerConnection, localStream: MediaStream, remoteStream: MediaStream, aesKey?: string): Promise<void> {
|
||||
const videos = getVideoElements()
|
||||
if (!videos) throw Error("no video elements")
|
||||
|
||||
let key: CryptoKey | undefined
|
||||
if (aesKey) {
|
||||
const keyData = decodeBase64(encodeAscii(aesKey))
|
||||
if (keyData) key = await crypto.subtle.importKey("raw", keyData, keyAlgorithm, false, keyUsages)
|
||||
}
|
||||
for (const track of localStream.getTracks()) {
|
||||
pc.addTrack(track, localStream)
|
||||
}
|
||||
if (key) {
|
||||
console.log("set up encryption for sending")
|
||||
for (const sender of pc.getSenders() as RTCRtpSenderWithEncryption[]) {
|
||||
setupPeerTransform(sender, encodeFunction(key))
|
||||
}
|
||||
}
|
||||
// Pull tracks from remote stream as they arrive add them to remoteStream video
|
||||
pc.ontrack = (event) => {
|
||||
if (key) {
|
||||
console.log("set up decryption for receiving")
|
||||
setupPeerTransform(event.receiver as RTCRtpReceiverWithEncryption, decodeFunction(key))
|
||||
}
|
||||
for (const track of event.streams[0].getTracks()) {
|
||||
remoteStream.addTrack(track)
|
||||
}
|
||||
}
|
||||
// We assume VP8 encoding in the decode/encode stages to get the initial
|
||||
// bytes to pass as plaintext so we enforce that here.
|
||||
// VP8 is supported by all supports of webrtc.
|
||||
// Use of VP8 by default may also reduce depacketisation issues.
|
||||
// We do not encrypt the first couple of bytes of the payload so that the
|
||||
// video elements can work by determining video keyframes and the opus mode
|
||||
// being used. This appears to be necessary for any video feed at all.
|
||||
// For VP8 this is the content described in
|
||||
// https://tools.ietf.org/html/rfc6386#section-9.1
|
||||
// which is 10 bytes for key frames and 3 bytes for delta frames.
|
||||
// For opus (where encodedFrame.type is not set) this is the TOC byte from
|
||||
// https://tools.ietf.org/html/rfc6716#section-3.1
|
||||
|
||||
const capabilities = RTCRtpSender.getCapabilities("video")
|
||||
if (capabilities) {
|
||||
const {codecs} = capabilities
|
||||
const selectedCodecIndex = codecs.findIndex((c) => c.mimeType === "video/VP8")
|
||||
const selectedCodec = codecs[selectedCodecIndex]
|
||||
codecs.splice(selectedCodecIndex, 1)
|
||||
codecs.unshift(selectedCodec)
|
||||
for (const t of pc.getTransceivers()) {
|
||||
if (t.sender.track?.kind === "video") {
|
||||
t.setCodecPreferences(codecs)
|
||||
}
|
||||
}
|
||||
}
|
||||
// setupVideoElement(videos.local)
|
||||
// setupVideoElement(videos.remote)
|
||||
videos.local.srcObject = localStream
|
||||
videos.remote.srcObject = remoteStream
|
||||
}
|
||||
|
||||
function callMediaContraints(mediaType: CallMediaType): MediaStreamConstraints {
|
||||
switch (mediaType) {
|
||||
case CallMediaType.Audio:
|
||||
return {audio: true, video: false}
|
||||
case CallMediaType.Video:
|
||||
return {
|
||||
audio: true,
|
||||
video: {
|
||||
frameRate: 24,
|
||||
width: {
|
||||
min: 480,
|
||||
ideal: 720,
|
||||
max: 1280,
|
||||
},
|
||||
aspectRatio: 1.33,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function supportsInsertableStreams(): boolean {
|
||||
return ("createEncodedStreams" in RTCRtpSender.prototype)
|
||||
&& ("createEncodedStreams" in RTCRtpReceiver.prototype)
|
||||
}
|
||||
|
||||
interface VideoElements {
|
||||
local: HTMLMediaElement
|
||||
remote: HTMLMediaElement
|
||||
}
|
||||
|
||||
function resetVideoElements() {
|
||||
const videos = getVideoElements()
|
||||
if (!videos) return
|
||||
videos.local.srcObject = null
|
||||
videos.remote.srcObject = null
|
||||
}
|
||||
|
||||
function getVideoElements(): VideoElements | undefined {
|
||||
const local = document.getElementById("local-video-stream")
|
||||
const remote = document.getElementById("remote-video-stream")
|
||||
if (!(local && remote && local instanceof HTMLMediaElement && remote instanceof HTMLMediaElement)) return
|
||||
return {local, remote}
|
||||
}
|
||||
|
||||
// function setupVideoElement(video: HTMLElement) {
|
||||
// // TODO use display: none
|
||||
// video.style.opacity = "0"
|
||||
// video.onplaying = () => {
|
||||
// video.style.opacity = "1"
|
||||
// }
|
||||
// }
|
||||
|
||||
// what does it do?
|
||||
// function toggleVideo(b) {
|
||||
// if (b == "true") {
|
||||
// localStream.getVideoTracks()[0].enabled = true
|
||||
// } else {
|
||||
// localStream.getVideoTracks()[0].enabled = false
|
||||
// }
|
||||
// }
|
||||
|
||||
function f() {
|
||||
console.log("Debug Function")
|
||||
return "Debugging Return"
|
||||
}
|
||||
|
||||
/* Stream Transforms */
|
||||
function setupPeerTransform(peer: RTCRtpSenderWithEncryption | RTCRtpReceiverWithEncryption, transform: (frame: RTCEncodedVideoFrame, controller: TransformStreamDefaultController) => void) {
|
||||
const streams = peer.createEncodedStreams()
|
||||
streams.readable.pipeThrough(new TransformStream({transform})).pipeTo(streams.writable)
|
||||
}
|
||||
|
||||
/* Cryptography */
|
||||
function encodeFunction(key: CryptoKey): (frame: RTCEncodedVideoFrame, controller: TransformStreamDefaultController) => void {
|
||||
return async (frame, controller) => {
|
||||
const data = new Uint8Array(frame.data)
|
||||
const n = frame instanceof RTCEncodedVideoFrame ? initialPlainTextRequired[frame.type] : 0
|
||||
const iv = randomIV()
|
||||
const initial = data.subarray(0, n)
|
||||
const plaintext = data.subarray(n, data.byteLength)
|
||||
try {
|
||||
const ciphertext = await crypto.subtle.encrypt({name: "AES-GCM", iv: iv.buffer}, key, plaintext)
|
||||
frame.data = concatN(initial, new Uint8Array(ciphertext), iv).buffer
|
||||
controller.enqueue(frame)
|
||||
} catch (e) {
|
||||
console.log(`encryption error ${e}`)
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function decodeFunction(key: CryptoKey): (frame: RTCEncodedVideoFrame, controller: TransformStreamDefaultController) => Promise<void> {
|
||||
return async (frame, controller) => {
|
||||
const data = new Uint8Array(frame.data)
|
||||
const n = frame instanceof RTCEncodedVideoFrame ? initialPlainTextRequired[frame.type] : 0
|
||||
const initial = data.subarray(0, n)
|
||||
const ciphertext = data.subarray(n, data.byteLength - IV_LENGTH)
|
||||
const iv = data.subarray(data.byteLength - IV_LENGTH, data.byteLength)
|
||||
try {
|
||||
const plaintext = await crypto.subtle.decrypt({name: "AES-GCM", iv}, key, ciphertext)
|
||||
frame.data = concatN(initial, new Uint8Array(plaintext)).buffer
|
||||
controller.enqueue(frame)
|
||||
} catch (e) {
|
||||
console.log(`decryption error ${e}`)
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class RTCEncodedVideoFrame {
|
||||
constructor(public type: "key" | "delta", public data: ArrayBuffer) {}
|
||||
}
|
||||
|
||||
function randomIV() {
|
||||
return crypto.getRandomValues(new Uint8Array(IV_LENGTH))
|
||||
}
|
||||
|
||||
const char_equal = "=".charCodeAt(0)
|
||||
|
||||
function concatN(...bs: Uint8Array[]): Uint8Array {
|
||||
const a = new Uint8Array(bs.reduce((size, b) => size + b.byteLength, 0))
|
||||
bs.reduce((offset, b: Uint8Array) => {
|
||||
a.set(b, offset)
|
||||
return offset + b.byteLength
|
||||
}, 0)
|
||||
return a
|
||||
}
|
||||
|
||||
function encodeAscii(s: string): Uint8Array {
|
||||
const a = new Uint8Array(s.length)
|
||||
let i = s.length
|
||||
while (i--) a[i] = s.charCodeAt(i)
|
||||
return a
|
||||
}
|
||||
|
||||
function decodeAscii(a: Uint8Array): string {
|
||||
let s = ""
|
||||
for (let i = 0; i < a.length; i++) s += String.fromCharCode(a[i])
|
||||
return s
|
||||
}
|
||||
|
||||
const base64chars = new Uint8Array(
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".split("").map((c) => c.charCodeAt(0))
|
||||
)
|
||||
|
||||
const base64lookup = new Array(256) as (number | undefined)[]
|
||||
base64chars.forEach((c, i) => (base64lookup[c] = i))
|
||||
|
||||
function encodeBase64(a: Uint8Array): Uint8Array {
|
||||
const len = a.length
|
||||
const b64len = Math.ceil(len / 3) * 4
|
||||
const b64 = new Uint8Array(b64len)
|
||||
|
||||
let j = 0
|
||||
for (let i = 0; i < len; i += 3) {
|
||||
b64[j++] = base64chars[a[i] >> 2]
|
||||
b64[j++] = base64chars[((a[i] & 3) << 4) | (a[i + 1] >> 4)]
|
||||
b64[j++] = base64chars[((a[i + 1] & 15) << 2) | (a[i + 2] >> 6)]
|
||||
b64[j++] = base64chars[a[i + 2] & 63]
|
||||
}
|
||||
|
||||
if (len % 3) b64[b64len - 1] = char_equal
|
||||
if (len % 3 === 1) b64[b64len - 2] = char_equal
|
||||
|
||||
return b64
|
||||
}
|
||||
|
||||
function decodeBase64(b64: Uint8Array): Uint8Array | undefined {
|
||||
let len = b64.length
|
||||
if (len % 4) return
|
||||
let bLen = (len * 3) / 4
|
||||
|
||||
if (b64[len - 1] === char_equal) {
|
||||
len--
|
||||
bLen--
|
||||
if (b64[len - 1] === char_equal) {
|
||||
len--
|
||||
bLen--
|
||||
}
|
||||
}
|
||||
|
||||
const bytes = new Uint8Array(bLen)
|
||||
|
||||
let i = 0
|
||||
let pos = 0
|
||||
while (i < len) {
|
||||
const enc1 = base64lookup[b64[i++]]
|
||||
const enc2 = i < len ? base64lookup[b64[i++]] : 0
|
||||
const enc3 = i < len ? base64lookup[b64[i++]] : 0
|
||||
const enc4 = i < len ? base64lookup[b64[i++]] : 0
|
||||
if (enc1 === undefined || enc2 === undefined || enc3 === undefined || enc4 === undefined) return
|
||||
bytes[pos++] = (enc1 << 2) | (enc2 >> 4)
|
||||
bytes[pos++] = ((enc2 & 15) << 4) | (enc3 >> 2)
|
||||
bytes[pos++] = ((enc3 & 3) << 6) | (enc4 & 63)
|
||||
}
|
||||
|
||||
return bytes
|
||||
}
|
||||
24
packages/simplex-chat-webrtc/src/style.css
Normal file
24
packages/simplex-chat-webrtc/src/style.css
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
video::-webkit-media-controls {
|
||||
display: none;
|
||||
}
|
||||
html, body {
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
}
|
||||
#remote-video-stream {
|
||||
position: absolute;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
#local-video-stream {
|
||||
position: absolute;
|
||||
width: 30%;
|
||||
max-width: 30%;
|
||||
object-fit: cover;
|
||||
margin: 16px;
|
||||
border-radius: 16px;
|
||||
bottom: 0;
|
||||
right: 0;
|
||||
}
|
||||
21
packages/simplex-chat-webrtc/tsconfig.json
Normal file
21
packages/simplex-chat-webrtc/tsconfig.json
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"include": ["src"],
|
||||
"compilerOptions": {
|
||||
"declaration": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"lib": ["ES2018", "DOM"],
|
||||
"module": "CommonJS",
|
||||
"moduleResolution": "Node",
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noImplicitAny": true,
|
||||
"noImplicitReturns": true,
|
||||
"noImplicitThis": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"outDir": "dist",
|
||||
"sourceMap": true,
|
||||
"strict": true,
|
||||
"strictNullChecks": true,
|
||||
"target": "ES2018"
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue