lovely-projects/simplex-chat
lovely-projects/simplex-chat
1
0
Fork 0
mirror of https://github.com/simplex-chat/simplex-chat.git synced 2025-03-14 09:45:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

android: refactor webrtc calls, compress webrtc session info, make compatible with Safari (with flag) (#642)

Browse source 
* use simplex.chat relay

* update webrtc settings

* WebRTCView to use command/response types

* compress WebRTC session descriptions, simple web UI for calls

* update webrtc ui

* use webworked in desktop browser

* use RTCRtpScriptTransform in safari

* update android type

* refactor

* add await
This commit is contained in:
Evgeny Poberezkin 2022-05-16 19:27:58 +01:00 • committed by GitHub
parent 36ef6df9fb
commit 82445ec8d5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 1612 additions and 1174 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
apps/android/app/src/main/assets/www/call.html
View file

@ -2,6 +2,7 @@
<html>
<head>
<link href="./style.css" rel="stylesheet" />
<script src="./lz-string.min.js"></script>
</head>
<body>
<video id="remote-video-stream" autoplay playsinline></video>

958
apps/android/app/src/main/assets/www/call.js
View file

@ -6,464 +6,536 @@ var CallMediaType;
CallMediaType["Audio"] = "audio";
CallMediaType["Video"] = "video";
})(CallMediaType || (CallMediaType = {}));
const keyAlgorithm = {
name: "AES-GCM",
length: 256,
};
const keyUsages = ["encrypt", "decrypt"];
let activeCall;
const IV_LENGTH = 12;
const initialPlainTextRequired = {
key: 10,
delta: 3,
undefined: 1,
};
function defaultCallConfig(encodedInsertableStreams) {
return {
peerConnectionConfig: {
iceServers: [{ urls: ["stun:stun.l.google.com:19302"] }],
iceCandidatePoolSize: 10,
encodedInsertableStreams,
},
iceCandidates: {
delay: 2000,
extrasInterval: 2000,
extrasTimeout: 8000,
},
};
}
async function initializeCall(config, mediaType, aesKey) {
const conn = new RTCPeerConnection(config.peerConnectionConfig);
const remoteStream = new MediaStream();
const localStream = await navigator.mediaDevices.getUserMedia(callMediaConstraints(mediaType));
await setUpMediaStreams(conn, localStream, remoteStream, aesKey);
conn.addEventListener("connectionstatechange", connectionStateChange);
const iceCandidates = new Promise((resolve, _) => {
let candidates = [];
let resolved = false;
let extrasInterval;
let extrasTimeout;
const delay = setTimeout(() => {
if (!resolved) {
resolveIceCandidates();
extrasInterval = setInterval(() => {
sendIceCandidates();
}, config.iceCandidates.extrasInterval);
extrasTimeout = setTimeout(() => {
clearInterval(extrasInterval);
sendIceCandidates();
}, config.iceCandidates.extrasTimeout);
}
}, config.iceCandidates.delay);
conn.onicecandidate = ({ candidate: c }) => c && candidates.push(c);
conn.onicegatheringstatechange = () => {
if (conn.iceGatheringState == "complete") {
if (resolved) {
if (extrasInterval)
clearInterval(extrasInterval);
if (extrasTimeout)
clearTimeout(extrasTimeout);
sendIceCandidates();
}
else {
// for debugging
// var sendMessageToNative = ({resp}: WVApiMessage) => console.log(JSON.stringify({command: resp}))
var sendMessageToNative = (msg) => console.log(JSON.stringify(msg));
// Global object with cryptrographic/encoding functions
const callCrypto = callCryptoFunction();
var TransformOperation;
(function (TransformOperation) {
TransformOperation["Encrypt"] = "encrypt";
TransformOperation["Decrypt"] = "decrypt";
})(TransformOperation || (TransformOperation = {}));
;
(function () {
let activeCall;
function defaultCallConfig(encodedInsertableStreams) {
return {
peerConnectionConfig: {
iceServers: [
{ urls: "stun:stun.simplex.chat:5349" },
// {urls: "turn:turn.simplex.chat:5349", username: "private", credential: "yleob6AVkiNI87hpR94Z"},
],
iceCandidatePoolSize: 10,
encodedInsertableStreams,
// iceTransportPolicy: "relay",
},
iceCandidates: {
delay: 2000,
extrasInterval: 2000,
extrasTimeout: 8000,
},
};
}
async function initializeCall(config, mediaType, aesKey, useWorker) {
const conn = new RTCPeerConnection(config.peerConnectionConfig);
const remoteStream = new MediaStream();
const localStream = await navigator.mediaDevices.getUserMedia(callMediaConstraints(mediaType));
await setUpMediaStreams(conn, localStream, remoteStream, aesKey, useWorker);
conn.addEventListener("connectionstatechange", connectionStateChange);
const iceCandidates = new Promise((resolve, _) => {
let candidates = [];
let resolved = false;
let extrasInterval;
let extrasTimeout;
const delay = setTimeout(() => {
if (!resolved) {
resolveIceCandidates();
extrasInterval = setInterval(() => {
sendIceCandidates();
}, config.iceCandidates.extrasInterval);
extrasTimeout = setTimeout(() => {
clearInterval(extrasInterval);
sendIceCandidates();
}, config.iceCandidates.extrasTimeout);
}
}, config.iceCandidates.delay);
conn.onicecandidate = ({ candidate: c }) => c && candidates.push(c);
conn.onicegatheringstatechange = () => {
if (conn.iceGatheringState == "complete") {
if (resolved) {
if (extrasInterval)
clearInterval(extrasInterval);
if (extrasTimeout)
clearTimeout(extrasTimeout);
sendIceCandidates();
}
else {
resolveIceCandidates();
}
}
};
function resolveIceCandidates() {
if (delay)
clearTimeout(delay);
resolved = true;
const iceCandidates = serialize(candidates);
candidates = [];
resolve(iceCandidates);
}
function sendIceCandidates() {
if (candidates.length === 0)
return;
const iceCandidates = serialize(candidates);
candidates = [];
sendMessageToNative({ resp: { type: "ice", iceCandidates } });
}
});
return { connection: conn, iceCandidates, localMedia: mediaType, localStream };
function connectionStateChange() {
sendMessageToNative({
resp: {
type: "connection",
state: {
connectionState: conn.connectionState,
iceConnectionState: conn.iceConnectionState,
iceGatheringState: conn.iceGatheringState,
signalingState: conn.signalingState,
},
},
});
if (conn.connectionState == "disconnected" || conn.connectionState == "failed") {
conn.removeEventListener("connectionstatechange", connectionStateChange);
sendMessageToNative({ resp: { type: "ended" } });
conn.close();
activeCall = undefined;
resetVideoElements();
}
}
}
function serialize(x) {
return LZString.compressToBase64(JSON.stringify(x));
}
function parse(s) {
return JSON.parse(LZString.decompressFromBase64(s));
}
Object.defineProperty(window, "processCommand", { value: processCommand });
async function processCommand(body) {
const { corrId, command } = body;
const pc = activeCall === null || activeCall === void 0 ? void 0 : activeCall.connection;
let resp;
try {
switch (command.type) {
case "capabilities":
const encryption = supportsInsertableStreams(command.useWorker);
resp = { type: "capabilities", capabilities: { encryption } };
break;
case "start":
console.log("starting call");
if (activeCall) {
resp = { type: "error", message: "start: call already started" };
}
else if (!supportsInsertableStreams(command.useWorker) && command.aesKey) {
resp = { type: "error", message: "start: encryption is not supported" };
}
else {
const { media, useWorker } = command;
const encryption = supportsInsertableStreams(useWorker);
const aesKey = encryption ? command.aesKey : undefined;
activeCall = await initializeCall(defaultCallConfig(encryption && !!aesKey), media, aesKey, useWorker);
const pc = activeCall.connection;
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
// for debugging, returning the command for callee to use
// resp = {
// type: "offer",
// offer: serialize(offer),
// iceCandidates: await activeCall.iceCandidates,
// media,
// aesKey,
// }
resp = {
type: "offer",
offer: serialize(offer),
iceCandidates: await activeCall.iceCandidates,
capabilities: { encryption },
};
}
break;
case "offer":
if (activeCall) {
resp = { type: "error", message: "accept: call already started" };
}
else if (!supportsInsertableStreams(command.useWorker) && command.aesKey) {
resp = { type: "error", message: "accept: encryption is not supported" };
}
else {
const offer = parse(command.offer);
const remoteIceCandidates = parse(command.iceCandidates);
const { media, aesKey, useWorker } = command;
activeCall = await initializeCall(defaultCallConfig(!!aesKey), media, aesKey, useWorker);
const pc = activeCall.connection;
await pc.setRemoteDescription(new RTCSessionDescription(offer));
const answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
addIceCandidates(pc, remoteIceCandidates);
// same as command for caller to use
resp = {
type: "answer",
answer: serialize(answer),
iceCandidates: await activeCall.iceCandidates,
};
}
break;
case "answer":
if (!pc) {
resp = { type: "error", message: "answer: call not started" };
}
else if (!pc.localDescription) {
resp = { type: "error", message: "answer: local description is not set" };
}
else if (pc.currentRemoteDescription) {
resp = { type: "error", message: "answer: remote description already set" };
}
else {
const answer = parse(command.answer);
const remoteIceCandidates = parse(command.iceCandidates);
await pc.setRemoteDescription(new RTCSessionDescription(answer));
addIceCandidates(pc, remoteIceCandidates);
resp = { type: "ok" };
}
break;
case "ice":
if (pc) {
const remoteIceCandidates = parse(command.iceCandidates);
addIceCandidates(pc, remoteIceCandidates);
resp = { type: "ok" };
}
else {
resp = { type: "error", message: "ice: call not started" };
}
break;
case "media":
if (!activeCall) {
resp = { type: "error", message: "media: call not started" };
}
else if (activeCall.localMedia == CallMediaType.Audio && command.media == CallMediaType.Video) {
resp = { type: "error", message: "media: no video" };
}
else {
enableMedia(activeCall.localStream, command.media, command.enable);
resp = { type: "ok" };
}
break;
case "end":
if (pc) {
pc.close();
activeCall = undefined;
resetVideoElements();
resp = { type: "ok" };
}
else {
resp = { type: "error", message: "end: call not started" };
}
break;
default:
resp = { type: "error", message: "unknown command" };
break;
}
}
catch (e) {
resp = { type: "error", message: e.message };
}
const apiResp = { corrId, resp, command };
sendMessageToNative(apiResp);
return apiResp;
}
function addIceCandidates(conn, iceCandidates) {
for (const c of iceCandidates) {
conn.addIceCandidate(new RTCIceCandidate(c));
}
}
async function setUpMediaStreams(pc, localStream, remoteStream, aesKey, useWorker) {
var _a;
const videos = getVideoElements();
if (!videos)
throw Error("no video elements");
let key;
let worker;
if (aesKey) {
key = await callCrypto.decodeAesKey(aesKey);
if (useWorker) {
const workerCode = `const callCrypto = (${callCryptoFunction.toString()})(); (${workerFunction.toString()})()`;
worker = new Worker(URL.createObjectURL(new Blob([workerCode], { type: "text/javascript" })));
}
}
for (const track of localStream.getTracks()) {
pc.addTrack(track, localStream);
}
if (aesKey && key) {
console.log("set up encryption for sending");
for (const sender of pc.getSenders()) {
setupPeerTransform(TransformOperation.Encrypt, sender, worker, aesKey, key);
}
}
// Pull tracks from remote stream as they arrive add them to remoteStream video
pc.ontrack = (event) => {
if (aesKey && key) {
console.log("set up decryption for receiving");
setupPeerTransform(TransformOperation.Decrypt, event.receiver, worker, aesKey, key);
}
remoteStream.addTrack(event.track);
};
// We assume VP8 encoding in the decode/encode stages to get the initial
// bytes to pass as plaintext so we enforce that here.
// VP8 is supported by all supports of webrtc.
// Use of VP8 by default may also reduce depacketisation issues.
// We do not encrypt the first couple of bytes of the payload so that the
// video elements can work by determining video keyframes and the opus mode
// being used. This appears to be necessary for any video feed at all.
// For VP8 this is the content described in
// https://tools.ietf.org/html/rfc6386#section-9.1
// which is 10 bytes for key frames and 3 bytes for delta frames.
// For opus (where encodedFrame.type is not set) this is the TOC byte from
// https://tools.ietf.org/html/rfc6716#section-3.1
const capabilities = RTCRtpSender.getCapabilities("video");
if (capabilities) {
const { codecs } = capabilities;
const selectedCodecIndex = codecs.findIndex((c) => c.mimeType === "video/VP8");
const selectedCodec = codecs[selectedCodecIndex];
codecs.splice(selectedCodecIndex, 1);
codecs.unshift(selectedCodec);
for (const t of pc.getTransceivers()) {
if (((_a = t.sender.track) === null || _a === void 0 ? void 0 : _a.kind) === "video") {
t.setCodecPreferences(codecs);
}
}
}
// setupVideoElement(videos.local)
// setupVideoElement(videos.remote)
videos.local.srcObject = localStream;
videos.remote.srcObject = remoteStream;
}
function setupPeerTransform(operation, peer, worker, aesKey, key) {
if (worker && "RTCRtpScriptTransform" in window) {
console.log(`${operation} with worker & RTCRtpScriptTransform`);
peer.transform = new RTCRtpScriptTransform(worker, { operation, aesKey });
}
else if ("createEncodedStreams" in peer) {
const { readable, writable } = peer.createEncodedStreams();
if (worker) {
console.log(`${operation} with worker`);
worker.postMessage({ operation, readable, writable, aesKey }, [readable, writable]);
}
else {
console.log(`${operation} without worker`);
const transform = callCrypto.transformFrame[operation](key);
readable.pipeThrough(new TransformStream({ transform })).pipeTo(writable);
}
}
else {
console.log(`no ${operation}`);
}
}
function callMediaConstraints(mediaType) {
switch (mediaType) {
case CallMediaType.Audio:
return { audio: true, video: false };
case CallMediaType.Video:
return {
audio: true,
video: {
frameRate: 24,
width: {
min: 480,
ideal: 720,
max: 1280,
},
aspectRatio: 1.33,
},
};
}
}
function supportsInsertableStreams(useWorker) {
return (("createEncodedStreams" in RTCRtpSender.prototype && "createEncodedStreams" in RTCRtpReceiver.prototype) ||
(!!useWorker && "RTCRtpScriptTransform" in window));
}
function resetVideoElements() {
const videos = getVideoElements();
if (!videos)
return;
videos.local.srcObject = null;
videos.remote.srcObject = null;
}
function getVideoElements() {
const local = document.getElementById("local-video-stream");
const remote = document.getElementById("remote-video-stream");
if (!(local && remote && local instanceof HTMLMediaElement && remote instanceof HTMLMediaElement))
return;
return { local, remote };
}
// function setupVideoElement(video: HTMLElement) {
// // TODO use display: none
// video.style.opacity = "0"
// video.onplaying = () => {
// video.style.opacity = "1"
// }
// }
function enableMedia(s, media, enable) {
const tracks = media == CallMediaType.Video ? s.getVideoTracks() : s.getAudioTracks();
for (const t of tracks)
t.enabled = enable;
}
})();
// Cryptography function - it is loaded both in the main window and in worker context (if the worker is used)
function callCryptoFunction() {
const initialPlainTextRequired = {
key: 10,
delta: 3,
};
const IV_LENGTH = 12;
function encryptFrame(key) {
return async (frame, controller) => {
const data = new Uint8Array(frame.data);
const n = initialPlainTextRequired[frame.type] || 1;
const iv = randomIV();
const initial = data.subarray(0, n);
const plaintext = data.subarray(n, data.byteLength);
try {
const ciphertext = await crypto.subtle.encrypt({ name: "AES-GCM", iv: iv.buffer }, key, plaintext);
frame.data = concatN(initial, new Uint8Array(ciphertext), iv).buffer;
controller.enqueue(frame);
}
catch (e) {
console.log(`encryption error ${e}`);
throw e;
}
};
function resolveIceCandidates() {
if (delay)
clearTimeout(delay);
resolved = true;
const iceCandidates = candidates.map((c) => JSON.stringify(c));
candidates = [];
resolve(iceCandidates);
}
function sendIceCandidates() {
if (candidates.length === 0)
return;
const iceCandidates = candidates.map((c) => JSON.stringify(c));
candidates = [];
sendMessageToNative({ resp: { type: "ice", iceCandidates } });
}
});
return { connection: conn, iceCandidates, localMedia: mediaType, localStream };
function connectionStateChange() {
sendMessageToNative({
resp: {
type: "connection",
state: {
connectionState: conn.connectionState,
iceConnectionState: conn.iceConnectionState,
iceGatheringState: conn.iceGatheringState,
signalingState: conn.signalingState,
},
},
});
if (conn.connectionState == "disconnected" || conn.connectionState == "failed") {
conn.removeEventListener("connectionstatechange", connectionStateChange);
sendMessageToNative({ resp: { type: "ended" } });
conn.close();
activeCall = undefined;
resetVideoElements();
}
}
}
var sendMessageToNative = (msg) => console.log(JSON.stringify(msg));
async function processCommand(body) {
const { corrId, command } = body;
const pc = activeCall === null || activeCall === void 0 ? void 0 : activeCall.connection;
let resp;
try {
switch (command.type) {
case "capabilities":
const encryption = supportsInsertableStreams();
resp = { type: "capabilities", capabilities: { encryption } };
break;
case "start":
console.log("starting call");
if (activeCall) {
resp = { type: "error", message: "start: call already started" };
}
else if (!supportsInsertableStreams() && command.aesKey) {
resp = { type: "error", message: "start: encryption is not supported" };
}
else {
const encryption = supportsInsertableStreams();
const { media, aesKey } = command;
activeCall = await initializeCall(defaultCallConfig(encryption && !!aesKey), media, encryption ? aesKey : undefined);
const pc = activeCall.connection;
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
// for debugging, returning the command for callee to use
// resp = {type: "accept", offer: JSON.stringify(offer), iceCandidates: await iceCandidates, media, aesKey}
resp = {
type: "offer",
offer: JSON.stringify(offer),
iceCandidates: await activeCall.iceCandidates,
capabilities: { encryption },
};
}
break;
case "accept":
if (activeCall) {
resp = { type: "error", message: "accept: call already started" };
}
else if (!supportsInsertableStreams() && command.aesKey) {
resp = { type: "error", message: "accept: encryption is not supported" };
}
else {
const offer = JSON.parse(command.offer);
const remoteIceCandidates = command.iceCandidates.map((c) => JSON.parse(c));
activeCall = await initializeCall(defaultCallConfig(!!command.aesKey), command.media, command.aesKey);
const pc = activeCall.connection;
await pc.setRemoteDescription(new RTCSessionDescription(offer));
const answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
addIceCandidates(pc, remoteIceCandidates);
// same as command for caller to use
resp = {
type: "answer",
answer: JSON.stringify(answer),
iceCandidates: await activeCall.iceCandidates,
};
}
break;
case "answer":
if (!pc) {
resp = { type: "error", message: "answer: call not started" };
}
else if (!pc.localDescription) {
resp = { type: "error", message: "answer: local description is not set" };
}
else if (pc.currentRemoteDescription) {
resp = { type: "error", message: "answer: remote description already set" };
}
else {
const answer = JSON.parse(command.answer);
const remoteIceCandidates = command.iceCandidates.map((c) => JSON.parse(c));
await pc.setRemoteDescription(new RTCSessionDescription(answer));
addIceCandidates(pc, remoteIceCandidates);
resp = { type: "ok" };
}
break;
case "ice":
if (pc) {
const remoteIceCandidates = command.iceCandidates.map((c) => JSON.parse(c));
addIceCandidates(pc, remoteIceCandidates);
resp = { type: "ok" };
}
else {
resp = { type: "error", message: "ice: call not started" };
}
break;
case "media":
if (!activeCall) {
resp = { type: "error", message: "media: call not started" };
}
else if (activeCall.localMedia == CallMediaType.Audio && command.media == CallMediaType.Video) {
resp = { type: "error", message: "media: no video" };
}
else {
enableMedia(activeCall.localStream, command.media, command.enable);
resp = { type: "ok" };
}
break;
case "end":
if (pc) {
pc.close();
activeCall = undefined;
resetVideoElements();
resp = { type: "ok" };
}
else {
resp = { type: "error", message: "end: call not started" };
}
break;
default:
resp = { type: "error", message: "unknown command" };
break;
}
}
catch (e) {
resp = { type: "error", message: e.message };
}
const apiResp = { corrId, resp, command };
sendMessageToNative(apiResp);
return apiResp;
}
function addIceCandidates(conn, iceCandidates) {
for (const c of iceCandidates) {
conn.addIceCandidate(new RTCIceCandidate(c));
}
}
async function setUpMediaStreams(pc, localStream, remoteStream, aesKey) {
var _a;
const videos = getVideoElements();
if (!videos)
throw Error("no video elements");
let key;
if (aesKey) {
const keyData = decodeBase64(encodeAscii(aesKey));
if (keyData)
key = await crypto.subtle.importKey("raw", keyData, keyAlgorithm, false, keyUsages);
}
for (const track of localStream.getTracks()) {
pc.addTrack(track, localStream);
}
if (key) {
console.log("set up encryption for sending");
for (const sender of pc.getSenders()) {
setupPeerTransform(sender, encodeFunction(key));
}
}
// Pull tracks from remote stream as they arrive add them to remoteStream video
pc.ontrack = (event) => {
if (key) {
console.log("set up decryption for receiving");
setupPeerTransform(event.receiver, decodeFunction(key));
}
for (const track of event.streams[0].getTracks()) {
remoteStream.addTrack(track);
}
};
// We assume VP8 encoding in the decode/encode stages to get the initial
// bytes to pass as plaintext so we enforce that here.
// VP8 is supported by all supports of webrtc.
// Use of VP8 by default may also reduce depacketisation issues.
// We do not encrypt the first couple of bytes of the payload so that the
// video elements can work by determining video keyframes and the opus mode
// being used. This appears to be necessary for any video feed at all.
// For VP8 this is the content described in
// https://tools.ietf.org/html/rfc6386#section-9.1
// which is 10 bytes for key frames and 3 bytes for delta frames.
// For opus (where encodedFrame.type is not set) this is the TOC byte from
// https://tools.ietf.org/html/rfc6716#section-3.1
const capabilities = RTCRtpSender.getCapabilities("video");
if (capabilities) {
const { codecs } = capabilities;
const selectedCodecIndex = codecs.findIndex((c) => c.mimeType === "video/VP8");
const selectedCodec = codecs[selectedCodecIndex];
codecs.splice(selectedCodecIndex, 1);
codecs.unshift(selectedCodec);
for (const t of pc.getTransceivers()) {
if (((_a = t.sender.track) === null || _a === void 0 ? void 0 : _a.kind) === "video") {
t.setCodecPreferences(codecs);
function decryptFrame(key) {
return async (frame, controller) => {
const data = new Uint8Array(frame.data);
const n = initialPlainTextRequired[frame.type] || 1;
const initial = data.subarray(0, n);
const ciphertext = data.subarray(n, data.byteLength - IV_LENGTH);
const iv = data.subarray(data.byteLength - IV_LENGTH, data.byteLength);
try {
const plaintext = await crypto.subtle.decrypt({ name: "AES-GCM", iv }, key, ciphertext);
frame.data = concatN(initial, new Uint8Array(plaintext)).buffer;
controller.enqueue(frame);
}
}
catch (e) {
console.log(`decryption error ${e}`);
throw e;
}
};
}
// setupVideoElement(videos.local)
// setupVideoElement(videos.remote)
videos.local.srcObject = localStream;
videos.remote.srcObject = remoteStream;
}
function callMediaConstraints(mediaType) {
switch (mediaType) {
case CallMediaType.Audio:
return { audio: true, video: false };
case CallMediaType.Video:
return {
audio: true,
video: {
frameRate: 24,
width: {
min: 480,
ideal: 720,
max: 1280,
},
aspectRatio: 1.33,
},
};
function decodeAesKey(aesKey) {
const keyData = callCrypto.decodeBase64(callCrypto.encodeAscii(aesKey));
return crypto.subtle.importKey("raw", keyData, { name: "AES-GCM", length: 256 }, false, ["encrypt", "decrypt"]);
}
}
function supportsInsertableStreams() {
return "createEncodedStreams" in RTCRtpSender.prototype && "createEncodedStreams" in RTCRtpReceiver.prototype;
}
function resetVideoElements() {
const videos = getVideoElements();
if (!videos)
return;
videos.local.srcObject = null;
videos.remote.srcObject = null;
}
function getVideoElements() {
const local = document.getElementById("local-video-stream");
const remote = document.getElementById("remote-video-stream");
if (!(local && remote && local instanceof HTMLMediaElement && remote instanceof HTMLMediaElement))
return;
return { local, remote };
}
// function setupVideoElement(video: HTMLElement) {
// // TODO use display: none
// video.style.opacity = "0"
// video.onplaying = () => {
// video.style.opacity = "1"
// }
// }
function enableMedia(s, media, enable) {
const tracks = media == CallMediaType.Video ? s.getVideoTracks() : s.getAudioTracks();
for (const t of tracks)
t.enabled = enable;
}
/* Stream Transforms */
function setupPeerTransform(peer, transform) {
const streams = peer.createEncodedStreams();
streams.readable.pipeThrough(new TransformStream({ transform })).pipeTo(streams.writable);
}
/* Cryptography */
function encodeFunction(key) {
return async (frame, controller) => {
const data = new Uint8Array(frame.data);
const n = frame instanceof RTCEncodedVideoFrame ? initialPlainTextRequired[frame.type] : 0;
const iv = randomIV();
const initial = data.subarray(0, n);
const plaintext = data.subarray(n, data.byteLength);
try {
const ciphertext = await crypto.subtle.encrypt({ name: "AES-GCM", iv: iv.buffer }, key, plaintext);
frame.data = concatN(initial, new Uint8Array(ciphertext), iv).buffer;
controller.enqueue(frame);
}
catch (e) {
console.log(`encryption error ${e}`);
throw e;
}
};
}
function decodeFunction(key) {
return async (frame, controller) => {
const data = new Uint8Array(frame.data);
const n = frame instanceof RTCEncodedVideoFrame ? initialPlainTextRequired[frame.type] : 0;
const initial = data.subarray(0, n);
const ciphertext = data.subarray(n, data.byteLength - IV_LENGTH);
const iv = data.subarray(data.byteLength - IV_LENGTH, data.byteLength);
try {
const plaintext = await crypto.subtle.decrypt({ name: "AES-GCM", iv }, key, ciphertext);
frame.data = concatN(initial, new Uint8Array(plaintext)).buffer;
controller.enqueue(frame);
}
catch (e) {
console.log(`decryption error ${e}`);
throw e;
}
};
}
class RTCEncodedVideoFrame {
constructor(type, data) {
this.type = type;
this.data = data;
function concatN(...bs) {
const a = new Uint8Array(bs.reduce((size, b) => size + b.byteLength, 0));
bs.reduce((offset, b) => {
a.set(b, offset);
return offset + b.byteLength;
}, 0);
return a;
}
}
function randomIV() {
return crypto.getRandomValues(new Uint8Array(IV_LENGTH));
}
const char_equal = "=".charCodeAt(0);
function concatN(...bs) {
const a = new Uint8Array(bs.reduce((size, b) => size + b.byteLength, 0));
bs.reduce((offset, b) => {
a.set(b, offset);
return offset + b.byteLength;
}, 0);
return a;
}
function encodeAscii(s) {
const a = new Uint8Array(s.length);
let i = s.length;
while (i--)
a[i] = s.charCodeAt(i);
return a;
}
function decodeAscii(a) {
let s = "";
for (let i = 0; i < a.length; i++)
s += String.fromCharCode(a[i]);
return s;
}
const base64chars = new Uint8Array("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".split("").map((c) => c.charCodeAt(0)));
const base64lookup = new Array(256);
base64chars.forEach((c, i) => (base64lookup[c] = i));
function encodeBase64(a) {
const len = a.length;
const b64len = Math.ceil(len / 3) * 4;
const b64 = new Uint8Array(b64len);
let j = 0;
for (let i = 0; i < len; i += 3) {
b64[j++] = base64chars[a[i] >> 2];
b64[j++] = base64chars[((a[i] & 3) << 4) | (a[i + 1] >> 4)];
b64[j++] = base64chars[((a[i + 1] & 15) << 2) | (a[i + 2] >> 6)];
b64[j++] = base64chars[a[i + 2] & 63];
function randomIV() {
return crypto.getRandomValues(new Uint8Array(IV_LENGTH));
}
if (len % 3)
b64[b64len - 1] = char_equal;
if (len % 3 === 1)
b64[b64len - 2] = char_equal;
return b64;
}
function decodeBase64(b64) {
let len = b64.length;
if (len % 4)
return;
let bLen = (len * 3) / 4;
if (b64[len - 1] === char_equal) {
len--;
bLen--;
const base64chars = new Uint8Array("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".split("").map((c) => c.charCodeAt(0)));
const base64lookup = new Array(256);
base64chars.forEach((c, i) => (base64lookup[c] = i));
const char_equal = "=".charCodeAt(0);
function encodeAscii(s) {
const a = new Uint8Array(s.length);
let i = s.length;
while (i--)
a[i] = s.charCodeAt(i);
return a;
}
function decodeAscii(a) {
let s = "";
for (let i = 0; i < a.length; i++)
s += String.fromCharCode(a[i]);
return s;
}
function encodeBase64(a) {
const len = a.length;
const b64len = Math.ceil(len / 3) * 4;
const b64 = new Uint8Array(b64len);
let j = 0;
for (let i = 0; i < len; i += 3) {
b64[j++] = base64chars[a[i] >> 2];
b64[j++] = base64chars[((a[i] & 3) << 4) | (a[i + 1] >> 4)];
b64[j++] = base64chars[((a[i + 1] & 15) << 2) | (a[i + 2] >> 6)];
b64[j++] = base64chars[a[i + 2] & 63];
}
if (len % 3)
b64[b64len - 1] = char_equal;
if (len % 3 === 1)
b64[b64len - 2] = char_equal;
return b64;
}
function decodeBase64(b64) {
let len = b64.length;
if (len % 4)
return;
let bLen = (len * 3) / 4;
if (b64[len - 1] === char_equal) {
len--;
bLen--;
if (b64[len - 1] === char_equal) {
len--;
bLen--;
}
}
const bytes = new Uint8Array(bLen);
let i = 0;
let pos = 0;
while (i < len) {
const enc1 = base64lookup[b64[i++]];
const enc2 = i < len ? base64lookup[b64[i++]] : 0;
const enc3 = i < len ? base64lookup[b64[i++]] : 0;
const enc4 = i < len ? base64lookup[b64[i++]] : 0;
if (enc1 === undefined || enc2 === undefined || enc3 === undefined || enc4 === undefined)
return;
bytes[pos++] = (enc1 << 2) | (enc2 >> 4);
bytes[pos++] = ((enc2 & 15) << 4) | (enc3 >> 2);
bytes[pos++] = ((enc3 & 3) << 6) | (enc4 & 63);
}
return bytes;
}
const bytes = new Uint8Array(bLen);
let i = 0;
let pos = 0;
while (i < len) {
const enc1 = base64lookup[b64[i++]];
const enc2 = i < len ? base64lookup[b64[i++]] : 0;
const enc3 = i < len ? base64lookup[b64[i++]] : 0;
const enc4 = i < len ? base64lookup[b64[i++]] : 0;
if (enc1 === undefined || enc2 === undefined || enc3 === undefined || enc4 === undefined)
return;
bytes[pos++] = (enc1 << 2) | (enc2 >> 4);
bytes[pos++] = ((enc2 & 15) << 4) | (enc3 >> 2);
bytes[pos++] = ((enc3 & 3) << 6) | (enc4 & 63);
return {
transformFrame: { encrypt: encryptFrame, decrypt: decryptFrame },
decodeAesKey,
encodeAscii,
decodeAscii,
encodeBase64,
decodeBase64,
};
}
// If the worker is used for decryption, this function code (as string) is used to load the worker via Blob
// We have to use worker optionally, as it crashes in Android web view, regardless of how it is loaded
function workerFunction() {
// encryption with createEncodedStreams support
self.addEventListener("message", async ({ data }) => {
await setupTransform(data);
});
// encryption using RTCRtpScriptTransform.
if ("RTCTransformEvent" in self) {
self.addEventListener("rtctransform", async ({ transformer }) => {
const { operation, aesKey } = transformer.options;
const { readable, writable } = transformer;
await setupTransform({ operation, aesKey, readable, writable });
});
}
async function setupTransform({ operation, aesKey, readable, writable }) {
const key = await callCrypto.decodeAesKey(aesKey);
const transform = callCrypto.transformFrame[operation](key);
readable.pipeThrough(new TransformStream({ transform })).pipeTo(writable);
}
return bytes;
}
//# sourceMappingURL=call.js.map

1
apps/android/app/src/main/assets/www/lz-string.min.js vendored Normal file
View file

@ -0,0 +1 @@
var LZString=function(){function o(o,r){if(!t[o]){t[o]={};for(var n=0;n<o.length;n++)t[o][o.charAt(n)]=n}return t[o][r]}var r=String.fromCharCode,n="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=",e="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-$",t={},i={compressToBase64:function(o){if(null==o)return"";var r=i._compress(o,6,function(o){return n.charAt(o)});switch(r.length%4){default:case 0:return r;case 1:return r+"===";case 2:return r+"==";case 3:return r+"="}},decompressFromBase64:function(r){return null==r?"":""==r?null:i._decompress(r.length,32,function(e){return o(n,r.charAt(e))})},compressToUTF16:function(o){return null==o?"":i._compress(o,15,function(o){return r(o+32)})+" "},decompressFromUTF16:function(o){return null==o?"":""==o?null:i._decompress(o.length,16384,function(r){return o.charCodeAt(r)-32})},compressToUint8Array:function(o){for(var r=i.compress(o),n=new Uint8Array(2*r.length),e=0,t=r.length;t>e;e++){var s=r.charCodeAt(e);n[2*e]=s>>>8,n[2*e+1]=s%256}return n},decompressFromUint8Array:function(o){if(null===o||void 0===o)return i.decompress(o);for(var n=new Array(o.length/2),e=0,t=n.length;t>e;e++)n[e]=256*o[2*e]+o[2*e+1];var s=[];return n.forEach(function(o){s.push(r(o))}),i.decompress(s.join(""))},compressToEncodedURIComponent:function(o){return null==o?"":i._compress(o,6,function(o){return e.charAt(o)})},decompressFromEncodedURIComponent:function(r){return null==r?"":""==r?null:(r=r.replace(/ /g,"+"),i._decompress(r.length,32,function(n){return o(e,r.charAt(n))}))},compress:function(o){return i._compress(o,16,function(o){return r(o)})},_compress:function(o,r,n){if(null==o)return"";var e,t,i,s={},p={},u="",c="",a="",l=2,f=3,h=2,d=[],m=0,v=0;for(i=0;i<o.length;i+=1)if(u=o.charAt(i),Object.prototype.hasOwnProperty.call(s,u)||(s[u]=f++,p[u]=!0),c=a+u,Object.prototype.hasOwnProperty.call(s,c))a=c;else{if(Object.prototype.hasOwnProperty.call(p,a)){if(a.charCodeAt(0)<256){for(e=0;h>e;e++)m<<=1,v==r-1?(v=0,d.push(n(m)),m=0):v++;for(t=a.charCodeAt(0),e=0;8>e;e++)m=m<<1|1&t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t>>=1}else{for(t=1,e=0;h>e;e++)m=m<<1|t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t=0;for(t=a.charCodeAt(0),e=0;16>e;e++)m=m<<1|1&t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t>>=1}l--,0==l&&(l=Math.pow(2,h),h++),delete p[a]}else for(t=s[a],e=0;h>e;e++)m=m<<1|1&t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t>>=1;l--,0==l&&(l=Math.pow(2,h),h++),s[c]=f++,a=String(u)}if(""!==a){if(Object.prototype.hasOwnProperty.call(p,a)){if(a.charCodeAt(0)<256){for(e=0;h>e;e++)m<<=1,v==r-1?(v=0,d.push(n(m)),m=0):v++;for(t=a.charCodeAt(0),e=0;8>e;e++)m=m<<1|1&t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t>>=1}else{for(t=1,e=0;h>e;e++)m=m<<1|t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t=0;for(t=a.charCodeAt(0),e=0;16>e;e++)m=m<<1|1&t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t>>=1}l--,0==l&&(l=Math.pow(2,h),h++),delete p[a]}else for(t=s[a],e=0;h>e;e++)m=m<<1|1&t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t>>=1;l--,0==l&&(l=Math.pow(2,h),h++)}for(t=2,e=0;h>e;e++)m=m<<1|1&t,v==r-1?(v=0,d.push(n(m)),m=0):v++,t>>=1;for(;;){if(m<<=1,v==r-1){d.push(n(m));break}v++}return d.join("")},decompress:function(o){return null==o?"":""==o?null:i._decompress(o.length,32768,function(r){return o.charCodeAt(r)})},_decompress:function(o,n,e){var t,i,s,p,u,c,a,l,f=[],h=4,d=4,m=3,v="",w=[],A={val:e(0),position:n,index:1};for(i=0;3>i;i+=1)f[i]=i;for(p=0,c=Math.pow(2,2),a=1;a!=c;)u=A.val&A.position,A.position>>=1,0==A.position&&(A.position=n,A.val=e(A.index++)),p|=(u>0?1:0)*a,a<<=1;switch(t=p){case 0:for(p=0,c=Math.pow(2,8),a=1;a!=c;)u=A.val&A.position,A.position>>=1,0==A.position&&(A.position=n,A.val=e(A.index++)),p|=(u>0?1:0)*a,a<<=1;l=r(p);break;case 1:for(p=0,c=Math.pow(2,16),a=1;a!=c;)u=A.val&A.position,A.position>>=1,0==A.position&&(A.position=n,A.val=e(A.index++)),p|=(u>0?1:0)*a,a<<=1;l=r(p);break;case 2:return""}for(f[3]=l,s=l,w.push(l);;){if(A.index>o)return"";for(p=0,c=Math.pow(2,m),a=1;a!=c;)u=A.val&A.position,A.position>>=1,0==A.position&&(A.position=n,A.val=e(A.index++)),p|=(u>0?1:0)*a,a<<=1;switch(l=p){case 0:for(p=0,c=Math.pow(2,8),a=1;a!=c;)u=A.val&A.position,A.position>>=1,0==A.position&&(A.position=n,A.val=e(A.index++)),p|=(u>0?1:0)*a,a<<=1;f[d++]=r(p),l=d-1,h--;break;case 1:for(p=0,c=Math.pow(2,16),a=1;a!=c;)u=A.val&A.position,A.position>>=1,0==A.position&&(A.position=n,A.val=e(A.index++)),p|=(u>0?1:0)*a,a<<=1;f[d++]=r(p),l=d-1,h--;break;case 2:return w.join("")}if(0==h&&(h=Math.pow(2,m),m++),f[l])v=f[l];else{if(l!==d)return null;v=s+s.charAt(0)}w.push(v),f[d++]=s+v.charAt(0),h--,s=v,0==h&&(h=Math.pow(2,m),m++)}}};return i}();"function"==typeof define&&define.amd?define(function(){return LZString}):"undefined"!=typeof module&&null!=module&&(module.exports=LZString);

6
apps/android/app/src/main/java/chat/simplex/app/model/SimpleXAPI.kt
View file

@ -338,20 +338,20 @@ open class ChatController(private val ctrl: ChatCtrl, private val ntfManager: Nt
return r is CR.CmdOk
}
suspend fun apiSendCallOffer(contact: Contact, rtcSession: String, rtcIceCandidates: List<String>, media: CallMediaType, capabilities: CallCapabilities): Boolean {
suspend fun apiSendCallOffer(contact: Contact, rtcSession: String, rtcIceCandidates: String, media: CallMediaType, capabilities: CallCapabilities): Boolean {
val webRtcSession = WebRTCSession(rtcSession, rtcIceCandidates)
val callOffer = WebRTCCallOffer(CallType(media, capabilities), webRtcSession)
val r = sendCmd(CC.ApiSendCallOffer(contact, callOffer))
return r is CR.CmdOk
}
suspend fun apiSendCallAnswer(contact: Contact, rtcSession: String, rtcIceCandidates: List<String>): Boolean {
suspend fun apiSendCallAnswer(contact: Contact, rtcSession: String, rtcIceCandidates: String): Boolean {
val answer = WebRTCSession(rtcSession, rtcIceCandidates)
val r = sendCmd(CC.ApiSendCallAnswer(contact, answer))
return r is CR.CmdOk
}
suspend fun apiSendCallExtraInfo(contact: Contact, rtcIceCandidates: List<String>): Boolean {
suspend fun apiSendCallExtraInfo(contact: Contact, rtcIceCandidates: String): Boolean {
val extraInfo = WebRTCExtraInfo(rtcIceCandidates)
val r = sendCmd(CC.ApiSendCallExtraInfo(contact, extraInfo))
return r is CR.CmdOk

211
apps/android/app/src/main/java/chat/simplex/app/views/call/CallView.kt
View file

@ -1,7 +1,6 @@
package chat.simplex.app.views.call
import android.Manifest
import android.annotation.SuppressLint
import android.content.ClipData
import android.content.ClipboardManager
import android.util.Log
@ -24,16 +23,67 @@ import androidx.lifecycle.LifecycleEventObserver
import androidx.webkit.WebViewAssetLoader
import androidx.webkit.WebViewClientCompat
import chat.simplex.app.TAG
import chat.simplex.app.model.json
import chat.simplex.app.views.helpers.TextEditor
import com.google.accompanist.permissions.rememberMultiplePermissionsState
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString
@SuppressLint("SetJavaScriptEnabled")
@Composable
fun VideoCallView(close: () -> Unit) {
val callCommand = remember { mutableStateOf<WCallCommand?>(null)}
val commandText = remember { mutableStateOf("{\"command\": {\"type\": \"start\", \"media\": \"video\", \"aesKey\": \"FwW+t6UbnwHoapYOfN4mUBUuqR7UtvYWxW16iBqM29U=\"}}") }
val clipboard = ContextCompat.getSystemService(LocalContext.current, ClipboardManager::class.java)
BackHandler(onBack = close)
Column(
horizontalAlignment = Alignment.CenterHorizontally,
verticalArrangement = Arrangement.spacedBy(12.dp),
modifier = Modifier
.background(MaterialTheme.colors.background)
.fillMaxSize()
) {
WebRTCView(callCommand) { resp ->
// for debugging
// commandText.value = resp
commandText.value = json.encodeToString(resp)
}
TextEditor(Modifier.height(180.dp), text = commandText)
Row(
Modifier
.fillMaxWidth()
.padding(bottom = 6.dp),
horizontalArrangement = Arrangement.SpaceBetween
) {
Button(onClick = {
val clip: ClipData = ClipData.newPlainText("js command", commandText.value)
clipboard?.setPrimaryClip(clip)
}) { Text("Copy") }
Button(onClick = {
try {
val apiCall: WVAPICall = json.decodeFromString(commandText.value)
commandText.value = ""
println("sending: ${commandText.value}")
callCommand.value = apiCall.command
} catch(e: Error) {
println("error parsing command: ${commandText.value}")
println(e)
}
}) { Text("Send") }
Button(onClick = {
commandText.value = ""
}) { Text("Clear") }
}
}
}
@Composable
// for debugging
// fun WebRTCView(callCommand: MutableState<WCallCommand?>, onResponse: (String) -> Unit) {
fun WebRTCView(callCommand: MutableState<WCallCommand?>, onResponse: (WVAPIMessage) -> Unit) {
lateinit var wv: WebView
val context = LocalContext.current
val clipboard = ContextCompat.getSystemService(context, ClipboardManager::class.java)
val permissionsState = rememberMultiplePermissionsState(
permissions = listOf(
Manifest.permission.CAMERA,
@ -42,6 +92,10 @@ fun VideoCallView(close: () -> Unit) {
Manifest.permission.INTERNET
)
)
fun processCommand(cmd: WCallCommand) {
val apiCall = WVAPICall(command = cmd)
wv.evaluateJavascript("processCommand(${json.encodeToString(apiCall)})", null)
}
val lifecycleOwner = LocalLifecycleOwner.current
DisposableEffect(lifecycleOwner) {
val observer = LifecycleEventObserver { _, event ->
@ -52,96 +106,83 @@ fun VideoCallView(close: () -> Unit) {
lifecycleOwner.lifecycle.addObserver(observer)
onDispose {
wv.evaluateJavascript("endCall()", null)
processCommand(WCallCommand.End())
lifecycleOwner.lifecycle.removeObserver(observer)
}
}
val localContext = LocalContext.current
val commandToShow = remember { mutableStateOf("processCommand({command: {type: 'start', media: 'video'}})") } //, aesKey: 'FwW+t6UbnwHoapYOfN4mUBUuqR7UtvYWxW16iBqM29U='})") }
LaunchedEffect(callCommand.value) {
val cmd = callCommand.value
if (cmd != null) {
callCommand.value = null
processCommand(cmd)
}
}
val assetLoader = WebViewAssetLoader.Builder()
.addPathHandler("/assets/www/", WebViewAssetLoader.AssetsPathHandler(localContext))
.addPathHandler("/assets/www/", WebViewAssetLoader.AssetsPathHandler(LocalContext.current))
.build()
Column(
horizontalAlignment = Alignment.CenterHorizontally,
verticalArrangement = Arrangement.spacedBy(12.dp),
modifier = Modifier
.background(MaterialTheme.colors.background)
.fillMaxSize()
) {
if (permissionsState.allPermissionsGranted) {
Box(
Modifier
.fillMaxWidth()
.aspectRatio(ratio = 1F)
) {
AndroidView(
factory = { AndroidViewContext ->
WebView(AndroidViewContext).apply {
layoutParams = ViewGroup.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.MATCH_PARENT,
)
this.webChromeClient = object: WebChromeClient() {
override fun onPermissionRequest(request: PermissionRequest) {
if (request.origin.toString().startsWith("file:/")) {
request.grant(request.resources)
} else {
Log.d(TAG, "Permission request from webview denied.")
request.deny()
}
}
override fun onConsoleMessage(consoleMessage: ConsoleMessage?): Boolean {
val rtnValue = super.onConsoleMessage(consoleMessage)
val msg = consoleMessage?.message() as String
if (msg.startsWith("{")) {
commandToShow.value = "processCommand($msg)"
}
return rtnValue
}
}
this.webViewClient = LocalContentWebViewClient(assetLoader)
this.clearHistory()
this.clearCache(true)
// this.addJavascriptInterface(JavascriptInterface(), "Android")
val webViewSettings = this.settings
webViewSettings.allowFileAccess = true
webViewSettings.allowContentAccess = true
webViewSettings.javaScriptEnabled = true
webViewSettings.mediaPlaybackRequiresUserGesture = false
webViewSettings.cacheMode = WebSettings.LOAD_NO_CACHE
this.loadUrl("file:android_asset/www/call.html")
}
}
) {
wv = it
}
}
} else {
Text("NEED PERMISSIONS")
}
TextEditor(Modifier.height(180.dp), text = commandToShow)
Row(
if (permissionsState.allPermissionsGranted) {
Box(
Modifier
.fillMaxWidth()
.padding(bottom = 6.dp),
horizontalArrangement = Arrangement.SpaceBetween
.aspectRatio(ratio = 1F)
) {
Button( onClick = {
val clip: ClipData = ClipData.newPlainText("js command", commandToShow.value)
clipboard?.setPrimaryClip(clip)
}) {Text("Copy")}
Button( onClick = {
println("sending: ${commandToShow.value}")
wv.evaluateJavascript(commandToShow.value, null)
commandToShow.value = ""
}) {Text("Send")}
Button( onClick = {
commandToShow.value = ""
}) {Text("Clear")}
AndroidView(
factory = { AndroidViewContext ->
WebView(AndroidViewContext).apply {
layoutParams = ViewGroup.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.MATCH_PARENT,
)
this.webChromeClient = object: WebChromeClient() {
override fun onPermissionRequest(request: PermissionRequest) {
if (request.origin.toString().startsWith("file:/")) {
request.grant(request.resources)
} else {
Log.d(TAG, "Permission request from webview denied.")
request.deny()
}
}
}
this.webViewClient = LocalContentWebViewClient(assetLoader)
this.clearHistory()
this.clearCache(true)
this.addJavascriptInterface(WebRTCInterface(onResponse), "WebRTCInterface")
val webViewSettings = this.settings
webViewSettings.allowFileAccess = true
webViewSettings.allowContentAccess = true
webViewSettings.javaScriptEnabled = true
webViewSettings.mediaPlaybackRequiresUserGesture = false
webViewSettings.allowFileAccessFromFileURLs = true;
webViewSettings.cacheMode = WebSettings.LOAD_NO_CACHE
this.loadUrl("file:android_asset/www/call.html")
}
}
) {
wv = it
// for debugging
// wv.evaluateJavascript("sendMessageToNative = ({resp}) => WebRTCInterface.postMessage(JSON.stringify({command: resp}))", null)
wv.evaluateJavascript("sendMessageToNative = (msg) => WebRTCInterface.postMessage(JSON.stringify(msg))", null)
}
}
} else {
Text("NEED PERMISSIONS")
}
}
// for debugging
// class WebRTCInterface(private val onResponse: (String) -> Unit) {
class WebRTCInterface(private val onResponse: (WVAPIMessage) -> Unit) {
@JavascriptInterface
fun postMessage(message: String) {
Log.d(TAG, "WebRTCInterface.postMessage")
try {
// for debugging
// onResponse(message)
onResponse(json.decodeFromString(message))
} catch (e: Error) {
Log.e(TAG, "failed parsing WebView message: $message")
}
}
}

35
apps/android/app/src/main/java/chat/simplex/app/views/call/WebRTC.kt
View file

@ -40,15 +40,15 @@ enum class CallState {
}
@Serializable class WVAPICall(val corrId: Int? = null, val command: WCallCommand)
@Serializable class WVAPIMessage(val corrId: Int? = null, val resp: WCallResponse, val command: WCallCommand?)
@Serializable class WVAPIMessage(val corrId: Int? = null, val resp: WCallResponse, val command: WCallCommand? = null)
@Serializable
sealed class WCallCommand {
@Serializable @SerialName("capabilities") class Capabilities(): WCallCommand()
@Serializable @SerialName("start") class Start(val media: CallMediaType, val aesKey: String? = null): WCallCommand()
@Serializable @SerialName("accept") class Accept(val offer: String, val iceCandidates: List<String>, val media: CallMediaType, val aesKey: String? = null): WCallCommand()
@Serializable @SerialName("answer") class Answer (val answer: String, val iceCandidates: List<String>): WCallCommand()
@Serializable @SerialName("ice") class Ice(val iceCandidates: List<String>): WCallCommand()
@Serializable @SerialName("offer") class Accept(val offer: String, val iceCandidates: String, val media: CallMediaType, val aesKey: String? = null): WCallCommand()
@Serializable @SerialName("answer") class Answer (val answer: String, val iceCandidates: String): WCallCommand()
@Serializable @SerialName("ice") class Ice(val iceCandidates: String): WCallCommand()
@Serializable @SerialName("media") class Media(val media: CallMediaType, val enable: Boolean): WCallCommand()
@Serializable @SerialName("end") class End(): WCallCommand()
}
@ -56,11 +56,11 @@ sealed class WCallCommand {
@Serializable
sealed class WCallResponse {
@Serializable @SerialName("capabilities") class Capabilities(val capabilities: CallCapabilities): WCallResponse()
@Serializable @SerialName("offer") class Offer(val offer: String, val iceCandidates: List<String>): WCallResponse()
@Serializable @SerialName("offer") class Offer(val offer: String, val iceCandidates: String): WCallResponse()
// TODO remove accept, it is needed for debugging
@Serializable @SerialName("accept") class Accept(val offer: String, val iceCandidates: List<String>, val media: CallMediaType, val aesKey: String? = null): WCallResponse()
@Serializable @SerialName("answer") class Answer(val answer: String, val iceCandidates: List<String>): WCallResponse()
@Serializable @SerialName("ice") class Ice(val iceCandidates: List<String>): WCallResponse()
@Serializable @SerialName("accept") class Accept(val offer: String, val iceCandidates: String, val media: CallMediaType, val aesKey: String? = null): WCallResponse()
@Serializable @SerialName("answer") class Answer(val answer: String, val iceCandidates: String): WCallResponse()
@Serializable @SerialName("ice") class Ice(val iceCandidates: String): WCallResponse()
@Serializable @SerialName("connection") class Connection(val state: ConnectionState): WCallResponse()
@Serializable @SerialName("ended") class Ended(): WCallResponse()
@Serializable @SerialName("ok") class Ok(): WCallResponse()
@ -69,22 +69,23 @@ sealed class WCallResponse {
}
@Serializable class WebRTCCallOffer(val callType: CallType, val rtcSession: WebRTCSession)
@Serializable class WebRTCSession(val rtcSession: String, val rtcIceCandidates: List<String>)
@Serializable class WebRTCExtraInfo(val rtcIceCandidates: List<String>)
@Serializable class WebRTCSession(val rtcSession: String, val rtcIceCandidates: String)
@Serializable class WebRTCExtraInfo(val rtcIceCandidates: String)
@Serializable class CallType(val media: CallMediaType, val capabilities: CallCapabilities)
@Serializable class CallInvitation(val peerMedia: CallMediaType?, val sharedKey: String?)
@Serializable class CallCapabilities(val encryption: Boolean)
enum class WebRTCCallStatus(val status: String) {
Connected("connected"),
Disconnected("disconnected"),
Failed("failed")
@Serializable
enum class WebRTCCallStatus {
@SerialName("connected") Connected,
@SerialName("disconnected") Disconnected,
@SerialName("failed") Failed
}
@Serializable
enum class CallMediaType(val media: String) {
Video("video"),
Audio("audio")
enum class CallMediaType {
@SerialName("video") Video,
@SerialName("audio") Audio
}
@Serializable

4
apps/android/app/src/main/java/chat/simplex/app/views/usersettings/SettingsView.kt
View file

@ -208,8 +208,8 @@ fun SettingsLayout(
Text(annotatedStringResource(R.string.install_simplex_chat_for_terminal))
}
Divider(Modifier.padding(horizontal = 8.dp))
// SettingsSectionView(showVideoChatPrototype) {
SettingsSectionView() {
SettingsSectionView(showVideoChatPrototype) {
// SettingsSectionView() {
Text("v${BuildConfig.VERSION_NAME} (${BuildConfig.VERSION_CODE})")
}
}

4
apps/ios/Shared/Model/Shared/CallTypes.swift
View file

@ -15,11 +15,11 @@ struct WebRTCCallOffer: Encodable {
struct WebRTCSession: Codable {
var rtcSession: String
var rtcIceCandidates: [String]
var rtcIceCandidates: String
}
struct WebRTCExtraInfo: Codable {
var rtcIceCandidates: [String]
var rtcIceCandidates: String
}
struct CallInvitation {

8
apps/ios/Shared/Model/SimpleXAPI.swift
View file

@ -382,18 +382,18 @@ func apiRejectCall(_ contact: Contact) async throws {
try await sendCommandOkResp(.apiRejectCall(contact: contact))
}
func apiSendCallOffer(_ contact: Contact, _ rtcSession: String, _ rtcIceCandidates: [String], media: CallMediaType, capabilities: CallCapabilities) async throws {
func apiSendCallOffer(_ contact: Contact, _ rtcSession: String, _ rtcIceCandidates: String, media: CallMediaType, capabilities: CallCapabilities) async throws {
let webRtcSession = WebRTCSession(rtcSession: rtcSession, rtcIceCandidates: rtcIceCandidates)
let callOffer = WebRTCCallOffer(callType: CallType(media: media, capabilities: capabilities), rtcSession: webRtcSession)
try await sendCommandOkResp(.apiSendCallOffer(contact: contact, callOffer: callOffer))
}
func apiSendCallAnswer(_ contact: Contact, _ rtcSession: String, _ rtcIceCandidates: [String]) async throws {
func apiSendCallAnswer(_ contact: Contact, _ rtcSession: String, _ rtcIceCandidates: String) async throws {
let answer = WebRTCSession(rtcSession: rtcSession, rtcIceCandidates: rtcIceCandidates)
try await sendCommandOkResp(.apiSendCallAnswer(contact: contact, answer: answer))
}
func apiSendCallExtraInfo(_ contact: Contact, _ rtcIceCandidates: [String]) async throws {
func apiSendCallExtraInfo(_ contact: Contact, _ rtcIceCandidates: String) async throws {
let extraInfo = WebRTCExtraInfo(rtcIceCandidates: rtcIceCandidates)
try await sendCommandOkResp(.apiSendCallExtraInfo(contact: contact, extraInfo: extraInfo))
}
@ -624,7 +624,7 @@ func processReceivedMsg(_ res: ChatResponse) {
// TODO check encryption is compatible
withCall(contact) { call in
m.activeCall = call.copy(callState: .offerReceived, peerMedia: callType.media, sharedKey: sharedKey)
m.callCommand = .accept(offer: offer.rtcSession, iceCandidates: offer.rtcIceCandidates, media: callType.media, aesKey: sharedKey)
m.callCommand = .offer(offer: offer.rtcSession, iceCandidates: offer.rtcIceCandidates, media: callType.media, aesKey: sharedKey)
}
case let .callAnswer(contact, answer):
withCall(contact) { call in

122
apps/ios/Shared/Views/Call/WebRTC.swift
View file

@ -97,18 +97,18 @@ struct WVAPICall: Encodable {
var command: WCallCommand
}
struct WVAPIMessage: Equatable, Decodable {
struct WVAPIMessage: Equatable, Decodable, Encodable {
var corrId: Int?
var resp: WCallResponse
var command: WCallCommand?
}
enum WCallCommand: Equatable, Encodable, Decodable {
case capabilities
case start(media: CallMediaType, aesKey: String? = nil)
case accept(offer: String, iceCandidates: [String], media: CallMediaType, aesKey: String? = nil)
case answer(answer: String, iceCandidates: [String])
case ice(iceCandidates: [String])
case capabilities(useWorker: Bool? = nil)
case start(media: CallMediaType, aesKey: String? = nil, useWorker: Bool? = nil)
case offer(offer: String, iceCandidates: String, media: CallMediaType, aesKey: String? = nil, useWorker: Bool? = nil)
case answer(answer: String, iceCandidates: String)
case ice(iceCandidates: String)
case media(media: CallMediaType, enable: Bool)
case end
@ -116,6 +116,7 @@ enum WCallCommand: Equatable, Encodable, Decodable {
case type
case media
case aesKey
case useWorker
case offer
case answer
case iceCandidates
@ -127,7 +128,7 @@ enum WCallCommand: Equatable, Encodable, Decodable {
switch self {
case .capabilities: return "capabilities"
case .start: return "start"
case .accept: return "accept"
case .offer: return "offer"
case .answer: return "answer"
case .ice: return "ice"
case .media: return "media"
@ -139,18 +140,21 @@ enum WCallCommand: Equatable, Encodable, Decodable {
func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
switch self {
case .capabilities:
case let .capabilities(useWorker):
try container.encode("capabilities", forKey: .type)
case let .start(media, aesKey):
try container.encode(useWorker, forKey: .useWorker)
case let .start(media, aesKey, useWorker):
try container.encode("start", forKey: .type)
try container.encode(media, forKey: .media)
try container.encode(aesKey, forKey: .aesKey)
case let .accept(offer, iceCandidates, media, aesKey):
try container.encode(useWorker, forKey: .useWorker)
case let .offer(offer, iceCandidates, media, aesKey, useWorker):
try container.encode("accept", forKey: .type)
try container.encode(offer, forKey: .offer)
try container.encode(iceCandidates, forKey: .iceCandidates)
try container.encode(media, forKey: .media)
try container.encode(aesKey, forKey: .aesKey)
try container.encode(useWorker, forKey: .useWorker)
case let .answer(answer, iceCandidates):
try container.encode("answer", forKey: .type)
try container.encode(answer, forKey: .answer)
@ -172,23 +176,26 @@ enum WCallCommand: Equatable, Encodable, Decodable {
let type = try container.decode(String.self, forKey: CodingKeys.type)
switch type {
case "capabilities":
self = .capabilities
let useWorker = try container.decode((Bool?).self, forKey: CodingKeys.useWorker)
self = .capabilities(useWorker: useWorker)
case "start":
let media = try container.decode(CallMediaType.self, forKey: CodingKeys.media)
let aesKey = try? container.decode(String.self, forKey: CodingKeys.aesKey)
self = .start(media: media, aesKey: aesKey)
case "accept":
let useWorker = try container.decode((Bool?).self, forKey: CodingKeys.useWorker)
self = .start(media: media, aesKey: aesKey, useWorker: useWorker)
case "offer":
let offer = try container.decode(String.self, forKey: CodingKeys.offer)
let iceCandidates = try container.decode([String].self, forKey: CodingKeys.iceCandidates)
let iceCandidates = try container.decode(String.self, forKey: CodingKeys.iceCandidates)
let media = try container.decode(CallMediaType.self, forKey: CodingKeys.media)
let aesKey = try? container.decode(String.self, forKey: CodingKeys.aesKey)
self = .accept(offer: offer, iceCandidates: iceCandidates, media: media, aesKey: aesKey)
let useWorker = try container.decode((Bool?).self, forKey: CodingKeys.useWorker)
self = .offer(offer: offer, iceCandidates: iceCandidates, media: media, aesKey: aesKey, useWorker: useWorker)
case "answer":
let answer = try container.decode(String.self, forKey: CodingKeys.answer)
let iceCandidates = try container.decode([String].self, forKey: CodingKeys.iceCandidates)
let iceCandidates = try container.decode(String.self, forKey: CodingKeys.iceCandidates)
self = .answer(answer: answer, iceCandidates: iceCandidates)
case "ice":
let iceCandidates = try container.decode([String].self, forKey: CodingKeys.iceCandidates)
let iceCandidates = try container.decode(String.self, forKey: CodingKeys.iceCandidates)
self = .ice(iceCandidates: iceCandidates)
case "media":
let media = try container.decode(CallMediaType.self, forKey: CodingKeys.media)
@ -205,11 +212,11 @@ enum WCallCommand: Equatable, Encodable, Decodable {
enum WCallResponse: Equatable, Decodable {
case capabilities(capabilities: CallCapabilities)
case offer(offer: String, iceCandidates: [String], capabilities: CallCapabilities)
case offer(offer: String, iceCandidates: String, capabilities: CallCapabilities)
// TODO remove accept, it is needed for debugging
// case accept(offer: String, iceCandidates: [String], media: CallMediaType, aesKey: String? = nil)
case answer(answer: String, iceCandidates: [String])
case ice(iceCandidates: [String])
// case offer(offer: String, iceCandidates: [String], media: CallMediaType, aesKey: String? = nil)
case answer(answer: String, iceCandidates: String)
case ice(iceCandidates: String)
case connection(state: ConnectionState)
case ended
case ok
@ -234,7 +241,6 @@ enum WCallResponse: Equatable, Decodable {
switch self {
case .capabilities: return("capabilities")
case .offer: return("offer")
// case .accept: return("accept")
case .answer: return("answer (TODO remove)")
case .ice: return("ice")
case .connection: return("connection")
@ -256,15 +262,15 @@ enum WCallResponse: Equatable, Decodable {
self = .capabilities(capabilities: capabilities)
case "offer":
let offer = try container.decode(String.self, forKey: CodingKeys.offer)
let iceCandidates = try container.decode([String].self, forKey: CodingKeys.iceCandidates)
let iceCandidates = try container.decode(String.self, forKey: CodingKeys.iceCandidates)
let capabilities = try container.decode(CallCapabilities.self, forKey: CodingKeys.capabilities)
self = .offer(offer: offer, iceCandidates: iceCandidates, capabilities: capabilities)
case "answer":
let answer = try container.decode(String.self, forKey: CodingKeys.answer)
let iceCandidates = try container.decode([String].self, forKey: CodingKeys.iceCandidates)
let iceCandidates = try container.decode(String.self, forKey: CodingKeys.iceCandidates)
self = .answer(answer: answer, iceCandidates: iceCandidates)
case "ice":
let iceCandidates = try container.decode([String].self, forKey: CodingKeys.iceCandidates)
let iceCandidates = try container.decode(String.self, forKey: CodingKeys.iceCandidates)
self = .ice(iceCandidates: iceCandidates)
case "connection":
let state = try container.decode(ConnectionState.self, forKey: CodingKeys.state)
@ -286,39 +292,39 @@ enum WCallResponse: Equatable, Decodable {
}
// This protocol is for debugging
//extension WCallResponse: Encodable {
// func encode(to encoder: Encoder) throws {
// var container = encoder.container(keyedBy: CodingKeys.self)
// switch self {
// case .capabilities:
// try container.encode("capabilities", forKey: .type)
// case let .offer(offer, iceCandidates, capabilities):
// try container.encode("offer", forKey: .type)
// try container.encode(offer, forKey: .offer)
// try container.encode(iceCandidates, forKey: .iceCandidates)
// try container.encode(capabilities, forKey: .capabilities)
// case let .answer(answer, iceCandidates):
// try container.encode("answer", forKey: .type)
// try container.encode(answer, forKey: .answer)
// try container.encode(iceCandidates, forKey: .iceCandidates)
// case let .ice(iceCandidates):
// try container.encode("ice", forKey: .type)
// try container.encode(iceCandidates, forKey: .iceCandidates)
// case let .connection(state):
// try container.encode("connection", forKey: .type)
// try container.encode(state, forKey: .state)
// case .ended:
// try container.encode("ended", forKey: .type)
// case .ok:
// try container.encode("ok", forKey: .type)
// case let .error(message):
// try container.encode("error", forKey: .type)
// try container.encode(message, forKey: .message)
// case let .invalid(type):
// try container.encode(type, forKey: .type)
// }
// }
//}
extension WCallResponse: Encodable {
func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
switch self {
case .capabilities:
try container.encode("capabilities", forKey: .type)
case let .offer(offer, iceCandidates, capabilities):
try container.encode("offer", forKey: .type)
try container.encode(offer, forKey: .offer)
try container.encode(iceCandidates, forKey: .iceCandidates)
try container.encode(capabilities, forKey: .capabilities)
case let .answer(answer, iceCandidates):
try container.encode("answer", forKey: .type)
try container.encode(answer, forKey: .answer)
try container.encode(iceCandidates, forKey: .iceCandidates)
case let .ice(iceCandidates):
try container.encode("ice", forKey: .type)
try container.encode(iceCandidates, forKey: .iceCandidates)
case let .connection(state):
try container.encode("connection", forKey: .type)
try container.encode(state, forKey: .state)
case .ended:
try container.encode("ended", forKey: .type)
case .ok:
try container.encode("ok", forKey: .type)
case let .error(message):
try container.encode("error", forKey: .type)
try container.encode(message, forKey: .message)
case let .invalid(type):
try container.encode(type, forKey: .type)
}
}
}
struct ConnectionState: Codable, Equatable {
var connectionState: String

139
apps/ios/Shared/Views/Call/WebRTCView.swift
View file

@ -31,6 +31,7 @@ class WebRTCCoordinator: NSObject, WKNavigationDelegate, WKScriptMessageHandler
didReceive message: WKScriptMessage
) {
logger.debug("WebRTCCoordinator.userContentController")
logger.debug("\(String(describing: message.body as? String))")
if let msgStr = message.body as? String,
let msg: WVAPIMessage = decodeJSON(msgStr) {
webViewMsg.wrappedValue = msg
@ -90,74 +91,76 @@ struct WebRTCView: UIViewRepresentable {
}
}
//struct CallViewDebug: View {
// @State private var coordinator: WebRTCCoordinator? = nil
// @State private var commandStr = ""
// @State private var webViewReady: Bool = false
// @State private var webViewMsg: WVAPIMessage? = nil
// @FocusState private var keyboardVisible: Bool
//
// var body: some View {
// VStack(spacing: 30) {
// WebRTCView(coordinator: $coordinator, webViewReady: $webViewReady, webViewMsg: $webViewMsg).frame(maxHeight: 260)
// .onChange(of: webViewMsg) { _ in
// if let resp = webViewMsg {
// commandStr = encodeJSON(resp)
// }
// }
// TextEditor(text: $commandStr)
// .focused($keyboardVisible)
// .disableAutocorrection(true)
// .textInputAutocapitalization(.never)
// .padding(.horizontal, 5)
// .padding(.top, 2)
// .frame(height: 112)
// .overlay(
// RoundedRectangle(cornerRadius: 10)
// .strokeBorder(.secondary, lineWidth: 0.3, antialiased: true)
// )
// HStack(spacing: 20) {
// Button("Copy") {
// UIPasteboard.general.string = commandStr
// }
// Button("Paste") {
// commandStr = UIPasteboard.general.string ?? ""
// }
// Button("Clear") {
// commandStr = ""
// }
// Button("Send") {
// if let c = coordinator,
// let command: WCallCommand = decodeJSON(commandStr) {
// c.sendCommand(command: command)
// }
// }
// }
// HStack(spacing: 20) {
// Button("Capabilities") {
//
// }
// Button("Start") {
// if let c = coordinator {
// c.sendCommand(command: .start(media: .video))
// }
// }
// Button("Accept") {
//
// }
// Button("Answer") {
//
// }
// Button("ICE") {
//
// }
// Button("End") {
//
// }
// }
// }
// }
//}
struct CallViewDebug: View {
@State private var coordinator: WebRTCCoordinator? = nil
@State private var commandStr = ""
@State private var webViewReady: Bool = false
@State private var webViewMsg: WVAPIMessage? = nil
@FocusState private var keyboardVisible: Bool
var body: some View {
VStack(spacing: 30) {
WebRTCView(coordinator: $coordinator, webViewReady: $webViewReady, webViewMsg: $webViewMsg).frame(maxHeight: 260)
.onChange(of: webViewMsg) { _ in
if let resp = webViewMsg {
commandStr = encodeJSON(resp)
}
}
TextEditor(text: $commandStr)
.focused($keyboardVisible)
.disableAutocorrection(true)
.textInputAutocapitalization(.never)
.padding(.horizontal, 5)
.padding(.top, 2)
.frame(height: 112)
.overlay(
RoundedRectangle(cornerRadius: 10)
.strokeBorder(.secondary, lineWidth: 0.3, antialiased: true)
)
HStack(spacing: 20) {
Button("Copy") {
UIPasteboard.general.string = commandStr
}
Button("Paste") {
commandStr = UIPasteboard.general.string ?? ""
}
Button("Clear") {
commandStr = ""
}
Button("Send") {
if let c = coordinator,
let command: WCallCommand = decodeJSON(commandStr) {
c.sendCommand(command: command)
}
}
}
HStack(spacing: 20) {
Button("Capabilities") {
if let c = coordinator {
c.sendCommand(command: .capabilities(useWorker: true))
}
}
Button("Start") {
if let c = coordinator {
c.sendCommand(command: .start(media: .video))
}
}
Button("Accept") {
}
Button("Answer") {
}
Button("ICE") {
}
Button("End") {
}
}
}
}
}
//
//struct CallViewDebug_Previews: PreviewProvider {
// static var previews: some View {

2
apps/ios/Shared/Views/Chat/ChatView.swift
View file

@ -126,7 +126,7 @@ struct ChatView: View {
localMedia: media
)
showCallView = true
chatModel.callCommand = .capabilities
chatModel.callCommand = .capabilities(useWorker: true)
} label: {
Image(systemName: imageName)
}

10
apps/ios/Shared/Views/UserSettings/SettingsView.swift
View file

@ -126,12 +126,12 @@ struct SettingsView: View {
// notificationsToggle(token)
// }
// }
// NavigationLink {
// CallViewDebug()
// .frame(maxHeight: .infinity, alignment: .top)
// } label: {
NavigationLink {
CallViewDebug()
.frame(maxHeight: .infinity, alignment: .top)
} label: {
Text("v\(appVersion ?? "?") (\(appBuild ?? "?"))")
// }
}
}
}
.navigationTitle("Your settings")

4
packages/simplex-chat-webrtc/copy
View file

@ -3,8 +3,12 @@
# it can be tested in the browser from dist folder
cp ./src/call.html ./dist/call.html
cp ./src/style.css ./dist/style.css
cp ./node_modules/lz-string/libs/lz-string.min.js ./dist/lz-string.min.js
cp ./src/webcall.html ./dist/webcall.html
cp ./src/ui.js ./dist/ui.js
# copy to android app
cp ./src/call.html ../../apps/android/app/src/main/assets/www/call.html
cp ./src/style.css ../../apps/android/app/src/main/assets/www/style.css
cp ./dist/call.js ../../apps/android/app/src/main/assets/www/call.js
cp ./node_modules/lz-string/libs/lz-string.min.js ../../apps/android/app/src/main/assets/www/lz-string.min.js

4
packages/simplex-chat-webrtc/package.json
View file

@ -14,6 +14,7 @@
"author": "",
"license": "AGPL-3.0-or-later",
"devDependencies": {
"@types/lz-string": "^1.3.34",
"husky": "^7.0.4",
"lint-staged": "^12.4.1",
"prettier": "^2.6.2",
@ -21,5 +22,8 @@
},
"lint-staged": {
"**/*": "prettier --write --ignore-unknown"
},
"dependencies": {
"lz-string": "^1.4.4"
}
}

1
packages/simplex-chat-webrtc/src/call.html
View file

@ -2,6 +2,7 @@
<html>
<head>
<link href="./style.css" rel="stylesheet" />
<script src="./lz-string.min.js"></script>
</head>
<body>
<video id="remote-video-stream" autoplay playsinline></video>

1099
packages/simplex-chat-webrtc/src/call.ts
View file

File diff suppressed because it is too large Load diff

115
packages/simplex-chat-webrtc/src/ui.js Normal file
View file

@ -0,0 +1,115 @@
;(async function run() {
const START_E2EE_CALL_BTN = "start-e2ee-call"
const START_CALL_BTN = "start-call"
const URL_FOR_PEER = "url-for-peer"
const COPY_URL_FOR_PEER_BTN = "copy-url-for-peer"
const DATA_FOR_PEER = "data-for-peer"
const COPY_DATA_FOR_PEER_BTN = "copy-data-for-peer"
const PASS_DATA_TO_PEER_TEXT = "pass-data-to-peer"
const CHAT_COMMAND_FOR_PEER = "chat-command-for-peer"
const COMMAND_TO_PROCESS = "command-to-process"
const PROCESS_COMMAND_BTN = "process-command"
const urlForPeer = document.getElementById(URL_FOR_PEER)
const dataForPeer = document.getElementById(DATA_FOR_PEER)
const passDataToPeerText = document.getElementById(PASS_DATA_TO_PEER_TEXT)
const chatCommandForPeer = document.getElementById(CHAT_COMMAND_FOR_PEER)
const commandToProcess = document.getElementById(COMMAND_TO_PROCESS)
const processCommandButton = document.getElementById(PROCESS_COMMAND_BTN)
const startE2EECallButton = document.getElementById(START_E2EE_CALL_BTN)
const {resp} = await processCommand({command: {type: "capabilities", useWorker: true}})
if (resp?.capabilities?.encryption) {
startE2EECallButton.onclick = startCall(true)
} else {
startE2EECallButton.style.display = "none"
}
const startCallButton = document.getElementById(START_CALL_BTN)
startCallButton.onclick = startCall()
const copyUrlButton = document.getElementById(COPY_URL_FOR_PEER_BTN)
copyUrlButton.onclick = () => {
navigator.clipboard.writeText(urlForPeer.innerText)
commandToProcess.style.display = ""
processCommandButton.style.display = ""
}
const copyDataButton = document.getElementById(COPY_DATA_FOR_PEER_BTN)
copyDataButton.onclick = () => {
navigator.clipboard.writeText(dataForPeer.innerText)
commandToProcess.style.display = ""
processCommandButton.style.display = ""
}
processCommandButton.onclick = () => {
sendCommand(JSON.parse(commandToProcess.value))
commandToProcess.value = ""
}
const parsed = new URLSearchParams(document.location.hash.substring(1))
let apiCallStr = parsed.get("command")
if (apiCallStr) {
startE2EECallButton.style.display = "none"
startCallButton.style.display = "none"
await sendCommand(JSON.parse(decodeURIComponent(apiCallStr)))
}
function startCall(encryption) {
return async () => {
let aesKey
if (encryption) {
const key = await crypto.subtle.generateKey({name: "AES-GCM", length: 256}, true, ["encrypt", "decrypt"])
const keyBytes = await crypto.subtle.exportKey("raw", key)
aesKey = callCrypto.decodeAscii(callCrypto.encodeBase64(new Uint8Array(keyBytes)))
}
sendCommand({command: {type: "start", media: "video", aesKey, useWorker: true}})
startE2EECallButton.style.display = "none"
startCallButton.style.display = "none"
}
}
async function sendCommand(apiCall) {
try {
console.log(apiCall)
const {command} = apiCall
const {resp} = await processCommand(apiCall)
console.log(resp)
switch (resp.type) {
case "offer": {
const {media, aesKey} = command
const {offer, iceCandidates, capabilities} = resp
const peerWCommand = {
command: {type: "offer", offer, iceCandidates, media, aesKey: capabilities.encryption ? aesKey : undefined, useWorker: true},
}
const url = new URL(document.location)
parsed.set("command", encodeURIComponent(JSON.stringify(peerWCommand)))
url.hash = parsed.toString()
urlForPeer.innerText = url.toString()
dataForPeer.innerText = JSON.stringify(peerWCommand)
copyUrlButton.style.display = ""
copyDataButton.style.display = ""
// const webRTCCallOffer = {callType: {media, capabilities}, rtcSession: {rtcSession: offer, rtcIceCandidates: iceCandidates}}
// const peerChatCommand = `/_call @${parsed.contact} offer ${JSON.stringify(webRTCCallOffer)}`
// chatCommandForPeer.innerText = peerChatCommand
return
}
case "answer": {
const {answer, iceCandidates} = resp
const peerWCommand = {command: {type: "answer", answer, iceCandidates}}
dataForPeer.innerText = JSON.stringify(peerWCommand)
copyUrlButton.style.display = "none"
copyDataButton.style.display = ""
// const webRTCSession = {rtcSession: answer, rtcIceCandidates: iceCandidates}
// const peerChatCommand = `/_call @${parsed.contact} answer ${JSON.stringify(webRTCSession)}`
// chatCommandForPeer.innerText = peerChatCommand
return
}
case "ok":
if ((command.type = "answer")) {
console.log("connecting")
commandToProcess.style.display = "none"
processCommandButton.style.display = "none"
}
return
}
} catch (e) {
console.log("error: ", e)
}
}
})()

51
packages/simplex-chat-webrtc/src/webcall.html Normal file
View file

@ -0,0 +1,51 @@
<!DOCTYPE html>
<html>
<head>
<link href="./style.css" rel="stylesheet" />
<script src="./lz-string.min.js"></script>
<style>
#data-for-peer,
#url-for-peer,
#chat-command-for-peer,
#pass-data-to-peer {
position: absolute;
color: white;
}
#ui-overlay {
position: absolute;
}
</style>
</head>
<body>
<video id="remote-video-stream" autoplay playsinline></video>
<video id="local-video-stream" muted autoplay playsinline></video>
<div id="ui-overlay">
<div>
<button id="start-e2ee-call" type="submit">Start call with e2ee</button>
<button id="start-call" type="submit">Start call</button>
<button id="copy-url-for-peer" type="submit" style="display: none">Copy url for your contact</button>
<button id="copy-data-for-peer" type="submit" style="display: none">Copy data for your contact</button>
<p id="pass-data-to-peer" style="display: none">Send copied data back to your contact</p>
</div>
<div>
<textarea
id="command-to-process"
cols="30"
rows="10"
style="display: none"
placeholder="Pass copied URL to your contact. Once they send you back the data, paste it here and click Connect"
></textarea>
</div>
<div>
<button id="process-command" type="submit" style="display: none">Connect</button>
</div>
<div id="url-for-peer" style="display: none"></div>
<div id="data-for-peer" style="display: none"></div>
<div id="chat-command-for-peer" style="display: none"></div>
</div>
</body>
<footer>
<script src="./call.js"></script>
<script src="./ui.js"></script>
</footer>
</html>

3
packages/simplex-chat-webrtc/tsconfig.json
View file

@ -16,6 +16,7 @@
"sourceMap": true,
"strict": true,
"strictNullChecks": true,
"target": "ES2018"
"target": "ES2018",
"types": ["lz-string"]
}
}

6
src/Simplex/Chat/Call.hs
View file

@ -177,8 +177,8 @@ instance ToJSON CallExtraInfo where
toEncoding = J.genericToEncoding J.defaultOptions
data WebRTCSession = WebRTCSession
{ rtcSession :: Text,
rtcIceCandidates :: [Text]
{ rtcSession :: Text, -- LZW compressed JSON encoding of offer or answer
rtcIceCandidates :: Text -- LZW compressed JSON encoding of array of ICE candidates
}
deriving (Eq, Show, Generic, FromJSON)
@ -187,7 +187,7 @@ instance ToJSON WebRTCSession where
toEncoding = J.genericToEncoding J.defaultOptions
data WebRTCExtraInfo = WebRTCExtraInfo
{ rtcIceCandidates :: [Text]
{ rtcIceCandidates :: Text -- LZW compressed JSON encoding of array of ICE candidates
}
deriving (Eq, Show, Generic, FromJSON)

2
tests/ChatTests.hs
View file

@ -1870,7 +1870,7 @@ testWebRTCSession :: WebRTCSession
testWebRTCSession =
WebRTCSession
{ rtcSession = "{}",
rtcIceCandidates = [""]
rtcIceCandidates = "[]"
}
testWebRTCCallOffer :: WebRTCCallOffer