Video conferencing skeleton.

This commit is contained in:
aforge
2023-05-03 22:50:51 -07:00
parent f9e7a71920
commit 0425bd4348
10 changed files with 808 additions and 33 deletions
+12 -5
View File
@@ -7,11 +7,17 @@ import { secondsToTime } from '../lib/strformat.js';
export default class CallMessage extends React.PureComponent {
render() {
const bigIcon = this.props.vc ? 'video_call' : 'call';
const isCallDropped = ['busy', 'declined', 'disconnected', 'missed'].includes(this.props.callState);
const successClass = 'material-icons medium ' + (isCallDropped ? 'red' : 'green');
const callIcon = this.props.incoming ?
(isCallDropped ? 'call_missed' : 'call_received') :
(isCallDropped ? 'call_missed_outgoing' : 'call_made');
const title = this.props.vc ?
<FormattedMessage id="calls_conference" defaultMessage="Conference call" description="Conference call label" /> :
this.props.incoming ?
<FormattedMessage id="calls_incoming" defaultMessage="Incoming call" description="Incoming call label" /> :
<FormattedMessage id="calls_outgoing" defaultMessage="Outgoing call" description="Outgoing call label" />;
let duration;
if (isCallDropped) {
switch (this.props.callState) {
@@ -33,14 +39,15 @@ export default class CallMessage extends React.PureComponent {
} else {
duration = <span>{secondsToTime(this.props.duration / 1000)}</span>;
}
const allowJoin = this.props.vc && this.props.callState == 'started';
return <div className="call-message">
<div><i className="material-icons big gray">call</i></div>
<div><i className="material-icons big gray">{bigIcon}</i></div>
<div className="flex-column narrow">
<div>{this.props.incoming ?
<FormattedMessage id="calls_incoming" defaultMessage="Incoming call" description="Incoming call label" /> :
<FormattedMessage id="calls_outgoing" defaultMessage="Outgoing call" description="Outgoing call label" />
}</div>
<div>{title}</div>
<div className="duration"><i className={successClass}>{callIcon}</i> {duration}</div>
{allowJoin ?
<a onClick={this.props.onCallJoin}>Join</a>
: null}
</div>
</div>;
}
+7 -1
View File
@@ -29,6 +29,7 @@ class BaseChatMessage extends React.PureComponent {
this.handleContextClick = this.handleContextClick.bind(this);
this.handleCancelUpload = this.handleCancelUpload.bind(this);
this.handleQuoteClick = this.handleQuoteClick.bind(this);
this.handleCallJoin = this.handleCallJoin.bind(this);
this.formatterContext = {
formatMessage: props.intl.formatMessage.bind(props.intl),
@@ -38,7 +39,8 @@ class BaseChatMessage extends React.PureComponent {
onImagePreview: this.handleExpandImage,
onVideoPreview: this.handlePlayVideo,
onFormButtonClick: this.handleFormButtonClick,
onQuoteClick: this.handleQuoteClick
onQuoteClick: this.handleQuoteClick,
onCallJoin: this.handleCallJoin
};
}
@@ -149,6 +151,10 @@ class BaseChatMessage extends React.PureComponent {
}
}
handleCallJoin() {
this.props.onAcceptCall(this.props.topic, true, this.props.seq);
}
render() {
const sideClass = this.props.sequence + ' ' + (this.props.response ? 'left' : 'right');
const bubbleClass = (this.props.sequence == 'single' || this.props.sequence == 'last') ? 'bubble tip' : 'bubble';
+2 -1
View File
@@ -493,11 +493,12 @@ class ContextMenu extends React.Component {
item = this.MenuItems[item];
}
if (item && item.title) {
const className = item.disabled ? 'disabled' : undefined;
menu.push(
item.title == '-' ?
<li className="separator" key={count} />
:
<li onClick={this.handleClick} data-id={count} key={count}>{item.title}</li>
<li className={className} onClick={this.handleClick} data-id={count} key={count}>{item.title}</li>
);
}
count++;
+94
View File
@@ -0,0 +1,94 @@
// Video conference carousel item.
import React from 'react';
import { injectIntl } from 'react-intl';
import {
Track,
} from 'livekit-client';
import LetterTile from './letter-tile.jsx';
class VCCarouselItem extends React.PureComponent {
constructor(props) {
super(props);
this.state = {
videoRef: undefined,
audioRef: undefined
};
this.handleVideoRefChange = this.handleVideoRefChange.bind(this);
this.handleAudioRefChange = this.handleAudioRefChange.bind(this);
}
componentDidUpdate(prevProps) {
if (this.state.videoRef && !this.state.videoRef.srcObject && this.props.cameraPub && this.props.cameraPub.videoTrack) {
this.props.cameraPub.videoTrack.attach(this.state.videoRef);
}
if (this.state.audioRef && !this.state.audioRef.srcObject && this.props.micPub && this.props.micPub.audioTrack) {
this.props.micPub.audioTrack.attach(this.state.audioRef);
}
}
handleVideoRefChange(node) {
if (!node) { return; }
this.setState({videoRef: node});
// Attach video track to video element.
const cameraPub = this.props.cameraPub;
const cameraEnabled = cameraPub && cameraPub.isSubscribed && !cameraPub.isMuted;
if (cameraEnabled && cameraPub.videoTrack) {
cameraPub.videoTrack.attach(node);
}
}
handleAudioRefChange(node) {
if (!node) { return; }
this.setState({audioRef: node});
// Attach audio track to audio element.
const micPub = this.props.micPub;
const micEnabled = micPub && micPub.isSubscribed && !micPub.isMuted;
if (micEnabled && this.state.audioTrack) {
this.state.audioTrack.attach(node);
}
}
render() {
const name = this.props.name;
let classes = "call-party carousel-item";
if (this.props.isSpeaking) {
classes += " call-party-speaking";
}
const micPub = this.props.micPub;
const cameraPub = this.props.cameraPub;
return (
<div className={classes} key={this.props.key}>
<>
<video ref={this.handleVideoRefChange}></video>
<audio ref={this.handleAudioRefChange}></audio>
{micPub && micPub.isMuted ?
<i className="material-icons muted">mic_off</i>
: null
}
{cameraPub && cameraPub.isMuted ?
<>
<div className="avatar-box carousel-item">
<LetterTile
tinode={this.props.tinode}
avatar={this.props.photo}
topic={this.props.identity}
title={this.props.name} />
</div>
</>
: null}
<div className="caller-name inactive">
{this.props.name}
</div>
</>
</div>
);
}
}
export default injectIntl(VCCarouselItem);
+102
View File
@@ -0,0 +1,102 @@
import React from 'react';
import { injectIntl } from 'react-intl';
import {
Track,
} from 'livekit-client';
import LetterTile from './letter-tile.jsx';
import VCCarouselItem from './vc-carousel-item.jsx';
class VCCarousel extends React.Component {
constructor(props) {
super(props);
this.handlePrevClick = this.handlePrevClick.bind(this);
this.handleNextClick = this.handleNextClick.bind(this);
this.carouselRef = React.createRef();
this.contentRef = React.createRef();
this.prevRef = React.createRef();
this.nextRef = React.createRef();
}
handlePrevClick() {
const gap = 16;
const width = this.carouselRef.current.offsetWidth;
this.carouselRef.current.scrollBy(-(width + gap), 0);
if (this.carouselRef.current.scrollLeft - width - gap <= 0) {
this.prevRef.current.style.display = "none";
}
if (!this.contentRef.current.scrollWidth - width - gap <= this.carouselRef.current.scrollLeft + width) {
this.nextRef.current.style.display = "flex";
}
}
handleNextClick() {
const gap = 16;
const width = this.carouselRef.current.offsetWidth;
this.carouselRef.current.scrollBy(width + gap, 0);
if (this.carouselRef.current.scrollWidth !== 0) {
this.prevRef.current.style.display = "flex";
}
if (this.contentRef.current.scrollWidth - width - gap <= this.carouselRef.current.scrollLeft + width) {
this.nextRef.current.style.display = "none";
}
}
render() {
let peers = [];
if (this.props.participants) {
Object.entries(this.props.participants).forEach(([identity, party], i) => {
const participant = party.participant;
const cameraPub = participant.getTrack(Track.Source.Camera);
const micPub = participant.getTrack(Track.Source.Microphone);
peers.push(<VCCarouselItem
tinode={this.props.tinode}
isSpeaking={participant.isSpeaking}
cameraPub={cameraPub ? cameraPub : null}
micPub={micPub ? micPub : null}
name={party.name}
identity={identity}
photo={party.photo}
key={i}
/>);
});
}
return (
<>
<div id="carousel-wrapper">
<div id="carousel" ref={this.carouselRef}>
<div id="carousel-content" ref={this.contentRef}>
{peers}
</div>
</div>
<button id="carousel-prev" ref={this.prevRef} onClick={this.handlePrevClick}>
<svg
xmlns="http://www.w3.org/2000/svg"
width="24"
height="24"
viewBox="0 0 24 24"
>
<path fill="none" d="M0 0h24v24H0V0z" />
<path d="M15.61 7.41L14.2 6l-6 6 6 6 1.41-1.41L11.03 12l4.58-4.59z" />
</svg>
</button>
<button id="carousel-next" ref={this.nextRef} onClick={this.handleNextClick}>
<svg
xmlns="http://www.w3.org/2000/svg"
width="24"
height="24"
viewBox="0 0 24 24"
>
<path fill="none" d="M0 0h24v24H0V0z" />
<path d="M10.02 6L8.61 7.41 13.19 12l-4.58 4.59L10.02 18l6-6-6-6z" />
</svg>
</button>
</div>
</>
);
}
}
export default injectIntl(VCCarousel);
+424
View File
@@ -0,0 +1,424 @@
// Video conference or broadcast: local and/or peer viewports and controls.
import React from 'react';
import { FormattedMessage, defineMessages, injectIntl } from 'react-intl';
import {
connect,
LocalParticipant,
MediaDeviceFailure,
Participant,
ParticipantEvent,
RemoteParticipant,
RemoteTrack,
RemoteTrackPublication,
Room,
RoomEvent,
Track,
VideoPresets,
} from 'livekit-client';
import LetterTile from './letter-tile.jsx';
import VCCarousel from './vc-carousel.jsx';
import { makeImageUrl } from '../lib/blob-helpers.js';
import { MAX_PEER_TITLE_LENGTH } from '../config.js';
import { CALL_STATE_OUTGOING_INITATED } from '../constants.js';
import { clipStr } from '../lib/utils.js'
class VCPanel extends React.PureComponent {
constructor(props) {
super(props);
this.state = {
room: undefined,
participants: {},
activeSpeaker: undefined,
previousOnInfo: undefined,
};
this.isOutgoingCall = props.callState == CALL_STATE_OUTGOING_INITATED;
const topic = this.props.tinode.getTopic(this.props.topic);
this.isBroadcast = topic && topic.chan;
this.localRef = React.createRef();
this.remoteRef = React.createRef();
this.onInfo = this.onInfo.bind(this);
this.start = this.start.bind(this);
this.stop = this.stop.bind(this);
this.reportError = this.reportError.bind(this);
this.handleCloseClick = this.handleCloseClick.bind(this);
this.handleToggleCameraClick = this.handleToggleCameraClick.bind(this);
this.handleToggleMicClick = this.handleToggleMicClick.bind(this);
this.handleRemoteHangup = this.handleRemoteHangup.bind(this);
this.handleTrackSubscribed2 = this.handleTrackSubscribed2.bind(this);
this.handleTrackUnsubscribed2 = this.handleTrackUnsubscribed2.bind(this);
this.handleActiveSpeakerChange2 = this.handleActiveSpeakerChange2.bind(this);
this.handleDisconnect2 = this.handleDisconnect2.bind(this);
this.handleLocalTrackUnpublished2 = this.handleLocalTrackUnpublished2.bind(this);
this.handleLocalTrackPublished2 = this.handleLocalTrackPublished2.bind(this);
this.handleParticipantConnected2 = this.handleParticipantConnected2.bind(this);
this.handleParticipantDisconnected2 = this.handleParticipantDisconnected2.bind(this);
this.handleData2 = this.handleData2.bind(this);
this.handleDevicesChanged = this.handleDevicesChanged.bind(this);
this.startRoom = this.startRoom.bind(this);
this.participants = {};
}
componentDidMount() {
const topic = this.props.tinode.getTopic(this.props.topic);
this.previousOnInfo = topic.onInfo;
topic.onInfo = this.onInfo;
this.start();
}
componentWillUnmount() {
const topic = this.props.tinode.getTopic(this.props.topic);
topic.onInfo = this.previousOnInfo;
this.stop();
}
onInfo(info) {
if (info.what != 'call') {
return;
}
switch (info.event) {
case 'vc-token':
const ts = info.payload;
this.setState({token: ts.token});
this.startRoom(ts.token);
break;
case 'hang-up':
this.handleRemoteHangup(info);
break;
default:
console.warn("Unknown call event", info.event);
break;
}
}
handleParticipantConnected2(participant) {
if (!participant) return;
if (participant instanceof RemoteParticipant) {
participant
.on(ParticipantEvent.TrackMuted, pub => {
console.log('track was muted', pub.trackSid, participant.identity);
this.forceUpdate(); // Redraw carousel.
})
.on(ParticipantEvent.TrackUnmuted, pub => {
console.log('track was unmuted', pub.trackSid, participant.identity);
this.forceUpdate(); // Redraw carousel.
})
.on(ParticipantEvent.IsSpeakingChanged, _ => {
this.forceUpdate(); // Redraw carousel.
})
.on(ParticipantEvent.ConnectionQualityChanged, _ => {
this.forceUpdate(); // Redraw carousel.
});
const me = this.props.tinode.getMeTopic();
const c = me.getContact(participant.identity);
const name = c && c.public && c.public.fn ? c.public.fn : participant.identity;
const photo = makeImageUrl(c && c.public ? c.public.photo : null);
const p = {
name: name,
photo: photo,
participant: participant
};
this.participants[participant.identity] = p
console.log('participant connected', this.participants);
this.setState({participants: this.participants}, _ => { this.forceUpdate(); });
}
}
handleParticipantDisconnected2(participant) {
console.log('Deleting participant ', participant.identity);
delete this.participants[participant.identity];
this.setState({participants: this.participants});
}
handleData2(msg, participant) {
console.log('Data message ', msg, participant);
}
handleTrackSubscribed2(track, publication, participant) {
if (participant instanceof RemoteParticipant) {
// Check if we're presenting a remote video stream.
if (this.remoteRef.current && !(this.remoteRef.current.src != '' || this.remoteRef.srcObject)) {
track.attach(this.remoteRef.current);
}
}
// Redraw the carousel and remote view.
this.forceUpdate();
}
handleTrackUnsubscribed2(track, publication, participant) {
console.log('handleTrackUnsubscribed2', track);
// remove tracks from all attached elements
track.detach();
this.forceUpdate();
}
handleLocalTrackPublished2(pub) {
console.log('Local track published', pub);
const track = pub.track;
if (this.localRef.current) {
const elem = this.localRef.current;
track.attach(elem);
// flip.
elem.style.transform = 'scale(-1, 1)';
}
}
handleLocalTrackUnpublished2(pub) {
const track = pub.track;
console.log('handleLocalTrackUnpublished2', track);
track.detach();
// Redraw carousel.
this.forceUpdate();
}
handleActiveSpeakerChange2(speakers) {
console.log('handleActiveSpeakerChange2', speakers);
speakers.forEach(s => {
if (s.isSpeaking) {
console.log('Speaking participant ', s);
// Display speaker in remote peer view.
const elem = this.remoteRef.current;
const me = this.props.tinode.getMeTopic();
const c = me.getContact(s.identity);
const name = c && c.public && c.public.fn ? c.public.fn : undefined;
this.setState({activeSpeaker: name});
const cameraPub = s.getTrack(Track.Source.Camera);
const cameraEnabled = cameraPub && cameraPub.isSubscribed && !cameraPub.isMuted;
if (cameraEnabled && cameraPub.videoTrack) {
cameraPub.videoTrack.attach(elem);
}
}
});
}
handleDevicesChanged() {
console.log('devices changed');
// TODO: implement.
}
handleDisconnect2() {
console.log('disconnected from room');
this.handleCloseClick();
}
startRoom(token) {
const roomOpts = {
adaptiveStream: true,
dynacast: true,
publishDefaults: {
simulcast: true,
videoSimulcastLayers: [VideoPresets.h90, VideoPresets.h216],
videoCodec: 'vp8'
},
videoCaptureDefaults: {
resolution: VideoPresets.h720.resolution
}
};
const connectOpts = {
autoSubscribe: true
};
// creates a new room with options
const room = new Room(roomOpts);
this.setState({room: room});
const startTime = Date.now();
const topic = this.props.tinode.getTopic(this.props.topic);
const shouldPublish = this.isOutgoingCall || !this.isBroadcast;
room
.on(RoomEvent.ParticipantConnected, this.handleParticipantConnected2)
.on(RoomEvent.ParticipantDisconnected, this.handleParticipantDisconnected2)
.on(RoomEvent.DataReceived, this.handleData2)
.on(RoomEvent.TrackSubscribed, this.handleTrackSubscribed2)
.on(RoomEvent.TrackUnsubscribed, this.handleTrackUnsubscribed2)
.on(RoomEvent.ActiveSpeakersChanged, this.handleActiveSpeakerChange2)
.on(RoomEvent.Disconnected, this.handleDisconnect2)
.on(RoomEvent.LocalTrackUnpublished, this.handleLocalTrackUnpublished2)
.on(RoomEvent.LocalTrackPublished, this.handleLocalTrackPublished2)
.on(RoomEvent.RoomMetadataChanged, metadata => {
console.log('VC room new metadata ', metadata);
})
.on(RoomEvent.MediaDevicesChanged, this.handleDevicesChanged)
.on(RoomEvent.AudioPlaybackStatusChanged, e => {
// TODO: handle it.
console.log('Audio playback status change: ', e);
})
.on(RoomEvent.MediaDevicesError, e => {
const failure = MediaDeviceFailure.getFailure(e);
this.reportError(failure);
})
.on(RoomEvent.ConnectionQualityChanged, (quality, participant) => {
const id = participant ? participant.identity : 'nil';
console.log('Connection quality changed', id, ' -> ', quality);
})
.on(RoomEvent.SignalConnected, _ => {
const signalConnectionTime = Date.now() - startTime;
console.log(`signal connection established in ${signalConnectionTime}ms`);
// speed up publishing by starting to publish before it's fully connected
// publishing is accepted as soon as signal connection has established
if (shouldPublish) {
room.localParticipant.enableCameraAndMicrophone();
console.log(`tracks published in ${Date.now() - startTime}ms`);
}
});
const vcEndpoint = this.props.tinode.getServerParam('vcEndpoint', null);
room.connect(vcEndpoint, token, connectOpts).then(_ => {
console.log('Connected to VC room', room.name, '. Access token', token);
room.participants.forEach((participant) => {
this.handleParticipantConnected2(participant);
});
this.handleParticipantConnected2(room.localParticipant);
});
}
start() {
if (this.state.localStream) {
this.props.onError(this.props.intl.formatMessage(messages.already_in_call), 'info');
return;
}
if (this.isOutgoingCall) {
this.props.onCreateCall(this.props.topic, this.props.seq, this.props.callState, this.props.callAudioOnly)
.then(ctrl => {
const token = ctrl.params.token;
const seq = ctrl.params.seq;
this.setState({callSeq: seq, token: token});
this.startRoom(token);
});
} else {
this.props.onJoin(this.props.topic, this.props.seq);
}
}
stop() {
if (this.state.room) {
this.state.room.disconnect();
}
}
reportError(err) {
this.props.onError(err.message, 'err');
}
// Call disconnected by remote.
handleRemoteHangup() {
this.handleCloseClick();
}
handleCloseClick() {
this.stop();
this.props.onHangup(this.props.topic, this.props.seq);
}
handleToggleCameraClick() {
if (!this.state.room) return;
const enabled = this.state.room.localParticipant.isCameraEnabled;
this.state.room.localParticipant.setCameraEnabled(!enabled)
.then(_ => this.forceUpdate());
this.forceUpdate();
}
handleToggleMicClick() {
if (!this.state.room) return;
const enabled = this.state.room.localParticipant.isMicrophoneEnabled;
this.state.room.localParticipant.setMicrophoneEnabled(!enabled)
.then(_ => this.forceUpdate());
// Make sure we redraw the mute/unmute icons (e.g. mic -> mic_off).
this.forceUpdate();
}
render() {
const isPublishing = this.isOutgoingCall || !this.isBroadcast;
const disabled = !isPublishing || !this.state.room || !this.state.room.localParticipant;
const audioIcon = !disabled && this.state.room.localParticipant.isMicrophoneEnabled ? 'mic' : 'mic_off';
const videoIcon = !disabled && this.state.room.localParticipant.isCameraEnabled ? 'videocam' : 'videocam_off';
const peerTitle = this.state.activeSpeaker ? this.state.activeSpeaker : clipStr(this.props.title, MAX_PEER_TITLE_LENGTH);
const remoteActive = this.remoteRef.current && (this.remoteRef.current.src != '' || this.remoteRef.current.srcObject)
return (
<>
<div id="video-container">
{ !this.isBroadcast && Object.keys(this.state.participants).length > 0 ?
<VCCarousel
tinode={this.props.tinode}
participants={this.state.participants} /> :
null
}
<div id="video-container-panel">
{ isPublishing ?
<div className="call-party self" disabled={this.state.audioOnly}>
<video ref={this.localRef} autoPlay muted playsInline />
<div className="caller-name inactive">
<FormattedMessage id="calls_you_label"
defaultMessage="You" description="Shown over the local video screen" />
</div>
</div> :
null }
<div className="call-party peer" disabled={!remoteActive}>
<video ref={this.remoteRef} autoPlay playsInline />
{remoteActive ?
<>
<div className="caller-name inactive">{peerTitle}</div>
</> :
<>
<div className="caller-card">
<div className="avatar-box">
<LetterTile
tinode={this.props.tinode}
avatar={this.props.avatar}
topic={this.props.topic}
title={this.props.title} />
</div>
<div className="caller-name">{peerTitle}</div>
</div>
</>
}
</div>
</div>
<div className="controls">
<button className="danger" onClick={this.handleCloseClick}>
<i className="material-icons">call_end</i>
</button>
<button className="secondary" onClick={this.handleToggleCameraClick} disabled={disabled}>
<i className="material-icons">{videoIcon}</i>
</button>
<button className="secondary" onClick={this.handleToggleMicClick} disabled={disabled}>
<i className="material-icons">{audioIcon}</i>
</button>
</div>
</div>
</>
);
}
};
export default injectIntl(VCPanel);