pro/im_webapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

calc audio preview at the source

Browse Source
This commit is contained in:
or-else
2022-03-13 15:36:30 -07:00
parent 025d4cfadb
commit cc3b1190fd
10 changed files with 199 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/views/messages-view.jsx
+42 -31
View File
@@ -871,38 +871,49 @@ class MessagesView extends React.Component {
}
// sendAudioAttachment sends audio bits inband as Drafty message (no preview).
sendAudioAttachment(url, duration) {
// Server-provided limit reduced for base64 encoding and overhead.
const maxInbandAttachmentSize = (this.props.tinode.getServerLimit(
'maxMessageSize', MAX_INBAND_ATTACHMENT_SIZE) * 0.75 - 1024) | 0;
fetch(url).then(result => {
return result.blob();
}).then(blob => {
if (blob.size > maxInbandAttachmentSize) {
// Too large to send inband - uploading out of band and sending as a link.
const uploader = this.props.tinode.getLargeFileHelper();
if (!uploader) {
this.props.onError(this.props.intl.formatMessage(messages.cannot_initiate_upload));
return;
sendAudioAttachment(url, preview, duration) {
let previewEncoded;
blobToBase64(new Blob(Uint8Array.from(preview)))
.then(pv64 => {
previewEncoded = pv64 ? pv64.bits : undefined;
return fetch(url);
})
.then(result => result.blob())
.then(blob => {
// Server-provided limit reduced for base64 encoding and overhead.
const maxInbandAttachmentSize = this.props.tinode.getServerLimit('maxMessageSize', MAX_INBAND_ATTACHMENT_SIZE) * 0.75 - 1024;
if (blob.size > maxInbandAttachmentSize) {
// Too large to send inband - uploading out of band and sending as a link.
const uploader = this.props.tinode.getLargeFileHelper();
if (!uploader) {
this.props.onError(this.props.intl.formatMessage(messages.cannot_initiate_upload));
return;
}
const uploadCompletionPromise = uploader.upload(blob);
const msg = Drafty.appendAudio(null, {
mime: blob.type,
size: blob.size,
duration: duration,
preview: previewEncoded,
urlPromise: uploadCompletionPromise
});
// Pass data and the uploader to the TinodeWeb.
this.sendMessage(msg, uploadCompletionPromise, uploader);
} else {
// Small enough to send inband.
blobToBase64(blob)
.then(b64 => {
this.sendMessage(Drafty.appendAudio(null, {
mime: b64.mime,
size: blob.size,
data: b64.bits,
duration: duration,
preview: previewEncoded,
}))
})
}
const uploadCompletionPromise = uploader.upload(blob);
const msg = Drafty.appendAudio(null, {
mime: blob.type,
size: blob.size,
duration: duration,
urlPromise: uploadCompletionPromise
});
// Pass data and the uploader to the TinodeWeb.
this.sendMessage(msg, uploadCompletionPromise, uploader);
} else {
// Small enough to send inband.
blobToBase64(blob)
.then(b64 => {
this.sendMessage(Drafty.appendAudio(null, {mime: b64.mime, data: b64.bits, duration: duration}))
})
.catch(err => {this.props.onError(err)});
}
});
})
.catch(err => {this.props.onError(err)});;
}
handleCancelUpload(seq, uploader) {
src/widgets/audio-player.jsx
+20 -16
View File
@@ -4,8 +4,6 @@ import React from 'react';
import { secondsToTime } from '../lib/strformat';
// FFT resolution.
const BUFFER_SIZE = 256;
// Thickness of a visualization bar.
const LINE_WIDTH = 4;
// Spacing between two visualization bars.
@@ -18,8 +16,6 @@ const BAR_COLOR = '#8fbed6';
const BAR_SCALE = 96.0;
// Background color
const BKG_COLOR = '#eeeeee';
// Maximum duration of a recording in milliseconds (10 min).
const MAX_DURATION = 600000;
export default class AudioPlayer extends React.PureComponent {
constructor(props) {
@@ -33,11 +29,13 @@ export default class AudioPlayer extends React.PureComponent {
};
this.visualize = this.visualize.bind(this);
this.initVisualizer = this.initVisualizer.bind(this);
this.handlePlay = this.handlePlay.bind(this);
this.handlePause = this.handlePause.bind(this);
this.handleError = this.handleError.bind(this);
this.audioPlayer = this.props.src ? new Audio(this.props.src) : null;
this.audioPlayer = null;
this.durationMillis = 0;
this.startedOn = null;
@@ -51,23 +49,21 @@ export default class AudioPlayer extends React.PureComponent {
// To reduce line blurring.
this.canvasContext.translate(0.5, 0.5);
if (this.audioPlayer) {
this.audioPlayer.addEventListener('canplay', _ => {
console.log('canplay', this.audioPlayer.duration);
this.setState({canPlay: true, duration: secondsToTime(this.audioPlayer.duration)})
});
if (this.props.src) {
this.audioPlayer = new Audio(this.props.src);
this.audioPlayer.addEventListener('loadedmetadata', _ => this.setState({duration: secondsToTime(this.audioPlayer.duration)}));
this.audioPlayer.addEventListener('canplay', _ => this.setState({canPlay: true}));
this.audioPlayer.addEventListener('timeupdate', _ => console.log(this.audioPlayer.currentTime));
this.audioPlayer.addEventListener('ended', _ => this.setState({playing: false, currentTime: secondsToTime(0)}));
}
if (this.props.src) {
this.initVisualizer();
}
}
// Draw amplitude of sound.
visualize() {
const pcmData = new Uint8Array(this.analyser.frequencyBinCount);
const width = this.canvasRef.current.width;
const height = this.canvasRef.current.height;
const viewLength = (width / (LINE_WIDTH + SPACING)) | 0;
const viewDuration = MILLIS_PER_BAR * viewLength;
this.canvasContext.fillStyle = BKG_COLOR;
this.canvasContext.lineWidth = LINE_WIDTH;
@@ -136,6 +132,10 @@ export default class AudioPlayer extends React.PureComponent {
drawFrame();
}
initVisualizer() {
console.log("Draw amplitude bars");
}
handlePause(e) {
e.preventDefault();
this.setState({recording: false, duration: secondsToTime(this.durationMillis / 1000)});
@@ -154,6 +154,10 @@ export default class AudioPlayer extends React.PureComponent {
this.setState({playing: !this.state.playing});
}
handleError(err) {
console.log(err);
}
render() {
const playClass = 'material-icons large' + (this.state.canPlay ? '' : ' disabled');
return (
@@ -162,7 +166,7 @@ export default class AudioPlayer extends React.PureComponent {
<i className={playClass}>{this.state.playing ? 'pause_circle' : 'play_circle'}</i>
</a>
<div>
<canvas className="visualiser" style={{backgroundColor: '#666'}} ref={this.canvasRef} />
<canvas className="visualiser" style={{border: '1px solid #666'}} ref={this.canvasRef} />
<div className="timer">{this.state.currentTime}/{this.state.duration}</div>
</div>
</div>
src/widgets/audio-recorder.jsx
+45 -9
View File
@@ -15,13 +15,15 @@ const MILLIS_PER_BAR = 100;
// Color of histogram bars
const BAR_COLOR = '#8fbed6';
// Scaling for visualization bars.
const BAR_SCALE = 96.0;
const BAR_SCALE = 64.0;
// Background color
const BKG_COLOR = '#eeeeee';
// Minimum duration of a recording in milliseconds.
const MIN_DURATION = 200;
// Maximum duration of a recording in milliseconds (10 min).
const MAX_DURATION = 600000;
// Number of bars in preview.
const VISUALIZATION_BARS = 96;
export default class AudioRecorder extends React.PureComponent {
constructor(props) {
@@ -85,6 +87,8 @@ export default class AudioRecorder extends React.PureComponent {
this.canvasContext.strokeStyle = BAR_COLOR;
let prevBarCount = 0;
let volume = 0.0;
let countPerBar = 0;
const drawFrame = () => {
if (!this.startedOn) {
return;
@@ -107,11 +111,14 @@ export default class AudioRecorder extends React.PureComponent {
// Get current waveform and calculate its amplitude.
this.analyser.getByteTimeDomainData(pcmData);
let volume = 0.0;
let amp = 0.0;
for (const amplitude of pcmData) {
volume += (amplitude - 127) ** 2;
amp += (amplitude - 127) ** 2;
}
volume = Math.sqrt(volume/pcmData.length);
// Sum the amplitude.
volume += Math.sqrt(amp/pcmData.length);
countPerBar ++;
let barCount = (duration / MILLIS_PER_BAR) | 0;
// Shift of the histogram along x-axis to make scrolling smooth. No need to shift if recording is too short.
@@ -121,7 +128,9 @@ export default class AudioRecorder extends React.PureComponent {
if (prevBarCount != barCount) {
prevBarCount = barCount;
// Add new amplitude visualization bar.
this.viewBuffer.push(volume);
this.viewBuffer.push(volume/countPerBar);
volume = 0.0;
countPerBar = 0;
if (this.viewBuffer.length > viewLength) {
// Keep at most 'viewLength' amplitude bars.
this.viewBuffer.shift();
@@ -135,7 +144,7 @@ export default class AudioRecorder extends React.PureComponent {
this.canvasContext.beginPath();
for (let i = 0; i < this.viewBuffer.length; i++) {
let x = i * (LINE_WIDTH + SPACING) - dx;
let y = this.viewBuffer[i] / BAR_SCALE * height;
let y = Math.min(this.viewBuffer[i] / BAR_SCALE, 0.9) * height;
this.canvasContext.moveTo(x, (height - y) * 0.5);
this.canvasContext.lineTo(x, height * 0.5 + y * 0.5);
@@ -198,15 +207,17 @@ export default class AudioRecorder extends React.PureComponent {
this.audioInput.connect(this.analyser);
this.mediaRecorder.onstop = _ => {
const blob = new Blob(this.audioChunks, { 'type' : 'audio/mp3; codecs=mp3' });
const blob = new Blob(this.audioChunks, {'type' : 'audio/mp3; codecs=mp3'});
this.audioChunks = [];
const url = window.URL.createObjectURL(blob);
this.cleanUp();
if (this.durationMillis > MIN_DURATION) {
this.props.onFinished(url, this.durationMillis);
blob.arrayBuffer().then(ab => this.audioContext.decodeAudioData(ab))
.then(data => this.createPreview(data))
.then(preview => this.props.onFinished(url, preview, this.durationMillis));
} else {
this.props.onDeleted();
}
this.cleanUp();
}
this.mediaRecorder.ondataavailable = (e) => {
@@ -221,6 +232,31 @@ export default class AudioRecorder extends React.PureComponent {
this.visualize();
}
// Preview must be calculated at the source: Chrome does not allow background AudioContext.
createPreview(audio) {
const data = audio.getChannelData(0);
// Number of amplitude bars in preview.
const viewLength = Math.min(data.length, VISUALIZATION_BARS);
// The number of samples in each bar. Limit the number of samples to something reasonable, no need to process all.
const spb = Math.min(100, (data.length / viewLength) | 0);
let buffer = [];
let max = -1;
for (let i = 0; i < viewLength; i++) {
let amplitude = 0;
for (let j = 0; j < spb; j++) {
amplitude += data[spb * i + j] ** 2;
}
const val = Math.sqrt(amplitude / spb);
buffer.push(val);
max = Math.max(max, val);
}
// Normalize amplitude to 0..100.
if (max > 0) {
buffer = buffer.map(a => (100 * a / max) | 0);
}
return buffer;
}
cleanUp() {
this.audioInput.disconnect();
this.stream.getTracks().forEach(track => track.stop());
src/widgets/send-message.jsx
+2 -2
View File
@@ -130,9 +130,9 @@ class SendMessage extends React.PureComponent {
e.target.value = '';
}
handleAttachAudio(url, duration) {
handleAttachAudio(url, preview, duration) {
this.setState({audioRec: false});
this.props.onAttachAudio(url, duration);
this.props.onAttachAudio(url, preview, duration);
}
handleSend(e) {