calc audio preview at the source

src/views/messages-view.jsx

@@ -871,13 +871,17 @@ class MessagesView extends React.Component {
   }
 
   // sendAudioAttachment sends audio bits inband as Drafty message (no preview).
-  sendAudioAttachment(url, duration) {
+  sendAudioAttachment(url, preview, duration) {
+    let previewEncoded;
+    blobToBase64(new Blob(Uint8Array.from(preview)))
+      .then(pv64 => {
+        previewEncoded = pv64 ? pv64.bits : undefined;
+        return fetch(url);
+      })
+      .then(result => result.blob())
+      .then(blob => {
         // Server-provided limit reduced for base64 encoding and overhead.
-    const maxInbandAttachmentSize = (this.props.tinode.getServerLimit(
+        const maxInbandAttachmentSize = this.props.tinode.getServerLimit('maxMessageSize', MAX_INBAND_ATTACHMENT_SIZE) * 0.75 - 1024;
-      'maxMessageSize', MAX_INBAND_ATTACHMENT_SIZE) * 0.75 - 1024) | 0;
-    fetch(url).then(result => {
-      return result.blob();
-    }).then(blob => {
         if (blob.size > maxInbandAttachmentSize) {
           // Too large to send inband - uploading out of band and sending as a link.
           const uploader = this.props.tinode.getLargeFileHelper();
@@ -890,6 +894,7 @@ class MessagesView extends React.Component {
             mime: blob.type,
             size: blob.size,
             duration: duration,
+            preview: previewEncoded,
             urlPromise: uploadCompletionPromise
           });
           // Pass data and the uploader to the TinodeWeb.
@@ -898,11 +903,17 @@ class MessagesView extends React.Component {
           // Small enough to send inband.
           blobToBase64(blob)
             .then(b64 => {
-            this.sendMessage(Drafty.appendAudio(null, {mime: b64.mime, data: b64.bits, duration: duration}))
+              this.sendMessage(Drafty.appendAudio(null, {
+                mime: b64.mime,
+                size: blob.size,
+                data: b64.bits,
+                duration: duration,
+                preview: previewEncoded,
+              }))
             })
-          .catch(err => {this.props.onError(err)});
         }
-    });
+      })
+      .catch(err => {this.props.onError(err)});;
   }
 
   handleCancelUpload(seq, uploader) {

src/widgets/audio-player.jsx

@@ -4,8 +4,6 @@ import React from 'react';
 
 import { secondsToTime } from '../lib/strformat';
 
-// FFT resolution.
-const BUFFER_SIZE = 256;
 // Thickness of a visualization bar.
 const LINE_WIDTH = 4;
 // Spacing between two visualization bars.
@@ -18,8 +16,6 @@ const BAR_COLOR = '#8fbed6';
 const BAR_SCALE = 96.0;
 // Background color
 const BKG_COLOR = '#eeeeee';
-// Maximum duration of a recording in milliseconds (10 min).
-const MAX_DURATION = 600000;
 
 export default class AudioPlayer extends React.PureComponent {
   constructor(props) {
@@ -33,11 +29,13 @@ export default class AudioPlayer extends React.PureComponent {
     };
 
     this.visualize = this.visualize.bind(this);
+    this.initVisualizer = this.initVisualizer.bind(this);
 
     this.handlePlay = this.handlePlay.bind(this);
     this.handlePause = this.handlePause.bind(this);
+    this.handleError = this.handleError.bind(this);
 
-    this.audioPlayer = this.props.src ? new Audio(this.props.src) : null;
+    this.audioPlayer = null;
 
     this.durationMillis = 0;
     this.startedOn = null;
@@ -51,23 +49,21 @@ export default class AudioPlayer extends React.PureComponent {
     // To reduce line blurring.
     this.canvasContext.translate(0.5, 0.5);
 
-    if (this.audioPlayer) {
+    if (this.props.src) {
-      this.audioPlayer.addEventListener('canplay', _ => {
+      this.audioPlayer = new Audio(this.props.src);
-        console.log('canplay', this.audioPlayer.duration);
+      this.audioPlayer.addEventListener('loadedmetadata', _ => this.setState({duration: secondsToTime(this.audioPlayer.duration)}));
-        this.setState({canPlay: true, duration: secondsToTime(this.audioPlayer.duration)})
+      this.audioPlayer.addEventListener('canplay', _ => this.setState({canPlay: true}));
-      });
       this.audioPlayer.addEventListener('timeupdate', _ => console.log(this.audioPlayer.currentTime));
       this.audioPlayer.addEventListener('ended', _ => this.setState({playing: false, currentTime: secondsToTime(0)}));
     }
+
+    if (this.props.src) {
+      this.initVisualizer();
+    }
   }
 
   // Draw amplitude of sound.
   visualize() {
-    const pcmData = new Uint8Array(this.analyser.frequencyBinCount);
-    const width = this.canvasRef.current.width;
-    const height = this.canvasRef.current.height;
-    const viewLength = (width / (LINE_WIDTH + SPACING)) | 0;
-    const viewDuration = MILLIS_PER_BAR * viewLength;
 
     this.canvasContext.fillStyle = BKG_COLOR;
     this.canvasContext.lineWidth = LINE_WIDTH;
@@ -136,6 +132,10 @@ export default class AudioPlayer extends React.PureComponent {
     drawFrame();
   }
 
+  initVisualizer() {
+    console.log("Draw amplitude bars");
+  }
+
   handlePause(e) {
     e.preventDefault();
     this.setState({recording: false, duration: secondsToTime(this.durationMillis / 1000)});
@@ -154,6 +154,10 @@ export default class AudioPlayer extends React.PureComponent {
     this.setState({playing: !this.state.playing});
   }
 
+  handleError(err) {
+    console.log(err);
+  }
+
   render() {
     const playClass = 'material-icons large' + (this.state.canPlay ? '' : ' disabled');
     return (
@@ -162,7 +166,7 @@ export default class AudioPlayer extends React.PureComponent {
           <i className={playClass}>{this.state.playing ? 'pause_circle' : 'play_circle'}</i>
         </a>
         <div>
-          <canvas className="visualiser" style={{backgroundColor: '#666'}} ref={this.canvasRef} />
+          <canvas className="visualiser" style={{border: '1px solid #666'}} ref={this.canvasRef} />
           <div className="timer">{this.state.currentTime}/{this.state.duration}</div>
         </div>
       </div>

src/widgets/audio-recorder.jsx

@@ -15,13 +15,15 @@ const MILLIS_PER_BAR = 100;
 // Color of histogram bars
 const BAR_COLOR = '#8fbed6';
 // Scaling for visualization bars.
-const BAR_SCALE = 96.0;
+const BAR_SCALE = 64.0;
 // Background color
 const BKG_COLOR = '#eeeeee';
 // Minimum duration of a recording in milliseconds.
 const MIN_DURATION = 200;
 // Maximum duration of a recording in milliseconds (10 min).
 const MAX_DURATION = 600000;
+// Number of bars in preview.
+const VISUALIZATION_BARS = 96;
 
 export default class AudioRecorder extends React.PureComponent {
   constructor(props) {
@@ -85,6 +87,8 @@ export default class AudioRecorder extends React.PureComponent {
     this.canvasContext.strokeStyle = BAR_COLOR;
 
     let prevBarCount = 0;
+    let volume = 0.0;
+    let countPerBar = 0;
     const drawFrame = () => {
       if (!this.startedOn) {
         return;
@@ -107,11 +111,14 @@ export default class AudioRecorder extends React.PureComponent {
 
       // Get current waveform and calculate its amplitude.
       this.analyser.getByteTimeDomainData(pcmData);
-      let volume = 0.0;
+      let amp = 0.0;
       for (const amplitude of pcmData) {
-        volume += (amplitude - 127) ** 2;
+        amp += (amplitude - 127) ** 2;
       }
-      volume = Math.sqrt(volume/pcmData.length);
+
+      // Sum the amplitude.
+      volume += Math.sqrt(amp/pcmData.length);
+      countPerBar ++;
 
       let barCount = (duration / MILLIS_PER_BAR) | 0;
       // Shift of the histogram along x-axis to make scrolling smooth. No need to shift if recording is too short.
@@ -121,7 +128,9 @@ export default class AudioRecorder extends React.PureComponent {
       if (prevBarCount != barCount) {
         prevBarCount = barCount;
         // Add new amplitude visualization bar.
-        this.viewBuffer.push(volume);
+        this.viewBuffer.push(volume/countPerBar);
+        volume = 0.0;
+        countPerBar = 0;
         if (this.viewBuffer.length > viewLength) {
           // Keep at most 'viewLength' amplitude bars.
           this.viewBuffer.shift();
@@ -135,7 +144,7 @@ export default class AudioRecorder extends React.PureComponent {
       this.canvasContext.beginPath();
       for (let i = 0; i < this.viewBuffer.length; i++) {
         let x = i * (LINE_WIDTH + SPACING) - dx;
-        let y = this.viewBuffer[i] / BAR_SCALE * height;
+        let y = Math.min(this.viewBuffer[i] / BAR_SCALE, 0.9) * height;
 
         this.canvasContext.moveTo(x, (height - y) * 0.5);
         this.canvasContext.lineTo(x, height * 0.5 + y * 0.5);
@@ -198,15 +207,17 @@ export default class AudioRecorder extends React.PureComponent {
     this.audioInput.connect(this.analyser);
 
     this.mediaRecorder.onstop = _ => {
-      const blob = new Blob(this.audioChunks, { 'type' : 'audio/mp3; codecs=mp3' });
+      const blob = new Blob(this.audioChunks, {'type' : 'audio/mp3; codecs=mp3'});
       this.audioChunks = [];
       const url = window.URL.createObjectURL(blob);
-      this.cleanUp();
       if (this.durationMillis > MIN_DURATION) {
-        this.props.onFinished(url, this.durationMillis);
+        blob.arrayBuffer().then(ab => this.audioContext.decodeAudioData(ab))
+          .then(data => this.createPreview(data))
+          .then(preview => this.props.onFinished(url, preview, this.durationMillis));
       } else {
         this.props.onDeleted();
       }
+      this.cleanUp();
     }
 
     this.mediaRecorder.ondataavailable = (e) => {
@@ -221,6 +232,31 @@ export default class AudioRecorder extends React.PureComponent {
     this.visualize();
   }
 
+  // Preview must be calculated at the source: Chrome does not allow background AudioContext.
+  createPreview(audio) {
+    const data = audio.getChannelData(0);
+    // Number of amplitude bars in preview.
+    const viewLength = Math.min(data.length, VISUALIZATION_BARS);
+    // The number of samples in each bar. Limit the number of samples to something reasonable, no need to process all.
+    const spb = Math.min(100, (data.length / viewLength) | 0);
+    let buffer = [];
+    let max = -1;
+    for (let i = 0; i < viewLength; i++) {
+      let amplitude = 0;
+      for (let j = 0; j < spb; j++) {
+        amplitude += data[spb * i + j] ** 2;
+      }
+      const val = Math.sqrt(amplitude / spb);
+      buffer.push(val);
+      max = Math.max(max, val);
+    }
+    // Normalize amplitude to 0..100.
+    if (max > 0) {
+      buffer = buffer.map(a => (100 * a / max) | 0);
+    }
+    return buffer;
+  }
+
   cleanUp() {
     this.audioInput.disconnect();
     this.stream.getTracks().forEach(track => track.stop());

src/widgets/send-message.jsx

@@ -130,9 +130,9 @@ class SendMessage extends React.PureComponent {
     e.target.value = '';
   }
 
-  handleAttachAudio(url, duration) {
+  handleAttachAudio(url, preview, duration) {
     this.setState({audioRec: false});
-    this.props.onAttachAudio(url, duration);
+    this.props.onAttachAudio(url, preview, duration);
   }
 
   handleSend(e) {

umd/index.dev.js

@@ -6197,11 +6197,14 @@ class MessagesView extends (react__WEBPACK_IMPORTED_MODULE_0___default().Compone
     });
   }
 
-  sendAudioAttachment(url, duration) {
+  sendAudioAttachment(url, preview, duration) {
-    const maxInbandAttachmentSize = this.props.tinode.getServerLimit('maxMessageSize', _config_js__WEBPACK_IMPORTED_MODULE_15__.MAX_INBAND_ATTACHMENT_SIZE) * 0.75 - 1024 | 0;
+    let previewEncoded;
-    fetch(url).then(result => {
+    (0,_lib_blob_helpers_js__WEBPACK_IMPORTED_MODULE_16__.blobToBase64)(new Blob(Uint8Array.from(preview))).then(pv64 => {
-      return result.blob();
+      previewEncoded = pv64 ? pv64.bits : undefined;
-    }).then(blob => {
+      return fetch(url);
+    }).then(result => result.blob()).then(blob => {
+      const maxInbandAttachmentSize = this.props.tinode.getServerLimit('maxMessageSize', _config_js__WEBPACK_IMPORTED_MODULE_15__.MAX_INBAND_ATTACHMENT_SIZE) * 0.75 - 1024;
+
       if (blob.size > maxInbandAttachmentSize) {
         const uploader = this.props.tinode.getLargeFileHelper();
 
@@ -6215,6 +6218,7 @@ class MessagesView extends (react__WEBPACK_IMPORTED_MODULE_0___default().Compone
           mime: blob.type,
           size: blob.size,
           duration: duration,
+          preview: previewEncoded,
           urlPromise: uploadCompletionPromise
         });
         this.sendMessage(msg, uploadCompletionPromise, uploader);
@@ -6222,14 +6226,17 @@ class MessagesView extends (react__WEBPACK_IMPORTED_MODULE_0___default().Compone
         (0,_lib_blob_helpers_js__WEBPACK_IMPORTED_MODULE_16__.blobToBase64)(blob).then(b64 => {
           this.sendMessage(tinode_sdk__WEBPACK_IMPORTED_MODULE_2__.Drafty.appendAudio(null, {
             mime: b64.mime,
+            size: blob.size,
             data: b64.bits,
-            duration: duration
+            duration: duration,
+            preview: previewEncoded
           }));
+        });
+      }
     }).catch(err => {
       this.props.onError(err);
     });
-      }
+    ;
-    });
   }
 
   handleCancelUpload(seq, uploader) {
@@ -9631,15 +9638,12 @@ __webpack_require__.r(__webpack_exports__);
 /* harmony import */ var _lib_strformat__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! ../lib/strformat */ "./src/lib/strformat.js");
 
 
-const BUFFER_SIZE = 256;
 const LINE_WIDTH = 4;
 const SPACING = 2;
 const MILLIS_PER_BAR = 100;
 const BAR_COLOR = '#8fbed6';
 const BAR_SCALE = 96.0;
 const BKG_COLOR = '#eeeeee';
-const MIN_DURATION = 200;
-const MAX_DURATION = 600000;
 class AudioPlayer extends (react__WEBPACK_IMPORTED_MODULE_0___default().PureComponent) {
   constructor(props) {
     super(props);
@@ -9647,12 +9651,14 @@ class AudioPlayer extends (react__WEBPACK_IMPORTED_MODULE_0___default().PureComp
       canPlay: false,
       playing: false,
       currentTime: '0:00',
-      duration: '0:00'
+      duration: (0,_lib_strformat__WEBPACK_IMPORTED_MODULE_1__.secondsToTime)(this.props.duration)
     };
     this.visualize = this.visualize.bind(this);
+    this.initVisualizer = this.initVisualizer.bind(this);
     this.handlePlay = this.handlePlay.bind(this);
     this.handlePause = this.handlePause.bind(this);
-    this.audioPlayer = this.props.src ? new Audio(this.props.src) : null;
+    this.handleError = this.handleError.bind(this);
+    this.audioPlayer = null;
     this.durationMillis = 0;
     this.startedOn = null;
     this.viewBuffer = [];
@@ -9664,28 +9670,27 @@ class AudioPlayer extends (react__WEBPACK_IMPORTED_MODULE_0___default().PureComp
     this.canvasContext.lineCap = 'round';
     this.canvasContext.translate(0.5, 0.5);
 
-    if (this.audioPlayer) {
+    if (this.props.src) {
-      this.audioPlayer.addEventListener('canplay', _ => {
+      this.audioPlayer = new Audio(this.props.src);
-        console.log('canplay', this.audioPlayer.duration);
+      this.audioPlayer.addEventListener('loadedmetadata', _ => this.setState({
-        this.setState({
-          canPlay: true,
         duration: (0,_lib_strformat__WEBPACK_IMPORTED_MODULE_1__.secondsToTime)(this.audioPlayer.duration)
-        });
+      }));
-      });
+      this.audioPlayer.addEventListener('canplay', _ => this.setState({
+        canPlay: true
+      }));
       this.audioPlayer.addEventListener('timeupdate', _ => console.log(this.audioPlayer.currentTime));
       this.audioPlayer.addEventListener('ended', _ => this.setState({
         playing: false,
         currentTime: (0,_lib_strformat__WEBPACK_IMPORTED_MODULE_1__.secondsToTime)(0)
       }));
     }
+
+    if (this.props.src) {
+      this.initVisualizer();
+    }
   }
 
   visualize() {
-    const pcmData = new Uint8Array(this.analyser.frequencyBinCount);
-    const width = this.canvasRef.current.width;
-    const height = this.canvasRef.current.height;
-    const viewLength = width / (LINE_WIDTH + SPACING) | 0;
-    const viewDuration = MILLIS_PER_BAR * viewLength;
     this.canvasContext.fillStyle = BKG_COLOR;
     this.canvasContext.lineWidth = LINE_WIDTH;
     this.canvasContext.strokeStyle = BAR_COLOR;
@@ -9749,6 +9754,10 @@ class AudioPlayer extends (react__WEBPACK_IMPORTED_MODULE_0___default().PureComp
     drawFrame();
   }
 
+  initVisualizer() {
+    console.log("Draw amplitude bars");
+  }
+
   handlePause(e) {
     e.preventDefault();
     this.setState({
@@ -9775,6 +9784,10 @@ class AudioPlayer extends (react__WEBPACK_IMPORTED_MODULE_0___default().PureComp
     });
   }
 
+  handleError(err) {
+    console.log(err);
+  }
+
   render() {
     const playClass = 'material-icons large' + (this.state.canPlay ? '' : ' disabled');
     return react__WEBPACK_IMPORTED_MODULE_0___default().createElement("div", {
@@ -9788,7 +9801,7 @@ class AudioPlayer extends (react__WEBPACK_IMPORTED_MODULE_0___default().PureComp
     }, this.state.playing ? 'pause_circle' : 'play_circle')), react__WEBPACK_IMPORTED_MODULE_0___default().createElement("div", null, react__WEBPACK_IMPORTED_MODULE_0___default().createElement("canvas", {
       className: "visualiser",
       style: {
-        backgroundColor: '#666'
+        border: '1px solid #666'
       },
       ref: this.canvasRef
     }), react__WEBPACK_IMPORTED_MODULE_0___default().createElement("div", {
@@ -9821,10 +9834,11 @@ const LINE_WIDTH = 4;
 const SPACING = 2;
 const MILLIS_PER_BAR = 100;
 const BAR_COLOR = '#8fbed6';
-const BAR_SCALE = 96.0;
+const BAR_SCALE = 64.0;
 const BKG_COLOR = '#eeeeee';
 const MIN_DURATION = 200;
 const MAX_DURATION = 600000;
+const VISUALIZATION_BARS = 96;
 class AudioRecorder extends (react__WEBPACK_IMPORTED_MODULE_0___default().PureComponent) {
   constructor(props) {
     super(props);
@@ -9879,6 +9893,8 @@ class AudioRecorder extends (react__WEBPACK_IMPORTED_MODULE_0___default().PureCo
     this.canvasContext.lineWidth = LINE_WIDTH;
     this.canvasContext.strokeStyle = BAR_COLOR;
     let prevBarCount = 0;
+    let volume = 0.0;
+    let countPerBar = 0;
 
     const drawFrame = () => {
       if (!this.startedOn) {
@@ -9903,19 +9919,22 @@ class AudioRecorder extends (react__WEBPACK_IMPORTED_MODULE_0___default().PureCo
       }
 
       this.analyser.getByteTimeDomainData(pcmData);
-      let volume = 0.0;
+      let amp = 0.0;
 
       for (const amplitude of pcmData) {
-        volume += (amplitude - 127) ** 2;
+        amp += (amplitude - 127) ** 2;
       }
 
-      volume = Math.sqrt(volume / pcmData.length);
+      volume += Math.sqrt(amp / pcmData.length);
+      countPerBar++;
       let barCount = duration / MILLIS_PER_BAR | 0;
       const dx = viewDuration > duration ? 0 : (duration - MILLIS_PER_BAR * barCount) / MILLIS_PER_BAR * (LINE_WIDTH + SPACING);
 
       if (prevBarCount != barCount) {
         prevBarCount = barCount;
-        this.viewBuffer.push(volume);
+        this.viewBuffer.push(volume / countPerBar);
+        volume = 0.0;
+        countPerBar = 0;
 
         if (this.viewBuffer.length > viewLength) {
           this.viewBuffer.shift();
@@ -9927,7 +9946,7 @@ class AudioRecorder extends (react__WEBPACK_IMPORTED_MODULE_0___default().PureCo
 
       for (let i = 0; i < this.viewBuffer.length; i++) {
         let x = i * (LINE_WIDTH + SPACING) - dx;
-        let y = this.viewBuffer[i] / BAR_SCALE * height;
+        let y = Math.min(this.viewBuffer[i] / BAR_SCALE, 0.9) * height;
         this.canvasContext.moveTo(x, (height - y) * 0.5);
         this.canvasContext.lineTo(x, height * 0.5 + y * 0.5);
       }
@@ -10003,13 +10022,14 @@ class AudioRecorder extends (react__WEBPACK_IMPORTED_MODULE_0___default().PureCo
       });
       this.audioChunks = [];
       const url = window.URL.createObjectURL(blob);
-      this.cleanUp();
 
       if (this.durationMillis > MIN_DURATION) {
-        this.props.onFinished(url, this.durationMillis);
+        blob.arrayBuffer().then(ab => this.audioContext.decodeAudioData(ab)).then(data => this.createPreview(data)).then(preview => this.props.onFinished(url, preview, this.durationMillis));
       } else {
         this.props.onDeleted();
       }
+
+      this.cleanUp();
     };
 
     this.mediaRecorder.ondataavailable = e => {
@@ -10024,6 +10044,32 @@ class AudioRecorder extends (react__WEBPACK_IMPORTED_MODULE_0___default().PureCo
     this.visualize();
   }
 
+  createPreview(audio) {
+    const data = audio.getChannelData(0);
+    const viewLength = Math.min(data.length, VISUALIZATION_BARS);
+    const spb = Math.min(100, data.length / viewLength | 0);
+    let buffer = [];
+    let max = -1;
+
+    for (let i = 0; i < viewLength; i++) {
+      let amplitude = 0;
+
+      for (let j = 0; j < spb; j++) {
+        amplitude += data[spb * i + j] ** 2;
+      }
+
+      const val = Math.sqrt(amplitude / spb);
+      buffer.push(val);
+      max = Math.max(max, val);
+    }
+
+    if (max > 0) {
+      buffer = buffer.map(a => 100 * a / max | 0);
+    }
+
+    return buffer;
+  }
+
   cleanUp() {
     this.audioInput.disconnect();
     this.stream.getTracks().forEach(track => track.stop());
@@ -14514,11 +14560,11 @@ class SendMessage extends (react__WEBPACK_IMPORTED_MODULE_0___default().PureComp
     e.target.value = '';
   }
 
-  handleAttachAudio(url, duration) {
+  handleAttachAudio(url, preview, duration) {
     this.setState({
       audioRec: false
     });
-    this.props.onAttachAudio(url, duration);
+    this.props.onAttachAudio(url, preview, duration);
   }
 
   handleSend(e) {