adding audio playback

src/lib/formatters.js

@@ -3,13 +3,14 @@ import { defineMessages } from 'react-intl';
 
 import { Drafty } from 'tinode-sdk';
 
+import AudioPlayer from '../widgets/audio-player.jsx'
 import LazyImage from '../widgets/lazy-image.jsx'
 import UploadingImage from '../widgets/uploading-image.jsx'
 
 import { IMAGE_THUMBNAIL_DIM, BROKEN_IMAGE_SIZE, REM_SIZE } from '../config.js';
 import { base64ToBlob, blobToBase64, fitImageSize, imageScaled } from './blob-helpers.js';
-import { idToColorClass, shortenFileName } from './strformat.js';
-import { cancelablePromise, sanitizeImageUrl } from './utils.js';
+import { idToColorClass, secondsToTime, shortenFileName } from './strformat.js';
+import { cancelablePromise, sanitizeUrlForMime } from './utils.js';
 
 const messages = defineMessages({
   drafty_form: {
@@ -34,7 +35,7 @@ const messages = defineMessages({
   }
 });
 
-// Size the already scaled image.
+// Additional processing of image data.
 function handleImageData(el, data, attr) {
   if (!data) {
     attr.src = 'img/broken_image.png';
@@ -58,7 +59,7 @@ function handleImageData(el, data, attr) {
     minHeight: dim.dstHeight + 'px'
   };
   if (!Drafty.isProcessing(data)) {
-    attr.src = this.authorizeURL(sanitizeImageUrl(attr.src));
+    attr.src = this.authorizeURL(sanitizeUrlForMime(attr.src, 'image'));
     attr.alt = data.name;
     if (attr.src) {
       if (Math.max(data.width || 0, data.height || 0) > IMAGE_THUMBNAIL_DIM) {
@@ -99,6 +100,16 @@ export function fullFormatter(style, data, values, key, stack) {
   let attr = Drafty.attrValue(style, data) || {};
   attr.key = key;
   switch (style) {
+    case 'AU':
+      // Show audio player.
+      if (attr.src) {
+        attr.src = this.authorizeURL(sanitizeUrlForMime(attr.src, 'audio'));
+        attr.loading = 'lazy';
+      }
+      el = AudioPlayer;
+      // Audio element cannot have content.
+      values = null;
+      break;
     case 'BR':
       values = null;
       break;
@@ -178,6 +189,11 @@ export function previewFormatter(style, data, values, key) {
   let el = Drafty.tagName(style);
   const attr = { key: key };
   switch (style) {
+    case 'AU':
+      // Voicemail as '[mic] 0:00'.
+      el = React.Fragment;
+      values = [<i key="au" className="material-icons">mic</i>, ' ', secondsToTime(data.duration/1000)];
+      break;
     case 'BR':
       // Replace new line with a space.
       el = React.Fragment;
@@ -195,7 +211,7 @@ export function previewFormatter(style, data, values, key) {
     case 'IM':
       // Replace image with '[icon] Image'.
       el = React.Fragment;
-      values = [<i key="im" className="material-icons">photo</i>, this.formatMessage(messages.drafty_image)];
+      values = [<i key="im" className="material-icons">photo</i>, ' ', this.formatMessage(messages.drafty_image)];
       break;
     case 'BN':
       el = 'span';
@@ -220,7 +236,7 @@ export function previewFormatter(style, data, values, key) {
         delete data.ref;
       }
       el = React.Fragment;
-      values = [<i key="ex" className="material-icons">attachment</i>, this.formatMessage(messages.drafty_attachment)];
+      values = [<i key="ex" className="material-icons">attachment</i>, ' ', this.formatMessage(messages.drafty_attachment)];
       break;
     case 'QQ':
     case 'HD':
@@ -229,7 +245,7 @@ export function previewFormatter(style, data, values, key) {
       break;
     default:
       el = React.Fragment;
-      values = [<i key="x0" className="material-icons gray">extension</i>, this.formatMessage(messages.drafty_unknown)];
+      values = [<i key="x0" className="material-icons gray">extension</i>, ' ', this.formatMessage(messages.drafty_unknown)];
       break;
   }
   if (!el) {
@@ -317,7 +333,7 @@ function quoteImage(data) {
     }
     promise = Promise.resolve(blob);
   } else if (data.ref) {
-    promise = fetch(this.authorizeURL(sanitizeImageUrl(data.ref)))
+    promise = fetch(this.authorizeURL(sanitizeUrlForMime(data.ref, 'image')))
       .then(evt => {
         if (evt.ok) {
           return evt.blob();

src/lib/strformat.js

@@ -19,7 +19,7 @@ export function shortDateFormat(then, locale) {
 // Convert seconds to minutes:seconds, i.e. 156 sec -> 2:36.
 // If <code>fixedMins</code> is true, then minutes are represented by at least two digits.
 export function secondsToTime(seconds, fixedMin) {
-  let min = Math.floor(seconds / 60);
+  let min = Math.floor(seconds / 60) | 0;
   if (fixedMin) {
     min = min < 10 ? `0${min}` : min;
   }

src/lib/utils.js

@@ -144,10 +144,10 @@ export function sanitizeUrl(url, allowedSchemes) {
   return url;
 }
 
-// Ensure URL is suitable for <img src="url"> field: the URL must be a relative URL or
+// Ensure URL is suitable as a source like <img src="url"> field: the URL must be a relative URL or
 // have http:, https:, blob: or data: scheme.
-// In case of data: scheme, the URL must start with a 'data:image/XXXX;base64,'.
-export function sanitizeImageUrl(url) {
+// In case of data: scheme, the URL must must be of the right MIME type such as 'data:{mimeMajor}/XXXX;base64,'.
+export function sanitizeUrlForMime(url, mimeMajor) {
   if (!url) {
     return null;
   }
@@ -157,8 +157,9 @@ export function sanitizeImageUrl(url) {
     return sanitizedUrl;
   }
 
-  // Is this a data: URL of an image?
-  if (/data:image\/[a-z0-9.-]+;base64,/i.test(url.trim())) {
+  // Is this a data: URL of the appropriate mime type?
+  const re = new RegExp(`data:${mimeMajor}\/[a-z0-9.-]+;base64,`, 'i');
+  if (re.test(url.trim())) {
     return url;
   }
 

src/widgets/audio-player.jsx

@@ -0,0 +1,171 @@
+// Audio recorder widget.
+
+import React from 'react';
+
+import { secondsToTime } from '../lib/strformat';
+
+// FFT resolution.
+const BUFFER_SIZE = 256;
+// Thickness of a visualization bar.
+const LINE_WIDTH = 4;
+// Spacing between two visualization bars.
+const SPACING = 2;
+// Duration represented by one visualization bar.
+const MILLIS_PER_BAR = 100;
+// Color of histogram bars
+const BAR_COLOR = '#8fbed6';
+// Scaling for visualization bars.
+const BAR_SCALE = 96.0;
+// Background color
+const BKG_COLOR = '#eeeeee';
+// Maximum duration of a recording in milliseconds (10 min).
+const MAX_DURATION = 600000;
+
+export default class AudioPlayer extends React.PureComponent {
+  constructor(props) {
+    super(props);
+
+    this.state = {
+      canPlay: false,
+      playing: false,
+      currentTime: '0:00',
+      duration: secondsToTime(this.props.duration)
+    };
+
+    this.visualize = this.visualize.bind(this);
+
+    this.handlePlay = this.handlePlay.bind(this);
+    this.handlePause = this.handlePause.bind(this);
+
+    this.audioPlayer = this.props.src ? new Audio(this.props.src) : null;
+
+    this.durationMillis = 0;
+    this.startedOn = null;
+    this.viewBuffer = [];
+    this.canvasRef = React.createRef();
+  }
+
+  componentDidMount() {
+    this.canvasContext = this.canvasRef.current.getContext('2d');
+    this.canvasContext.lineCap = 'round';
+    // To reduce line blurring.
+    this.canvasContext.translate(0.5, 0.5);
+
+    if (this.audioPlayer) {
+      this.audioPlayer.addEventListener('canplay', _ => {
+        console.log('canplay', this.audioPlayer.duration);
+        this.setState({canPlay: true, duration: secondsToTime(this.audioPlayer.duration)})
+      });
+      this.audioPlayer.addEventListener('timeupdate', _ => console.log(this.audioPlayer.currentTime));
+      this.audioPlayer.addEventListener('ended', _ => this.setState({playing: false, currentTime: secondsToTime(0)}));
+    }
+  }
+
+  // Draw amplitude of sound.
+  visualize() {
+    const pcmData = new Uint8Array(this.analyser.frequencyBinCount);
+    const width = this.canvasRef.current.width;
+    const height = this.canvasRef.current.height;
+    const viewLength = (width / (LINE_WIDTH + SPACING)) | 0;
+    const viewDuration = MILLIS_PER_BAR * viewLength;
+
+    this.canvasContext.fillStyle = BKG_COLOR;
+    this.canvasContext.lineWidth = LINE_WIDTH;
+    this.canvasContext.strokeStyle = BAR_COLOR;
+
+    let prevBarCount = 0;
+    const drawFrame = () => {
+      if (!this.startedOn) {
+        return;
+      }
+      window.requestAnimationFrame(drawFrame);
+
+      const duration = this.durationMillis + (Date.now() - this.startedOn);
+      // Update record length timer.
+      this.setState({duration: secondsToTime(duration / 1000)});
+
+      // Check if record is too long.
+      if (duration > MAX_DURATION) {
+        this.mediaRecorder.pause();
+        this.durationMillis += Date.now() - this.startedOn;
+        this.startedOn = null;
+        this.setState({enabled: false, recording: false, duration: secondsToTime(this.durationMillis / 1000)});
+      }
+
+      // Draw histogram.
+
+      // Get current waveform and calculate its amplitude.
+      this.analyser.getByteTimeDomainData(pcmData);
+      let volume = 0.0;
+      for (const amplitude of pcmData) {
+        volume += (amplitude - 127) ** 2;
+      }
+      volume = Math.sqrt(volume/pcmData.length);
+
+      let barCount = (duration / MILLIS_PER_BAR) | 0;
+      // Shift of the histogram along x-axis to make scrolling smooth. No need to shift if recording is too short.
+      const dx = viewDuration > duration ? 0 :
+        (duration - MILLIS_PER_BAR * barCount) / MILLIS_PER_BAR * (LINE_WIDTH + SPACING);
+
+      if (prevBarCount != barCount) {
+        prevBarCount = barCount;
+        // Add new amplitude visualization bar.
+        this.viewBuffer.push(volume);
+        if (this.viewBuffer.length > viewLength) {
+          // Keep at most 'viewLength' amplitude bars.
+          this.viewBuffer.shift();
+        }
+      }
+
+      // Clear canvas.
+      this.canvasContext.fillRect(0, 0, width, height);
+
+      // Draw amplitude bars.
+      this.canvasContext.beginPath();
+      for (let i = 0; i < this.viewBuffer.length; i++) {
+        let x = i * (LINE_WIDTH + SPACING) - dx;
+        let y = this.viewBuffer[i] / BAR_SCALE * height;
+
+        this.canvasContext.moveTo(x, (height - y) * 0.5);
+        this.canvasContext.lineTo(x, height * 0.5 + y * 0.5);
+      }
+      // Actually draw the bars on canvas.
+      this.canvasContext.stroke();
+    }
+
+    drawFrame();
+  }
+
+  handlePause(e) {
+    e.preventDefault();
+    this.setState({recording: false, duration: secondsToTime(this.durationMillis / 1000)});
+  }
+
+  handlePlay(e) {
+    e.preventDefault();
+    if (!this.state.canPlay) {
+      return;
+    }
+    if (this.state.playing) {
+      this.audioPlayer.pause();
+    } else {
+      this.audioPlayer.play();
+    }
+    this.setState({playing: !this.state.playing});
+  }
+
+  render() {
+    const playClass = 'material-icons large' + (this.state.canPlay ? '' : ' disabled');
+    return (
+      <div className="audio-player">
+        <a href="#" onClick={this.handlePlay} title="Play">
+          <i className={playClass}>{this.state.playing ? 'pause_circle' : 'play_circle'}</i>
+        </a>
+        <div>
+          <canvas className="visualiser" style={{backgroundColor: '#666'}} ref={this.canvasRef} />
+          <div className="timer">{this.state.currentTime}/{this.state.duration}</div>
+        </div>
+      </div>
+    );
+  }
+}

src/widgets/audio-recorder.jsx

@@ -233,7 +233,7 @@ export default class AudioRecorder extends React.PureComponent {
         <a href="#" onClick={this.handleDelete} title="Delete">
           <i className="material-icons">delete_outline</i>
         </a>
-        <canvas className="visualiser" width="200" height="60" ref={this.canvasRef} />
+        <canvas className="visualiser" ref={this.canvasRef} />
         <div className="duration">{this.state.duration}</div>
         {this.state.recording ?
           <a href="#" onClick={this.handlePause} title="Pause">

src/widgets/avatar-upload.jsx

@@ -3,7 +3,7 @@ import React from 'react';
 import LetterTile from './letter-tile.jsx';
 import LoadSpinner from './load-spinner.jsx';
 
-import { sanitizeImageUrl } from '../lib/utils.js';
+import { sanitizeUrlForMime } from '../lib/utils.js';
 
 import { AVATAR_SIZE } from '../config.js';
 
@@ -44,7 +44,7 @@ export default class AvatarUpload extends React.Component {
             <i className="material-icons">clear</i>
           </a>}
         {this.state.source ?
-          <img src={this.props.tinode.authorizeURL(sanitizeImageUrl(this.state.source))} className="preview" /> :
+          <img src={this.props.tinode.authorizeURL(sanitizeUrlForMime(this.state.source, 'image'))} className="preview" /> :
           this.props.readOnly && this.props.uid ?
             <div className="avatar-box">
               <LetterTile

src/widgets/letter-tile.jsx

@@ -3,7 +3,7 @@ import React from 'react';
 import { Tinode } from 'tinode-sdk';
 
 import { idToColorClass } from '../lib/strformat.js';
-import { sanitizeImageUrl } from '../lib/utils.js';
+import { sanitizeUrlForMime } from '../lib/utils.js';
 
 export default class LetterTile extends React.PureComponent {
   render() {
@@ -20,7 +20,7 @@ export default class LetterTile extends React.PureComponent {
         avatar = isGroup ? <i className={className}>group</i> : <i className={className}>person</i>;
       }
     } else if (this.props.avatar) {
-      const url = this.props.tinode.authorizeURL(sanitizeImageUrl(this.props.avatar));
+      const url = this.props.tinode.authorizeURL(sanitizeUrlForMime(this.props.avatar, 'image'));
       // If avatar image is invalid, show a placeholder.
       const className = 'avatar' + (this.props.deleted ? ' deleted' : '');
       avatar = <img className={className} alt="avatar" src={url}

src/widgets/side-navbar.jsx

@@ -6,7 +6,7 @@ import ContactBadges from './contact-badges.jsx';
 import MenuContacts from './menu-contacts.jsx';
 import MenuStart from './menu-start.jsx';
 
-import { sanitizeImageUrl } from '../lib/utils.js';
+import { sanitizeUrlForMime } from '../lib/utils.js';
 
 export default class SideNavbar extends React.PureComponent {
   render() {
@@ -18,7 +18,7 @@ export default class SideNavbar extends React.PureComponent {
     }
     let avatar = null;
     if (this.props.tinode) {
-      avatar = this.props.tinode.authorizeURL(sanitizeImageUrl(this.props.avatar));
+      avatar = this.props.tinode.authorizeURL(sanitizeUrlForMime(this.props.avatar, 'image'));
     }
     return (
       <div id="side-caption-panel" className="caption-panel">