fractional fetching, wip

src/config.js

@@ -121,4 +121,3 @@ export const LINK_PRIVACY_POLICY = 'https://tinode.co/privacy.html';
 
 // Link to Terms of Service.
 export const LINK_TERMS_OF_SERVICE = 'https://tinode.co/terms.html';
-

src/views/messages-view.jsx

@@ -157,8 +157,9 @@ class MessagesView extends React.Component {
     this.handleDrag = this.handleDrag.bind(this);
     this.handleDrop = this.handleDrop.bind(this);
 
-    this.chatMessageRefs = {};
+    this.chatMessageRefs = [];
     this.getOrCreateMessageRef = this.getOrCreateMessageRef.bind(this);
+    this.getVisibleMessageRange = this.getVisibleMessageRange.bind(this);
 
     // Keeps track of the drag event.
     // Need a counter b/c the browser's 'drag' events may fire multiple times
@@ -174,7 +175,7 @@ class MessagesView extends React.Component {
   }
 
   getOrCreateMessageRef(seqId) {
-    if (this.chatMessageRefs.hasOwnProperty(seqId)) {
+    if (this.chatMessageRefs[seqId]) {
       return this.chatMessageRefs[seqId];
     }
     const ref = React.createRef();
@@ -182,6 +183,21 @@ class MessagesView extends React.Component {
     return ref;
   }
 
+  getVisibleMessageRange(holderRect) {
+    let min = Number.MAX_SAFE_INTEGER, max = -1;
+    this.chatMessageRefs.forEach((ref, seq) => {
+      if (ref.current) {
+        const { top, bottom, height } = ref.current.getBoundingClientRect();
+        const visible = top <= holderRect.top ? holderRect.top - top <= height : bottom - holderRect.bottom <= height;
+        if (visible) {
+          min = Math.min(min, seq);
+          max = Math.max(max, seq);
+        }
+      }
+    });
+    return max >= min ? {min: min, max: max} : {min: 0, max: 0};
+  }
+
   componentDidMount() {
     if (this.messagesScroller) {
       this.messagesScroller.addEventListener('scroll', this.handleScrollEvent);
@@ -565,15 +581,20 @@ class MessagesView extends React.Component {
 
     if (event.target.scrollTop <= FETCH_PAGE_TRIGGER) {
       const topic = this.props.tinode.getTopic(this.state.topic);
-      if (topic && topic.isSubscribed() && topic.msgHasMoreMessages()) {
+      if (topic && topic.isSubscribed()) {
+        const {min, max} = this.getVisibleMessageRange(event.target.getBoundingClientRect());
+        const gaps = topic.msgHasMoreMessages(min, max, false);
+        if (gaps.length > 0) {
+          console.log("Fetch triggered", min, max, gaps);
           this.setState({fetchingMessages: true}, _ => {
-          topic.getMessagesPage(MESSAGES_PAGE)
+            topic.getMessagesPage(MESSAGES_PAGE, gaps)
               .catch(err => this.props.onError(err.message, 'err'))
               .finally(_ => this.setState({fetchingMessages: false}));
             });
         }
       }
     }
+  }
 
   /* Mount drag and drop events */
   mountDnDEvents(dnd) {

umd/index.dev.js

@@ -5226,8 +5226,9 @@ class MessagesView extends (react__WEBPACK_IMPORTED_MODULE_0___default().Compone
     this.handleDragOut = this.handleDragOut.bind(this);
     this.handleDrag = this.handleDrag.bind(this);
     this.handleDrop = this.handleDrop.bind(this);
-    this.chatMessageRefs = {};
+    this.chatMessageRefs = [];
     this.getOrCreateMessageRef = this.getOrCreateMessageRef.bind(this);
+    this.getVisibleMessageRange = this.getVisibleMessageRange.bind(this);
     this.dragCounter = 0;
     this.dndRef = null;
     this.readNotificationQueue = [];
@@ -5235,13 +5236,38 @@ class MessagesView extends (react__WEBPACK_IMPORTED_MODULE_0___default().Compone
     this.keyPressTimer = null;
   }
   getOrCreateMessageRef(seqId) {
-    if (this.chatMessageRefs.hasOwnProperty(seqId)) {
+    if (this.chatMessageRefs[seqId]) {
       return this.chatMessageRefs[seqId];
     }
     const ref = react__WEBPACK_IMPORTED_MODULE_0___default().createRef();
     this.chatMessageRefs[seqId] = ref;
     return ref;
   }
+  getVisibleMessageRange(holderRect) {
+    let min = Number.MAX_SAFE_INTEGER,
+      max = -1;
+    this.chatMessageRefs.forEach((ref, seq) => {
+      if (ref.current) {
+        const {
+          top,
+          bottom,
+          height
+        } = ref.current.getBoundingClientRect();
+        const visible = top <= holderRect.top ? holderRect.top - top <= height : bottom - holderRect.bottom <= height;
+        if (visible) {
+          min = Math.min(min, seq);
+          max = Math.max(max, seq);
+        }
+      }
+    });
+    return max >= min ? {
+      min: min,
+      max: max
+    } : {
+      min: 0,
+      max: 0
+    };
+  }
   componentDidMount() {
     if (this.messagesScroller) {
       this.messagesScroller.addEventListener('scroll', this.handleScrollEvent);
@@ -5558,17 +5584,25 @@ class MessagesView extends (react__WEBPACK_IMPORTED_MODULE_0___default().Compone
     }
     if (event.target.scrollTop <= FETCH_PAGE_TRIGGER) {
       const topic = this.props.tinode.getTopic(this.state.topic);
-      if (topic && topic.isSubscribed() && topic.msgHasMoreMessages()) {
+      if (topic && topic.isSubscribed()) {
+        const {
+          min,
+          max
+        } = this.getVisibleMessageRange(event.target.getBoundingClientRect());
+        const gaps = topic.msgHasMoreMessages(min, max, false);
+        if (gaps.length > 0) {
+          console.log("Fetch triggered", min, max, gaps);
           this.setState({
             fetchingMessages: true
           }, _ => {
-          topic.getMessagesPage(_config_js__WEBPACK_IMPORTED_MODULE_18__.MESSAGES_PAGE).catch(err => this.props.onError(err.message, 'err')).finally(_ => this.setState({
+            topic.getMessagesPage(_config_js__WEBPACK_IMPORTED_MODULE_18__.MESSAGES_PAGE, gaps).catch(err => this.props.onError(err.message, 'err')).finally(_ => this.setState({
               fetchingMessages: false
             }));
           });
         }
       }
     }
+  }
   mountDnDEvents(dnd) {
     if (dnd) {
       dnd.addEventListener('dragstart', this.handleDragStart);

umd/tinode.dev.js

@@ -782,6 +782,8 @@ __webpack_require__.r(__webpack_exports__);
 /* harmony export */ __webpack_require__.d(__webpack_exports__, {
 /* harmony export */   "default": () => (/* binding */ DB)
 /* harmony export */ });
+/* harmony import */ var _utils_js__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ./utils.js */ "./src/utils.js");
+
 
 const DB_VERSION = 3;
 const DB_NAME = 'tinode-web';
@@ -1136,13 +1138,16 @@ class DB {
         let count = 0;
         query.ranges.forEach(range => {
           const key = range.hi ? IDBKeyRange.bound([topicName, range.low], [topicName, range.Hi], false, true) : IDBKeyRange.only([topicName, range.low]);
-          const req = trx.objectStore('message').get(key);
+          trx.objectStore('message').get(key).onsuccess = event => {
-          req.onsuccess = event => {
+            const msgs = event.target.result;
+            if (callback) {
+              callback.call(context, msgs);
+            }
             count++;
-            if (Array.isArray(event.target.result)) {
+            if (Array.isArray(msgs)) {
-              result.concat(event.target.result);
+              result.concat(msgs);
             } else {
-              result.push(event.target.result);
+              result.push(msgs);
             }
             if (count == query.ranges.length) {
               resolve(result);
@@ -1238,7 +1243,7 @@ class DB {
           const delrange = event2.target.result && event2.target.result.value;
           if (delrange) {
             const result = [];
-            clipped.forEach(r => result.push.apply(result, DB.#clipRange(r, delrange)));
+            clipped.forEach(r => result.push.apply(result, (0,_utils_js__WEBPACK_IMPORTED_MODULE_0__.clipRange)(r, delrange)));
             clipped = result;
             if (clipped.length > 0) {
               event2.target.result.continue();
@@ -1283,6 +1288,42 @@ class DB {
       trx.commit();
     });
   }
+  readDelLog(topicName, query) {
+    query = query || {};
+    const since = query.since > 0 ? query.since : 0;
+    const before = query.before > 0 ? query.before : Number.MAX_SAFE_INTEGER;
+    const limit = query.limit | 0;
+    if (!this.isReady()) {
+      return this.disabled ? Promise.resolve([]) : Promise.reject(new Error("not initialized"));
+    }
+    return new Promise((resolve, reject) => {
+      const trx = this.db.transaction(['dellog']);
+      trx.onerror = event => {
+        this.#logger('PCache', 'readDelLog', event.target.error);
+        reject(event.target.error);
+      };
+      let count = 0;
+      const result = [];
+      const range = IDBKeyRange.bound([topicName, 0, since], [topicName, before, Number.MAX_SAFE_INTEGER], false, true);
+      trx.objectStore('dellog').openCursor(range, 'prev').onsuccess = event => {
+        const cursor = event.target.result;
+        if (cursor) {
+          result.push({
+            low: cursor.value.low,
+            hi: cursor.value.hi
+          });
+          count += cursor.value.hi - cursor.value.low;
+          if (limit <= 0 || count < limit) {
+            cursor.continue();
+          } else {
+            resolve(result);
+          }
+        } else {
+          resolve(result);
+        }
+      };
+    });
+  }
   maxDelId(topicName) {
     if (!this.isReady()) {
       return this.disabled ? Promise.resolve(0) : Promise.reject(new Error("not initialized"));
@@ -1358,31 +1399,6 @@ class DB {
     });
     return res;
   }
-  static #clipRange(src, clip) {
-    if (clip.hi < src.low || clip.low >= src.hi) {
-      return [src];
-    }
-    if (clip.low <= src.low) {
-      if (clip.hi >= src.hi) {
-        return [];
-      }
-      return [{
-        low: src.low,
-        hi: clip.hi
-      }];
-    }
-    const result = [{
-      low: src.low,
-      hi: clip.low
-    }];
-    if (clip.hi < src.hi) {
-      result.push({
-        low: clip.hi,
-        hi: src.hi
-      });
-    }
-    return result;
-  }
   static setDatabaseProvider(idbProvider) {
     IDBProvider = idbProvider;
   }
@@ -4060,20 +4076,8 @@ class Topic {
   getMeta(params) {
     return this._tinode.getMeta(this.name, params);
   }
-  getMessagesPage(limit, forward) {
+  getMessagesPage(limit, gaps, forward) {
-    const gaps = [];
+    console.log("getMessagesPage", gaps, forward);
-    this._messages.forEach((msg, prev) => {
-      const p = prev || {
-        seq: 0
-      };
-      if (msg.seq > p.seq + 1) {
-        gaps.push({
-          low: p.seq + 1,
-          hi: msg.seq
-        });
-      }
-    });
-    console.log("Gaps in cache found:", gaps);
     let query = forward ? this.startMetaQuery().withLaterData(limit) : this.startMetaQuery().withEarlierData(limit);
     return this._loadMessages(this._tinode._db, query.extract('data')).then(count => {
       if (count == limit) {
@@ -4456,6 +4460,9 @@ class Topic {
           if (Topic.#isReplacementMsg(msg)) {
             return;
           }
+          if (msg._deleted) {
+            return;
+          }
           const latest = this.latestMsgVersion(msg.seq) || msg;
           if (!latest._origTs) {
             latest._origTs = latest.ts;
@@ -4527,8 +4534,39 @@ class Topic {
   msgRecvCount(seq) {
     return this.msgReceiptCount('recv', seq);
   }
-  msgHasMoreMessages(newer) {
+  msgHasMoreMessages(min, max, newer) {
-    return newer ? this.seq > this._maxSeq : this._minSeq > 1 && !this._noEarlierMsgs;
+    const gaps = [];
+    let maxSeq = 0;
+    let gap;
+    this._messages.forEach((msg, prev) => {
+      const p = prev || {
+        seq: 0
+      };
+      const expected = p._deleted ? p.hi : p.seq + 1;
+      if (msg.seq > expected) {
+        gap = {
+          low: expected,
+          hi: msg.seq
+        };
+      } else {
+        gap = null;
+      }
+      if (gap && (newer ? gap.hi >= min : gap.low < max)) {
+        gaps.push(gap);
+      }
+      maxSeq = expected;
+    });
+    if (maxSeq < this.seq) {
+      gap = {
+        low: maxSeq + 1,
+        hi: this.seq + 1
+      };
+      if (newer ? gap.hi >= min : gap.low < max) {
+        gaps.push(gap);
+      }
+    }
+    console.log("Gaps in cache found:", gaps, "min/max:", min, max, "haveMore:", gaps.length > 0);
+    return gaps;
   }
   isNewMessage(seqId) {
     return this._maxSeq <= seqId;
@@ -4953,17 +4991,10 @@ class Topic {
   _getQueuedSeqId() {
     return this._queuedSeqId++;
   }
-  _loadMessages(db, params) {
+  _loadMessages(db, query) {
-    const {
+    query = query || {};
-      since,
+    query.limit = query.limit || _config_js__WEBPACK_IMPORTED_MODULE_3__.DEFAULT_MESSAGES_PAGE;
-      before,
+    return db.readMessages(this.name, query).then(msgs => {
-      limit
-    } = params || {};
-    return db.readMessages(this.name, {
-      since: since,
-      before: before,
-      limit: limit || _config_js__WEBPACK_IMPORTED_MODULE_3__.DEFAULT_MESSAGES_PAGE
-    }).then(msgs => {
       msgs.forEach(data => {
         if (data.seq > this._maxSeq) {
           this._maxSeq = data.seq;
@@ -4975,6 +5006,17 @@ class Topic {
         this._maybeUpdateMessageVersionsCache(data);
       });
       return msgs.length;
+    }).then(_ => db.readDelLog(this.name, query)).then(dellog => {
+      return dellog.forEach(rec => {
+        this._messages.put({
+          seq: rec.low,
+          low: rec.low,
+          hi: rec.hi,
+          _deleted: true
+        });
+      });
+    }).then(_ => {
+      this._messages.forEach(msg => console.log(msg));
     });
   }
   _updateReceived(seq, act) {
@@ -4999,6 +5041,7 @@ class Topic {
 
 __webpack_require__.r(__webpack_exports__);
 /* harmony export */ __webpack_require__.d(__webpack_exports__, {
+/* harmony export */   "clipRange": () => (/* binding */ clipRange),
 /* harmony export */   "isUrlRelative": () => (/* binding */ isUrlRelative),
 /* harmony export */   "jsonParseHelper": () => (/* binding */ jsonParseHelper),
 /* harmony export */   "listToRanges": () => (/* binding */ listToRanges),
@@ -5173,6 +5216,31 @@ function listToRanges(list) {
     return out;
   }, []);
 }
+function clipRange(src, clip) {
+  if (clip.hi < src.low || clip.low >= src.hi) {
+    return [src];
+  }
+  if (clip.low <= src.low) {
+    if (clip.hi >= src.hi) {
+      return [];
+    }
+    return [{
+      low: src.low,
+      hi: clip.hi
+    }];
+  }
+  const result = [{
+    low: src.low,
+    hi: clip.low
+  }];
+  if (clip.hi < src.hi) {
+    result.push({
+      low: clip.hi,
+      hi: src.hi
+    });
+  }
+  return result;
+}
 
 /***/ }),