Hard heap size limit for the LRU cache

lru-cache.js

@@ -1,9 +1,15 @@
 // Cache any data with a timestamp,
 // remove only the oldest data.
 
-function Cache(size) {
-  if (!this instanceof Cache) { return new Cache(size); }
+var typeEnum = {
+  unit: 0,
+  heap: 1,
+};
+
+function Cache(size, type) {
+  if (!this instanceof Cache) { return new Cache(size, type); }
   this.size = size;
+  this.type = typeEnum[type];
   // `cache` contains {content, index}.
   // - content: the actual data that is cached.
   // - index: the position in `order` of the data.
@@ -11,45 +17,76 @@ function Cache(size) {
   this.order = [];  // list of cache keys from oldest to newest.
 }
 
-Cache.prototype.set =
-function addToCache(cacheIndex, cached) {
-  if (this.cache[cacheIndex] !== undefined) {
-    this.order.splice(this.cache[cacheIndex].index, 1);
-    // Put the new element at the end of `order`.
-    this.cache[cacheIndex].index = this.order.length;
-    this.cache[cacheIndex].content = cached;
-    this.order.push(cacheIndex);
-  } else {
-    // If the cache is full, remove the oldest data
-    // (ie, the data requested longest ago.)
-    if (this.order.length >= this.size) {
-      // Remove `order`'s oldest element, the first.
-      delete this.cache[this.order[0]];
-      this.order.shift();
-    }
+Cache.prototype = {
+  set: function addToCache(cacheIndex, cached) {
+    if (this.cache[cacheIndex] !== undefined) {
+      this.order.splice(this.cache[cacheIndex].index, 1);
+      // Put the new element at the end of `order`.
+      this.cache[cacheIndex].index = this.order.length;
+      this.cache[cacheIndex].content = cached;
+      this.order.push(cacheIndex);
+    } else {
+      // If the cache is full, remove the oldest data
+      // (ie, the data requested longest ago.)
+      var numberToRemove = this.limitReached();
+      for (var i = 0; i < numberToRemove; i++) {
+        // Remove `order`'s oldest element, the first.
+        delete this.cache[this.order[0]];
+        this.order.shift();
+      }
 
-    this.cache[cacheIndex] = {
-      index: this.order.length,
-      content: cached
+      this.cache[cacheIndex] = {
+        index: this.order.length,
+        content: cached,
+      }
+      this.order.push(cacheIndex);
     }
-    this.order.push(cacheIndex);
+  },
+
+  get: function getFromCache(cacheIndex) {
+    if (this.cache[cacheIndex] !== undefined) {
+      this.order.splice(this.cache[cacheIndex].index, 1);
+      // Put the new element at the end of `order`.
+      this.cache[cacheIndex].index = this.order.length;
+      this.order.push(cacheIndex);
+      return this.cache[cacheIndex].content;
+    } else { return; }
+  },
+
+  has: function hasInCache(cacheIndex) {
+    return this.cache[cacheIndex] !== undefined;
+  },
+
+  // Returns true if we're past the limit.
+  limitReached: function heuristic() {
+    if (this.type === typeEnum.unit) {
+      return Math.max(0, (this.order.length - this.size));
+    } else if (this.type === typeEnum.heap) {
+      if (getHeapSize() >= this.size) {
+        // Remove a quarter of them.
+        return (this.order.length >> 2);
+      } else { return 0; }
+    } else {
+      console.error("Unknown heuristic for LRU cache.");
+      return 1;
+    }
+  },
+};
+
+// In bytes.
+var heapSize;
+var heapSizeTimeout;
+function getHeapSize() {
+  if (heapSizeTimeout == null) {
+    // Compute the heap size every 60 seconds.
+    heapSizeTimeout = setInterval(computeHeapSize, 60 * 1000);
+    return computeHeapSize();
+  } else {
+    return heapSize;
   }
 }
-
-Cache.prototype.get =
-function getFromCache(cacheIndex) {
-  if (this.cache[cacheIndex] !== undefined) {
-    this.order.splice(this.cache[cacheIndex].index, 1);
-    // Put the new element at the end of `order`.
-    this.cache[cacheIndex].index = this.order.length;
-    this.order.push(cacheIndex);
-    return this.cache[cacheIndex].content;
-  } else { return; }
-}
-
-Cache.prototype.has =
-function hasInCache(cacheIndex) {
-  return this.cache[cacheIndex] !== undefined;
+function computeHeapSize() {
+  return heapSize = process.memoryUsage().heapTotal;
 }
 
 module.exports = Cache;

server.js

@@ -147,8 +147,8 @@ var minAccuracy = 0.75;
 //       = 1 - max(1, df) / rf
 var freqRatioMax = 1 - minAccuracy;
 
-// Request cache size of size 500_000 (~512MB, 1kB/image).
-var requestCache = new LruCache(500000);
+// Request cache size of 500MB heap limit.
+var requestCache = new LruCache(500000000, 'heap');
 
 // Deep error handling for vendor hooks.
 var vendorDomain = domain.create();