external data -> separate queues

wikizimmer.js

@@ -113,107 +113,108 @@ function pooledRequest( request, referenceUri, maxTokens = 1, interval = 10 ) {
     const retryLimit = 10
     const retryExternal = command.retryExternal == null ? retryLimit : command.retryExternal
     const requestTimeout = 5 * 60 * 1000
-    const queue = []
     const refHost = urlconv.parse( referenceUri ).host
-    let timer = null
+    const hostQueues = {}
-    let supressTimer = null
-    let supressTimeout = 60 * 1000
-    let tokenCounter = 0
 
-    function setTimer () {
+    class Queue {
-        if ( supressTimer )
+        constructor () {
+            this.queue = []
+            this.timer = null
+            this.supressTimer = null
+            this.supressTimeout = 60 * 1000
+            this.tokenCounter = 0
+            this.interval = interval
+        }
+
+        reshedule () {
+            if ( this.supressTimer )
                 return
-        timer = setTimeout(
+            this.timer = setTimeout(
-            () => ( timer = null, schedule() ),
+                () => ( this.timer = null, this.run() ),
-            interval
+                this.interval
             )
         }
 
-    function pause ( query ) {
+        pause ( query ) {
-        if ( timer ) {
+            if ( this.timer ) {
-            clearTimeout( timer )
+                clearTimeout( this.timer )
             }
-        if ( supressTimer ) {
+            if ( this.supressTimer ) {
-            clearTimeout( supressTimer )
+                clearTimeout( this.supressTimer )
             }
-        supressTimer = setTimeout(
+            this.supressTimer = setTimeout(
-            () => ( supressTimer = false, setTimer()),
+                () => ( this.supressTimer = false, this.reshedule()),
-            query.retries * supressTimeout
+                query.retries * this.supressTimeout
             )
             if ( ! query.external ) {
-            interval = interval * 4
+                this.interval = this.interval * 2
             }
         }
 
-    function release () {
+        release () {
-        tokenCounter --
+            this.tokenCounter --
-        schedule()
+            this.run()
         }
 
-    function retry ( query, error ) {
+        retry ( query, error ) {
-        pause( query )
+            this.pause( query )
             if ( ++ query.retries <= ( query.external ? retryExternal : retryLimit )) {
-            queue.push( query )
+                this.queue.push( query )
-            return
+            } else {
-        }
                 query.reject( error )
-        return
+            }
         }
 
-    function acquire () {
+        acquire () {
             let query
-        if ( timer || supressTimer || tokenCounter >= maxTokens || ! ( query = queue.shift()))
+            if ( this.timer || this.supressTimer || this.tokenCounter >= maxTokens || ! ( query = this.queue.shift()))
                 return false
-        tokenCounter ++
+            this.tokenCounter ++
-        return {
+            return query
-            query,
-            release,
-            retry,
-        }
         }
 
-    function schedule () {
+        submit ( query ) {
-        let token = acquire()
-        if ( ! token )
-            return
-        runRequest( token )
-        setTimer()
-    }
-
-    function runRequest( token ) {
-        let query = token.query
             return request( query )
             .catch( error => {
                 const retryCause = retryStatusCodes.includes( error.statusCode ) ? error.statusCode :
                     error.cause && retryErrorCodes.includes( error.cause.code ) ? error.cause.code : false
                 if ( retryCause ) {
                     log( 'retry request', interval, error.name, retryCause, query )
-                token.retry( query, error )
+                    this.retry( query, error )
                     return
                 }
                 query.reject( error )
                 return
             })
             .then( reply => {
-            token.release()
+                this.release()
-            return reply ? query.resolve( reply ) : null
+                return reply ? query.resolve( reply ) : query.reject( )
             })
         }
 
-    function append ( query, priority ) {
+        run () {
+            let query = this.acquire()
+            if ( query ) {
+                this.submit( query )
+                this.reshedule()
+            }
+        }
+
+        append ( query, priority ) {
             return new Promise(( resolve, reject ) => {
                 query.resolve = resolve
                 query.reject = reject
                 query.retries = 0
 
                 if ( priority )
-                queue.unshift( query )
+                    this.queue.unshift( query )
                 else
-                queue.push( query )
+                    this.queue.push( query )
 
-            schedule()
+                this.run()
             })
         }
+    }
 
     function processOptions ( query ) {
         let url
@@ -226,8 +227,8 @@ function pooledRequest( request, referenceUri, maxTokens = 1, interval = 10 ) {
             delete query.uri
         }
         query.url = urlconv.resolve( referenceUri, url )
-        const queryHost = urlconv.parse( query.url ).host
+        query.host = urlconv.parse( query.url ).host
-        query.external = queryHost != refHost
+        query.external = query.host != refHost
 
         if ( ! query.headers )
             query.headers = {}
@@ -243,7 +244,13 @@ function pooledRequest( request, referenceUri, maxTokens = 1, interval = 10 ) {
     }
 
     return function ( query, priority = false ) {
-        return append( processOptions( query ), priority )
+        processOptions( query )
+        let queue = hostQueues[ query.host ]
+        if ( ! queue ) {
+            queue = new Queue
+            hostQueues[ query.host ] = queue
+        }
+        return queue.append( query , priority )
     }
 }