diff --git a/zimmer.js b/zimmer.js index e185560..b8c57b5 100755 --- a/zimmer.js +++ b/zimmer.js @@ -426,42 +426,54 @@ class Cluster { } // -// ClusterWriter +// ClusterPool // -var ClusterWriter = { - true: new Cluster( true ), // compressible cluster - false: new Cluster( false ), // uncompressible cluster - pool: genericPool.createPool( - { - create () { return Promise.resolve( Symbol() ) }, - destroy ( resource ) { return Promise.resolve() }, - }, - { max: 8, } - ), +class ClusterPool { + constructor () { + this.holder = {} + this.pool = genericPool.createPool( + { + create () { return Promise.resolve( Symbol() ) }, + destroy ( resource ) { return Promise.resolve() }, + }, + { max: 8, } + ) + } - append: async function ( mimeType, data, id /* for debugging */ ) { + removeCluster ( type ) { + delete this.holder[ type ] + } + + getCluster ( type ) { + let cluster = this.holder[ type ] + if ( ! cluster ) + cluster = this.holder[ type ] = new Cluster( type ) + return cluster + } + + async append ( mimeType, data, id /* for debugging */ ) { //~ log( 'ClusterWriter.append', arguments ) - var compressible = ClusterWriter.isCompressible( mimeType, data, id ) - var cluster = ClusterWriter[ compressible ] + var compressible = this.isCompressible( mimeType, data, id ) + var cluster = this.getCluster( compressible ) var clusterNum = cluster.id var blobNum = cluster.append( data ) if ( blobNum === false ) { // store to a new cluster - ClusterWriter[ compressible ] = new Cluster( compressible ) - const token = await ClusterWriter.pool.acquire() + this.removeCluster( compressible ) + const token = await this.pool.acquire() cluster.save() - .then( () => ClusterWriter.pool.release( token )) + .then( () => this.pool.release( token )) - return ClusterWriter.append( mimeType, data, id ) + return this.append( mimeType, data, id ) } log( 'ClusterWriter.append', compressible, clusterNum, blobNum, data.length, id ) - return Promise.resolve([ clusterNum, blobNum ]) - }, + return [ clusterNum, blobNum ] + } - isCompressible: function ( mimeType, data, id ) { + isCompressible ( mimeType, data, id ) { if ( data == null || data.length == 0 ) return false if ( !mimeType ) { @@ -471,7 +483,7 @@ var ClusterWriter = { if ( mimeType == 'image/svg+xml' || mimeType.split( '/' )[ 0 ] == 'text' ) return true return !! ( mimeDb[ mimeType ] && mimeDb[ mimeType ].compressible ) - }, + } // The cluster pointer list is a list of 8 byte offsets which point to all data clusters in a ZIM file. // Field Name Type Offset Length Description @@ -480,8 +492,8 @@ var ClusterWriter = { // integer (n-1)*8 8 pointer to the // ... integer ... 8 ... - storeIndex: async function () { - return saveIndex ({ + async storeIndex () { + header.clusterPtrPos = await saveIndex ({ query: 'SELECT ' + 'offset ' + @@ -493,17 +505,17 @@ var ClusterWriter = { count: header.clusterCount, logInfo: 'storeClusterIndex', }) - .then( offset => header.clusterPtrPos = offset ) - }, + } - finish: async function () { + async finish () { //~ log( 'ClusterWriter.finish', ClusterWriter ) - await ClusterWriter[ true ].save() // save last compressible cluster - await ClusterWriter[ false ].save() // save last uncompressible cluster - await ClusterWriter.pool.drain() - await ClusterWriter.pool.clear() - return ClusterWriter.storeIndex() - }, + for ( let i in this.holder ) { // save last clusters + await this.holder[ i ].save() + } + await this.pool.drain() + await this.pool.clear() + return this.storeIndex() + } } class NoProcessingRequired extends Error { @@ -777,7 +789,7 @@ class DataItem extends Item { if ( !( data instanceof Buffer )) { data = Buffer.from( data ) } - const [ clusterIdx, blobIdx ] = await ClusterWriter.append( this.mimeType, data, this.path ) + const [ clusterIdx, blobIdx ] = await clusterWriter.append( this.mimeType, data, this.path ) Object.assign( this, { clusterIdx, blobIdx }) } @@ -1381,6 +1393,8 @@ function calculateFileHash () { } async function initialise () { + clusterWriter = new ClusterPool + var stat = await fs.stat( srcPath ) if ( ! stat.isDirectory() ) { throw new Error( srcPath + ' is not a directory' ) @@ -1491,7 +1505,7 @@ async function loadRawArticles () { } async function postProcess () { - await ClusterWriter.finish() + await clusterWriter.finish() await sortArticles() await resolveRedirects() await storeUrlIndex()