suzanne.soy/wiki.gruntnetwork.com--zimmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

zimlib v4.0.4 compatibility mode

Browse Source
This commit is contained in:
vss-devel 2019-03-11 14:31:57 +03:00
parent fa9c552e5e
commit 1618900aaf
1 changed files with 29 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
zimmer.js
View File

@ -454,12 +454,17 @@ class ClusterPool {
async save ( cluster ) { async save ( cluster ) {
const data = await cluster.getData() const data = await cluster.getData()
await fs.outputFile( osPath.join( this.savePrefix, `${cluster.id}` ), data ) let offset
if ( argv.zimlib4Fix ) { // zimlib4Fix stores clusters in separate files
await fs.outputFile( osPath.join( this.savePrefix, `${cluster.id}` ), data )
} else {
offset = await out.write( data )
}
await wikiDb.run( await wikiDb.run(
'INSERT INTO clusters ( id, size ) VALUES ( ?,? )', 'INSERT INTO clusters ( id, offset ) VALUES ( ?,? )',
[ [
cluster.id, cluster.id,
data.length ( argv.zimlib4Fix ? data.length : offset ).toString() // zimlib4Fix stores cluster sizes instead of offsets
] ]
) )
log( 'Cluster saved', cluster.id, data.length ) log( 'Cluster saved', cluster.id, data.length )
@ -509,13 +514,12 @@ class ClusterPool {
async storeIndex () { async storeIndex () {
const byteLength = 8 const byteLength = 8
const count = header.clusterCount const count = header.clusterCount
const start = await out.write( Buffer.alloc( 0 )) let offsetZimlib4 = await out.write( Buffer.alloc( 0 )) + BigInt( count * byteLength ) // zimlib4Fix
let offset = start + BigInt( count * byteLength )
header.clusterPtrPos = await saveIndex ({ header.clusterPtrPos = await saveIndex ({
query:` query:`
SELECT SELECT
size CAST( offset AS TEXT ) AS offset -- to prevent casting to JS Number
FROM clusters FROM clusters
ORDER BY id ORDER BY id
;`, ;`,
@ -524,22 +528,29 @@ class ClusterPool {
count, count,
logPrefix: 'storeClusterIndex', logPrefix: 'storeClusterIndex',
rowCb: ( row, index ) => { rowCb: ( row, index ) => {
const val = offset const offset = BigInt( row.offset )
offset += BigInt( row.size ) if ( ! argv.zimlib4Fix ) {
return val return offset
} else { // zimlib4Fix stores cluster sizes instead of offsets
const val = offsetZimlib4
offsetZimlib4 += offset
return val
}
}, },
}) })
} }
async storeClusters () { async storeClusters () {
for ( let i = 0; i < header.clusterCount; i++ ) { if ( argv.zimlib4Fix ) { // zimlib4Fix stores clusters in separate files
const fname = osPath.join( this.savePrefix, `${i}` ) for ( let i = 0; i < header.clusterCount; i++ ) {
const data = await fs.readFile( fname ) const fname = osPath.join( this.savePrefix, `${i}` )
const pos = await out.write( data ) const data = await fs.readFile( fname )
log( 'storeClusters', i, pos ) const pos = await out.write( data )
await fs.remove( fname ) log( 'storeClusters', i, pos )
await fs.remove( fname )
}
await fs.remove( this.savePrefix )
} }
await fs.remove( this.savePrefix )
} }
async finish () { async finish () {
@ -1136,7 +1147,7 @@ async function openWikiDb( dbName ) {
); );
CREATE TABLE clusters ( CREATE TABLE clusters (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
size INTEGER offset INTEGER
); );
` `
) )
@ -1608,6 +1619,7 @@ async function main () {
//~ .option( '-i, --withFullTextIndex', 'index the content and add it to the ZIM' ) //~ .option( '-i, --withFullTextIndex', 'index the content and add it to the ZIM' )
// Extra arguments: // Extra arguments:
.option( '--optimg', 'optimise images' ) .option( '--optimg', 'optimise images' )
.option( '-4, --no-zimlib4-fix', 'no zimlib v4.0.4 compatibility mode' )
.option( '--jpegquality <factor>', 'JPEG quality', parseInt, 60 ) .option( '--jpegquality <factor>', 'JPEG quality', parseInt, 60 )
.option( '--no-compress', "do not compress clusters" ) .option( '--no-compress', "do not compress clusters" )
.parse( osProcess.argv ) .parse( osProcess.argv )