suzanne.soy/wiki.gruntnetwork.com--zimmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cosmetic

Browse Source
This commit is contained in:
vss-devel 2019-07-16 18:22:08 +03:00
parent 55d3ff5972
commit 62f5849305
2 changed files with 36 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
wikizimmer.js
View File

@ -336,7 +336,7 @@ class NameSpaceSet {
} }
const wiki = { const wiki = {
saveDir: null, outPath: null,
apiUrl: null, apiUrl: null,
metadata: {}, metadata: {},
nameSpaces: null, nameSpaces: null,
@ -468,7 +468,7 @@ class WikiItem {
if ( data == null ) if ( data == null )
return return
const savePath = osPath.join( wiki.saveDir, this.localPath()) const savePath = osPath.join( wiki.outPath, this.localPath())
log( '+', savePath ) log( '+', savePath )
return fs.outputFile( savePath, data ) return fs.outputFile( savePath, data )
@ -918,9 +918,9 @@ ${data}
} }
} }
async function processSamplePage ( sampleURLUrl ) { async function processSamplePage ( url ) {
const resp = await requestPromise({ const resp = await requestPromise({
url: encodeurl( sampleURLUrl ), url: encodeurl( url ),
resolveWithFullResponse: true, resolveWithFullResponse: true,
}) })
//~log(resp) //~log(resp)
@ -1206,7 +1206,7 @@ async function loadCss( dom ) {
async function initWikiDb () { async function initWikiDb () {
let dbName = osPath.join( wiki.saveDir, 'metadata.db' ) let dbName = osPath.join( wiki.outPath, 'metadata.db' )
try { try {
await fs.unlink( dbName ) await fs.unlink( dbName )
@ -1250,22 +1250,20 @@ function closeMetadataStorage () {
return wiki.db.close() return wiki.db.close()
} }
async function initDir ( sampleURL, outPath ) { async function initDir ( url, path ) {
const purl = urlconv.parse( sampleURL ) wiki.outPath = path || sanitizeFN( urlconv.parse( url ).hostname )
wiki.saveDir = outPath || sanitizeFN( purl.hostname )
let done = true let done = true
if ( command.rmdir ) { if ( command.rmdir ) {
const oldDir = wiki.saveDir + '$' const oldDir = wiki.outPath + '$'
try { try {
await fs.move( wiki.saveDir, oldDir, { overwrite: true }) await fs.move( wiki.outPath, oldDir, { overwrite: true })
} catch ( err ) { } catch ( err ) {
log( 'initDir', err ) log( 'initDir', err )
} }
done = fs.remove( oldDir ) done = fs.remove( oldDir )
} }
await fs.mkdirs( wiki.saveDir ) await fs.mkdirs( wiki.outPath )
return { done } return { done }
} }
@ -1277,9 +1275,9 @@ async function core ( sampleURL, outPath ) {
try { try {
const oldDir = await initDir( sampleURL, outPath ) const oldDir = await initDir( sampleURL, outPath )
await initWikiDb()
await loadPreRequisites() await loadPreRequisites()
const sampleDom = await processSamplePage( sampleURL ) const sampleDom = await processSamplePage( sampleURL )
await initWikiDb()
await loadCss( sampleDom ) await loadCss( sampleDom )
await getSiteInfo() await getSiteInfo()
await getPages() await getPages()

50
zimmer.js
View File

@ -36,7 +36,7 @@ const osProcess = require( 'process' )
const url = require( 'url' ) const url = require( 'url' )
const crypto = require( "crypto" ) const crypto = require( "crypto" )
const argv = require('commander') const command = require('commander')
const fs = require( 'fs-extra' ) const fs = require( 'fs-extra' )
const expandHomeDir = require( 'expand-home-dir' ) const expandHomeDir = require( 'expand-home-dir' )
const lzma = require( 'lzma-native' ) const lzma = require( 'lzma-native' )
@ -161,7 +161,7 @@ function mimeTypeIndex ( mimeType ) {
} }
function getNameSpace ( mimeType ) { function getNameSpace ( mimeType ) {
if ( argv.uniqueNamespace ) if ( command.uniqueNamespace )
return 'A' return 'A'
if ( !mimeType ) if ( !mimeType )
return null return null
@ -455,7 +455,7 @@ class ClusterPool {
async save ( cluster ) { async save ( cluster ) {
const data = await cluster.getData() const data = await cluster.getData()
const row = [ cluster.id ] const row = [ cluster.id ]
if ( ! argv.zimlib4Fix ) { if ( ! command.zimlib4Fix ) {
const offset = await out.write( data ) const offset = await out.write( data )
row.push( offset.toString() ) // convert BigInt to String row.push( offset.toString() ) // convert BigInt to String
} else { // zimlib4Fix stores clusters in separate files } else { // zimlib4Fix stores clusters in separate files
@ -487,7 +487,7 @@ class ClusterPool {
} }
isCompressible ( mimeType, data, id ) { isCompressible ( mimeType, data, id ) {
if ( ! argv.compress ) if ( ! command.compress )
return false return false
if ( data == null || data.length == 0 ) if ( data == null || data.length == 0 )
return false return false
@ -511,7 +511,7 @@ class ClusterPool {
const count = header.clusterCount const count = header.clusterCount
let rowCb = ( row, index ) => BigInt( row.offset ) let rowCb = ( row, index ) => BigInt( row.offset )
if ( argv.zimlib4Fix ) { if ( command.zimlib4Fix ) {
let offsetZimlib4 = await out.write( Buffer.alloc( 0 )) + BigInt( count * byteLength ) let offsetZimlib4 = await out.write( Buffer.alloc( 0 )) + BigInt( count * byteLength )
rowCb = ( row, index ) => { rowCb = ( row, index ) => {
const val = offsetZimlib4 const val = offsetZimlib4
@ -535,7 +535,7 @@ class ClusterPool {
} }
async storeClusters () { async storeClusters () {
if ( argv.zimlib4Fix ) { // zimlib4Fix stores clusters in separate files if ( command.zimlib4Fix ) { // zimlib4Fix stores clusters in separate files
for ( let i = 0; i < header.clusterCount; i++ ) { for ( let i = 0; i < header.clusterCount; i++ ) {
const fname = osPath.join( this.savePrefix, `${i}` ) const fname = osPath.join( this.savePrefix, `${i}` )
const data = await fs.readFile( fname ) const data = await fs.readFile( fname )
@ -874,13 +874,13 @@ class File extends DataItem {
} }
async processJpeg ( data ) { async processJpeg ( data ) {
if ( ! argv.optimg ) if ( ! command.optimg )
return data return data
this.mimeType = 'image/jpeg' this.mimeType = 'image/jpeg'
try { try {
return await spawn( return await spawn(
mozjpeg, mozjpeg,
[ '-quality', argv.jpegquality, data.length < 20000 ? '-baseline' : '-progressive' ], [ '-quality', command.jpegquality, data.length < 20000 ? '-baseline' : '-progressive' ],
data data
) )
} catch ( err ) { } catch ( err ) {
@ -890,7 +890,7 @@ class File extends DataItem {
} }
async processImage ( data ) { async processImage ( data ) {
if ( ! argv.optimg ) if ( ! command.optimg )
return data return data
try { try {
const image = sharp( data ) const image = sharp( data )
@ -953,7 +953,7 @@ class RawFile extends File {
this.mimeType = await mimeFromData( data ) this.mimeType = await mimeFromData( data )
this.nameSpace = this.nameSpace || getNameSpace( this.mimeType ) this.nameSpace = this.nameSpace || getNameSpace( this.mimeType )
} }
if ( argv.inflateHtml && this.mimeType == 'text/html' ) { if ( command.inflateHtml && this.mimeType == 'text/html' ) {
data = await zlib.inflate( data ) // inflateData data = await zlib.inflate( data ) // inflateData
} }
await this.preProcessHtml( data ) await this.preProcessHtml( data )
@ -1078,14 +1078,14 @@ function fillInMetadata () {
const outParsed = osPath.parse( outPath ) const outParsed = osPath.parse( outPath )
const metadata = [ const metadata = [
[ 'Name', outParsed.base ], [ 'Name', outParsed.base ],
[ 'Title', argv.title ], [ 'Title', command.title ],
[ 'Creator', argv.creator ], [ 'Creator', command.creator ],
[ 'Publisher', argv.publisher ], [ 'Publisher', command.publisher ],
[ 'Date', new Date().toISOString().split( 'T' )[ 0 ]], [ 'Date', new Date().toISOString().split( 'T' )[ 0 ]],
[ 'Description', argv.description ], [ 'Description', command.description ],
[ 'Language', argv.language ], [ 'Language', command.language ],
[ 'logo', argv.favicon ], [ 'logo', command.favicon ],
//~ [ 'mainpage', argv.welcome ], //~ [ 'mainpage', command.welcome ],
] ]
const done = [] const done = []
@ -1149,7 +1149,7 @@ async function openWikiDb( dbName ) {
async function newWikiDb() { async function newWikiDb() {
var dbName = '' var dbName = ''
if ( argv.verbose ) { if ( command.verbose ) {
var parsed = osPath.parse( outPath ) var parsed = osPath.parse( outPath )
dbName = osPath.join( parsed.dir, parsed.base + '.db' ) dbName = osPath.join( parsed.dir, parsed.base + '.db' )
} }
@ -1209,8 +1209,8 @@ async function loadRedirects () {
var redirectsFile var redirectsFile
if ( preProcessed ) if ( preProcessed )
redirectsFile = osPath.join( srcPath, 'redirects.csv' ) redirectsFile = osPath.join( srcPath, 'redirects.csv' )
else if ( argv.redirects ) else if ( command.redirects )
redirectsFile = expandHomeDir( argv.redirects ) redirectsFile = expandHomeDir( command.redirects )
else else
return return
@ -1587,7 +1587,7 @@ async function core () {
async function main () { async function main () {
argv command
.version( packageInfo.version ) .version( packageInfo.version )
.arguments( '<source-directory> [<output-file>]' ) .arguments( '<source-directory> [<output-file>]' )
.description( `Pack a directory into a zim file .description( `Pack a directory into a zim file
@ -1618,9 +1618,9 @@ async function main () {
.option( '--no-compress', "do not compress clusters" ) .option( '--no-compress', "do not compress clusters" )
.parse( osProcess.argv ) .parse( osProcess.argv )
log( argv ) log( command )
const args = argv.args const args = command.args
while ( args[ 0 ] == '' ) // if mwoffliner prepends with empty extra parameter(s) while ( args[ 0 ] == '' ) // if mwoffliner prepends with empty extra parameter(s)
args.shift() args.shift()
@ -1633,8 +1633,8 @@ async function main () {
outPath = parsed.base + '.zim' outPath = parsed.base + '.zim'
} }
//~ if ( argv.minChunkSize ) { //~ if ( command.minChunkSize ) {
//~ ClusterSizeThreshold = argv.minChunkSize * 1024 //~ ClusterSizeThreshold = command.minChunkSize * 1024
//~ } //~ }
await core () await core ()