more async/await
This commit is contained in:
parent
481ddaa58e
commit
e09f2809d6
92
zimmer.js
92
zimmer.js
|
@ -231,7 +231,7 @@ function chunksToBuffer( list ) {
|
||||||
return Buffer.concat( chunks )
|
return Buffer.concat( chunks )
|
||||||
}
|
}
|
||||||
|
|
||||||
function spawn ( command, args, input ) { // after https://github.com/panosoft/spawn-promise
|
async function spawn ( command, args, input ) { // after https://github.com/panosoft/spawn-promise
|
||||||
var child = childProcess.spawn( command, args )
|
var child = childProcess.spawn( command, args )
|
||||||
|
|
||||||
// Capture errors
|
// Capture errors
|
||||||
|
@ -255,19 +255,21 @@ function spawn ( command, args, input ) { // after https://github.com/panosoft/s
|
||||||
child.stdin.end()
|
child.stdin.end()
|
||||||
|
|
||||||
// Run
|
// Run
|
||||||
return new Promise( resolve => {
|
await new Promise(( resolve, reject ) => {
|
||||||
child.on( 'close', ( code, signal ) => resolve( code ))
|
child.on( 'close', ( code, signal ) => {
|
||||||
|
if ( code !== 0 ) {
|
||||||
|
reject( new Error( `Command failed: ${ code } ${ JSON.stringify( errors ) }` ))
|
||||||
|
} else {
|
||||||
|
resolve()
|
||||||
|
}
|
||||||
|
})
|
||||||
child.stdin.end( input )
|
child.stdin.end( input )
|
||||||
})
|
})
|
||||||
.then( exitCode => {
|
|
||||||
if ( exitCode !== 0 )
|
|
||||||
return Promise.reject( new Error( `Command failed: ${ exitCode } ${ JSON.stringify( errors ) }` ))
|
|
||||||
|
|
||||||
//~ if ( Object.keys( errors ).length !== 0 )
|
//~ if ( Object.keys( errors ).length !== 0 )
|
||||||
//~ return Promise.reject( new Error( JSON.stringify( errors )))
|
//~ return Promise.reject( new Error( JSON.stringify( errors )))
|
||||||
|
|
||||||
return Buffer.concat( buffers )
|
return Buffer.concat( buffers )
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function cvsReader ( path, options ) {
|
function cvsReader ( path, options ) {
|
||||||
|
@ -323,25 +325,23 @@ class Writer {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
write ( data ) {
|
async write ( data ) {
|
||||||
return this.queue.acquire()
|
const token = await this.queue.acquire()
|
||||||
.then( token => {
|
const startPosition = this.position
|
||||||
const startPosition = this.position
|
this.position += data.length
|
||||||
this.position += data.length
|
|
||||||
|
|
||||||
const saturated = ! this.stream.write( data )
|
const saturated = ! this.stream.write( data )
|
||||||
if ( saturated ) {
|
if ( saturated ) {
|
||||||
this.stream.once( 'drain', () => this.queue.release( token ))
|
this.stream.once( 'drain', () => this.queue.release( token ))
|
||||||
} else {
|
} else {
|
||||||
this.queue.release( token )
|
this.queue.release( token )
|
||||||
}
|
}
|
||||||
return startPosition
|
return startPosition
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
close () {
|
async close () {
|
||||||
return this.queue.drain()
|
await this.queue.drain()
|
||||||
.then( () => new Promise( resolve => {
|
return await new Promise( resolve => {
|
||||||
this.queue.clear()
|
this.queue.clear()
|
||||||
this.stream.once( 'close', () => {
|
this.stream.once( 'close', () => {
|
||||||
log( this.stream.path, 'closed', this.position, this.stream.bytesWritten )
|
log( this.stream.path, 'closed', this.position, this.stream.bytesWritten )
|
||||||
|
@ -349,7 +349,7 @@ class Writer {
|
||||||
})
|
})
|
||||||
log( 'closing', this.stream.path )
|
log( 'closing', this.stream.path )
|
||||||
this.stream.end()
|
this.stream.end()
|
||||||
}))
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -599,27 +599,23 @@ class Item {
|
||||||
return Promise.resolve( this.id )
|
return Promise.resolve( this.id )
|
||||||
}
|
}
|
||||||
|
|
||||||
saveItemIndex () {
|
async saveItemIndex () {
|
||||||
if ( ! this.path ) {
|
if ( ! this.path ) {
|
||||||
fatal( 'Item no url', this )
|
fatal( 'Item no url', this )
|
||||||
}
|
}
|
||||||
|
|
||||||
const row = [
|
const row = [
|
||||||
this.urlKey(),
|
this.urlKey(),
|
||||||
this.titleKey(),
|
this.titleKey(),
|
||||||
this.revision,
|
this.revision,
|
||||||
this.mimeId(),
|
this.mimeId(),
|
||||||
]
|
]
|
||||||
|
const result = await wikiDb.run(
|
||||||
return wikiDb.run(
|
|
||||||
'INSERT INTO articles ( urlKey, titleKey, revision, mimeId ) VALUES ( ?,?,?,? )',
|
'INSERT INTO articles ( urlKey, titleKey, revision, mimeId ) VALUES ( ?,?,?,? )',
|
||||||
row
|
row
|
||||||
)
|
)
|
||||||
.then( res => {
|
const id = result.stmt.lastID
|
||||||
const id = res.stmt.lastID
|
log( 'saveItemIndex', id, this )
|
||||||
log( 'saveItemIndex', id, this )
|
return id
|
||||||
return id
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Article Entry
|
// Article Entry
|
||||||
|
@ -842,12 +838,12 @@ class File extends DataItem {
|
||||||
//~ revision ,
|
//~ revision ,
|
||||||
//~ urlKey ,
|
//~ urlKey ,
|
||||||
//~ titleKey
|
//~ titleKey
|
||||||
getData () {
|
async getData () {
|
||||||
if ( this.data == null )
|
if ( this.data == null ) {
|
||||||
this.data = fs.readFile( this.srcPath())
|
this.data = fs.readFile( this.srcPath())
|
||||||
.then( data => this.preProcess( data ))
|
}
|
||||||
|
const data = await this.data
|
||||||
return Promise.resolve( this.data )
|
return await this.preProcess( data )
|
||||||
}
|
}
|
||||||
|
|
||||||
srcPath () {
|
srcPath () {
|
||||||
|
@ -953,7 +949,7 @@ class RawFile extends File {
|
||||||
return super.preProcess( data )
|
return super.preProcess( data )
|
||||||
}
|
}
|
||||||
|
|
||||||
preProcessHtml ( data ) {
|
async preProcessHtml ( data ) {
|
||||||
const dom = ( this.mimeType == 'text/html' ) && cheerio.load( data.toString())
|
const dom = ( this.mimeType == 'text/html' ) && cheerio.load( data.toString())
|
||||||
if ( dom ) {
|
if ( dom ) {
|
||||||
const title = dom( 'title' ).text()
|
const title = dom( 'title' ).text()
|
||||||
|
@ -966,13 +962,13 @@ class RawFile extends File {
|
||||||
title: this.title,
|
title: this.title,
|
||||||
to: redirectTarget,
|
to: redirectTarget,
|
||||||
})
|
})
|
||||||
return redirect.process()
|
await redirect.process()
|
||||||
.then( () => Promise.reject( new NoProcessingRequired()))
|
return Promise.reject( new NoProcessingRequired())
|
||||||
}
|
}
|
||||||
if ( this.alterLinks( dom ))
|
if ( this.alterLinks( dom ))
|
||||||
data = Buffer.from( dom.html())
|
data = Buffer.from( dom.html())
|
||||||
}
|
}
|
||||||
return Promise.resolve( data )
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
alterLinks ( dom ) {
|
alterLinks ( dom ) {
|
||||||
|
@ -1577,7 +1573,7 @@ async function core () {
|
||||||
// -r, --redirects path to the CSV file with the list of redirects (url, title, target_url tab separated).
|
// -r, --redirects path to the CSV file with the list of redirects (url, title, target_url tab separated).
|
||||||
// -i, --withFullTextIndex index the content and add it to the ZIM.
|
// -i, --withFullTextIndex index the content and add it to the ZIM.
|
||||||
|
|
||||||
function main () {
|
async function main () {
|
||||||
|
|
||||||
argv
|
argv
|
||||||
.version( packageInfo.version )
|
.version( packageInfo.version )
|
||||||
|
@ -1628,8 +1624,8 @@ function main () {
|
||||||
//~ ClusterSizeThreshold = argv.minChunkSize * 1024
|
//~ ClusterSizeThreshold = argv.minChunkSize * 1024
|
||||||
//~ }
|
//~ }
|
||||||
|
|
||||||
core ()
|
await core ()
|
||||||
.then( () => log( 'Done...' ))
|
log( 'Done...' )
|
||||||
}
|
}
|
||||||
|
|
||||||
main ()
|
main ()
|
||||||
|
|
Loading…
Reference in New Issue
Block a user