process -> save

This commit is contained in:
v 2018-11-28 18:27:04 +03:00
parent 7e34635653
commit e4ec93b5b2

View File

@ -67,7 +67,6 @@ const mimeIds = []
let articleCount = 0 let articleCount = 0
let redirectCount = 0 let redirectCount = 0
let http // http request let http // http request
let indexerDb
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247 // https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247
// just in case https://www.mediawiki.org/wiki/Manual:Page_title // just in case https://www.mediawiki.org/wiki/Manual:Page_title
@ -442,7 +441,7 @@ class WikiItem {
return id return id
} }
store ( data ) { storeData ( data ) {
if ( data == null ) if ( data == null )
return Promise.reject( new Error( 'data == null' )) return Promise.reject( new Error( 'data == null' ))
@ -460,7 +459,7 @@ class WikiItem {
this.revision, this.revision,
this.mimeId(), this.mimeId(),
] ]
return indexerDb.run( return wiki.db.run(
'INSERT INTO articles ( urlKey, titleKey, revision, mimeId ) VALUES ( ?,?,?,? )', 'INSERT INTO articles ( urlKey, titleKey, revision, mimeId ) VALUES ( ?,?,?,? )',
row row
) )
@ -472,12 +471,12 @@ class WikiItem {
}) })
} }
process () { save () {
if ( this.blackListed() ) if ( this.blackListed() )
return '' return ''
return Promise.resolve() return Promise.resolve()
.then( () => this.getData()) .then( () => this.getData())
.then( data => this.store( data )) .then( data => this.storeData( data ))
.then( () => this.storeMetadata() ) .then( () => this.storeMetadata() )
.then( () => this.localPath() ) .then( () => this.localPath() )
.catch( err => { .catch( err => {
@ -657,7 +656,7 @@ class Article extends ArticleStub {
async saveImage ( url ) { async saveImage ( url ) {
const image = new Image( url ) const image = new Image( url )
const localPath = await image.process() const localPath = await image.save()
return encodeURI( this.pathToTop() + '../' + localPath ) return encodeURI( this.pathToTop() + '../' + localPath )
} }
} }
@ -694,7 +693,7 @@ class Redirect extends ArticleStub {
log( '>', this.title || this.url, row) log( '>', this.title || this.url, row)
indexerDb.run( wiki.db.run(
'INSERT INTO redirects (id, targetKey, fragment) VALUES (?,?,?)', 'INSERT INTO redirects (id, targetKey, fragment) VALUES (?,?,?)',
row row
) )
@ -753,10 +752,10 @@ class PageComponent extends WikiItem {
return sanitizeFN( decodeURIComponent( name )) return sanitizeFN( decodeURIComponent( name ))
} }
process () { save () {
let saved = urlCache.get( this.url ) let saved = urlCache.get( this.url )
if (! saved ) { if (! saved ) {
saved = super.process() saved = super.save()
urlCache.set( this.url, saved ) urlCache.set( this.url, saved )
saved.then( localPath => { // keep item's data in the cache saved.then( localPath => { // keep item's data in the cache
@ -779,10 +778,10 @@ class Image extends PageComponent {
return super.getData() return super.getData()
} }
*/ */
process () { save () {
if (! command.images ) if (! command.images )
return Promise.resolve( this.localPath() ) return Promise.resolve( this.localPath() )
return super.process() return super.save()
} }
basePath () { basePath () {
if ( ! this.url ) if ( ! this.url )
@ -858,7 +857,7 @@ class Style extends LayoutItem {
src.replace( urlre, ( match, start, url, end ) => { src.replace( urlre, ( match, start, url, end ) => {
if ( ! url.startsWith( 'data:' )) { if ( ! url.startsWith( 'data:' )) {
const styleItem = new LayoutItem( urlconv.resolve( this.url, url )) const styleItem = new LayoutItem( urlconv.resolve( this.url, url ))
requests.push( styleItem.process() ) requests.push( styleItem.save() )
} }
return match return match
}) })
@ -1018,17 +1017,17 @@ async function saveMetadata () {
Source: wiki.uri, Source: wiki.uri,
} }
await new MainPage().process() await new MainPage().save()
await new FavIcon().process() await new FavIcon().save()
for ( let i in metadata ) { for ( let i in metadata ) {
await new Metadata( i, metadata[i] ).process() await new Metadata( i, metadata[i] ).save()
} }
} }
async function saveMimeTypes () { async function saveMimeTypes () {
for ( let i=0, li=mimeIds.length; i < li; i++ ) { for ( let i=0, li=mimeIds.length; i < li; i++ ) {
await indexerDb.run( await wiki.db.run(
'INSERT INTO mimeTypes (id, value) VALUES (?,?)', 'INSERT INTO mimeTypes (id, value) VALUES (?,?)',
[ i + 1, mimeIds[ i ]] [ i + 1, mimeIds[ i ]]
) )
@ -1077,7 +1076,7 @@ function batchRedirects ( pageInfos ) {
return null return null
item.to = target item.to = target
item.toFragment = rdr.tofragment item.toFragment = rdr.tofragment
return new Redirect( item ).process() return new Redirect( item ).save()
}) })
return Promise.all( done ) return Promise.all( done )
}) })
@ -1109,14 +1108,14 @@ async function batchPages ( nameSpace ) {
let continueFrom = '' let continueFrom = ''
while ( true ) { while ( true ) {
await indexerDb.run( await wiki.db.run(
'INSERT OR REPLACE INTO continue (id, "from") VALUES (1, ?)', 'INSERT OR REPLACE INTO continue (id, "from") VALUES (1, ?)',
[ continueFrom ] [ continueFrom ]
) )
if ( continueFrom == null ) if ( continueFrom == null )
break break
await indexerDb.run( 'BEGIN' ) await wiki.db.run( 'BEGIN' )
const resp = await api( query ) const resp = await api( query )
let pages = {} let pages = {}
@ -1148,12 +1147,12 @@ async function batchPages ( nameSpace ) {
return null return null
} }
log( '#', pageInfo.title ) log( '#', pageInfo.title )
return new Article( pageInfo ).process() return new Article( pageInfo ).save()
}) })
done.push( batchRedirects( redirects )) done.push( batchRedirects( redirects ))
await Promise.all( done ) await Promise.all( done )
await indexerDb.run( 'COMMIT' ) await wiki.db.run( 'COMMIT' )
continueFrom = null continueFrom = null
try { try {
@ -1186,7 +1185,7 @@ function loadCss( dom ) {
if (! command.css ) if (! command.css )
return Promise.resolve() return Promise.resolve()
const css = new GlobalCss( dom ) const css = new GlobalCss( dom )
return css.process() return css.save()
} }
function initMetadataStorage ( samplePageDOM ) { function initMetadataStorage ( samplePageDOM ) {
@ -1197,8 +1196,8 @@ function initMetadataStorage ( samplePageDOM ) {
.catch( () => null ) .catch( () => null )
.then( () => sqlite.open( dbName )) .then( () => sqlite.open( dbName ))
.then( db => { .then( db => {
indexerDb = db wiki.db = db
return indexerDb.exec( return wiki.db.exec(
'PRAGMA synchronous = OFF;' + 'PRAGMA synchronous = OFF;' +
//~ 'PRAGMA journal_mode = OFF;' + //~ 'PRAGMA journal_mode = OFF;' +
'PRAGMA journal_mode = WAL;' + 'PRAGMA journal_mode = WAL;' +
@ -1236,7 +1235,7 @@ function initMetadataStorage ( samplePageDOM ) {
} }
function closeMetadataStorage () { function closeMetadataStorage () {
return indexerDb.close() return wiki.db.close()
} }
function core ( samplePage ) { function core ( samplePage ) {