process -> save
This commit is contained in:
parent
7e34635653
commit
e4ec93b5b2
|
@ -67,7 +67,6 @@ const mimeIds = []
|
||||||
let articleCount = 0
|
let articleCount = 0
|
||||||
let redirectCount = 0
|
let redirectCount = 0
|
||||||
let http // http request
|
let http // http request
|
||||||
let indexerDb
|
|
||||||
|
|
||||||
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247
|
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247
|
||||||
// just in case https://www.mediawiki.org/wiki/Manual:Page_title
|
// just in case https://www.mediawiki.org/wiki/Manual:Page_title
|
||||||
|
@ -442,7 +441,7 @@ class WikiItem {
|
||||||
return id
|
return id
|
||||||
}
|
}
|
||||||
|
|
||||||
store ( data ) {
|
storeData ( data ) {
|
||||||
if ( data == null )
|
if ( data == null )
|
||||||
return Promise.reject( new Error( 'data == null' ))
|
return Promise.reject( new Error( 'data == null' ))
|
||||||
|
|
||||||
|
@ -460,7 +459,7 @@ class WikiItem {
|
||||||
this.revision,
|
this.revision,
|
||||||
this.mimeId(),
|
this.mimeId(),
|
||||||
]
|
]
|
||||||
return indexerDb.run(
|
return wiki.db.run(
|
||||||
'INSERT INTO articles ( urlKey, titleKey, revision, mimeId ) VALUES ( ?,?,?,? )',
|
'INSERT INTO articles ( urlKey, titleKey, revision, mimeId ) VALUES ( ?,?,?,? )',
|
||||||
row
|
row
|
||||||
)
|
)
|
||||||
|
@ -472,12 +471,12 @@ class WikiItem {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
process () {
|
save () {
|
||||||
if ( this.blackListed() )
|
if ( this.blackListed() )
|
||||||
return ''
|
return ''
|
||||||
return Promise.resolve()
|
return Promise.resolve()
|
||||||
.then( () => this.getData())
|
.then( () => this.getData())
|
||||||
.then( data => this.store( data ))
|
.then( data => this.storeData( data ))
|
||||||
.then( () => this.storeMetadata() )
|
.then( () => this.storeMetadata() )
|
||||||
.then( () => this.localPath() )
|
.then( () => this.localPath() )
|
||||||
.catch( err => {
|
.catch( err => {
|
||||||
|
@ -657,7 +656,7 @@ class Article extends ArticleStub {
|
||||||
|
|
||||||
async saveImage ( url ) {
|
async saveImage ( url ) {
|
||||||
const image = new Image( url )
|
const image = new Image( url )
|
||||||
const localPath = await image.process()
|
const localPath = await image.save()
|
||||||
return encodeURI( this.pathToTop() + '../' + localPath )
|
return encodeURI( this.pathToTop() + '../' + localPath )
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -694,7 +693,7 @@ class Redirect extends ArticleStub {
|
||||||
|
|
||||||
log( '>', this.title || this.url, row)
|
log( '>', this.title || this.url, row)
|
||||||
|
|
||||||
indexerDb.run(
|
wiki.db.run(
|
||||||
'INSERT INTO redirects (id, targetKey, fragment) VALUES (?,?,?)',
|
'INSERT INTO redirects (id, targetKey, fragment) VALUES (?,?,?)',
|
||||||
row
|
row
|
||||||
)
|
)
|
||||||
|
@ -753,10 +752,10 @@ class PageComponent extends WikiItem {
|
||||||
return sanitizeFN( decodeURIComponent( name ))
|
return sanitizeFN( decodeURIComponent( name ))
|
||||||
}
|
}
|
||||||
|
|
||||||
process () {
|
save () {
|
||||||
let saved = urlCache.get( this.url )
|
let saved = urlCache.get( this.url )
|
||||||
if (! saved ) {
|
if (! saved ) {
|
||||||
saved = super.process()
|
saved = super.save()
|
||||||
urlCache.set( this.url, saved )
|
urlCache.set( this.url, saved )
|
||||||
|
|
||||||
saved.then( localPath => { // keep item's data in the cache
|
saved.then( localPath => { // keep item's data in the cache
|
||||||
|
@ -779,10 +778,10 @@ class Image extends PageComponent {
|
||||||
return super.getData()
|
return super.getData()
|
||||||
}
|
}
|
||||||
*/
|
*/
|
||||||
process () {
|
save () {
|
||||||
if (! command.images )
|
if (! command.images )
|
||||||
return Promise.resolve( this.localPath() )
|
return Promise.resolve( this.localPath() )
|
||||||
return super.process()
|
return super.save()
|
||||||
}
|
}
|
||||||
basePath () {
|
basePath () {
|
||||||
if ( ! this.url )
|
if ( ! this.url )
|
||||||
|
@ -858,7 +857,7 @@ class Style extends LayoutItem {
|
||||||
src.replace( urlre, ( match, start, url, end ) => {
|
src.replace( urlre, ( match, start, url, end ) => {
|
||||||
if ( ! url.startsWith( 'data:' )) {
|
if ( ! url.startsWith( 'data:' )) {
|
||||||
const styleItem = new LayoutItem( urlconv.resolve( this.url, url ))
|
const styleItem = new LayoutItem( urlconv.resolve( this.url, url ))
|
||||||
requests.push( styleItem.process() )
|
requests.push( styleItem.save() )
|
||||||
}
|
}
|
||||||
return match
|
return match
|
||||||
})
|
})
|
||||||
|
@ -1018,17 +1017,17 @@ async function saveMetadata () {
|
||||||
Source: wiki.uri,
|
Source: wiki.uri,
|
||||||
}
|
}
|
||||||
|
|
||||||
await new MainPage().process()
|
await new MainPage().save()
|
||||||
await new FavIcon().process()
|
await new FavIcon().save()
|
||||||
|
|
||||||
for ( let i in metadata ) {
|
for ( let i in metadata ) {
|
||||||
await new Metadata( i, metadata[i] ).process()
|
await new Metadata( i, metadata[i] ).save()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function saveMimeTypes () {
|
async function saveMimeTypes () {
|
||||||
for ( let i=0, li=mimeIds.length; i < li; i++ ) {
|
for ( let i=0, li=mimeIds.length; i < li; i++ ) {
|
||||||
await indexerDb.run(
|
await wiki.db.run(
|
||||||
'INSERT INTO mimeTypes (id, value) VALUES (?,?)',
|
'INSERT INTO mimeTypes (id, value) VALUES (?,?)',
|
||||||
[ i + 1, mimeIds[ i ]]
|
[ i + 1, mimeIds[ i ]]
|
||||||
)
|
)
|
||||||
|
@ -1077,7 +1076,7 @@ function batchRedirects ( pageInfos ) {
|
||||||
return null
|
return null
|
||||||
item.to = target
|
item.to = target
|
||||||
item.toFragment = rdr.tofragment
|
item.toFragment = rdr.tofragment
|
||||||
return new Redirect( item ).process()
|
return new Redirect( item ).save()
|
||||||
})
|
})
|
||||||
return Promise.all( done )
|
return Promise.all( done )
|
||||||
})
|
})
|
||||||
|
@ -1109,14 +1108,14 @@ async function batchPages ( nameSpace ) {
|
||||||
|
|
||||||
let continueFrom = ''
|
let continueFrom = ''
|
||||||
while ( true ) {
|
while ( true ) {
|
||||||
await indexerDb.run(
|
await wiki.db.run(
|
||||||
'INSERT OR REPLACE INTO continue (id, "from") VALUES (1, ?)',
|
'INSERT OR REPLACE INTO continue (id, "from") VALUES (1, ?)',
|
||||||
[ continueFrom ]
|
[ continueFrom ]
|
||||||
)
|
)
|
||||||
if ( continueFrom == null )
|
if ( continueFrom == null )
|
||||||
break
|
break
|
||||||
|
|
||||||
await indexerDb.run( 'BEGIN' )
|
await wiki.db.run( 'BEGIN' )
|
||||||
|
|
||||||
const resp = await api( query )
|
const resp = await api( query )
|
||||||
let pages = {}
|
let pages = {}
|
||||||
|
@ -1148,12 +1147,12 @@ async function batchPages ( nameSpace ) {
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
log( '#', pageInfo.title )
|
log( '#', pageInfo.title )
|
||||||
return new Article( pageInfo ).process()
|
return new Article( pageInfo ).save()
|
||||||
})
|
})
|
||||||
done.push( batchRedirects( redirects ))
|
done.push( batchRedirects( redirects ))
|
||||||
await Promise.all( done )
|
await Promise.all( done )
|
||||||
|
|
||||||
await indexerDb.run( 'COMMIT' )
|
await wiki.db.run( 'COMMIT' )
|
||||||
|
|
||||||
continueFrom = null
|
continueFrom = null
|
||||||
try {
|
try {
|
||||||
|
@ -1186,7 +1185,7 @@ function loadCss( dom ) {
|
||||||
if (! command.css )
|
if (! command.css )
|
||||||
return Promise.resolve()
|
return Promise.resolve()
|
||||||
const css = new GlobalCss( dom )
|
const css = new GlobalCss( dom )
|
||||||
return css.process()
|
return css.save()
|
||||||
}
|
}
|
||||||
|
|
||||||
function initMetadataStorage ( samplePageDOM ) {
|
function initMetadataStorage ( samplePageDOM ) {
|
||||||
|
@ -1197,8 +1196,8 @@ function initMetadataStorage ( samplePageDOM ) {
|
||||||
.catch( () => null )
|
.catch( () => null )
|
||||||
.then( () => sqlite.open( dbName ))
|
.then( () => sqlite.open( dbName ))
|
||||||
.then( db => {
|
.then( db => {
|
||||||
indexerDb = db
|
wiki.db = db
|
||||||
return indexerDb.exec(
|
return wiki.db.exec(
|
||||||
'PRAGMA synchronous = OFF;' +
|
'PRAGMA synchronous = OFF;' +
|
||||||
//~ 'PRAGMA journal_mode = OFF;' +
|
//~ 'PRAGMA journal_mode = OFF;' +
|
||||||
'PRAGMA journal_mode = WAL;' +
|
'PRAGMA journal_mode = WAL;' +
|
||||||
|
@ -1236,7 +1235,7 @@ function initMetadataStorage ( samplePageDOM ) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function closeMetadataStorage () {
|
function closeMetadataStorage () {
|
||||||
return indexerDb.close()
|
return wiki.db.close()
|
||||||
}
|
}
|
||||||
|
|
||||||
function core ( samplePage ) {
|
function core ( samplePage ) {
|
||||||
|
|
Loading…
Reference in New Issue
Block a user