feat: xkcd cloner
License: MIT Signed-off-by: Henrique Dias <hacdias@gmail.com>
This commit is contained in:
Henrique Dias
parent
abae3a22a3
commit
89b2517ffc
125
bin/index.js
Normal file
125
bin/index.js
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
const fs = require('fs-extra')
|
||||
const yargs = require('yargs')
|
||||
const { basename, join } = require('path')
|
||||
const { getLatestId, getComic } = require('../lib/xkcd')
|
||||
const { homePage, comicPage } = require('../lib/html')
|
||||
const { pad, progress } = require('../lib/helpers')
|
||||
|
||||
const argv = yargs
|
||||
.usage('$0', 'Clones XKCD comics. By default it only downloads the missing comics.')
|
||||
.scriptName('xkcd-clone')
|
||||
.option('dir', {
|
||||
alias: 'd',
|
||||
describe: 'Output directory',
|
||||
type: 'string',
|
||||
demandOption: true
|
||||
}).option('empty', {
|
||||
alias: 'e',
|
||||
describe: 'Redownload all comics',
|
||||
type: 'boolean'
|
||||
})
|
||||
.help()
|
||||
.argv
|
||||
|
||||
async function write ({ data, img }, dir) {
|
||||
try {
|
||||
await fs.outputJSON(join(dir, 'info.json'), data, { spaces: '\t' })
|
||||
const dest = fs.createWriteStream(join(dir, basename(data.img)))
|
||||
img.body.pipe(dest)
|
||||
await fs.outputFile(join(dir, 'index.html'), comicPage(data))
|
||||
} catch (err) {
|
||||
await fs.remove(dir)
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
async function run () {
|
||||
console.log(`😊 Going to clone XKCD to ${argv.dir}`)
|
||||
|
||||
let added = []
|
||||
let errored = []
|
||||
|
||||
try {
|
||||
console.log(`🔍 Finding the latest comic`)
|
||||
const latest = await getLatestId()
|
||||
console.log(`😁 Found! We're on comic number ${latest}!`)
|
||||
|
||||
await fs.ensureDir(argv.dir)
|
||||
if (argv.empty) {
|
||||
await fs.emptyDir(argv.dir)
|
||||
}
|
||||
|
||||
for (let i = 1; i <= latest; i++) {
|
||||
const num = pad(i, 4)
|
||||
const dir = join(argv.dir, num)
|
||||
|
||||
if (await fs.pathExists(dir)) {
|
||||
const data = await fs.readJSON(join(dir, 'info.json'))
|
||||
added.push({ id: i, title: data.title, num })
|
||||
await fs.outputFile(join(dir, 'index.html'), comicPage(data))
|
||||
continue
|
||||
} else if (i === 404) {
|
||||
progress(`📦 404 not found 😵`)
|
||||
continue
|
||||
} else {
|
||||
progress(`📦 Fetching ${i} out of ${latest}`)
|
||||
}
|
||||
|
||||
let comic = null
|
||||
|
||||
try {
|
||||
comic = await getComic(i)
|
||||
} catch (err) {
|
||||
progress(`😢 Could not fetch ${i}, will try again later\n`)
|
||||
errored.push(i)
|
||||
}
|
||||
|
||||
await write(comic, dir)
|
||||
added.push({
|
||||
id: i,
|
||||
title: comic.data.title,
|
||||
num: num
|
||||
})
|
||||
}
|
||||
} catch (err) {
|
||||
console.log(`🐉 ${err.stack}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
// TODO: redownload errored
|
||||
|
||||
if (errored.length === 0) {
|
||||
progress(`📦 All comics fetched\n`)
|
||||
} else {
|
||||
progress(`📦 Some comics fetched\n`)
|
||||
}
|
||||
|
||||
added = added.sort((a, b) => a.num - b.num)
|
||||
await fs.copyFile(join(__dirname, '../node_modules/tachyons/css/tachyons.min.css'), join(argv.dir, 'tachyons.css'))
|
||||
await fs.outputFile(join(argv.dir, 'index.html'), homePage(added))
|
||||
}
|
||||
/*
|
||||
|
||||
const clone = async ({ baseDir, empty, onlyMissing }) => {
|
||||
/*
|
||||
|
||||
for (const num of errored) {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
try {
|
||||
const comic = await getComic(i)
|
||||
await write(comic, dir)
|
||||
break
|
||||
} catch (err) {
|
||||
if (i === 2) {
|
||||
console.log(`😢 ${num} could not be fetched: ${err.toString()}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
*/
|
||||
|
||||
run()
|
||||
150
index.js
150
index.js
|
|
@ -1,150 +0,0 @@
|
|||
const fetch = require('node-fetch')
|
||||
const fs = require('fs-extra')
|
||||
const { basename, join } = require('path')
|
||||
|
||||
const progress = (str) => {
|
||||
process.stdout.clearLine()
|
||||
process.stdout.cursorTo(0)
|
||||
process.stdout.write(str)
|
||||
}
|
||||
|
||||
const pad = (str, max) => {
|
||||
str = str.toString()
|
||||
return str.length < max ? pad('0' + str, max) : str
|
||||
}
|
||||
|
||||
const fetchLatest = () => {
|
||||
return fetch(`https://xkcd.com/info.0.json`)
|
||||
.then(res => res.json())
|
||||
}
|
||||
|
||||
const fetchId = (id) => {
|
||||
return fetch(`https://xkcd.com/${id}/info.0.json`)
|
||||
.then(res => res.json())
|
||||
}
|
||||
|
||||
const htmlPage = ({ alt, title, transcript, num, img }) => {
|
||||
const btnClass = 'dib navy mh2 pa2 bg-light-blue bg-animate hover-bg-lightest-blue br2 ba bw1 b--navy no-underline'
|
||||
|
||||
return `<html>
|
||||
<head>
|
||||
<title>${num} - ${title}</title>
|
||||
<link rel="stylesheet" href="https://unpkg.com/tachyons@4.10.0/css/tachyons.min.css"/>
|
||||
</head>
|
||||
<body class="tc bg-washed-blue navy sans-serif ml-auto mr-auto mw7 w-90">
|
||||
<h1 class="mh0 mt4 mb3 f2 small-caps tracked">${title} <span class="light-blue">#${num}</span></h1>
|
||||
|
||||
<nav class="mv3">
|
||||
<a class="${btnClass}" href="../${pad(num - 1, 4)}/index.html"><span class="gray">←</span> Prev</a>
|
||||
<a class="${btnClass}" href="../index.html">Home</a>
|
||||
<a class="${btnClass}" href="../${pad(num + 1, 4)}/index.html">Next <span class="gray">→</span></a>
|
||||
</nav>
|
||||
|
||||
<img src="./${basename(img)}" alt="${alt}">
|
||||
<p class="dn">${transcript}</p>
|
||||
</body>
|
||||
</html>`
|
||||
}
|
||||
|
||||
const download = async (baseDir, i) => {
|
||||
const num = pad(i, 4)
|
||||
const res = await fetchId(i)
|
||||
const dir = join(baseDir, num)
|
||||
|
||||
try {
|
||||
await fs.outputJSON(join(dir, 'info.json'), res, { spaces: '\t' })
|
||||
|
||||
const img = await fetch(res.img)
|
||||
const dest = fs.createWriteStream(join(dir, basename(res.img)))
|
||||
|
||||
await fs.outputFile(join(dir, 'index.html'), htmlPage(res))
|
||||
img.body.pipe(dest)
|
||||
} catch (err) {
|
||||
await fs.remove(dir)
|
||||
throw err
|
||||
}
|
||||
|
||||
return {
|
||||
id: i,
|
||||
title: res.title,
|
||||
num: num
|
||||
}
|
||||
}
|
||||
|
||||
const homePage = (list) => `<html>
|
||||
<head>
|
||||
<title>XKCD</title>
|
||||
<link rel="stylesheet" href="https://unpkg.com/tachyons@4.10.0/css/tachyons.min.css"/>
|
||||
</head>
|
||||
<body class="tc bg-washed-blue navy sans-serif ml-auto mr-auto mw7 w-90">
|
||||
<h1 class="mh0 mt4 mb3 f2 small-caps tracked">XKCD</h1>
|
||||
|
||||
${list.map(({ id, title, num }) => `<a href="./${num}/index.html">${id} - ${title}</a>`).join('\n')}
|
||||
</body>
|
||||
</html>`
|
||||
|
||||
const clone = async ({ baseDir, empty, onlyMissing }) => {
|
||||
console.log(`😊 Going to clone XKCD to ${baseDir}`)
|
||||
|
||||
let errored = []
|
||||
let added = []
|
||||
|
||||
try {
|
||||
console.log(`🔍 Finding the latest comic`)
|
||||
const latest = (await fetchLatest()).num
|
||||
console.log(`😁 Found! Will download ${latest} comics 🥶`)
|
||||
|
||||
await fs.ensureDir(baseDir)
|
||||
if (empty) await fs.emptyDir(baseDir)
|
||||
|
||||
let existent = []
|
||||
if (onlyMissing) {
|
||||
existent = fs.readdirSync(baseDir).map(v => Number(v.split('-')[0].trim()))
|
||||
}
|
||||
|
||||
for (let i = 1; i <= 10 + 1; i++) {
|
||||
if (existent.includes(i)) {
|
||||
continue
|
||||
} else if (i === 404) {
|
||||
progress(`📦 404 not found 😵`)
|
||||
continue
|
||||
} else if (i === latest + 1) {
|
||||
progress(`📦 All ${latest} comics fetched\n`)
|
||||
continue
|
||||
} else {
|
||||
progress(`📦 Fetching ${i} out of ${latest}`)
|
||||
}
|
||||
|
||||
try {
|
||||
added.push(await download(baseDir, i))
|
||||
} catch (err) {
|
||||
errored.push(i)
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.log(`🐉 ${err.stack}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
for (const num of errored) {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
try {
|
||||
added.push(await download(baseDir, i))
|
||||
break
|
||||
} catch (err) {
|
||||
if (i === 2) {
|
||||
console.log(`😢 ${num} could not be fetched: ${err.toString()}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
added = added.sort((a, b) => a.num - b.num)
|
||||
await fs.outputFile(join(baseDir, 'index.html'), homePage(added))
|
||||
}
|
||||
|
||||
clone({
|
||||
baseDir: './clone',
|
||||
empty: true,
|
||||
onlyMissing: false
|
||||
})
|
||||
15
lib/helpers.js
Normal file
15
lib/helpers.js
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
const progress = (str) => {
|
||||
process.stdout.clearLine()
|
||||
process.stdout.cursorTo(0)
|
||||
process.stdout.write(str)
|
||||
}
|
||||
|
||||
const pad = (str, max) => {
|
||||
str = str.toString()
|
||||
return str.length < max ? pad('0' + str, max) : str
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
progress,
|
||||
pad
|
||||
}
|
||||
44
lib/html.js
Normal file
44
lib/html.js
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
const { basename } = require('path')
|
||||
const { pad } = require('./helpers')
|
||||
|
||||
const comicPage = ({ alt, title, transcript, num, img }) => {
|
||||
const btnClass = 'dib navy mh2 pa2 bg-light-blue hover-bg-lightest-blue br2 ba bw1 b--navy no-underline'
|
||||
|
||||
return `<html>
|
||||
<head>
|
||||
<title>${num} - ${title}</title>
|
||||
<link rel="stylesheet" href="../tachyons.css"/>
|
||||
</head>
|
||||
<body class="tc bg-washed-blue navy sans-serif ml-auto mr-auto mw7 w-90">
|
||||
<h1 class="mh0 mt4 mb3 f2 small-caps tracked">${title} <span class="light-blue">#${num}</span></h1>
|
||||
|
||||
<nav class="mv3">
|
||||
<a class="${btnClass}" href="../${pad(num - 1, 4)}/index.html"><span class="gray">←</span> Prev</a>
|
||||
<a class="${btnClass}" href="../index.html">Home</a>
|
||||
<a class="${btnClass}" href="../${pad(num + 1, 4)}/index.html">Next <span class="gray">→</span></a>
|
||||
</nav>
|
||||
|
||||
<img src="./${basename(img)}" alt="${alt}">
|
||||
<p class="dn">${transcript}</p>
|
||||
</body>
|
||||
</html>`
|
||||
}
|
||||
|
||||
const homePage = (list) => `<html>
|
||||
<head>
|
||||
<title>XKCD</title>
|
||||
<link rel="stylesheet" href="./tachyons.css"/>
|
||||
</head>
|
||||
<body class="bg-washed-blue navy sans-serif ml-auto mr-auto mw7 w-90">
|
||||
<h1 class="tc mh0 mt4 mb3 f2 small-caps tracked">XKCD</h1>
|
||||
|
||||
<ul class="list pa0 ma0">
|
||||
${list.map(({ id, title, num }) => `<li><a class="blue hover-dark-blue no-underline" href="./${num}/index.html">${id} - ${title}</a></li>`).join('\n')}
|
||||
</ul>
|
||||
</body>
|
||||
</html>`
|
||||
|
||||
module.exports = {
|
||||
comicPage,
|
||||
homePage
|
||||
}
|
||||
20
lib/xkcd.js
Normal file
20
lib/xkcd.js
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
const fetch = require('node-fetch')
|
||||
|
||||
async function getLatestId () {
|
||||
const raw = await fetch(`https://xkcd.com/info.0.json`)
|
||||
const data = await raw.json()
|
||||
return data.num
|
||||
}
|
||||
|
||||
async function getComic (id) {
|
||||
const raw = await fetch(`https://xkcd.com/${id}/info.0.json`)
|
||||
const data = await raw.json()
|
||||
const img = await fetch(data.img)
|
||||
|
||||
return { data, img }
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getLatestId,
|
||||
getComic
|
||||
}
|
||||
|
|
@ -2,9 +2,12 @@
|
|||
"name": "xkcd-clone",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"main": "./lib/index.js",
|
||||
"bin": {
|
||||
"xkcd-clone": "./bin/index.js"
|
||||
},
|
||||
"license": "MIT",
|
||||
"author": "Henrique Dias <hacdias@gmail.com>",
|
||||
"dependencies": {
|
||||
"fs-extra": "^7.0.1",
|
||||
"node-fetch": "^2.4.0",
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user