diff --git a/.github/files-to-cache.lst b/.github/files-to-cache.lst new file mode 100644 index 0000000..8b1edff --- /dev/null +++ b/.github/files-to-cache.lst @@ -0,0 +1,15 @@ +codemirror-5.60.0/lib/codemirror.css +codemirror-5.60.0/lib/codemirror.js +codemirror-5.60.0/mode/javascript/javascript.js +sha1.js/sha1.js +pako/pako.min.js +Viz.js/viz.js +FileSaver.js/FileSaver.js +Blob.js/Blob.js +JSZip/jszip.min.js +git-tutorial.css +git-tutorial.js +sha256.js +micro_ipfs.js +directory_hashes.js +favicon.ico \ No newline at end of file diff --git a/.github/pin-using-ipfs.sh b/.github/pin-using-ipfs.sh index 0c09568..b84a129 100755 --- a/.github/pin-using-ipfs.sh +++ b/.github/pin-using-ipfs.sh @@ -55,22 +55,5 @@ if test -n "${IPFS_REMOTE_API_ENDPOINT:-}" && test -n "${IPFS_REMOTE_TOKEN:-}" & done) fi -# warm up cache, twice (a few files in the first attempt would likely fail as the DHT propagation is not instant) -for i in `seq 2`; do - ipfs add --progress=false --ignore-rules-path "result/www/.ipfsignore" --pin=false --hidden -r result/www \ - | cut -d ' ' -f 3- \ - | sed -e 's~^www/*~~' \ - | while read f; do - if (printf %s\\n "$IPFS_REMOTE_API_ENDPOINT" | grep pinata) >/dev/null 2>&1; then - printf "Warming up pinata cache for %s (attempt %d)...\n" "$f" "$i" - wget --tries=1 --timeout=10 -O- "https://gateway.pinata.cloud/ipfs/$h/$f" > /dev/null || true - fi - printf "Warming up Cloudflare cache for %s (attempt %d)...\n" "$f" "$i" - wget --tries=1 --timeout=10 -O- "https://cloudflare-ipfs.com/ipfs/$h/$f" > /dev/null || true - printf "Warming up dweb.link cache for %s (attempt %d)...\n" "$f" "$i" - wget --tries=1 --timeout=10 -O- "https://$h.ipfs.dweb.link/$f" > /dev/null || true - done -done - # Fail job if one of the pinning services didn't work exit "$(cat ipfs-pin-global-exitcode)" diff --git a/.github/warm-up-gateway-caches.sh b/.github/warm-up-gateway-caches.sh new file mode 100755 index 0000000..072176a --- /dev/null +++ b/.github/warm-up-gateway-caches.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -euET -o pipefail + +echo "Warm up cache on a couple of IPFS gateways" + +h="$(result/www/ipfs-add.sh --pin=true)" + +#wget --reject-regex ".*\?.*" -r -np --timeout=2 --tries=1 "https://cloudflare-ipfs.com/ipfs/$h" 2>&1 | grep '^--' & pid_cloudflare="$!" +#wget --reject-regex ".*\?.*" -r -np --timeout=2 --tries=1 "https://$h.ipfs.dweb.link/" 2>&1 | grep '^--' & pid_dweb="$!" +#wait "$pid_cloudflare" || true +#wait "$pid_dweb" || true + +# Download the files, twice (a few files in the first attempt would likely fail as the DHT propagation is not instantaneous?) +for i in `seq 2`; do + #ipfs add --progress=false --ignore-rules-path "result/www/.ipfsignore" --pin=false --hidden -r result/www \ + #| cut -d ' ' -f 3- \ + #| sed -e 's~^www/*~~' \ + cat .github/files-to-cache.lst \ + | while read f; do + if (printf %s\\n "$IPFS_REMOTE_API_ENDPOINT" | grep pinata) >/dev/null 2>&1; then + printf "Warming up pinata cache for %s (attempt %d)...\n" "$f" "$i" + wget --tries=1 --timeout=10 -O- "https://gateway.pinata.cloud/ipfs/$h/$f" > /dev/null || true + fi + printf "Warming up Cloudflare cache for %s (attempt %d)...\n" "$f" "$i" + wget --tries=1 --timeout=10 -O- "https://cloudflare-ipfs.com/ipfs/$h/$f" > /dev/null || true + printf "Warming up dweb.link cache for %s (attempt %d)...\n" "$f" "$i" + wget --tries=1 --timeout=10 -O- "https://$h.ipfs.dweb.link/$f" > /dev/null || true + done +done diff --git a/.github/workflows/upload-to-ipfs-and-update-dns.yml b/.github/workflows/upload-to-ipfs-and-update-dns.yml index f31192c..59e5a6c 100644 --- a/.github/workflows/upload-to-ipfs-and-update-dns.yml +++ b/.github/workflows/upload-to-ipfs-and-update-dns.yml @@ -41,6 +41,17 @@ jobs: run: ./.github/github_install_ipfs.sh - name: Print and compare IPFS hahes run: export PATH="/tmp/kubo:$PATH"; .github/print-and-compare-ipfs.sh + - name: Make tarball of website + run: mkdir -p "$RUNNER_TEMP" && tar --directory result/www/ -cvf "$RUNNER_TEMP/artifact.tar" . + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: github-pages + path: ${{ runner.temp }}/artifact.tar + if-no-files-found: error + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 - name: Upload to IPFS run: PATH="/tmp/kubo:$PATH" ./.github/pin-using-ipfs.sh continue-on-error: true @@ -50,7 +61,7 @@ jobs: IPFS_REMOTE_TOKEN: ${{ secrets.IPFS_REMOTE_TOKEN }} - name: Install OVH pip package run: pip install ovh - - name: Install OVH pip package + - name: Update OVH _dnslink run: python ./.github/update-ovh.py >/dev/null 2>&1 env: API_OVH_APPLICATION_KEY: ${{ secrets.API_OVH_APPLICATION_KEY }} @@ -59,17 +70,11 @@ jobs: API_OVH_ENDPOINT: ${{ secrets.API_OVH_ENDPOINT }} OVH_DNS_DOMAIN: ${{ secrets.OVH_DNS_DOMAIN }} OVH_DNS_RECORD_ID: ${{ secrets.OVH_DNS_RECORD_ID }} + - name: Warm up IPFS gateway caches + run: PATH="/tmp/kubo:$PATH" ./.github/warm-up-gateway-caches.sh # - name: Update homepage URL # run: PATH="/tmp/kubo:$PATH" ./.github/github_update_homepage.sh # env: # API_TOKEN_FOR_UPDATE_HOMEPAGE: ${{ secrets.API_TOKEN_FOR_UPDATE_HOMEPAGE }} # - name: Setup Pages # uses: actions/configure-pages@v3 -# - name: Upload artifact -# uses: actions/upload-pages-artifact@v2 -# with: -# # Upload entire repository -# path: 'result/www/' -# - name: Deploy to GitHub Pages -# id: deployment -# uses: actions/deploy-pages@v2 diff --git a/build-and-update.sh b/build-and-update.sh new file mode 100755 index 0000000..53c66cc --- /dev/null +++ b/build-and-update.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euET -o pipefail + +nix build +cp result/www/directory_hashes.js directory_hashes.js +cp result/www/favicon.ico favicon.ico +cp result/www/sitemap.html sitemap.html +if test -n "$(git status --short)"; then git commit -a --amend; fi +nix build +diff result/www/directory_hashes.js directory_hashes.js +diff result/www/favicon.ico favicon.ico +diff result/www/sitemap.html sitemap.html + +./.github/print-and-compare-ipfs.sh \ No newline at end of file diff --git a/deploy.sh b/deploy.sh index 767ce1e..ef5e2b9 100755 --- a/deploy.sh +++ b/deploy.sh @@ -17,17 +17,7 @@ if ! grep 'Sitemap . ./.github +./.github/files-to-cache.lst ./.github/github_install_ipfs.sh ./.github/github_update_homepage.sh ./.github/pin-using-ipfs.sh ./.github/print-and-compare-ipfs.sh ./.github/update-ovh.py +./.github/warm-up-gateway-caches.sh ./.github/workflows ./.github/workflows/upload-to-ipfs-and-update-dns.yml ./.gitignore @@ -27,6 +29,7 @@ ./Viz.js/LICENSE ./Viz.js/viz-lite.js ./Viz.js/viz.js +./build-and-update.sh ./codemirror-5.60.0 ./codemirror-5.60.0/.editorconfig ./codemirror-5.60.0/.gitattributes