From ff957af67690d0dd41a789da98fbd4c3e6de8854 Mon Sep 17 00:00:00 2001 From: Suzanne Soy Date: Sun, 31 Jan 2021 19:41:21 +0000 Subject: [PATCH 01/13] Nightly builds --- .github/workflows/main.yml | 102 +++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..2e2d79a --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,102 @@ +name: Build-Documentation + +on: + # Trigger the workflow every night + schedule: + - cron: '0 1 * * *' + + # Trigger on push to master and for pull requests targetting master + push: + branches: [ master ] + + pull_request: + branches: [ master ] + + # Trigger manually from the Actions tab + workflow_dispatch: + +jobs: + build-documentation: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + steps: + - name: Check-out the repository under $GITHUB_WORKSPACE + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install prerequisites from the README + run: sudo apt install python qttools5-dev-tools wkhtmltopdf python-pypdf2 + + - name: Print versions + run: | + python2 --version || true + wkhtmltopdf --version --version || true + qhelpgenerator --qt=qt4 --version || true + + # GitHub Actions currently uses 18.04 as of 02021-01-31, but will likely update in the future + - name: Try to install prerequisites for Ubuntu 20.04 from the README + run: sudo apt install qhelpgenerator-qt5 || true + + - name: Install prerequisites for testing + run: sudo apt install xvfb + + - name: Remove existing downloads (for now, because the update.py script seems to not work well) + run: | + rm -rf localwiki revisions.txt updates.txt wikifiles.txt + mkdir localwiki + touch revisions.txt updates.txt wikifiles.txt + + - name: Build an index file containing a list of all the files to download + run: python2 ./buildwikiindex.py + + - name: Download wiki pages + # Redirect log because it seems we exceed the amount of data GitHub is willing + # to show in the interacive logs and there seems to be no UI to access the raw + # logs at this time + #run: python2 ./downloadwiki.py > ./downloadwiki.py.log + run: python2 ./downloadwiki.py + +# - name: Create a list of revision IDs for each page +# run: python2 ./update.py +# +# - name: Get a list of pages that have changed +# run: python2 ./update.py +# +# - name: Download the changed pages (and all their dependencies) again +# run: python2 ./update.py + + - name: Generate freecad.qhc and freecad.qch files + run: python2 ./buildqhelp.py + + - name: Generate freecad.pdf + run: python2 ./buildpdf.py + + - name: Split the generated freecad.qch into parts that are smaller than 50Mb (github limit) + run: split -d --byte=49M localwiki/freecad.qch localwiki/freecad.qch.part + + - name: Reassemble the previously-split freecad.qch + run: cat localwiki/freecad.qch.part* >> test.qch + + - name: Check that the reassembled test.qch is identical to localwiki/freecad.qch + run: diff -q test.qch localwiki/freecad.qch + + - name: Try to open the .qch file + run: | + xvfb-run assistant -collectionFile localwiki/freecad.qhc & + sleep 5 + killall assistant + + - name: Upload freecad.qch + uses: actions/upload-artifact@v2 + with: + name: freecad.qch + path: localwiki/freecad.qch + + - name: Upload freecad.qhc + uses: actions/upload-artifact@v2 + with: + name: freecad.qhc + path: localwiki/freecad.qhc + From 6a37ab405b532070d86895958c982ed533edd428 Mon Sep 17 00:00:00 2001 From: Suzanne Soy Date: Sun, 31 Jan 2021 19:27:47 +0000 Subject: [PATCH 02/13] Fix qhelpgenerator: could not find a Qt installation of '' --- buildqhelp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildqhelp.py b/buildqhelp.py index 177d4e1..7679392 100755 --- a/buildqhelp.py +++ b/buildqhelp.py @@ -39,7 +39,7 @@ from urllib2 import urlopen, HTTPError FOLDER = "./localwiki" INDEX = "Online_Help_Toc" # the start page from where to crawl the wiki VERBOSE = True # to display what's going on. Otherwise, runs totally silent. -QHELPCOMPILER = 'qhelpgenerator' +QHELPCOMPILER = 'qhelpgenerator -qt=qt5' QCOLLECTIOMGENERATOR = 'qcollectiongenerator' RELEASE = '0.17' From d1a7903efb63f2511caee5c784ebbbb55350c395 Mon Sep 17 00:00:00 2001 From: Suzanne Soy Date: Sun, 31 Jan 2021 16:02:22 +0000 Subject: [PATCH 03/13] Don't store compiled .pyc files in version control --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a74b07a --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/*.pyc From 3093572100aa0970dd660de1c1a55a3c312a1baf Mon Sep 17 00:00:00 2001 From: Suzanne Soy Date: Sun, 31 Jan 2021 19:30:59 +0000 Subject: [PATCH 04/13] fixup! Don't store compiled .pyc files in version control --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a74b07a..0491981 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /*.pyc +/__pycache__ From d0803db4fee04f77e3e1cf7308bb3fc5a0643301 Mon Sep 17 00:00:00 2001 From: Suzanne Soy Date: Sun, 31 Jan 2021 20:05:49 +0000 Subject: [PATCH 05/13] fixup! Fix qhelpgenerator: could not find a Qt installation of '' --- buildqhelp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildqhelp.py b/buildqhelp.py index 7679392..bf29346 100755 --- a/buildqhelp.py +++ b/buildqhelp.py @@ -40,7 +40,7 @@ FOLDER = "./localwiki" INDEX = "Online_Help_Toc" # the start page from where to crawl the wiki VERBOSE = True # to display what's going on. Otherwise, runs totally silent. QHELPCOMPILER = 'qhelpgenerator -qt=qt5' -QCOLLECTIOMGENERATOR = 'qcollectiongenerator' +QCOLLECTIOMGENERATOR = 'qcollectiongenerator -qt=qt5' RELEASE = '0.17' # END CONFIGURATION ############################################## From 77d08bb399619f9f9c2c883804e0f1e3ce362f47 Mon Sep 17 00:00:00 2001 From: Suzanne Soy Date: Sun, 31 Jan 2021 20:13:33 +0000 Subject: [PATCH 06/13] Do not ask about copying files in the source directory in the github action; Added --non-interactive --- .github/workflows/main.yml | 2 +- buildqhelp.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2e2d79a..2ce6849 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -68,7 +68,7 @@ jobs: # run: python2 ./update.py - name: Generate freecad.qhc and freecad.qch files - run: python2 ./buildqhelp.py + run: python2 ./buildqhelp.py --non-interactive - name: Generate freecad.pdf run: python2 ./buildpdf.py diff --git a/buildqhelp.py b/buildqhelp.py index bf29346..a973d77 100755 --- a/buildqhelp.py +++ b/buildqhelp.py @@ -68,7 +68,15 @@ def crawl(): print("Error at compiling") return 1 if VERBOSE: print("All done!") - i=raw_input("Copy the files to their correct location in the source tree? y/n (default=no) ") + if "--yes-copy" in sys.argv: + i="yes" + elif "--no-copy" in sys.argv: + i="no" + else: + try: + i=raw_input("Copy the files to their correct location in the source tree? y/n (default=no) ") + except: + i="no" if i.upper() in ["Y","YES"]: shutil.copy("localwiki/freecad.qch","../../Doc/freecad.qch") shutil.copy("localwiki/freecad.qhc","../../Doc/freecad.qhc") From e5f5213d65f5e2590029391beb727c88f715d85a Mon Sep 17 00:00:00 2001 From: Suzanne Soy Date: Sun, 31 Jan 2021 20:15:33 +0000 Subject: [PATCH 07/13] Do not loose exit code of crawl() when an error occurs --- buildqhelp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildqhelp.py b/buildqhelp.py index a973d77..92d23f4 100755 --- a/buildqhelp.py +++ b/buildqhelp.py @@ -242,5 +242,5 @@ def buildtoc(): return qfilename if __name__ == "__main__": - crawl() + exit(crawl()) From a762491d78755cfb0cf102d50cf5e02f607e20c1 Mon Sep 17 00:00:00 2001 From: Suzanne Soy Date: Sun, 31 Jan 2021 20:21:06 +0000 Subject: [PATCH 08/13] inkscape dependency for SVG to PDF --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2ce6849..68f0704 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,7 +27,7 @@ jobs: fetch-depth: 0 - name: Install prerequisites from the README - run: sudo apt install python qttools5-dev-tools wkhtmltopdf python-pypdf2 + run: sudo apt install python qttools5-dev-tools wkhtmltopdf python-pypdf2 inkscape - name: Print versions run: | From 57521f129a2f7e9cd3708ee9c5de6a02007db251 Mon Sep 17 00:00:00 2001 From: Suzanne Soy Date: Sun, 31 Jan 2021 20:21:38 +0000 Subject: [PATCH 09/13] Use os.path.join --- buildpdf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buildpdf.py b/buildpdf.py index 0d987a3..f694f6d 100755 --- a/buildpdf.py +++ b/buildpdf.py @@ -472,7 +472,7 @@ def joinpdf(): result = PdfFileWriter() createCover() - inputfile = PdfFileReader(open(FOLDER+os.sep+'Cover.pdf','rb')) + inputfile = PdfFileReader(open(os.path.join(FOLDER,'Cover.pdf'),'rb')) result.addPage(inputfile.getPage(0)) count = 1 @@ -547,11 +547,11 @@ def createCover(): "downloads and creates a cover page" if VERBOSE: print("fetching " + COVER) data = (urlopen(COVER).read()) - path = FOLDER + os.sep + "Cover.svg" + path = os.path.join(FOLDER, "Cover.svg") fil = open(path,'wb') fil.write(data) fil.close() - os.system('inkscape --export-pdf='+FOLDER+os.sep+'Cover.pdf'+' '+FOLDER+os.sep+'Cover.svg') + os.system('inkscape --export-pdf='+os.path.join(FOLDER,'Cover.pdf')+' '+os.path.join(FOLDER,'Cover.svg')) if __name__ == "__main__": From 15e98ff2cf4776c993ef8bfb3b4c552cdccaf015 Mon Sep 17 00:00:00 2001 From: Suzanne Soy Date: Sun, 31 Jan 2021 20:22:16 +0000 Subject: [PATCH 10/13] Do not ignore error from calling Inkscape --- buildpdf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/buildpdf.py b/buildpdf.py index f694f6d..1d4f595 100755 --- a/buildpdf.py +++ b/buildpdf.py @@ -551,7 +551,10 @@ def createCover(): fil = open(path,'wb') fil.write(data) fil.close() - os.system('inkscape --export-pdf='+os.path.join(FOLDER,'Cover.pdf')+' '+os.path.join(FOLDER,'Cover.svg')) + if os.system('inkscape --export-pdf='+os.path.join(FOLDER,'Cover.pdf')+' '+os.path.join(FOLDER,'Cover.svg')) == 0: + return + else: + raise Exception('Conversion of Cover.svg to Cover.pdf failed. Is Inkscape installed?') if __name__ == "__main__": From 6a88c9fe78841b4695528e5370bac821edd07f6a Mon Sep 17 00:00:00 2001 From: Suzanne Soy Date: Sun, 31 Jan 2021 20:26:53 +0000 Subject: [PATCH 11/13] fixup! Nightly builds --- .github/workflows/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68f0704..1d21f7d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -100,3 +100,9 @@ jobs: name: freecad.qhc path: localwiki/freecad.qhc + - name: Upload freecad.pdf + uses: actions/upload-artifact@v2 + with: + name: freecad.pdf + path: localwiki/freecad.pdf + From c9d627351e22fc8bd2f800d62d703f53a115519b Mon Sep 17 00:00:00 2001 From: Suzanne Soy Date: Sun, 31 Jan 2021 20:30:30 +0000 Subject: [PATCH 12/13] Do not ignore error from Qt tools --- buildqhelp.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/buildqhelp.py b/buildqhelp.py index 92d23f4..66191a6 100755 --- a/buildqhelp.py +++ b/buildqhelp.py @@ -90,6 +90,9 @@ def compile(qhpfile): if not os.system(QHELPCOMPILER + ' '+qhpfile+' -o '+qchfile): if VERBOSE: print("Successfully created",qchfile) return 0 + else: + os.system('cat -v ' + qhpfile) + raise "Error during generation of freecad.qch" def generate(qhcpfile): "generates qassistant-specific settings like icon, title, ..." @@ -104,6 +107,9 @@ def generate(qhcpfile): if not os.system(QCOLLECTIOMGENERATOR+' '+qhcpfile+' -o '+qhcfile): if VERBOSE: print("Successfully created ",qhcfile) return 0 + else: + os.system('cat -v ' + qhcpfile) + raise "Error during generation of freecad.qhc" def createCollProjectFile(): qprojectfile = ''' From 119b100a16c61faa5a11184ae7ee3de292ed82c1 Mon Sep 17 00:00:00 2001 From: Suzanne Soy Date: Sun, 31 Jan 2021 20:34:11 +0000 Subject: [PATCH 13/13] re-enabled update.py, needs debuging --- .github/workflows/main.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1d21f7d..c7bcf91 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -42,11 +42,11 @@ jobs: - name: Install prerequisites for testing run: sudo apt install xvfb - - name: Remove existing downloads (for now, because the update.py script seems to not work well) - run: | - rm -rf localwiki revisions.txt updates.txt wikifiles.txt - mkdir localwiki - touch revisions.txt updates.txt wikifiles.txt +# - name: Remove existing downloads (for now, because the update.py script seems to not work well) +# run: | +# rm -rf localwiki revisions.txt updates.txt wikifiles.txt +# mkdir localwiki +# touch revisions.txt updates.txt wikifiles.txt - name: Build an index file containing a list of all the files to download run: python2 ./buildwikiindex.py @@ -58,14 +58,14 @@ jobs: #run: python2 ./downloadwiki.py > ./downloadwiki.py.log run: python2 ./downloadwiki.py -# - name: Create a list of revision IDs for each page -# run: python2 ./update.py -# -# - name: Get a list of pages that have changed -# run: python2 ./update.py -# -# - name: Download the changed pages (and all their dependencies) again -# run: python2 ./update.py + - name: Create a list of revision IDs for each page + run: python2 ./update.py + + - name: Get a list of pages that have changed + run: python2 ./update.py + + - name: Download the changed pages (and all their dependencies) again + run: python2 ./update.py - name: Generate freecad.qhc and freecad.qch files run: python2 ./buildqhelp.py --non-interactive