re-enabled update.py, needs debuging

Use print() function in both Python 2 and Python 3

.github/workflows/main.yml

@@ -0,0 +1,108 @@
+name: Build-Documentation
+
+on:
+  # Trigger the workflow every night
+  schedule:
+    - cron: '0 1 * * *'
+
+  # Trigger on push to master and for pull requests targetting master
+  push:
+    branches: [ master ]
+
+  pull_request:
+    branches: [ master ]
+
+  # Trigger manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  build-documentation:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check-out the repository under $GITHUB_WORKSPACE
+        uses: actions/checkout@v2
+        with:
+            fetch-depth: 0
+
+      - name: Install prerequisites from the README
+        run: sudo apt install python qttools5-dev-tools wkhtmltopdf python-pypdf2 inkscape
+
+      - name: Print versions
+        run: |
+            python2 --version || true
+            wkhtmltopdf --version --version || true
+            qhelpgenerator --qt=qt4 --version || true
+
+      # GitHub Actions currently uses 18.04 as of 02021-01-31, but will likely update in the future
+      - name: Try to install prerequisites for Ubuntu 20.04 from the README
+        run: sudo apt install qhelpgenerator-qt5 || true
+
+      - name: Install prerequisites for testing
+        run: sudo apt install xvfb
+
+#      - name: Remove existing downloads (for now, because the update.py script seems to not work well)
+#        run: |
+#            rm -rf localwiki revisions.txt updates.txt wikifiles.txt
+#            mkdir localwiki
+#            touch revisions.txt updates.txt wikifiles.txt
+
+      - name: Build an index file containing a list of all the files to download
+        run: python2 ./buildwikiindex.py
+
+      - name: Download wiki pages
+        # Redirect log because it seems we exceed the amount of data GitHub is willing
+        # to show in the interacive logs and there seems to be no UI to access the raw
+        # logs at this time
+        #run: python2 ./downloadwiki.py > ./downloadwiki.py.log
+        run: python2 ./downloadwiki.py
+
+      - name: Create a list of revision IDs for each page
+        run: python2 ./update.py
+
+      - name: Get a list of pages that have changed
+        run: python2 ./update.py
+
+      - name: Download the changed pages (and all their dependencies) again
+        run: python2 ./update.py
+
+      - name: Generate freecad.qhc and freecad.qch files
+        run: python2 ./buildqhelp.py --non-interactive
+
+      - name: Generate freecad.pdf
+        run: python2 ./buildpdf.py
+
+      - name: Split the generated freecad.qch into parts that are smaller than 50Mb (github limit)
+        run: split -d --byte=49M localwiki/freecad.qch localwiki/freecad.qch.part
+
+      - name: Reassemble the previously-split freecad.qch 
+        run: cat localwiki/freecad.qch.part* >> test.qch
+
+      - name: Check that the reassembled test.qch is identical to localwiki/freecad.qch
+        run: diff -q test.qch localwiki/freecad.qch
+
+      - name: Try to open the .qch file
+        run: |
+          xvfb-run assistant -collectionFile localwiki/freecad.qhc &
+          sleep 5
+          killall assistant
+
+      - name: Upload freecad.qch
+        uses: actions/upload-artifact@v2
+        with:
+          name: freecad.qch
+          path: localwiki/freecad.qch
+
+      - name: Upload freecad.qhc
+        uses: actions/upload-artifact@v2
+        with:
+          name: freecad.qhc
+          path: localwiki/freecad.qhc
+
+      - name: Upload freecad.pdf
+        uses: actions/upload-artifact@v2
+        with:
+          name: freecad.pdf
+          path: localwiki/freecad.pdf
+

.gitignore

@@ -0,0 +1,2 @@
+/*.pyc
+/__pycache__

buildpdf.py

@@ -22,6 +22,7 @@
 #*                                                                         *
 #***************************************************************************
 
+from __future__ import print_function
 __title__="buildpdf"
 __author__ = "Yorik van Havre <yorik@uncreated.net>"
 __url__ = "http://www.freecadweb.org"
@@ -63,7 +64,7 @@ Part_RegularPolygon
 Part_Booleans
 # Part_Common
 # Part_Cut
-Part_Fuse
+Part_Union
 # Part_Shapebuilder
 Part_Extrude
 Part_Fillet
@@ -170,6 +171,7 @@ Draft_Dimension
 Draft_BSpline
 Draft_Point
 Draft_ShapeString
+Draft_Drawing
 Draft_Facebinder
 Draft_BezCurve
 Draft_Move
@@ -223,7 +225,6 @@ Drawing_Annotation
 Drawing_Clip
 Drawing_Openbrowser
 Drawing_Symbol
-Drawing_DraftView
 Drawing_Save
 Drawing_ProjectShape
 # Drawing_Othoviews
@@ -363,19 +364,19 @@ def crawl():
             return 1
     elif PDFCONVERTOR == 'htmldoc':
         if os.system('htmldoc --version'):
-            print "Error: Htmldoc not found, exiting."
+            print("Error: Htmldoc not found, exiting.")
             return 1
     try:
         from PyPDF2 import PdfFileReader,PdfFileWriter
     except:
-        print "Error: Python-pypdf2 not installed, exiting."
+        print("Error: Python-pypdf2 not installed, exiting.")
 
     # run ########################################################
     
     buildpdffiles()
     joinpdf()
 
-    if VERBOSE: print "All done!"
+    if VERBOSE: print("All done!")
     return 0
 
 
@@ -389,10 +390,10 @@ def buildpdffiles():
     for i in templist:
         if i[-5:] == '.html':
             fileslist.append(i)
-    print "converting ",len(fileslist)," pages"
+    print("converting ",len(fileslist)," pages")
     i = 1
     for f in fileslist:
-        print i," : ",f
+        print(i," : ",f)
         if PDFCONVERTOR == 'pisa':
             createpdf_pisa(f[:-5])
         elif PDFCONVERTOR == 'wkhtmltopdf': 
@@ -421,7 +422,7 @@ def createpdf_pisa(pagename):
     if (not exists(pagename+".pdf",image=True)) or OVERWRTIE:
         infile = open(FOLDER + os.sep + pagename+'.html','ro')
         outfile = open(FOLDER + os.sep + pagename+'.pdf','wb')
-        if VERBOSE: print "Converting " + pagename + " to pdf..."
+        if VERBOSE: print("Converting " + pagename + " to pdf...")
         pdf = pisa.CreatePDF(infile,outfile,FOLDER,link_callback=fetch_resources)
         outfile.close()
         if pdf.err: 
@@ -441,7 +442,7 @@ def createpdf_firefox(pagename):
         if os.path.exists(FIREFOXPDFFOLDER + os.sep + pagename + ".pdf"):
             shutil.move(FIREFOXPDFFOLDER+os.sep+pagename+".pdf",outfile)
         else:
-            print "-----------------------------------------> Couldn't find print output!"
+            print("-----------------------------------------> Couldn't find print output!")
 
 
 def createpdf_htmldoc(pagename):
@@ -458,20 +459,20 @@ def createpdf_wkhtmltopdf(pagename):
         infile = FOLDER + os.sep + pagename+'.html'
         outfile = FOLDER + os.sep + pagename+'.pdf'
         cmd = 'wkhtmltopdf -L 5mm --user-style-sheet '+FOLDER+os.sep+'wkhtmltopdf.css '+infile+' '+outfile
-        print cmd
+        print(cmd)
         #return os.system(cmd)
     else:
-        print "skipping"
+        print("skipping")
 
 
 def joinpdf():
     "creates one pdf file from several others, following order from the cover"
     from PyPDF2 import PdfFileReader,PdfFileWriter
-    if VERBOSE: print "Building table of contents..."
+    if VERBOSE: print("Building table of contents...")
     
     result = PdfFileWriter()
     createCover()
-    inputfile = PdfFileReader(open(FOLDER+os.sep+'Cover.pdf','rb'))
+    inputfile = PdfFileReader(open(os.path.join(FOLDER,'Cover.pdf'),'rb'))
     result.addPage(inputfile.getPage(0))
     count = 1
 
@@ -488,11 +489,11 @@ def joinpdf():
             if page == "end":
                 parent = False
                 continue
-            if VERBOSE: print 'Appending',page, "at position",count
+            if VERBOSE: print('Appending',page, "at position",count)
             title = page.replace("_"," ")
             pdffile = page + ".pdf"
             if exists(pdffile,True):
-                inputfile = PdfFileReader(open(FOLDER + os.sep + pdffile,'rb'))
+                inputfile = PdfFileReader(open(FOLDER + os.sep + pdffile,'rb'), strict=False)
                 numpages = inputfile.getNumPages()
                 for i in range(numpages):
                     result.addPage(inputfile.getPage(i))
@@ -504,16 +505,16 @@ def joinpdf():
                     result.addBookmark(title,count,parent)
                 count += numpages
             else:
-                print "page",pdffile,"not found, aborting."
+                print("page",pdffile,"not found, aborting.")
                 sys.exit()
 
-    if VERBOSE: print "Writing..."
+    if VERBOSE: print("Writing...")
     outputfile = open(FOLDER+os.sep+"freecad.pdf",'wb')
     result.write(outputfile)
     outputfile.close()
     if VERBOSE: 
-        print ' '
+        print(' ')
-        print 'Successfully created '+FOLDER+os.sep+'freecad.pdf'
+        print('Successfully created '+FOLDER+os.sep+'freecad.pdf')
 
 
 def local(page,image=False):
@@ -544,13 +545,16 @@ def makeStyleSheet():
 
 def createCover():
     "downloads and creates a cover page"
-    if VERBOSE: print "fetching " + COVER
+    if VERBOSE: print("fetching " + COVER)
     data = (urlopen(COVER).read())
-    path = FOLDER + os.sep + "Cover.svg"
+    path = os.path.join(FOLDER, "Cover.svg")
     fil = open(path,'wb')
     fil.write(data)
     fil.close()
-    os.system('inkscape --export-pdf='+FOLDER+os.sep+'Cover.pdf'+' '+FOLDER+os.sep+'Cover.svg')
+    if os.system('inkscape --export-pdf='+os.path.join(FOLDER,'Cover.pdf')+' '+os.path.join(FOLDER,'Cover.svg')) == 0:
+        return
+    else:
+        raise Exception('Conversion of Cover.svg to Cover.pdf failed. Is Inkscape installed?')
 
 
 if __name__ == "__main__":

buildqhelp.py

@@ -22,6 +22,7 @@
 #*                                                                         *
 #***************************************************************************
 
+from __future__ import print_function
 __title__="wiki2qhelp"
 __author__ = "Yorik van Havre <yorik@uncreated.net>"
 __url__ = "http://www.freecadweb.org"
@@ -38,8 +39,8 @@ from urllib2 import urlopen, HTTPError
 FOLDER = "./localwiki"
 INDEX = "Online_Help_Toc" # the start page from where to crawl the wiki
 VERBOSE = True # to display what's going on. Otherwise, runs totally silent.
-QHELPCOMPILER = 'qhelpgenerator'
+QHELPCOMPILER = 'qhelpgenerator -qt=qt5'
-QCOLLECTIOMGENERATOR = 'qcollectiongenerator'
+QCOLLECTIOMGENERATOR = 'qcollectiongenerator -qt=qt5'
 RELEASE = '0.17'
 
 #    END CONFIGURATION      ##############################################
@@ -52,10 +53,10 @@ def crawl():
     # tests ###############################################
     
     if os.system(QHELPCOMPILER +' -v'):
-        print "Error: QAssistant not fully installed, exiting."
+        print("Error: QAssistant not fully installed, exiting.")
         return 1
     if os.system(QCOLLECTIOMGENERATOR +' -v'):
-        print "Error: QAssistant not fully installed, exiting."
+        print("Error: QAssistant not fully installed, exiting.")
         return 1
 
     # run ########################################################
@@ -64,23 +65,34 @@ def crawl():
     qhcp = createCollProjectFile()
     shutil.copy("freecad-icon-64.png","localwiki/freecad-icon-64.png")
     if generate(qhcp) or compile(qhp):
-        print "Error at compiling"
+        print("Error at compiling")
         return 1
-    if VERBOSE: print "All done!"
+    if VERBOSE: print("All done!")
-    i=raw_input("Copy the files to their correct location in the source tree? y/n (default=no) ")
+    if "--yes-copy" in sys.argv:
+        i="yes"
+    elif "--no-copy" in sys.argv:
+        i="no"
+    else:
+        try:
+            i=raw_input("Copy the files to their correct location in the source tree? y/n (default=no) ")
+        except:
+            i="no"
     if i.upper() in ["Y","YES"]:
         shutil.copy("localwiki/freecad.qch","../../Doc/freecad.qch")
         shutil.copy("localwiki/freecad.qhc","../../Doc/freecad.qhc")
     else:
-        print 'Files are in localwiki. Test with "assistant -collectionFile localwiki/freecad.qhc"'
+        print('Files are in localwiki. Test with "assistant -collectionFile localwiki/freecad.qhc"')
     return 0
     
 def compile(qhpfile):
     "compiles the whole html doc with qassistant"
     qchfile = FOLDER + os.sep + "freecad.qch"
     if not os.system(QHELPCOMPILER + ' '+qhpfile+' -o '+qchfile):
-        if VERBOSE: print "Successfully created",qchfile
+        if VERBOSE: print("Successfully created",qchfile)
         return 0
+    else:
+        os.system('cat -v ' + qhpfile)
+        raise "Error during generation of freecad.qch"
 
 def generate(qhcpfile):
     "generates qassistant-specific settings like icon, title, ..."
@@ -93,8 +105,11 @@ def generate(qhcpfile):
     about.close()
     qhcfile = FOLDER + os.sep + "freecad.qhc"
     if not os.system(QCOLLECTIOMGENERATOR+' '+qhcpfile+' -o '+qhcfile):
-        if VERBOSE: print "Successfully created ",qhcfile
+        if VERBOSE: print("Successfully created ",qhcfile)
         return 0
+    else:
+        os.system('cat -v ' + qhcpfile)
+        raise "Error during generation of freecad.qhc"
 
 def createCollProjectFile():
     qprojectfile = '''<?xml version="1.0" encoding="UTF-8"?>
@@ -131,12 +146,12 @@ def createCollProjectFile():
     </docFiles>
 </QHelpCollectionProject>
 '''
-    if VERBOSE: print "Building project file..."
+    if VERBOSE: print("Building project file...")
     qfilename = FOLDER + os.sep + "freecad.qhcp"
     f = open(qfilename,'w')
     f.write(qprojectfile)
     f.close()
-    if VERBOSE: print "Done writing qhcp file",qfilename
+    if VERBOSE: print("Done writing qhcp file",qfilename)
     return qfilename
 
 def buildtoc():
@@ -182,7 +197,7 @@ def buildtoc():
         if not link: link = 'default.html'
         return title,link
 
-    if VERBOSE: print "Building table of contents..."
+    if VERBOSE: print("Building table of contents...")
     f = open(FOLDER+os.sep+INDEX+'.html')
     html = ''
     for line in f: html += line
@@ -229,9 +244,9 @@ def buildtoc():
     f = open(qfilename,'wb')
     f.write(qhelpfile)
     f.close()
-    if VERBOSE: print "Done writing qhp file",qfilename
+    if VERBOSE: print("Done writing qhp file",qfilename)
     return qfilename
     
 if __name__ == "__main__":
-	crawl()
+	exit(crawl())
       

buildwikiindex.py

@@ -22,6 +22,7 @@
 #*                                                                         *
 #***************************************************************************
 
+from __future__ import print_function
 __title__="buildwikiindex.py"
 __author__ = "Yorik van Havre <yorik@uncreated.net>"
 __url__ = "http://www.freecadweb.org"
@@ -39,7 +40,7 @@ from urllib2 import urlopen, HTTPError
 URL = "https://www.freecadweb.org/wiki" #default URL if no URL is passed
 INDEX = "Online_Help_Toc" # the start page from where to crawl the wiki
 NORETRIEVE = ['Manual','Developer_hub','Power_users_hub','Users_hub','Source_documentation', 'User_hub','Main_Page','About_this_site','Interesting_links','Syndication_feeds','FreeCAD:General_disclaimer','FreeCAD:About','FreeCAD:Privacy_policy','WikiPages'] # pages that won't be fetched (kept online)
-GETTRANSLATIONS = False # Set true if you want to get the translations too.
+GETTRANSLATIONS = True # Set true if you want to get the translations too.
 MAXFAIL = 3 # max number of retries if download fails
 VERBOSE = True # to display what's going on. Otherwise, runs totally silent.
 WRITETHROUGH = True # if true, fetched files are constantly written to disk, in case of failure.
@@ -60,15 +61,15 @@ def crawl(pagename=[]):
     else:
         if os.path.exists("wikifiles.txt"):
             f = open("wikifiles.txt","r")
-            if VERBOSE: print "Reading existing list..."
+            if VERBOSE: print("Reading existing list...")
             for l in f.readlines():
                 if l.strip() != "":
-                    if VERBOSE: print "Adding ",l
+                    if VERBOSE: print("Adding ",l)
                     processed.append(l.strip())
             f.close()
         if os.path.exists("todolist.txt"):
             f = open("todolist.txt","r")
-            if VERBOSE: print "Reading existing todo list..."
+            if VERBOSE: print("Reading existing todo list...")
             for l in f.readlines():
                 if l.strip() != "":
                     todolist.append(l.strip())
@@ -79,19 +80,19 @@ def crawl(pagename=[]):
     while todolist:
         targetpage = todolist.pop()
         if (not targetpage in NORETRIEVE):
-            if VERBOSE: print count, ": Scanning ", targetpage
+            if VERBOSE: print(count, ": Scanning ", targetpage)
             pages,images = get(targetpage)
             count += 1
             processed.append(targetpage)
             processed.extend(images)
-            if VERBOSE: print "got",len(pages),"links"
+            if VERBOSE: print("got",len(pages),"links")
             for p in pages:
                 if (not (p in todolist)) and (not (p in processed)):
                     todolist.append(p)
             if WRITETHROUGH:
                 writeList(processed)
                 writeList(todolist,"todolist.txt")
-    if VERBOSE: print "Fetched ", count, " pages"
+    if VERBOSE: print("Fetched ", count, " pages")
     if not WRITETHROUGH:
         writeList(processed)
     if pagename:
@@ -156,7 +157,7 @@ def getlinks(html):
                     NORETRIEVE.append(rg)
             if not rg in NORETRIEVE:
                 pages.append(rg)
-                print "got link: ",rg
+                print("got link: ",rg)
     return pages
 
 def getimagelinks(html):
@@ -167,7 +168,7 @@ def getimagelinks(html):
 
 def fetchpage(page):
     "retrieves given page from the wiki"
-    print "fetching: ",page
+    print("fetching: ",page)
     failcount = 0
     while failcount < MAXFAIL:
         try:
@@ -175,7 +176,7 @@ def fetchpage(page):
             return html
         except HTTPError:
             failcount += 1
-    print 'Error: unable to fetch page ' + page
+    print('Error: unable to fetch page ' + page)
     sys.exit()
 
 def cleanList(pagelist):
@@ -193,7 +194,7 @@ def writeList(pages,filename="wikifiles.txt"):
     for p in pages:
         f.write(p+"\n")
     f.close()
-    if VERBOSE: print "written ",filename
+    if VERBOSE: print("written ",filename)
 
 if __name__ == "__main__":
 	crawl(sys.argv[1:])

downloadwiki.py

@@ -22,6 +22,7 @@
 #*                                                                         *
 #***************************************************************************
 
+from __future__ import print_function
 __title__="downloadwiki"
 __author__ = "Yorik van Havre <yorik@uncreated.net>"
 __url__ = "http://www.freecadweb.org"
@@ -137,7 +138,7 @@ def crawl():
     "downloads an entire wiki site"
     global processed
     processed = []
-    if VERBOSE: print "crawling ", URL, ", saving in ", FOLDER
+    if VERBOSE: print("crawling ", URL, ", saving in ", FOLDER)
     if not os.path.isdir(FOLDER): os.mkdir(FOLDER)
     file = open(FOLDER + os.sep + "wiki.css",'wb')
     file.write(css)
@@ -151,16 +152,16 @@ def crawl():
     for l in lfile: locallist.append(l.replace("\n",""))
     lfile.close()
     todolist = locallist[:]
-    print "getting",len(todolist),"files..."
+    print("getting",len(todolist),"files...")
     count = 1
     indexpages = get(INDEX)
     while todolist:
         targetpage = todolist.pop()
-        if VERBOSE: print count, ": Fetching ", targetpage
+        if VERBOSE: print(count, ": Fetching ", targetpage)
         get(targetpage)
         count += 1
-    if VERBOSE: print "Fetched ", count, " pages"
+    if VERBOSE: print("Fetched ", count, " pages")
-    if VERBOSE: print "All done!"
+    if VERBOSE: print("All done!")
     return 0
 
 def get(page):
@@ -180,7 +181,7 @@ def get(page):
         html = cleanimagelinks(html)
         output(html,page)
     else:
-        if VERBOSE: print "    skipping",page
+        if VERBOSE: print("    skipping",page)
 
 def getlinks(html):
     "returns a list of wikipage links in html file"
@@ -268,7 +269,7 @@ def cleanimagelinks(html,links=None):
 
 def fetchpage(page):
     "retrieves given page from the wiki"
-    print "    fetching: ",page
+    print("    fetching: ",page)
     failcount = 0
     while failcount < MAXFAIL:
         try:
@@ -276,19 +277,19 @@ def fetchpage(page):
             return html
         except HTTPError:
             failcount += 1
-    print 'Error: unable to fetch page ' + page
+    print('Error: unable to fetch page ' + page)
 
 def fetchimage(imagelink):
     "retrieves given image from the wiki and saves it"
     if imagelink[0:5] == "File:":
-        print "Skipping file page link"
+        print("Skipping file page link")
         return
     filename = re.findall('.*/(.*)',imagelink)[0]
     if not exists(filename,image=True):
         failcount = 0
         while failcount < MAXFAIL:
             try:
-                if VERBOSE: print "    fetching " + filename
+                if VERBOSE: print("    fetching " + filename)
                 data = (urlopen(URL + imagelink).read())
                 path = local(filename,image=True)
                 file = open(path,'wb')
@@ -298,11 +299,11 @@ def fetchimage(imagelink):
                 failcount += 1
             else:
                 processed.append(filename)
-                if VERBOSE: print "    saving",local(filename,image=True)
+                if VERBOSE: print("    saving",local(filename,image=True))
                 return
-        print 'Error: unable to fetch file ' + filename
+        print('Error: unable to fetch file ' + filename)
     else:
-        if VERBOSE: print "    skipping",filename
+        if VERBOSE: print("    skipping",filename)
 
 def local(page,image=False):
     "returns a local path for a given page/image"
@@ -337,7 +338,7 @@ def output(html,page):
         filename = filename.replace("&pagefrom=","+")
         filename = filename.replace("#mw-pages","")
         filename = filename.replace(".html.html",".html")
-    print "    saving",filename
+    print("    saving",filename)
     file = open(filename,'wb')
     file.write(html)
     file.close()