raco pkg create, raco setup: add support for built (source+binary) packages

This change removes `rendered-scribblings' as a recognized "info.rkt"
definition (which was added for binary packages, but now `scribblings'
serves both roles).

The `raco setup' changes involve support for moving pre-rendered
documentation into place (where "moving" may actually mean
leaving it in place but updating the "local-redirect.js" path),
but also allowing documentation to be re-rendered.

collects/pkg/lib.rkt

@@ -1509,7 +1509,7 @@
       (lambda ()
         (define dest-dir (build-path tmp-dir name))
         (make-directory dest-dir)
-        (generate-stripped-directory (eq? mode 'binary) dir dest-dir)
+        (generate-stripped-directory mode dir dest-dir)
         (create-as-is create:format name dest-dir dir 
                       #:hide-src? #t 
                       #:quiet? quiet?
@@ -1549,12 +1549,11 @@
     [(as-is) (create-as-is create:format pkg-name dir dir
                            #:dest dest-dir
                            #:quiet? quiet?)]
-    [(source binary) (stripped-create mode pkg-name dir
+    [else (stripped-create mode pkg-name dir
                            #:dest dest-dir
                            #:format create:format
                            #:quiet? quiet?)]))
 
-
 (define (pkg-catalog-copy srcs dest
                         #:from-config? [from-config? #f]
                         #:merge? [merge? #f]
@@ -2012,7 +2011,7 @@
    (->* ((or/c 'zip 'tgz 'plt 'MANIFEST)
          path-string?)
         (#:source (or/c 'dir 'name)
-                  #:mode (or/c 'as-is 'source 'binary)
+                  #:mode (or/c 'as-is 'source 'binary 'built)
                   #:quiet? boolean?
                   #:dest (or/c (and/c path-string? complete-path?) #f))
         void?)]

collects/pkg/main.rkt

@@ -208,6 +208,7 @@
   [#:bool as-is () "Bundle the directory/package as-is (the default)"]
   [#:bool source () "Bundle sources only"]
   [#:bool binary () "Bundle bytecode and rendered documentation without sources"]
+  [#:bool built () "Bundle sources, bytecode and rendered documentation"]
   #:once-each
   [(#:str dest-dir #f) dest () "Create output files in <dest-dir>"]
   #:args (directory-or-package)
@@ -222,6 +223,7 @@
                 #:mode (cond
                         [source 'source]
                         [binary 'binary]
+                        [built 'built]
                         [else 'as-is])))]
  [config
   "View and modify the package manager's configuration"

collects/pkg/scribblings/pkg.scrbl

@@ -364,10 +364,10 @@ removing any of the @nonterm{pkg}s.
     GitHub repository or other non-archive formats. The @exec{create}
     sub-command can create an archive from a directory (the default) or
     from an installed package. It can also adjust the archive's content
-    to include only sources (which is the recommended mode, although not
+    to include only sources, only compiled bytecode and rendered documentation,
-    the default) or as a ``binary'' package (but packages are
+    or both---but packages are
     normally provided as source and converted to binary form by an
-    automatic service, instead of by a package author).
+    automatic service, instead of by a package author.
 
  The @exec{create} sub-command accepts 
  the following @nonterm{option}s:
@@ -382,34 +382,11 @@ removing any of the @nonterm{pkg}s.
  @item{@DFlag{manifest} --- Creates a manifest file for a directory, rather than an archive.}
  @item{@DFlag{as-is} --- Bundle all content of the package directory as is, with no filtering
        of sources, compiled files, or repository elements.}
- @item{@DFlag{source} --- Bundle only sources in the package directory, pruning (by default) 
+ @item{@DFlag{source} --- Bundle only sources in the package directory; see @secref["strip"].}
-       @filepath{compiled} directories (that normally hold compiled
-       bytecode), @filepath{doc} directories (that normally hold rendered documentation),
-       directories named @filepath{.svn}, directories and files whose names start with @filepath{.git},
-       and files whose name ends with @litchar{~} or starts and ends with @litchar{#}.
-       Override the default pruning rules with @racket[source-omit-files] and/or
-       @racket[source-keep-files] definitions in @filepath{info.rkt} files within the
-       package directory.}
  @item{@DFlag{binary} --- Bundle compiled bytecode and rendered
-       documentation in the package directory. Normally, this option is sensible for
+       documentation in the package directory; see @secref["strip"].}
-       a package that is installed from source in a user-specific scope. Bundling prunes (by default)
+ @item{@DFlag{built} --- Bundle compiled sources, bytecode, and rendered
-       @filepath{.rkt} and @filepath{.ss} files for which compiled bytecode is present, files with
+       documentation in the package directory, filtering repository elements; see @secref["strip"].}
-       a @filepath{.scrbl} suffix and their compiled files, files with a @filepath{.dep} suffix,
-       @filepath{tests} directories, @filepath{scribblings}
-       directories, @filepath{.svn} directories, directories and files whose names 
-       start with @filepath{.git}, and files whose name ends with @litchar{~} or starts and ends
-       with @litchar{#}. For each @filepath{.zo} file, submodules named @racketidfont{test},
-       @racketidfont{doc}, or @racketidfont{srcdoc} are removed. For each @filepath{.html} file that
-       refers to a @filepath{local-redirect.js} script, the path to the script is removed.
-       In addition, bundling updates any @filepath{info.rkt} as follows: it
-       adds a @racket[assume-virtual-sources] entry,
-       converts a @racket[scribblings] entry to a @racket[rendered-scribblings] entry,
-       changes a @racket[copy-foreign-libs] entry to a @racket[move-foreign-libs] entry,
-       changes a @racket[copy-man-pages] entry to a @racket[move-man-pages] entry,
-       and removes any @racket[build-deps] entry. Override
-       the default pruning rules with @racket[binary-omit-files]
-       and/or @racket[binary-keep-files] definitions in
-       @filepath{info.rkt} files within the package directory.}
   @item{@DFlag{dest} @nonterm{dest-dir} --- Writes generated bundles to @nonterm{dest-dir}.}
   ]
 }
@@ -751,11 +728,12 @@ The following @filepath{info.rkt} fields are used by the package manager:
        on the version of the Racket installation.}
 
  @item{@racketidfont{build-deps} --- like @racketidfont{deps}, but for
-       dependencies that can be dropped in a ``binary'' variant of the
+       dependencies that can be dropped in a @tech{binary package},
-       package that does not include sources. The
+       which does not include sources; see @secref["strip"]. The
-       @racketidfont{build-deps} and @racketidfont{deps} lists are appended,
+       @racketidfont{build-deps} and @racketidfont{deps} lists are
-       while @command-ref["create"] strips away @racketidfont{build-deps}
+       appended, while @command-ref["create"] strips away
-       when converting a package for @DFlag{binary} mode.}
+       @racketidfont{build-deps} when converting a package for
+       @DFlag{binary} mode.}
 
  @item{@racketidfont{setup-collects} --- a list of path strings and/or
        lists of path strings, which are used as collection names to
@@ -769,6 +747,10 @@ The following @filepath{info.rkt} fields are used by the package manager:
 
 @; ----------------------------------------
 
+@include-section["strip.scrbl"]
+
+@; ----------------------------------------
+
 @section{@|Planet1| Compatibility}
 
 PLT maintains a @tech{package catalog} to serve packages that

collects/pkg/scribblings/strip.scrbl

@@ -0,0 +1,161 @@
+#lang scribble/manual
+@(require "common.rkt"
+          (for-label pkg/strip))
+
+@title[#:tag "strip"]{Source, Binary, and Built Packages}
+
+A package, especially in a repository format, normally provides module
+implementations and documentation in source form. Such @deftech{source
+packages} can work with multiple Racket versions, and modules are
+compiled to bytecode and documentation is rendered when the package is
+installed.
+
+A @deftech{binary package} provides only compiled bytecode and
+rendered documentation, instead of package and documentation
+sources. Since compiled bytecode is specific to a version of Racket, a
+@tech{binary package} is specific to a version of Racket. The benefit
+of a binary package is that it can have less dependencies (e.g., on
+Scribble to implement the documentation or on other
+packages whose documentation is referenced) and it can install
+faster. A drawback of a binary package is that it is version-specific
+and the source code may be less immediately accessible to other
+programmers.
+
+A @deftech{built package} combines source and compiled elements. A
+@tech{built package} can be installed more quickly than source, as
+long as it is installed for a suitable Racket version, but the source
+remains available as a back-up for other programmers to consult or to
+re-build for a different Racket version.
+
+A package is not specifically tagged as a @tech{source package},
+@tech{binary package}, or @tech{built package}. The different kinds of
+packages are just conventions based on the content of the package. All
+forms of packages can be mixed in an installation, and a package can
+be updated from any form to any other form.
+
+Programmers normally supply only @tech{source packages}, while a
+package catalog service may convert each source package to a
+@tech{binary package} or @tech{built package}. Alternatively,
+programmers can create @tech{binary packages} or @tech{built packages}
+by using the @command-ref{create} subcommand with @DFlag{binary} or
+@DFlag{built}. As a convenience, the @command-ref{create} subcommand
+can also create a @tech{source package} from an installed package or
+repository checkout, dropping repository elements (such as a
+@filepath{.git} directory) and compiled code. Note that
+@command-ref{create} by default bundles a package directory as-is,
+with no filtering at all.
+
+Creating a @tech{source package}, @tech{binary package}, or
+@tech{built package} from a directory or package installation prunes
+the following files and directories:
+
+@itemlist[
+
+ @item{directories/files named @filepath{.svn};}
+
+ @item{directories/files whose names start with @filepath{.git};}
+
+ @item{directories/files whose names end with @filepath{~}; and}
+
+ @item{directories/files whose names start and end with @filepath{#}.}
+
+]
+
+Any of the above can be suppressed, however, by a
+@racket[source-keep-files] (for @tech{source package} and @tech{built
+package} bundling) or @racket[binary-keep-files] (for @tech{binary
+package} and @tech{built package} bundling) definition in an
+@filepath{info.rkt} in the package or any subdirectory. A
+@racket[binary-keep-files] or @racket[binary-keep-files] definition
+should bind the name to a list of paths relative to the
+@filepath{info.rkt} file.
+
+Creating a @tech{source package} prunes the following additional files
+and directories:
+
+@itemlist[
+
+ @item{directories/files named @filepath{compiled};}
+
+ @item{directories/files named @filepath{doc};}
+
+ @item{directories/files named @filepath{synced.rktd}, which can appear
+       as a marker in rendered documentation;}
+               
+ @item{directories/files named in an @filepath{info.rkt} file's
+       @racket[source-omit-files] definition.}
+
+]
+
+Any of the above removals can be suppressed through
+@racketidfont{source-keep-files}.
+
+Creating a @tech{binary package} prunes the following addition files
+and directories:
+
+@itemlist[
+
+ @item{directories/files with names ending in @filepath{.rkt} or
+ @filepath{.ss} for which a corresponding compiled bytecode file is
+ present (in a @filepath{compiled} subdirectory);}
+
+ @item{directories/files with names ending in @filepath{.scrbl},
+       @filepath{_scrbl.zo}, or @filepath{.dep};}
+
+ @item{directories/files named in an @filepath{info.rkt} file's
+       @racket[source-omit-files] definition.}
+
+]
+
+Any of the above removals can be suppressed through
+@racketidfont{binary-keep-files}.
+
+Creating a @tech{binary package} further adjusts the following files:
+
+@itemlist[
+
+ @item{for any file whose name ends in @filepath{.zo}, submodules
+       named @racketidfont{test}, @racketidfont{doc}, or
+       @racketidfont{srcdoc} are removed;}
+
+ @item{for each @filepath{.html} file that refers to a
+       @filepath{local-redirect.js} script, the path to the script is
+       removed; and}
+
+ @item{each @filepath{info.rkt} is adjusted as follows: an
+       @racket[assume-virtual-sources] definition is added, any
+       @racket[copy-foreign-libs] definition is changed to
+       @racket[move-foreign-libs], any @racket[copy-man-pages]
+       definition is changed to @racket[move-man-pages] entry, and any
+       @racket[build-deps] definition is removed.}
+
+]
+
+Finally, creating a @tech{built package} removes any file or directory
+that would be removed for a @tech{source package} and @tech{binary
+package}, and it performs the @filepath{.html} file updating of a
+@tech{binary package}.
+
+@defmodule[pkg/strip]{The @racketmodname[pkg/strip] module provides
+support for copying a package-style directory to a given destination
+with the same file/directory omissions and updates as
+@command-ref{create}.}
+
+@defproc[(generate-stripped-directory [mode (or/c 'source 'binary 'built)]
+                                      [src-dir path-string?]
+                                      [dest-dir path-string?])
+          void?]{
+
+Copies @racket[src-dir] to @racket[dest-dir] with file/directory
+omissions and updates corresponding the creation of a @tech{source
+package}, @tech{binary package}, or @tech{built package} as indicated
+by @racket[mode].}
+
+
+@defproc[(fixup-local-redirect-reference [file path-string?]
+                                         [js-path string?])
+         void?]{
+
+Assuming that @racket[file] is an HTML file for documentation, adjusts
+the URL reference to @filepath{local-redirect.js}, if any, to use the
+prefix @racket[js-path].}

collects/pkg/strip.rkt

@@ -4,32 +4,44 @@
          syntax/modread
          racket/match
          racket/file
-         racket/list)
+         racket/list
+         racket/set)
 
 (provide generate-stripped-directory
          fixup-local-redirect-reference)
 
-(define (generate-stripped-directory binary? dir dest-dir)
+(define (generate-stripped-directory mode dir dest-dir)
   (define drop-keep-ns (make-base-namespace))
   (define (add-drop+keeps dir base drops keeps)
     (define get-info (get-info/full dir #:namespace drop-keep-ns))
-    (define drop-tag (if binary? 'binary-omit-files 'source-omit-files))
+    (define (get-paths tag)
-    (define more-drops (if get-info
+      (define l (if get-info
-                           (get-info drop-tag (lambda () null))
+                    (get-info tag (lambda () null))
                     null))
-    (define keep-tag (if binary? 'binary-keep-files 'source-keep-files))
-    (define more-keeps (if get-info
-                           (get-info keep-tag (lambda () null))
-                           null))
-    (define (check tag l)
       (unless (and (list? l) (andmap (lambda (p)
                                        (and (path-string? p)
                                             (relative-path? p)))
                                      l))
-        (error 'strip "bad ~a value from \"info.rkt\": ~e" tag l)))
+        (error 'strip "bad ~a value from \"info.rkt\": ~e" tag l))
-    (check drop-tag more-drops)
+      l)
-    (check keep-tag more-keeps)
+    (define (intersect l1 l2)
-
+      (set->list (set-intersect (list->set l1) (list->set l2))))
+    (define (union l1 l2)
+      (set->list (set-union (list->set l1) (list->set l2))))
+    (define more-drops
+      (case mode
+        [(source) (get-paths 'source-omit-files)]
+        [(binary) (get-paths 'binary-omit-files)]
+        [(built) 
+         (intersect (get-paths 'source-omit-files)
+                    (get-paths 'binary-omit-files))]))
+    (define more-keeps
+      (case mode
+        [(source) (get-paths 'source-keep-files)]
+        [(binary) (get-paths 'binary-keep-files)]
+        [(built) 
+         (union (get-paths 'source-keep-files)
+                (get-paths 'binary-keep-files))]))
     (define (add ht l)
       (for/fold ([ht ht]) ([i (in-list l)])
         (hash-set ht 
@@ -44,32 +56,36 @@
     (define bstr (path->bytes path))
     (or (regexp-match? #rx#"^(?:[.]git.*|[.]svn|.*~|#.*#)$"
                        bstr)
-        (regexp-match? (if binary?
+        ;; can appear as a marker in rendered documentation:
-                           #rx#"^(?:tests|scribblings|.*(?:[.]scrbl|[.]dep|_scrbl[.]zo))$"
+        (equal? #"synced.rktd" bstr)
-                           #rx#"^(?:compiled|doc)$")
+        (case mode
+          [(source)
+           (regexp-match? #rx#"^(?:compiled|doc)$" bstr)]
+          [(binary)
+           (or (regexp-match? #rx#"^(?:tests|scribblings|.*(?:[.]scrbl|[.]dep|_scrbl[.]zo))$"
                               bstr)
-        (and binary?
+               (and (regexp-match? #rx"[.](?:ss|rkt)$" bstr)
-             (regexp-match? #rx"[.](?:ss|rkt)$" bstr)
                     (not (equal? #"info.rkt" bstr))
                     (file-exists? (let-values ([(base name dir?) (split-path (get-p))])
                                     (build-path base "compiled" (path-add-suffix name #".zo")))))
-        (and binary?
                ;; drop these, because they're recreated on fixup:
-             (or
                (equal? #"info_rkt.zo" bstr)
-              (equal? #"info_rkt.dep" bstr)))))
+               (equal? #"info_rkt.dep" bstr))]
+          [(built)
+           #f])))
   
   (define (fixup new-p path src-base)
-    (when binary?
+    (unless (eq? mode 'source)
       (define bstr (path->bytes path))
       (cond
        [(regexp-match? #rx"[.]html$" bstr)
         (fixup-html new-p)]
-       [(equal? #"info.rkt" bstr)
+       [(and (eq? mode 'binary)
+             (equal? #"info.rkt" bstr))
         (fixup-info new-p src-base)]
-       [(regexp-match? #rx"[.]zo$" bstr)
+       [(and (eq? mode 'binary)
-        (fixup-zo new-p)]
+             (regexp-match? #rx"[.]zo$" bstr))
-       [else (void)])))
+        (fixup-zo new-p)])))
   
   (define (explore base paths drops keeps)
     (for ([path (in-list paths)])
@@ -201,8 +217,6 @@
 (define ((fixup-info-definition get-info) defn)
   (match defn
     [`(define build-deps . ,v) #f]
-    [`(define scribblings . ,v)
-     `(define rendered-scribblings . ,v)]
     [`(define copy-foreign-libs . ,v)
      `(define move-foreign-libs . ,v)]
     [`(define copy-man-pages . ,v)

collects/scribblings/raco/setup.scrbl

@@ -261,7 +261,16 @@ Optional @filepath{info.rkt} fields trigger additional actions by
    A list of documents to build. Each document in the list is itself
    represented as a list, where each document's list starts with a
    string that is a collection-relative path to the document's source
-   file.
+   file. A directory for pre-rendered documentation is computed from the
+   source file name by starting with the directory of the @filepath{info.rkt}
+   file, adding @filepath{doc}, and then using the source file's name without
+   a suffix; if such a directory exists and does not have a
+   @filepath{synced.rktd} file, then it is treated as pre-rendered
+   documentation and moved into place, in which case the documentation source file
+   need not be present. (Moving documentation into place may require no movement
+   at all, depending on the way that the enclosing collection is installed, but 
+   movement includes adding a @filepath{synced.rktd} file to represent
+   the installation.)
 
    More precisely a @racketidfont{scribblings} entry must be a value
    that can be generated from an expression matching the following

collects/setup/private/cc-struct.rkt

@@ -3,5 +3,5 @@
 (provide (struct-out cc))
 
 (define-struct cc
-  (collection path name info omit-root info-root info-path info-path-mode shadowing-policy main?)
+  (collection path name info parent-cc omit-root info-root info-path info-path-mode shadowing-policy main?)
   #:inspector #f)

collects/setup/scribble.rkt

@@ -48,8 +48,7 @@
                                  under-main?
                                  pkg?
                                  category
-                                 out-count
+                                 out-count)
-                                 pre-rendered?)
   #:transparent)
 (define-serializable-struct info (doc       ; doc structure above
                                   undef     ; unresolved requires
@@ -147,10 +146,8 @@
                                   (apply validate i)))
                            infos)])
            (and (not (memq #f infos)) infos))))
-  (define ((get-docs main-dirs pre-rendered?) i rec)
+  (define ((get-docs main-dirs) i rec)
-    (let* ([pre-s (and i (i (if pre-rendered?
+    (let* ([pre-s (and i (i 'scribblings))]
-                                'rendered-scribblings
-                                'scribblings)))]
            [s (validate-scribblings-infos pre-s)]
            [dir (directory-record-path rec)])
       (if s
@@ -178,30 +175,23 @@
                              dest
                              flags under-main? (and (path->pkg src) #t)
                              (caddr d)
-                             (list-ref d 4)
+                             (list-ref d 4))))
-                             pre-rendered?)))
                s)
           (begin (setup-printf
                   "WARNING"
-                  "bad '~ascribblings info: ~e from: ~e" 
+                  "bad 'scribblings info: ~e from: ~e" 
-                  (if pre-rendered? "rendered-" "")
                   pre-s dir)
                  null))))
   (log-setup-info "getting documents")
   (define docs
     (let* ([recs (find-relevant-directory-records '(scribblings) 'all-available)]
-           [r-recs (find-relevant-directory-records '(rendered-scribblings) 'all-available)]
            [main-dirs (parameterize ([current-library-collection-paths
                                       (list (find-collects-dir))])
                         (for/hash ([k (in-list
-                                       (append
+                                       (find-relevant-directories '(scribblings) 'no-planet))])
-                                        (find-relevant-directories '(rendered-scribblings) 'no-planet)
-                                        (find-relevant-directories '(scribblings) 'no-planet)))])
                           (values k #t)))]
-           [infos (map get-info/full (map directory-record-path recs))]
+           [infos (map get-info/full (map directory-record-path recs))])
-           [r-infos (map get-info/full (map directory-record-path r-recs))])
+      (filter-user-docs (append-map (get-docs main-dirs) infos recs)
-      (filter-user-docs (append (append-map (get-docs main-dirs #f) infos recs) 
-                                (append-map (get-docs main-dirs #t) r-infos r-recs))
                         make-user?)))
   (define-values (main-docs user-docs) (partition doc-under-main? docs))
 
@@ -759,13 +749,16 @@
                        only-fast? force-out-of-date? lock)
          doc)
 
-  ;; First, move pre-rendered documentation into place
+  ;; First, move pre-rendered documentation, if any, into place
-  (when (and (doc-pre-rendered? doc)
+  (let ([rendered-dir (let-values ([(base name dir?) (split-path (doc-src-file doc))])
-             (can-build? only-dirs doc)
+                        (build-path (doc-src-dir doc) "doc" (path-replace-suffix name #"")))])
+    (when (and (can-build? only-dirs doc)
+               (directory-exists? rendered-dir)
+               (not (file-exists? (build-path rendered-dir "synced.rktd")))
                (or (not (directory-exists? (doc-dest-dir doc)))
                    force-out-of-date?
                    (not (file-exists? (build-path (doc-dest-dir doc) "synced.rktd")))))
-    (move-documentation-into-place doc setup-printf workerid lock))
+      (move-documentation-into-place doc rendered-dir setup-printf workerid lock)))
 
   (let* ([info-out-files (for/list ([i (add1 (doc-out-count doc))])
                            (sxref-path latex-dest doc (format "out~a.sxref" i)))]
@@ -773,13 +766,16 @@
          [db-file (find-db-file doc latex-dest)]
          [stamp-file  (sxref-path latex-dest doc "stamp.sxref")]
          [out-file (build-path (doc-dest-dir doc) "index.html")]
-         [src-zo (and 
+         [src-zo (let-values ([(base name dir?) (split-path (doc-src-file doc))])
-                  (not (doc-pre-rendered? doc))
-                  (let-values ([(base name dir?) (split-path (doc-src-file doc))])
                    (define path (build-path base "compiled" (path-add-suffix name ".zo")))
                    (or (for/or ([root (in-list (current-compiled-file-roots))])
                          (define p (reroot-path* path root))
                          (and (file-exists? p) p))
+                       (if (and (not (file-exists? path))
+                                (file-exists? out-file))
+                           ;; assume installed as pre-rendered:
+                           #f
+                           ;; need to render, so complain if no source is available:
                            path)))]
          [renderer (make-renderer latex-dest doc)]
          [can-run? (can-build? only-dirs doc)]
@@ -815,13 +811,13 @@
          [info-in-time (file-or-directory-modify-seconds info-in-file #f (lambda () #f))]
          [info-time (min (or info-out-time -inf.0) (or info-in-time -inf.0))]
          [vers (send renderer get-serialize-version)]
-         [src-time (and (not (doc-pre-rendered? doc))
+         [src-time (and src-zo
                         (file-or-directory-modify-seconds/stamp
                          src-zo
                          stamp-time stamp-data 0
                          get-compiled-file-sha1))]
          [up-to-date?
-          (or (doc-pre-rendered? doc)
+          (or (not src-zo)
               (and (not force-out-of-date?)
                    info-out-time
                    info-in-time
@@ -832,7 +828,7 @@
                        ;; this is mostly useful if we interrupt setup-plt after
                        ;; it runs some documents without rendering them:
                        (info-time . >= . src-time))))]
-         [can-run? (and (not (doc-pre-rendered? doc))
+         [can-run? (and src-zo
                         (or (not latex-dest)
                             (not (omit? (doc-category doc))))
                         (or can-run?
@@ -966,9 +962,7 @@
              (lambda () #f))
             #f))))
 
-(define (move-documentation-into-place doc setup-printf workerid lock)
+(define (move-documentation-into-place doc src-dir setup-printf workerid lock)
-  (define src-dir (let-values ([(base name dir?) (split-path (doc-src-file doc))])
-                    (build-path base "doc" (path-replace-suffix name #""))))
   (define dest-dir (doc-dest-dir doc))
   (define move? (not (equal? (file-or-directory-identity src-dir)
                              (and (directory-exists? dest-dir)
@@ -1157,6 +1151,12 @@
            (doc-src-file doc)
            (lambda () (send renderer render (list v) (list dest-dir) ri))
            void))
+         (unless (or latex-dest (main-doc? doc))
+           ;; Since dest dir is the same place as pre-built documentation,
+           ;; mark it so that it is not treated as needing an install:
+           (let ([synced (build-path (doc-dest-dir doc) "synced.rktd")])
+             (unless (file-exists? synced)
+               (close-output-port (open-output-file synced)))))
          (gc-point)
          (list in-delta? out-delta? undef searches))))
    (lambda () #f)))

collects/setup/setup-unit.rkt

@@ -192,7 +192,7 @@
   ;;              Find Collections                 ;;
   ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-  (define (make-cc* collection path omit-root info-root 
+  (define (make-cc* collection parent path omit-root info-root 
                     info-path info-path-mode shadowing-policy 
                     main?)
     (define info
@@ -218,6 +218,7 @@
                       (format "~a (~a)" path-name name)
                       path-name)
                   info
+                  parent
                   omit-root
                   info-root info-path info-path-mode
                   shadowing-policy
@@ -232,6 +233,7 @@
 
   ;; collection-cc! : listof-path .... -> cc
   (define (collection-cc! collection-p
+                          #:parent [parent-cc #f]
                           #:path [dir (apply collection-path collection-p)]
                           #:omit-root [omit-root #f]
                           #:info-root [given-info-root #f]
@@ -258,6 +260,7 @@
              dir))
     (define new-cc
       (make-cc* collection-p
+                parent-cc
                 dir
                 (if (eq? omit-root 'dir)
                     dir
@@ -302,6 +305,7 @@
       (error 'planet-cc! "non-path when building package ~e" pkg-file))
     (and (directory-exists? path)
          (make-cc* #f
+                   #f
                    path
                    omit-root
                    #f ; don't need info-root; absolute paths in cache.rktd will be ok
@@ -404,6 +408,7 @@
     (define (build-collection-tree cc)
       (define (make-child-cc parent-cc name)
         (collection-cc! (append (cc-collection parent-cc) (list name))
+                        #:parent parent-cc
                         #:path (build-path (cc-path parent-cc) name)
                         #:info-root (cc-info-root cc)
                         #:info-path (cc-info-path cc)
@@ -440,6 +445,7 @@
     (define (make-children-ccs cc children)
       (map (lambda (child)
              (collection-cc! (append (cc-collection cc) (list child))
+                             #:parent cc
                              #:path (build-path (cc-path cc) child)
                              #:info-root (cc-info-root cc)
                              #:info-path (cc-info-path cc)
@@ -589,6 +595,12 @@
                            "encountered ~a, neither a file nor a directory"
                            path)]))))
 
+  (define (assume-virtual-sources? cc)
+    (or ((cc-info cc) 'assume-virtual-sources (lambda () #f))
+        (let ([cc (cc-parent-cc cc)])
+          (and cc
+               (assume-virtual-sources? cc)))))
+
   (define (clean-collection cc dependencies)
     (begin-record-error cc "Cleaning"
       (define info (cc-info cc))
@@ -597,7 +609,7 @@
          info
          'clean
          (lambda ()
-           (if (info 'assume-virtual-sources (lambda () #f))
+           (if (assume-virtual-sources? cc)
                null
                (list mode-dir
                      (build-path mode-dir "native")
@@ -745,9 +757,9 @@
                        [compile-notify-handler doing-path])
           (thunk)))))
 
-  (define (clean-cc dir info)
+  (define (clean-cc cc dir info)
     ;; Clean up bad .zos:
-    (unless (info 'assume-virtual-sources (lambda () #f))
+    (unless (assume-virtual-sources? cc)
       (define c (build-path dir "compiled"))
       (when (directory-exists? c)
         (define ok-zo-files
@@ -807,7 +819,7 @@
          (lambda ()
            (define dir  (cc-path cc))
            (define info (cc-info cc))
-           (clean-cc dir info)
+           (clean-cc cc dir info)
            (compile-directory-zos dir info
                                   #:omit-root (cc-omit-root cc)
                                   #:managed-compile-zo caching-managed-compile-zo
@@ -853,7 +865,7 @@
             (iterate-cct (lambda (cc)
                            (define dir (cc-path cc))
                            (define info (cc-info cc))
-                           (clean-cc dir info))
+                           (clean-cc cc dir info))
                          cct)
             (parallel-compile (parallel-workers) setup-fprintf handle-error cct)
             (for/fold ([gcs 0]) ([cc planet-dirs-to-compile])

collects/tests/pkg/test-pkgs/.gitignore

@@ -4,4 +4,5 @@ MANIFEST
 *plt
 *CHECKSUM
 pkg-test1b*
-/src-pkg/
+/src-pkgs/
+/built-pkgs/

collects/tests/pkg/test-pkgs/pkg-y/y/sub/s.rkt

@@ -0,0 +1,3 @@
+#lang racket/base
+(provide s)
+(define (s) 's)

collects/tests/pkg/tests-binary.rkt

@@ -21,6 +21,7 @@
 
  $ "racket -l racket/base -l x -e '(x)'" =stdout> "'x\n"
  $ "racket -l racket/base -l y -e '(y)'" =stdout> "'y\n"
+ $ "racket -l racket/base -l y/sub/s -e '(s)'" =stdout> "'s\n"
 
  $ "racket -l racket/base -e '(require (submod x test))'"
  $ "racket -l racket/base -e '(require (submod y/other doc))'"
@@ -29,6 +30,7 @@
  $ "racket -l racket/base -t test-docs.rkt -e '(test-docs-y #t)'"
 
  (make-directory* "test-pkgs/src-pkgs")
+ (make-directory* "test-pkgs/built-pkgs")
  $ "raco pkg create --from-install --source --dest test-pkgs/src-pkgs pkg-x"
  $ "raco pkg create --from-install --source --dest test-pkgs/src-pkgs pkg-y"
  $ "raco pkg create --from-install --source --dest test-pkgs/src-pkgs pkg-z"
@@ -37,6 +39,10 @@
  $ "raco pkg create --from-install --binary --dest test-pkgs pkg-y"
  $ "raco pkg create --from-install --binary --dest test-pkgs pkg-z"
 
+ $ "raco pkg create --from-install --built --dest test-pkgs/built-pkgs pkg-x"
+ $ "raco pkg create --from-install --built --dest test-pkgs/built-pkgs pkg-y"
+ $ "raco pkg create --from-install --built --dest test-pkgs/built-pkgs pkg-z"
+
  (putenv "PLT_PKG_NOSETUP" "")
  $ "raco pkg remove pkg-x pkg-y pkg-z"
  (putenv "PLT_PKG_NOSETUP" "1")
@@ -45,6 +51,9 @@
  $ "racket -l racket/base -t test-docs.rkt -e '(test-docs-y #f)'"
 
  (define tmp-dir (make-temporary-file "unpack-~a" 'directory))
+
+ (shelly-case
+  "check content of source package"
   (make-directory* tmp-dir)
   (define (unpack name)
     (define orig-d (current-directory))
@@ -74,8 +83,10 @@
   (unpack "x")
   (unpack "y")
   (unpack "z")
- (delete-directory/files tmp-dir)
+  (delete-directory/files tmp-dir))
 
+ (shelly-case
+   "check content of binary package"
    (make-directory* tmp-dir)
    (define (unpack-bin name)
      (let ([z (path->complete-path (format "test-pkgs/pkg-~a.zip" name))])
@@ -92,7 +103,57 @@
    (unpack-bin "x")
    (unpack-bin "y")
    (unpack-bin "z")
- (delete-directory/files tmp-dir)
+   (delete-directory/files tmp-dir))
+
+  (shelly-case
+  "check content of built package"
+  (make-directory* tmp-dir)
+  (define (unpack-built name)
+    (let ([src (path->complete-path (format "test-pkgs/src-pkgs/pkg-~a.zip" name))]
+          [bin (path->complete-path (format "test-pkgs/pkg-~a.zip" name))]
+          [blt (path->complete-path (format "test-pkgs/built-pkgs/pkg-~a.zip" name))])
+      (define sd (build-path tmp-dir name "src"))
+      (define bd (build-path tmp-dir name "bin"))
+      (define td (build-path tmp-dir name "blt"))
+      (make-directory* sd)
+      (make-directory* bd)
+      (make-directory* td)
+      (parameterize ([current-directory sd])
+        (unzip src))
+      (parameterize ([current-directory bd])
+        (unzip bin))
+      (parameterize ([current-directory td])
+        (unzip blt)
+        (for ([f (in-directory)])
+          (when (file-exists? f)
+            (unless (or (file-exists? (build-path sd f))
+                        (file-exists? (build-path bd f)))
+              (unless (regexp-match? #rx#"(?:[.](?:dep)|_scrbl[.]zo)$" (path->bytes f))
+                (error 'built "extra ~s" f))))
+          (when (regexp-match? #rx#"[.]zo$" (path->bytes f))
+            (unless (file-exists? (path-replace-suffix f #".dep"))
+              (error 'build "dep missing for ~s" f)))
+          (when (regexp-match? #rx#"[.](rkt|scrbl|ss)$" (path->bytes f))
+            (let-values ([(base name dir?) (split-path f)])
+              (unless (file-exists? (build-path (if (eq? base 'relative) 'same base)
+                                                "compiled" 
+                                                (path-add-suffix name #".zo")))
+                (unless (regexp-match? #rx#"^(?:info.rkt|x/keep.scrbl)$" (path->bytes f))
+                  (error 'build "compiled file missing for ~s" f)))))))
+      (parameterize ([current-directory sd])
+        (for ([f (in-directory)])
+          (when (file-exists? f)
+            (unless (file-exists? (build-path td f))
+              (error 'built "missing source ~s" f)))))
+      (parameterize ([current-directory bd])
+        (for ([f (in-directory)])
+          (when (file-exists? f)
+            (unless (file-exists? (build-path td f))
+              (error 'built "missing binary ~s" f)))))))
+  (unpack-built "x")
+  (unpack-built "y")
+  (unpack-built "z")
+  (delete-directory/files tmp-dir))
 
  (shelly-case
   "source-package dependencies like original"
@@ -116,6 +177,9 @@
  $ "raco pkg install --deps fail test-pkgs/pkg-y.zip"
  (putenv "PLT_PKG_NOSETUP" "1")
 
+ $ "racket -l racket/base -l y -e '(y)'" =stdout> "'y\n"
+ $ "racket -l racket/base -l y/sub/s -e '(s)'" =stdout> "'s\n"
+
  $ "racket -l racket/base -t test-docs.rkt -e '(test-docs-x #f)'"
  $ "racket -l racket/base -t test-docs.rkt -e '(test-docs-y #t)'"
 
@@ -128,6 +192,25 @@
   $ "raco setup -c -l y"
   $ "racket -l y")
 
+ (putenv "PLT_PKG_NOSETUP" "")
  $ "raco pkg remove pkg-y"
+ (putenv "PLT_PKG_NOSETUP" "1")
+
+ $ "raco pkg install --deps fail test-pkgs/built-pkgs/pkg-x.zip" =exit> 1
+ $ "raco pkg install --deps fail test-pkgs/built-pkgs/pkg-y.zip" =exit> 1
+ $ "raco pkg install --deps fail test-pkgs/built-pkgs/pkg-x.zip test-pkgs/built-pkgs/pkg-z.zip" =exit> 1
+
+ (putenv "PLT_PKG_NOSETUP" "")
+ $ "raco pkg install --deps fail test-pkgs/built-pkgs/pkg-x.zip test-pkgs/built-pkgs/pkg-y.zip test-pkgs/built-pkgs/pkg-z.zip"
+ =stdout> #rx"syncing: [^\n]*x\n[^\n]*syncing: [^\n]*y"
+ (putenv "PLT_PKG_NOSETUP" "1")
+
+ $ "racket -l racket/base -t test-docs.rkt -e '(test-docs-x #t)'"
+ $ "racket -l racket/base -t test-docs.rkt -e '(test-docs-y #t)'"
+ $ "racket -l racket/base -l x -e '(x)'" =stdout> "'x\n"
+ $ "racket -l racket/base -l y -e '(y)'" =stdout> "'y\n"
+ $ "racket -l racket/base -l y/sub/s -e '(s)'" =stdout> "'s\n"
+
+ $ "raco pkg remove pkg-x pkg-y pkg-z"
 
  )