break cross-reference info for document into pieces

For example, the cross-reference information for the
Reference is now broken into about 16 pieces, so that
resolving a cross-reference into the Reference doesn't
require loading all cross-reference information for
the Reference.

Every document is split into two pieces, so that the title
of a document is roughly in its own piece. That way,
re-building the page of all installed documentation can be more
scalable (after some further changes).

original commit: a73dc502242d5a61efe526ebb19fc2588f30372a

collects/scribble/base-render.rkt

@@ -256,10 +256,49 @@
     (define/public (get-serialize-version)
       4)
 
+    (define/public (serialize-infos ri n d)
+      (if (= n 1)
+          (list (serialize-info ri))
+          (map (lambda (ht) (serialize-one-ht ri ht))
+               (partition-info (resolve-info-ci ri) n d))))
+
+    (define/private (partition-info all-ci n d)
+      ;; partition information in `all-ci' based on `d's:
+      (let ([prefix (part-tag-prefix d)]
+            [new-hts (for/list ([i (in-range n)])
+                       (make-hash))]
+            [covered (make-hash)])
+        ;; Fill in new-hts from parts:
+        (for ([sub-d (in-list (part-parts d))]
+              [i (in-naturals)])
+          (define ht (list-ref new-hts (min (add1 i) (sub1 n))))
+          (define cdi (hash-ref (collect-info-parts all-ci) sub-d #f))
+          (define sub-prefix (part-tag-prefix sub-d))
+          (when cdi
+            (for ([(k v) (in-hash (collected-info-info cdi))])
+              (when (cadr k)
+                (define sub-k (if sub-prefix
+                                  (convert-key sub-prefix k)
+                                  k))
+                (define full-k (if prefix
+                                   (convert-key prefix sub-k)
+                                   sub-k))
+                (hash-set! ht full-k v)
+                (hash-set! covered full-k #t)))))
+        ;; Anything not covered in the new-hts must go in the main hts:
+        (let ([ht0 (car new-hts)])
+          (for ([(k v) (in-hash (collect-info-ht all-ci))])
+            (unless (hash-ref covered k #f)
+              (hash-set! ht0 k v))))
+        ;; Return hts:
+        new-hts))
+
     (define/public (serialize-info ri)
+      (serialize-one-ht ri (collect-info-ht (resolve-info-ci ri))))
+
+    (define/public (serialize-one-ht ri ht)
       (parameterize ([current-serialize-resolve-info ri])
-        (serialize (cons root
+        (serialize (cons root ht))))
-                         (collect-info-ht (resolve-info-ci ri))))))
 
     (define/public (deserialize-info v ci #:root [root-path #f])
       (let ([root+ht (deserialize v)]
@@ -272,6 +311,10 @@
     (define/public (get-defined ci)
       (hash-map (collect-info-ht ci) (lambda (k v) k)))
 
+    (define/public (get-defineds ci n d)
+      (for/list ([ht (partition-info ci n d)])
+        (hash-map ht (lambda (k v) k))))
+
     (define/public (get-external ri)
       (hash-map (resolve-info-undef ri) (lambda (k v) k)))
 

collects/scribblings/scribble/renderer.scrbl

@@ -168,12 +168,25 @@ directory; normally, they should indicates a path within the
 @racket[_dest-dir] supplied on initialization of the @racket[render%]
 object.}
 
+
 @defmethod[(serialize-info [ri resolve-info?])
            any/c]{
 
 Serializes the collected info in @racket[ri].}
 
 
+@defmethod[(serialize-infos [ri resolve-info?] 
+                            [count exact-positive-integer?] 
+                            [doc part?])
+           list?]{
+
+Like @method[render<%> serialize-info], but produces @racket[count] results
+that together have the same information as produced by
+@method[render<%> serialize-info]. The structure of @racket[doc] is used to
+drive the partitioning (on the assumption that @racket[ri] is derived
+from @racket[doc]).}
+
+
 @defmethod[(deserialize-info [v any/c]
                              [ci collect-info?]
                              [#:root root-path (or/c path-string? false/c) #f])
@@ -193,6 +206,17 @@ Returns a list of tags that were defined within the documents
 represented by @racket[ci].}
 
 
+@defmethod[(get-defineds [ci collect-info?] 
+                         [count exact-positive-integer?] 
+                         [doc part?])
+           (listof (listof tag?))]{
+
+Analogous to @method[render<%> serialize-infos]: returns a list of
+tags for each of @racket[count] partitions of the result of
+@method[render<%> get-defined], using the structure of @racket[doc] to
+drive the partitioning.}
+
+
 @defmethod[(get-external [ri resolve-info?]) (listof tag?)]{
 
 Returns a list of tags that were referenced but not defined within the

collects/tests/scribble/xref.rkt

@@ -4,13 +4,15 @@
          setup/dirs
          tests/eli-tester)
 
+;; FIXME: need to look for out<i>.sxref files
+
 (provide xref-tests)
 (module+ main (xref-tests))
 (define (xref-tests)
   (define sxref (build-path (find-doc-dir) "reference" "out.sxref"))
   (when (file-exists? sxref)
     (define xref
-      (load-xref (list (λ() (cadr (call-with-input-file* sxref fasl->s-exp))))))
+      (load-xref (list (λ () (cadr (call-with-input-file* sxref fasl->s-exp))))))
     (test (xref-binding->definition-tag
            xref (list '(lib "contract.rkt" "racket") '->) #f)
           => '(form ((lib "racket/contract/base.rkt") ->)))))