proprocessor doc, merge scribblings and doc-categories fields

svn: r9414

collects/algol60/info.ss

@@ -2,5 +2,4 @@
 
 (define tools '(("tool.ss")))
 (define tool-names '("Algol 60"))
-(define scribblings '(("algol60.scrbl")))
+(define scribblings '(("algol60.scrbl" () (other))))
-(define doc-categories '(other))

collects/browser/info.ss

@@ -2,4 +2,4 @@
 
 (define tools (list (list "tool.ss")))
 (define tool-names (list "Browser"))
-(define scribblings '(("browser.scrbl")))
+(define scribblings '(("browser.scrbl" () (gui-library))))

collects/compiler/info.ss

@@ -1,7 +1,6 @@
 #lang setup/infotab
 
-(define scribblings '(("cffi.scrbl")))
+(define scribblings '(("cffi.scrbl" () (foreign -9))))
-(define doc-categories '((foreign -1)))
 
 (define mzscheme-launcher-libraries (list "main.ss"))
 (define mzscheme-launcher-names (list "mzc"))

collects/config/info.ss

@@ -1,3 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("config.scrbl")))
+(define scribblings '(("config.scrbl" () (tool-library))))

collects/dynext/info.ss

@@ -1,3 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("dynext.scrbl" (multi-page))))
+(define scribblings '(("dynext.scrbl" (multi-page) (tool-library))))

collects/embedded-gui/info.ss

@@ -1,3 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("scribblings/embedded-gui.scrbl" (multi-page))))
+(define scribblings '(("scribblings/embedded-gui.scrbl" (multi-page) (gui-library))))

collects/eopl/info.ss

@@ -7,8 +7,7 @@
 (define tool-names (list "Essentials of Programming Languages"))
 (define tool-urls (list "http://www.cs.indiana.edu/eopl/"))
 
-(define scribblings '(("eopl.scrbl")))
+(define scribblings '(("eopl.scrbl" () (language -11))))
-(define doc-categories '((language -10)))
 
 (define textbook-pls
   (list (list '("eopl-small.gif" "eopl")

collects/errortrace/scribblings/info.ss

@@ -1,3 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("errortrace.scrbl" (multi-page))))
+(define scribblings '(("errortrace.scrbl" (multi-page) (tool-library))))

collects/file/scribblings/info.ss

@@ -1,3 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("file.scrbl" (multi-page))))
+(define scribblings '(("file.scrbl" (multi-page) (parsing-library))))

collects/frtime/info.ss

@@ -2,7 +2,7 @@
 
 (define compile-omit-paths '("demos"))
 
-(define scribblings '(("frtime.scrbl" ())))
+(define scribblings '(("frtime.scrbl" () (language))))
 (define tools      '("frtime-tool.ss"))
 (define tool-icons '(("clock.png" "frtime")))
 (define tool-names '("FrTime Languages"))

collects/games/cards/info.ss

@@ -1,3 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("cards.scrbl")))
+(define scribblings '(("cards.scrbl" () (gui-library))))

collects/games/gl-board-game/info.ss

@@ -1,3 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("gl-board-game.scrbl")))
+(define scribblings '(("gl-board-game.scrbl" () (gui-library))))

collects/games/info.ss

@@ -1,6 +1,6 @@
 #lang setup/infotab
 
-(define scribblings '(("scribblings/games.scrbl" (multi-page))))
+(define scribblings '(("scribblings/games.scrbl" (multi-page) (gui-library))))
 
 (define mred-launcher-libraries (list "main.ss"))
 (define mred-launcher-names (list "PLT Games"))

collects/graphics/scribblings/info.ss

@@ -1,5 +1,5 @@
 #lang setup/infotab
 
-(define scribblings '(("graphics.scrbl" (multi-page))
+(define scribblings '(("graphics.scrbl" (multi-page) (legacy))
-                      ("turtles.scrbl" (multi-page))))
+                      ("turtles.scrbl" (multi-page) (gui-library))))
 (define doc-categories '(legacy library))

collects/html/info.ss

@@ -1,6 +1,6 @@
 #lang setup/infotab
 
-(define scribblings '(("html.scrbl")))
+(define scribblings '(("html.scrbl" () (parsing-library))))
 (define compile-omit-paths
   '("dtd.ss" "dtdr.ss" "dtds.ss" "dtd-ast.ss" "case.ss" "html-structs.ss"
     "entity-expander.ss" "generate-code.ss" "sgml.ss"))

collects/lazy/info.ss

@@ -2,7 +2,7 @@
 
 (require string-constants)
 (define name "Lazy Scheme")
-(define scribblings '(("lazy.scrbl")))
+(define scribblings '(("lazy.scrbl" () (language))))
 (define drscheme-language-modules '(("lazy.ss" "lazy")))
 (define drscheme-language-positions
   `((,(string-constant experimental-languages) "Lazy Scheme")))

collects/macro-debugger/info.ss

@@ -2,4 +2,4 @@
 
 (define tools '(["tool.ss"]))
 (define tool-names '("Macro Stepper"))
-(define scribblings '(("macro-debugger.scrbl")))
+(define scribblings '(("macro-debugger.scrbl" () (tool-library))))

collects/macro-debugger/macro-debugger.scrbl

@@ -15,7 +15,7 @@
                          macro-debugger/stepper-text))
      the-eval))
 
-@title{Macro debugger}
+@title{@bold{Macro Debugger}}
 
 The macro-debugger collection contains two tools: a stepper for macro
 expansion and a standalone syntax browser. The macro stepper shows the

collects/make/info.ss

@@ -1,3 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("make.scrbl" (multi-page))))
+(define scribblings '(("make.scrbl" (multi-page) (tool-library))))

collects/mrlib/scribblings/info.ss

@@ -1,3 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("mrlib.scrbl" (multi-page))))
+(define scribblings '(("mrlib.scrbl" (multi-page) (gui-library))))

collects/mysterx/info.ss

@@ -2,4 +2,4 @@
 
 (define post-install-collection "installer.ss")
 
-(define scribblings '(("scribblings/mysterx.scrbl" (multi-page))))
+(define scribblings '(("scribblings/mysterx.scrbl" (multi-page) (interop))))

collects/mzcom/info.ss

@@ -2,4 +2,5 @@
 
 (define post-install-collection "installer.ss")
 
-(define scribblings '(("mzcom.scrbl" ())))
+(define scribblings '(("mzcom.scrbl" () (interop))))
+

collects/mzlib/scribblings/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("mzlib.scrbl" (multi-page))))
+(define scribblings '(("mzlib.scrbl" (multi-page) (legacy))))
-(define doc-categories '(legacy))

collects/mzscheme/info.ss

@@ -2,7 +2,6 @@
 
 (define version '(400))
 
-(define scribblings '(("mzscheme.scrbl" (multi-page))))
+(define scribblings '(("mzscheme.scrbl" (multi-page) (legacy))))
-(define doc-categories '(legacy))
 
 (define compile-omit-paths '("examples"))

collects/net/scribblings/info.ss

@@ -1,3 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("net.scrbl" (multi-page))))
+(define scribblings '(("net.scrbl" (multi-page) (net-library))))

collects/openssl/info.ss

@@ -1,3 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("openssl.scrbl")))
+(define scribblings '(("openssl.scrbl" ()  (net-library))))

collects/parser-tools/info.ss

@@ -1,5 +1,5 @@
 #lang setup/infotab
 
-(define scribblings '(("parser-tools.scrbl" (multi-page))))
+(define scribblings '(("parser-tools.scrbl" (multi-page) (parsing-library))))
 
 (define compile-omit-paths '("examples"))

collects/plot/info.ss

@@ -2,6 +2,6 @@
 
 (define pre-install-collection "pre-installer.ss")
 
-(define scribblings '(("plot.scrbl" (multi-page))))
+(define scribblings '(("plot.scrbl" (multi-page) (gui-library 50))))
 
 (define compile-omit-paths '("demos"))

collects/preprocessor/info.ss

@@ -2,3 +2,5 @@
 
 (define mzscheme-launcher-names     '("mzpp"        "mztext"))
 (define mzscheme-launcher-libraries '("mzpp-run.ss" "mztext-run.ss"))
+
+(define scribblings '(("scribblings/preprocessor.scrbl" (multi-page) (legacy))))

collects/preprocessor/scribblings/mzpp.scrbl

@@ -0,0 +1,317 @@
+#lang scribble/doc
+@(require scribble/manual
+          (for-label scheme/base
+                     scheme/contract
+                     scheme/port
+                     preprocessor/mzpp))
+
+@title[#:tag "mzpp"]{@exec{mzpp}}
+
+@exec{mzpp} is a simple preprocessor that allows mixing Scheme code with text
+files in a similar way to PHP or BRL.  Processing of input files works
+by translating the input file to Scheme code that prints the contents,
+except for marked portions that contain Scheme code.  The Scheme parts
+of a file are marked with @litchar{<<} and @litchar{>>} tokens by default.  The Scheme
+code is then passed through a read-eval-print loop that is similar to a
+normal REPL with a few differences in how values are printed.
+
+@section{Invoking mzpp}
+
+Use the @Flag{-h} flag to get the available flags.  See above for an
+explanation of the @DFlag{run} flag.
+
+@section{mzpp files}
+
+Here is a sample file that @exec{mzpp} can process, using the default beginning
+and ending markers:
+
+@verbatim[#:indent 2]|{
+  << (define bar "BAR") >>
+  foo1
+  foo2 << bar newline* bar >> baz
+  foo3
+}|
+
+First, this file is converted to the following Scheme code:
+
+@verbatim[#:indent 2]|{
+  (thunk (cd "tmp/") (current-file "foo"))
+  (thunk (push-indentation ""))
+   (define bar "BAR") (thunk (pop-indentation))
+  newline*
+  "foo1"
+  newline*
+  "foo2 "
+  (thunk (push-indentation "     "))
+   bar newline* bar (thunk (pop-indentation))
+  " baz"
+  newline*
+  "foo3"
+  newline*
+  (thunk (cd "/home/eli") (current-file #f))
+}|
+
+which is then fed to the REPL, resulting in the following output:
+
+@verbatim[#:indent 2]|{
+  foo1
+  foo2 BAR
+       BAR baz
+  foo3
+}|
+
+To see the processed input that the REPL receives, use the @DFlag{debug}
+flag.  Note that the processed code contains expressions that have no
+side-effects, only values---see below for an explanation of the REPL
+printing behavior.  Some expressions produce values that change the REPL
+environment, for example, the indentation commands are used to keep
+track of the column where the Scheme marker was found, and @exec{cd} is used
+to switch to the directory where the file is (here it was in
+@filepath["/home/foo/tmp"]) so including a relative file works.  Also, note that
+the first @scheme[newline*] did not generate a newline, and that the one in the
+embedded Scheme code added the appropriate spaces for indentation.
+
+It is possible to temporarily switch from Scheme to text-mode and back
+in a way that does not respect a complete Scheme expression, but you
+should be aware that text is converted to a @italic{sequence} of side-effect
+free expressions (not to a single string, and not expression that uses
+side effects).  For example:
+
+@verbatim[#:indent 2]|{
+  << (if (zero? (random 2))
+       (list >>foo1<<)
+       (list >>foo2<<))
+  >>
+  << (if (zero? (random 2)) (list >>
+  foo1
+  <<) (list >>
+  foo2
+  <<)) >>
+}|
+
+will print two lines, each containing @litchar{foo1} or @litchar{foo} (the first
+approach plays better with the smart space handling).  The @scheme[show] function can be
+used instead of @scheme[list] with the same results, since it will print out the
+values in the same way the REPL does.  The conversion process does not
+transform every continuous piece of text into a single Scheme string
+because doing this:
+
+@itemize{
+
+  @item{the Scheme process will need to allocating big strings which makes
+  this unfeasible for big files,}
+
+  @item{it will not play well with ``interactive'' input feeding, for example,
+  piping in the output of some process will show results only on Scheme
+  marker boundaries,}
+
+  @item{special treatment for newlines in these strings will become expensive.}
+
+}
+
+(Note that this is different from the BRL approach.)
+
+@section{Raw preprocessing directives}
+
+Some preprocessing directives happen at the "raw level"---the stage
+where text is transformed into Scheme expressions.  These directives
+cannot be changed from withing transformed text because they change the
+way this transformation happens.  Some of these transformation
+
+@itemize{
+
+  @item{Skipping input:
+
+  First, the processing can be modified by specifying a @scheme[skip-to] string
+  that disables any output until a certain line is seen.  This is useful
+  for script files that use themselves for input.  For example, the
+  following script:
+
+@verbatim[#:indent 2]|{
+    #!/bin/sh
+    echo shell output
+    exec mzpp -s "---TEXT-START---" "$0"
+    exit 1
+    ---TEXT-START---
+    Some preprocessed text
+    123*456*789 = << (* 123 456 789) >>
+}|
+
+  will produce this output:
+
+@verbatim[#:indent 2]|{
+    shell output
+    Some preprocessed text
+    123*456*789 = 44253432}
+}|}
+
+  @item{Quoting the markers:
+
+  In case you need to use the actual text of the markers, you can quote
+  them.  A backslash before a beginning or an ending marker will make
+  the marker treated as text, it can also quote a sequence of
+  backslashes and a marker.  For example, using the default markers,
+  @litchar{\<<\>>} will output @litchar{<<>>}, @litchar{\\<<\\\>>} will output @litchar{\<<\\>>} and
+  @litchar{\a\b\<<} will output @litchar{\a\b<<}.}
+
+  @item{Modifying the markers:
+
+  Finally, if the markers collide with a certain file contents, it is
+  possible to change them.  This is done by a line with a special
+  structure---if the current Scheme markers are @litchar{<beg1>} and @litchar{<end1>}
+  then a line that contains exactly:
+
+@verbatim[#:indent 2]|{
+    <beg1><beg2><beg1><end1><end2><end1>
+}|
+
+  will change the markers to @litchar{<beg2>} and @litchar{<end2>}.  It is possible to
+  change the markers from the Scheme side (see below), but this will not
+  change already-transformed text, which is the reason for this special
+  format.}
+
+}
+
+@section{The mzpp read-eval-print loop}
+
+The REPL is initialized by requiring @scheme[preprocessor/mzpp], so the same module
+provides both the preprocessor functionality as well as bindings for
+embedded Scheme code in processed files.  The REPL is then fed the
+transformed Scheme code that is generated from the source text (the same
+code that @DFlag{debug} shows).  Each expression is evaluated and its result
+is printed using the @scheme[show] function (multiple values are all printed), where
+@scheme[show] works in the following way:
+
+@itemize{
+
+  @item{@|void-const| and @scheme[#f] values are ignored.}
+
+  @item{Structures of pairs are recursively scanned and their parts printed
+  (no spaces are used, so to produce Scheme code as output you must use
+  format strings---again, this is not intended for preprocessing Scheme
+  code).}
+
+  @item{Procedures are applied to zero arguments (so a procedure that doesn't
+  accept zero arguments will cause an error) and the result is sent back
+  to @scheme[show].  This is useful for using thunks to wrap side-effects as
+  values (e.g, the @scheme[thunk] wraps shown by the debug output above).}
+
+  @item{Promises are forced and the result is sent again to @scheme[show].}
+
+  @item{All other values are printed with @scheme[display].  No newlines are used
+  after printing values.}
+
+}
+
+@section[#:tag "mzpp-lib"]{Provided bindings}
+
+@defmodule[preprocessor/mzpp]
+
+First, bindings that are mainly useful for invoking the preprocessor:
+
+@defproc[(preprocess [in (or/c path-string? input-port?)] ...) void?]{
+
+  This is the main entry point to the preprocessor---invoking it on the
+  given list of files and input ports.  This is quite similar to
+  @scheme[include], but it adds some setup of the preprocessed code environment
+  (like requiring the @exec{mzpp} module).}
+
+@defparam[skip-to str string?]{
+
+  A string parameter---when the preprocessor is started, it ignores
+  everything until a line that contains exactly this string is
+  encountered.  This is primarily useful through a command-line flag for
+  scripts that extract some text from their own body.}
+
+@defboolparam[debug? on?]{
+
+  A boolean parameter.  If true, then the REPL is not invoked, instead,
+  the converted Scheme code is printed as is.}
+
+@defboolparam[no-spaces? on?]{
+
+  A boolean parameter.  If true, then the "smart" preprocessing of
+  spaces is turned off.}
+
+@deftogether[(
+@defparam[beg-mark str string?]
+@defparam[end-mark str string?]
+)]{
+
+  These two parameters are used to specify the Scheme beginning and end
+  markers.}
+
+All of the above are accessible in preprocessed texts, but the only one
+that might make any sense to use is @scheme[preprocess] and @scheme[include] is a
+better choice.  When @scheme[include] is used, it can be wrapped with parameter
+settings, which is why they are available.  Note in particular that
+these parameters change the way that the text transformation works and
+have no effect over the current preprocessed document (for example, the
+Scheme marks are used in a different thread, and @scheme[skip-to] cannot be
+re-set when processing has already began).  The only one that could be
+used is @scheme[no-spaces?] but even that makes little sense on selected parts.
+
+The following are bindings that are used in preprocessed texts:
+
+@deftogether[(
+@defproc[(push-indentation [str string?]) void?]
+@defproc[(pop-indentation) void?]
+)]{
+
+  These two calls are used to save the indentation column where the
+  Scheme beginning mark was found, and will be used by @scheme[newline*]
+  (unless smart space handling mode is disabled).}
+
+@defproc[(show [v any/c]) void?]{
+
+  The arguments are displayed as specified above.}
+
+@defproc[(newline*) void?]{
+
+  This is similar to @scheme[newline] except that it tries to handle spaces in
+  a ``smart'' way---it will print a newline and then spaces to reach the
+  left margin of the opening @litchar{<<}.  (Actually, it tries a bit more, for
+  example, it won't print the spaces if nothing is printed before
+  another newline.)  Setting @scheme[no-spaces?] to true disable this leaving
+  it equivalent to @scheme[newline].}
+
+@defproc[(include [file path-string?] ...) void?]{
+
+  This is the preferred way of including another file in the processing.
+  File names are searched relatively to the current preprocessed file,
+  and during processing the current directory is temporarily changed to
+  make this work.  In addition to file names, the arguments can be input
+  ports (the current directory is not changed in this case).  The files
+  that will be incorporated can use any current Scheme bindings etc, and
+  will use the current markers---but the included files cannot change
+  any of the parameter settings for the current processing
+  (specifically, the marks and the working directory will be restored
+  when the included files are processed).}
+
+  Note that when a sequence of files are processed (through command-line
+  arguments or through a single @scheme[include] expression), then they are all
+  taken as one textual unit---so changes to the markers, working
+  directory etc in one file can modify the way sequential files are
+  processed.  This means that including two files in a single @scheme[include]
+  expression can be different than using two expressions.
+
+@deftogether[(
+@defthing[stdin parameter?]
+@defthing[stdout  parameter?]
+@defthing[stderr  parameter?]
+@defthing[cd  parameter?]
+)]{
+
+  These are shorter names for the corresponding port parameters and
+  @scheme[current-directory].}
+
+@defparam[current-file path path-string?]{
+
+  This is a parameter that holds the name of the currently processed
+  file, or #f if none.}
+
+@defform[(thunk expr ...)]{
+
+  Expands to @scheme[(lambda () expr ...)].
+
+}

collects/preprocessor/scribblings/mztext.scrbl

@@ -0,0 +1,443 @@
+#lang scribble/doc
+@(require scribble/manual
+          (for-label scheme/base
+                     scheme/contract
+                     scheme/port
+                     preprocessor/mztext))
+
+
+@title[#:tag "mztext"]{@exec{mztext}}
+
+@exec{mztext} is another Scheme-based preprocessing language.  It can
+be used as a preprocessor in a similar way to @exec{mzpp} since it
+also uses @schememodname[preprocessor/pp-run] functionality.  However,
+@exec{mztext} uses a completely different processing principle, it is
+similar to TeX rather than the simple interleaving of text and Scheme
+code done by @exec{mzpp}.
+
+Text is being input from file(s), and by default copied to the
+standard output.  However, there are some magic sequences that trigger
+handlers that can take over this process---these handlers gain
+complete control over what is being read and what is printed, and at
+some point they hand control back to the main loop.  On a high-level
+point of view, this is similar to ``programming'' in TeX, where macros
+accept as input the current input stream.  The basic mechanism that
+makes this programming is a @deftech{composite input port} which is a
+prependable input port---so handlers are not limited to processing
+input and printing output, they can append their output back on the
+current input which will be reprocessed.
+
+The bottom line of all this is that @exec{mztext} is can perform more
+powerful preprocessing than the @exec{mzpp}, since you can define your own
+language as the file is processed.
+
+@section{Invoking mztext}
+
+Use the @Flag{h} flag to get the available flags.  SEE above for an
+explanation of the @DFlag{run} flag.
+
+@section{mztext processing: the standard command dispatcher}
+
+@exec{mztext} can use arbitrary magic sequences, but for convenience,
+there is a default built-in dispatcher that connects Scheme code with
+the preprocessed text---by default, it is triggered by @litchar["@"].
+When file processing encounters this marker, control is transfered to
+the command dispatcher.  In its turn, the command dispatcher reads a
+Scheme expression (using @scheme[read]), evaluates it, and decides
+what to do next.  In case of a simple Scheme value, it is converted to
+a string and pushed back on the preprocessed input.  For example, the
+following text:
+
+@verbatim[#:indent 2]|{
+  foo
+  @"bar"
+  @(+ 1 2)
+  @"@(* 3 4)"
+  @(/ (read) 3)12
+}|
+
+generates this output:
+
+@verbatim[#:indent 2]|{
+  foo
+  bar
+  3
+  12
+  4
+}|
+
+An explanation of a few lines:
+
+@itemize{
+
+  @item{@litchar|{@"bar"}|, @litchar|{[@(+ 1 2)}|---the Scheme objects
+  that is read is evaluated and displayed back on the input port which
+  is then printed.}
+
+  @item{@litchar|{@"@(* 3 4)"}| --- demonstrates that the results
+  are ``printed'' back on the input: the string that in this case
+  contains another use of @litchar["@"] which will then get read back
+  in, evaluated, and displayed.}
+
+  @item{@litchar|{@(/ (read) 3)12}| --- demonstrates that the Scheme
+  code can do anything with the current input.}
+
+}
+
+The complete behavior of the command dispatcher follows:
+
+@itemize{
+
+  @item{If the marker sequence is followed by itself, then it is simply
+  displayed, using the default, @litchar["@@"] outputs a @litchar["@"].}
+
+  @item{Otherwise a Scheme expression is read and evaluated, and the result is
+  processed as follows:
+
+    @itemize{
+
+      @item{If the result consists of multiple values, each one is processed,}
+
+      @item{If it is @|void-const| or @scheme[#f], nothing is done,}
+
+      @item{If it is a structure of pairs, this structure is processed
+      recursively,}
+
+      @item{If it is a promise, it is forced and its value is used instead,}
+
+      @item{Strings, bytes, and paths are pushed back on the input stream,}
+
+      @item{Symbols, numbers, and characters are converted to strings and pushed
+      back on the input,}
+
+      @item{An input port will be perpended to the input, both processed as a
+      single input,}
+
+      @item{Procedures of one or zero arity are treated in a special way---see
+      below, other procedures cause an error}
+
+      @item{All other values are ignored.}
+
+    }
+  }
+
+  @item{When this processing is done, and printable results have been re-added
+  to the input port, control is returned to the main processing loop.}
+
+}
+
+A built-in convenient behavior is that if the evaluation of the Scheme
+expression returned a @|void-const| or @scheme[#f] value (or multiple values that are
+all @|void-const| or @scheme[#f]), then the next newline is swallowed using
+@scheme[swallow-newline] (see below) if there is just white spaces before it.
+
+During evaluation, printed output is displayed as is, without
+re-processing.  It is not hard to do that, but it is a little expensive,
+so the choice is to ignore it.  (A nice thing to do is to redesign this
+so each evaluation is taken as a real filter, which is done in its own
+thread, so when a Scheme expression is about to evaluated, it is done in
+a new thread, and the current input is wired to that thread's output.
+However, this is much too heavy for a "simple" preprocesser...)
+
+So far, we get a language that is roughly the same as we get from @exec{mzpp}
+(with the added benefit of reprocessing generated text, which could be
+done in a better way using macros).  The special treatment of procedure
+values is what allows more powerful constructs.  There are handled by
+their arity (preferring a the nullary treatment over the unary one):
+
+@itemize{
+
+@item{A procedure of arity 0 is simply invoked, and its resulting value is
+  used.  The procedure can freely use the input stream to retrieve
+  arguments.  For example, here is how to define a standard C function
+  header for use in a MzScheme extension file:
+
+@verbatim[#:indent 2]|{
+    @(define (cfunc)
+       (format
+        "Scheme_Object *~a(int argc, Scheme_Object *argv[])\n"
+        (read-line)))
+    @cfunc foo
+    @cfunc bar
+
+  ==>
+
+    Scheme_Object * foo(int argc, Scheme_Object *argv[])
+    Scheme_Object * bar(int argc, Scheme_Object *argv[])
+}|
+
+  Note how @scheme[read-line] is used to retrieve an argument, and how this
+  results in an extra space in the actual argument value.  Replacing
+  this with @scheme[read] will work slightly better, except that input will
+  have to be a Scheme token (in addition, this will not consume the
+  final newline so the extra one in the format string should be
+  removed).  The @scheme[get-arg] function can be used to retrieve arguments
+  more easily---by default, it will return any text enclosed by
+  parenthesis, brackets, braces, or angle brackets (see below).  For
+  example:
+
+@verbatim[#:indent 2]|{
+    @(define (tt)
+       (format "<tt>~a</tt>" (get-arg)))
+    @(define (ref)
+       (format "<a href=~s>~a</a>" (get-arg) (get-arg)))
+    @(define (ttref)
+       (format "<a href=~s>@tt{~a}</a>" (get-arg) (get-arg)))
+    @(define (reftt)
+       (format "<a href=~s>~a</a>" (get-arg) (tt)))
+    @ttref{www.plt-scheme.edu}{PLT Scheme}
+    @reftt{www.plt-scheme.edu}{PLT Scheme}
+
+  ==>
+
+    <a href="www.plt-scheme.edu"><tt>PLT Scheme</tt></a>
+    <a href="www.plt-scheme.edu"><tt>PLT Scheme</tt></a>
+}|
+
+  Note that in @scheme[reftt] we use @scheme[tt] without arguments since it will
+  retrieve its own arguments.  This makes @scheme[ttref]'s approach more
+  natural, except that "calling" @scheme[tt] through a Scheme string doesn't
+  seem natural.  For this there is a @scheme[defcommand] command (see below)
+  that can be used to define such functions without using Scheme code:
+
+@verbatim[#:indent 2]|{
+    @defcommand{tt}{X}{<tt>X</tt>}
+    @defcommand{ref}{url text}{<a href="url">text</a>}
+    @defcommand{ttref}{url text}{<a href="url">@tt{text}</a>}
+    @ttref{www.plt-scheme.edu}{PLT Scheme}
+
+  ==>
+
+    <a href="www.plt-scheme.edu"><tt>PLT Scheme</tt></a>
+}|}
+
+  @item{A procedure of arity 1 is invoked differently---it is applied on a
+  thunk that holds the "processing continuation".  This application is
+  not expected to return, instead, the procedure can decide to hand over
+  control back to the main loop by using this thunk.  This is a powerful
+  facility that is rarely needed, similarly to the fact that @scheme[call/cc]
+  is rarely needed in Scheme.}
+
+}
+
+Remember that when procedures are used, generated output is not
+reprocessed, just like evaluating other expressions.
+
+@section[#:tag "mztext-lib"]{Provided bindings}
+
+@defmodule[preprocessor/mztext]
+
+Similarly to @exec{mzpp}, @schememodname[preprocessor/mztext] contains
+both the implementation as well as user-visible bindings.
+
+Dispatching-related bindings:
+
+@defproc*[([(command-marker) string?]
+           [(command-marker [str string?]) void?])]{
+
+
+  A string parameter-like procedure that can be used to set a
+  different command marker string.  Defaults to @litchar["@"].  It can
+  also be set to @scheme[#f] which will disable the command dispatcher
+  altogether.  Note that this is a procedure---it cannot be used with
+  @scheme[parameterize].}
+
+@defproc*[([(dispatchers) (listof list?)]
+           [(dispatchers [disps (listof list?)]) void?])]{
+
+  A parameter-like procedure (same as @scheme[command-marker]) holding a list
+  of lists---each one a dispatcher regexp and a handler function.  The
+  regexp should not have any parenthesized subgroups, use @scheme["(?:...)"] for
+  grouping.  The handler function is invoked whenever the regexp is seen
+  on the input stream: it is invoked on two arguments---the matched
+  string and a continuation thunk.  It is then responsible for the rest
+  of the processing, usually invoking the continuation thunk to resume
+  the default preprocessing.  For example:
+
+@verbatim[#:indent 2]|{
+    @(define (foo-handler str cont)
+       (add-to-input (list->string 
+                      (reverse (string->list (get-arg)))))
+       (cont))
+    @(dispatchers (cons (list "foo" foo-handler) (dispatchers)))
+    foo{>Foo<oof}
+
+  ==>
+
+    Foo
+}|
+
+  Note that the standard command dispatcher uses the same facility, and
+  it is added by default to the dispatcher list unless @scheme[command-marker]
+  is set to @scheme[#f].}
+
+
+@defproc[(make-composite-input [v any/c] ...) input-port?]{
+
+  Creates a composite input port, initialized by the given values
+  (input ports, strings, etc).  The resulting port will read data from
+  each of the values in sequence, appending them together to form a
+  single input port.  This is very similar to
+  @scheme[input-port-append], but it is extended to allow prepending
+  additional values to the beginning of the port using
+  @scheme[add-to-input].  The @exec{mztext} executable relies on this
+  functionality to be able to push text back on the input when it is
+  supposed to be reprocessed, so use only such ports for the current
+  input port.}
+
+@defproc[(add-to-input [v any/c] ...) void?]{
+
+  This should be used to ``output'' a string (or an input port) back
+  on the current composite input port.  As a special case, thunks can
+  be added to the input too---they will be executed when the ``read
+  header'' goes past them, and their output will be added back
+  instead.  This is used to plant handlers that happen when reading
+  beyond a specific point (for example, this is how the directory is
+  changed to the processed file to allow relative includes).  Other
+  simple values are converted to strings using @scheme[format], but
+  this might change.}
+
+@defparam[paren-pairs pairs (listof (list/c string? string?))]{
+
+  This is a parameter holding a list of lists, each one holding two
+  strings which are matching open/close tokens for @scheme[get-arg].}
+
+@defboolparam[get-arg-reads-word? on?]{
+
+  A parameter that holds a boolean value defaulting to @scheme[#f].  If true,
+  then @scheme[get-arg] will read a whole word (non-whitespace string delimited
+  by whitespaces) for arguments that are not parenthesized with a pair
+  in @scheme[paren-pairs].}
+
+@defproc[(get-arg) (or/c string? eof-object?)]{
+
+  This function will retrieve a text argument surrounded by a paren
+  pair specified by @scheme[paren-pairs].  First, an open-pattern is
+  searched, and then text is assembled making sure that open-close
+  patterns are respected, until a matching close-pattern is found.
+  When this scan is performed, other parens are ignored, so if the
+  input stream has @litchar|{{[(}}|, the return value will be
+  @scheme["[("].  It is possible for both tokens to be the same, which
+  will have no nesting possible.  If no open-pattern is found, the
+  first non-whitespace character is used, and if that is also not
+  found before the end of the input, an @scheme[eof] value is
+  returned.  For example (using @scheme[defcommand] which uses
+  @scheme[get-arg]):
+
+@verbatim[#:indent 2]|{
+    @(paren-pairs (cons (list "|" "|") (paren-pairs)))
+    @defcommand{verb}{X}{<tt>X</tt>}
+    @verb abc
+    @(get-arg-reads-word? #t)
+    @verb abc
+    @verb |FOO|
+    @verb
+
+  ==>
+
+    <tt>a</tt>bc
+    <tt>abc</tt>
+    <tt>FOO</tt>
+    verb: expecting an argument for `X'
+}|
+
+}
+
+@defproc[(get-arg*) (or/c string? eof-object?)]{
+
+  Similar to @scheme[get-arg], except that the resulting text is first
+  processed.  Since arguments are usually text strings,
+  ``programming'' can be considered as lazy evaluation, which
+  sometimes can be too inefficient (TeX suffers from the same
+  problem).  The @scheme[get-arg*] function can be used to reduce some
+  inputs immediately after they have been read.}
+
+@defproc[(swallow-newline) void?]{
+
+  This is a simple command that simply does this:
+
+@schemeblock[
+    (regexp-try-match #rx"^[ \t]*\r?\n" (stdin))
+]
+
+  The result is that a newline will be swallowed if there is only
+  whitespace from the current location to the end of the line.  Note
+  that as a general principle @scheme[regexp-try-match] should be
+  preferred over @scheme[regexp-match] for @exec{mztext}'s
+  preprocessing.
+
+}
+
+
+@defproc[(defcommand [name any/c][args list?][text string?]) void?]{
+
+  This is a command that can be used to define simple template
+  commands.  It should be used as a command, not from Scheme code
+  directly, and it should receive three arguments:
+
+  @itemize{
+
+    @item{The name for the new command (the contents of this argument is
+    converted to a string),}
+
+    @item{The list of arguments (the contents of this is turned to a list of
+    identifiers),}
+
+    @item{Arbitrary text, with @bold{textual} instances of the variables that
+    denote places they are used.}
+
+  }
+
+  For example, the sample code above:
+
+@verbatim[#:indent 2]|{
+    @defcommand{ttref}{url text}{<a href="url">@tt{text}</a>}
+}|
+
+  is translated to the following definition expression:
+
+@schemeblock[
+    (define (ttref)
+      (let ((url (get-arg)) (text (get-arg)))
+        (list "<a href=\"" url "\">@tt{" text "}</a>")))
+]
+
+  which is then evaluated.  Note that the arguments play a role as both
+  Scheme identifiers and textual markers.
+
+}
+
+
+@defproc[(include [file path-string?] ...) void?]{
+
+  This will add all of the given inputs to the composite port and run
+  the preprocessor loop.  In addition to the given inputs, some thunks
+  are added to the input port (see @scheme[add-to-input] above) to change
+  directory so relative includes work.
+
+  If it is called with no arguments, it will use @scheme[get-arg] to get an
+  input filename, therefore making it possible to use this as a
+  dispatcher command as well.}
+
+@defproc[(preprocess [in (or/c path-string? input-port?)]) void?]{
+
+  This is the main entry point to the preprocessor---creating a new
+  composite port, setting internal parameters, then calling @scheme[include] to
+  start the preprocessing.}
+
+
+@deftogether[(
+@defthing[stdin parameter?]
+@defthing[stdout  parameter?]
+@defthing[stderr  parameter?]
+@defthing[cd  parameter?]
+)]{
+
+  These are shorter names for the corresponding port parameters and
+  @scheme[current-directory].}
+
+@defparam[current-file path path-string?]{
+
+  This is a parameter that holds the name of the currently processed
+  file, or #f if none.}
+

collects/preprocessor/scribblings/preprocessor.scrbl

@@ -0,0 +1,72 @@
+#lang scribble/doc
+@(require scribble/manual
+          (for-label scheme/base
+                     scheme/contract
+                     scheme/port))
+
+@title{@exec{mzpp} and @exec{mztext}: Preprocessors}
+
+The @filepath{preprocessor} collection defines two Scheme-based
+preprocessors for texts that can have embedded Scheme code.  The two
+processors share a few features, like several command-line flags and
+the fact that embedded Scheme code is case-sensitive by default.
+
+Note that these processors are @bold{not} intended as preprocessors for
+Scheme code, since you have macros to do that.
+
+@table-of-contents[]
+
+@; ----------------------------------------
+
+@section[#:tag "overview"]{Overview}
+
+@defmodule[preprocessor/pp-run]
+
+The preprocessors can be invoked from Scheme programs, but the main
+usage should be through the launchers.  Both launchers use code from
+@schememodname[preprocessor/pp-run] that allows a special invocation
+mode through the @DFlag{run} flag.
+
+The @DFlag{run} is a convenient way of making the preprocessors cooperate
+with some other command, making it possible to use preprocessed text
+without an additional glue script or a makefile.  The following examples
+use @exec{mzpp}, but they work with @exec{mztext} too.  @DFlag{run} uses a single
+argument which is a string specifying a command to run:
+
+@itemize{
+
+@item{1. In its simplest form, the command string specifies some shell command
+   which will be executed with its standard input piped in from the
+   preprocessor's output.  For example, @exec{mzpp --run pr foo} is the same
+   as @exec{mzpp foo | pr}.  An error is raised if an output file is
+   specified with such an argument.}
+
+@item{2. If the command string contains a @scheme[*] and an output file is specified,
+   then the command will be executed on this output file after it is
+   generated.  For example, @exec{mzpp --run 'pr *' -o foo x y z} is the same
+   as @exec{mzpp -o foo x y z; pr foo}.}
+
+@item{3. If the command string contains a @scheme[*], and no output file is
+   specified, and there is exactly one input file, then a temporary file
+   will be used to save the original while the command is running.  For
+   example, @exec{mzpp --run 'pr *' foo} is the same as @exec{mv foo
+   foo-mzpp-temporary; mzpp -o foo foo-mzpp-temporary; pr foo; rm foo;
+   mv foo-mzpp-temporary foo}.  If there is an error while @exec{mzpp} is
+   running, the working file will be erased and the original will be
+   renamed back.}
+
+@item{4. Any other cases where the command string contains a @litchar{*} are invalid.}
+
+}
+
+If an executed command fails with a return status different than 0, the
+preprocessor execution will signal a failure by returning 1.
+
+
+@; ----------------------------------------
+@include-section["mzpp.scrbl"]
+
+@; ----------------------------------------
+@include-section["mztext.scrbl"]
+
+@; ----------------------------------------

collects/r5rs/info.ss

@@ -1,7 +1,6 @@
 #lang setup/infotab
 
-(define scribblings '(("r5rs.scrbl" (multi-page))))
+(define scribblings '(("r5rs.scrbl" (multi-page) (legacy 50))))
-(define doc-categories '((legacy 1)))
 
 (define mzscheme-launcher-names '("PLT R5RS"))
 (define mzscheme-launcher-libraries '("run.ss"))

collects/r6rs/info.ss

@@ -1,7 +1,6 @@
 #lang setup/infotab
 
-(define scribblings '(("scribblings/r6rs.scrbl" (multi-page))))
+(define scribblings '(("scribblings/r6rs.scrbl" (multi-page) (language 1))))
-(define doc-categories '((language -1)))
 
 (define mzscheme-launcher-names '("PLT R6RS"))
 (define mzscheme-launcher-libraries '("run.ss"))

collects/readline/info.ss

@@ -1,3 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("readline.scrbl")))
+(define scribblings '(("readline.scrbl" () (tool-library))))

collects/scribble/scribble.css

@@ -196,6 +196,14 @@ font-weight: bold;
   margin: 0.2em 0.2em 0.2em 0.2em;
 }
 
+.sepspace { 
+  font-size: 40%;
+}
+
+.septitle { 
+  font-size: 70%;
+}
+
 /* ---------------------------------------- */
 /* Inherited methods, left margin */
 

collects/scribblings/acks/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("acks.scrbl")))
+(define scribblings '(("acks.scrbl" () (omit))))
-(define doc-categories '(omit))

collects/scribblings/drscheme/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("drscheme.scrbl" (multi-page))))
+(define scribblings '(("drscheme.scrbl" (multi-page) (tool 100))))
-(define doc-categories '((tool 1)))

collects/scribblings/foreign/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("foreign.scrbl" (multi-page))))
+(define scribblings '(("foreign.scrbl" (multi-page) (foreign 9))))
-(define doc-categories '((foreign 1)))

collects/scribblings/framework/info.ss

@@ -1,3 +1,4 @@
 #lang setup/infotab
 
-(define scribblings '(("framework.scrbl" (#;multi-page))))
+(define scribblings '(("framework.scrbl" (#;multi-page) (gui-library 100))))
+

collects/scribblings/gui/info.ss

@@ -1,3 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("gui.scrbl" (multi-page))))
+(define scribblings '(("gui.scrbl" (multi-page) (gui-library 101))))

collects/scribblings/guide/info.ss

@@ -1,6 +1,5 @@
 #lang setup/infotab
 
-(define scribblings '(("guide.scrbl" (multi-page))))
+(define scribblings '(("guide.scrbl" (multi-page) (getting-started))))
-(define doc-categories '(getting-started))
 
 (define compile-omit-paths '("contracts-examples"))

collects/scribblings/honu/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("honu.scrbl" (multi-page))))
+(define scribblings '(("honu.scrbl" (multi-page) (other))))
-(define doc-categories '(other))

collects/scribblings/htdp-langs/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("htdp-langs.scrbl" (multi-page))))
+(define scribblings '(("htdp-langs.scrbl" (multi-page) (language -10))))
-(define doc-categories '((language -1)))

collects/scribblings/inside/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("inside.scrbl" (multi-page))))
+(define scribblings '(("inside.scrbl" (multi-page) (foreign))))
-(define doc-categories '(foreign))

collects/scribblings/license/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("license.scrbl")))
+(define scribblings '(("license.scrbl" () (omit))))
-(define doc-categories '(omit))

collects/scribblings/master-index/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("master-index.scrbl" (depends-all-main no-depend-on))))
+(define scribblings '(("master-index.scrbl" (depends-all-main no-depend-on) (omit))))
-(define doc-categories '(omit))

collects/scribblings/more/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("more.scrbl" ())))
+(define scribblings '(("more.scrbl" () (getting-started 5))))
-(define doc-categories '((getting-started 1)))

collects/scribblings/mzc/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("mzc.scrbl" (multi-page))))
+(define scribblings '(("mzc.scrbl" (multi-page) (tool 50))))
-(define doc-categories '(tool))

collects/scribblings/quick/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("quick.scrbl" ())))
+(define scribblings '(("quick.scrbl" () (getting-started 9))))
-(define doc-categories '((getting-started 2)))

collects/scribblings/reference/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("reference.scrbl" (multi-page))))
+(define scribblings '(("reference.scrbl" (multi-page) (language 100))))
-(define doc-categories '(language))

collects/scribblings/release/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("release.scrbl")))
+(define scribblings '(("release.scrbl" () (omit))))
-(define doc-categories '(omit))

collects/scribblings/scribble/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("scribble.scrbl" (multi-page))))
+(define scribblings '(("scribble.scrbl" (multi-page) (tool))))
-(define doc-categories '(tool))

collects/scribblings/setup-plt/info.ss

@@ -1,4 +1,4 @@
 #lang setup/infotab
 
-(define scribblings '(("setup-plt.scrbl" (multi-page))))
+(define scribblings '(("setup-plt.scrbl" (multi-page) (tool 50))))
-(define doc-categories '(tool))
+

collects/scribblings/setup-plt/setup-plt.scrbl

@@ -114,21 +114,38 @@ Optional @filepath{info.ss} fields trigger additional actions by
 
 @itemize{
 
- @item{@scheme[scribblings] : @scheme[(listof (cons/c string? list?))] ---
+ @item{@as-index{@schemeidfont{scribblings}} : @scheme[(listof (cons/c string? list?))] ---
    A list of documents to build. Each document in the list is itself
    represented as a list, where each document's list starts with a
    string that is a collection-relative path to the document's source
    file.
 
+   More precisely a @schemeidfont{scribblings} entry must be a value
+   that can be generated from an expression matching the following
+   @scheme[entry] grammar:
+
+   @schemegrammar*[
+     #:literals (list)
+     [entry (list doc ...)]
+     [doc (list src-string)
+          (list src-string flags)
+          (list src-string flags category)
+          (list src-string flags category name-string)]
+     [flags (list mode-symbol ...)]
+     [category (list category-symbol)
+               (list category-symbol sort-number)]
+   ]
+
    A document's list optionally continues with information on how to
    build the document. If a document's list contains a second item, it
    must be a list of mode symbols (described below). If a document's
-   list contains a third item, it is a name to use for the generated
+   list contains a third item, it must be a list that categorizes the
-   documentation, instead of defaulting to the source file's name sans
+   document (described further below). If a document's list contains a
-   extension.
+   fourth item, it is a name to use for the generated documentation,
+   instead of defaulting to the source file's name (sans extension).
 
-   Each mode symbol can be one of the following, where only
+   Each mode symbol in @scheme[_flags] can be one of the following,
-   @scheme['multi-page] is commonly used:
+   where only @scheme['multi-page] is commonly used:
 
    @itemize{
 
@@ -174,17 +191,12 @@ Optional @filepath{info.ss} fields trigger additional actions by
            that currently has this mode should be the only one with
            the mode.}
 
-    }}
+    }
 
- @item{@scheme[doc-categories] : @scheme[(listof (or/c symbol? (list/c
+    The @scheme[_category] list specifies how to show the document in
-   symbol? exact-integer?)))]  --- A list in parallel to the
+    the root table of contents. The list must start with a symbol,
-   @scheme[scribblings] list that specifies how to show the document
+    usually one of the following categories, which are ordered as
-   in the root table of contents. For each document, the symbol is one
+    below in the root documentation page:
-   of the fixed categories below, and the number determines sorting
-   within the category; documents with the same sorting number are
-   further sorted using the document title.
-
-   A category symbol is one of the following:
 
    @itemize{
 
@@ -196,8 +208,25 @@ Optional @filepath{info.ss} fields trigger additional actions by
 
      @item{@scheme['tool] : Documentation for an executable.}
 
+     @item{@scheme['gui-library] : Documentation for GUI and graphics
+           libraries.}
+
+     @item{@scheme['net-library] : Documentation for networking
+           libraries.}
+
+     @item{@scheme['parsing-library] : Documentation for parsing
+           libraries.}
+
+     @item{@scheme['tool-library] : Documentation for programming-tool
+           libraries (i.e., not important enough for the more
+           prominent @scheme['tool] category).}
+
+     @item{@scheme['interop] : Documentation for interoperability
+           tools and libraries.}
+
      @item{@scheme['library] : Documentation for libraries; this
-           category is the default.}
+           category is the default and used for unrecognized category
+           symbols.}
 
      @item{@scheme['legacy] : Documentation for deprecated libraries,
            languages, and tools.}
@@ -207,7 +236,15 @@ Optional @filepath{info.ss} fields trigger additional actions by
      @item{@scheme['omit] : Documentation that should not be listed on
            the root page.}
 
-   }}
+   }
+
+   If the category list has a second element, it must be a real number
+   that designates the manual's sorting position with the category;
+   manuals with the same sorting position are ordered
+   alphabetically. For a pair of manuals with sorting numbers
+   @scheme[_n] and @scheme[_m], the groups for the manuals are
+   separated by space or a label if @scheme[(truncate (/ _n 10))]and
+   @scheme[(truncate (/ _m 10))] are different.}
 
  @item{@scheme[mzscheme-launcher-names] : @scheme[(listof string?)]
    --- @elemtag["mzscheme-launcher-names"] A list of executable names

collects/scribblings/slideshow/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("slideshow.scrbl" (multi-page))))
+(define scribblings '(("slideshow.scrbl" (multi-page) (tool))))
-(define doc-categories '(tool))

collects/scribblings/start/info.ss

@@ -1,5 +1,4 @@
 #lang setup/infotab
 
 (define scribblings
-  '(("start.scrbl" (main-doc-root always-run depends-all-main no-depend-on))))
+  '(("start.scrbl" (main-doc-root always-run depends-all-main no-depend-on) (omit))))
-(define doc-categories '(omit))

collects/scribblings/start/manuals.ss

@@ -26,10 +26,20 @@
                   "Languages")
         (make-sec 'tool
                   "Tools")
-        (make-sec 'library
+        (make-sec 'gui-library
-                  "Libraries")
+                  "GUI and Graphics Libraries")
+        (make-sec 'net-library
+                  "Network Libraries")
+        (make-sec 'parsing-library
+                  "GUI and Graphics Libraries")
+        (make-sec 'tool-library
+                  "Tool Libraries")
         (make-sec 'foreign
                   "Low-Level APIs")
+        (make-sec 'interop
+                  "Interoperability")
+        (make-sec 'library
+                  "Miscellaneous Libraries")
         (make-sec 'legacy
                   "Legacy Languages and Libraries")
         (make-sec 'other
@@ -53,6 +63,25 @@
    null
    (list 'nbsp)))
 
+(define (add-sections cat mk-sep l)
+  (if (null? l)
+      null
+      (let loop ([l l][key (if (equal? "" (caddar l))
+                               (caar l)
+                               +inf.0)])
+        (cond
+         [(null? l) null]
+         [(equal? (caar l) key)
+          (cons (cadar l) (loop (cdr l) key))]
+         [else
+          (let ([lbl (caddar l)] ; currently always ""
+                [l (cons (cadar l) (loop (cdr l) (caar l)))]
+                [sep? (not (= (truncate (/ key 10))
+                              (truncate (/ (caar l) 10))))])
+            (if sep?
+                (cons (mk-sep lbl) l)
+                l))]))))
+
 (define (build-contents all?)
   (let* ([dirs (find-relevant-directories '(scribblings))]
          [infos (map get-info/full dirs)]
@@ -63,44 +92,42 @@
                                  (let ([s (i 'scribblings)])
                                    (apply
                                     append
-                                    (map (lambda (d cat)
+                                    (map (lambda (d)
                                            (if (and (not all?)
                                                     (pair? (cdr d))
                                                     (or (memq 'user-doc (cadr d))
                                                         (memq 'user-doc-root (cadr d))))
                                                null
-                                               (let ([new-cat (if (or (symbol? cat)
+                                               (let* ([new-cat (if ((length d) . > . 2)
-                                                                      (and (list? cat)
+                                                                   (caddr d)
-                                                                           (= 2 (length cat))
+                                                                   '(library))]
-                                                                           (symbol? (car cat))
+                                                      [sub-cat (and (list? new-cat)
-                                                                           (real? (cadr cat))))
+                                                                    ((length new-cat) . > . 1)
-                                                                  cat
+                                                                    (cadr new-cat))])
-                                                                  'unknown)])
                                                  (list
                                                   (list 
                                                    ;; Category
-                                                   (let ([the-cat (if (list? new-cat)
+                                                   (let ([the-cat (if (pair? new-cat)
                                                                       (car new-cat)
-                                                                      new-cat)])
+                                                                      'unknown)])
-                                                     (case the-cat
+                                                     (or (and (eq? the-cat 'omit) the-cat)
-                                                       [(getting-started language tool library foreign legacy other omit)
+                                                         (ormap (lambda (sec)
-                                                        the-cat]
+                                                                  (and (eq? the-cat (sec-cat sec))
-                                                       [else 
+                                                                       the-cat))
-                                                        (fprintf (current-error-port)
+                                                                sections)
-                                                                 "WARNING: bad base category: ~e from: ~e\n"
+                                                         'library))
-                                                                 cat
-                                                                 dir)]))
                                                    ;; Priority
-                                                   (if (list? new-cat)
+                                                   (if (and sub-cat
-                                                       (cadr new-cat)
+                                                            (real? sub-cat))
+                                                       sub-cat
                                                        0)
+                                                   ;; Priority label (not used):
+                                                   ""
                                                    ;; Path
                                                    (if (pair? d)
                                                        (build-path dir (car d))
                                                        (build-path dir "???")))))))
-                                         s
+                                         s)))
-                                         (i 'doc-categories (lambda ()
-                                                              (map (lambda (i) 'library) s))))))
                                  null))
                            infos
                            dirs))]
@@ -126,15 +153,21 @@
                            (list*
                             (plain-line (hspace 1))
                             (plain-line (sec-label sec))
-                            (map
+                            (add-sections
-                             cdr
+                             (sec-cat sec)
+                             (lambda (str)
+                               (plain-line (make-element (if (string=? str "")
+                                                             "sepspace" 
+                                                             "septitle")
+                                                         (list 'nbsp str))))
                              (sort
-                              (map (lambda (doc) (cons (cadr doc)
+                              (map (lambda (doc) (list (cadr doc)
-                                                       (line  (caddr doc))))
+                                                       (line (cadddr doc))
+                                                       (caddr doc)))
                                    docs)
                               (lambda (ad bd)
-                                (let ([a (cadr (paragraph-content (car (flow-paragraphs (cadr ad)))))]
+                                (let ([a (cadr (paragraph-content (car (flow-paragraphs (caadr ad)))))]
-                                      [b (cadr (paragraph-content (car (flow-paragraphs (cadr bd)))))])
+                                      [b (cadr (paragraph-content (car (flow-paragraphs (caadr bd)))))])
                                   (if (= (car ad) (car bd))
                                       (begin
                                         (string-ci<? (element->string a renderer part resolve-info)

collects/scribblings/tools/info.ss

@@ -1,4 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("tools.scrbl" (multi-page))))
+(define scribblings '(("tools.scrbl" (multi-page) (tool -100))))
-(define doc-categories '(other))

collects/scribblings/user-master-index/info.ss

@@ -1,5 +1,4 @@
 #lang setup/infotab
 
 (define scribblings
-  '(("master-index.scrbl" (user-doc depends-all no-depend-on))))
+  '(("master-index.scrbl" (user-doc depends-all no-depend-on) (omit))))
-(define doc-categories '(omit))

collects/scribblings/user-start/info.ss

@@ -1,5 +1,4 @@
 #lang setup/infotab
 
 (define scribblings
-  '(("user-start.scrbl" (user-doc-root depends-all always-run no-depend-on))))
+  '(("user-start.scrbl" (user-doc-root depends-all always-run no-depend-on) (omit))))
-(define doc-categories '(omit))

collects/setup/scribble.ss

@@ -55,23 +55,28 @@
     (memq sym '(main-doc main-doc-root user-doc-root user-doc multi-page
                 depends-all depends-all-main no-depend-on always-run)))
   (define (validate-scribblings-infos infos dir)
-    (define (validate path [flags '()] [name #f])
+    (define (validate path [flags '()] [cat '(library)] [name #f])
       (and (string? path) (relative-path? path)
            (list? flags) (andmap scribblings-flag? flags)
            (or (not name) (and (path-string? name) (relative-path? name) name))
-           (list path flags
+           (and (list? cat)
+                (<= 1 (length cat) 2)
+                (symbol? (car cat))
+                (or (null? (cdr cat))
+                    (real? (cadr cat))))
+           (list path flags cat
                  (or name (let-values ([(_1 name _2) (split-path path)])
                             (path-replace-suffix name #""))))))
     (and (list? infos)
          (let ([infos (map (lambda (i)
-                             (and (list? i) (<= 1 (length i) 3)
+                             (and (list? i) (<= 1 (length i) 4)
                                   (apply validate i)))
                            infos)])
            (and (not (memq #f infos)) infos))))
   (define (get-docs i dir)
     (let ([s (validate-scribblings-infos (i 'scribblings) dir)])
       (if s
-        (map (lambda (d cat)
+        (map (lambda (d)
                (let* ([flags (cadr d)]
                       [under-main?
                        (and (not (memq 'main-doc-root flags))
@@ -81,10 +86,9 @@
                                 (pair? (path->main-collects-relative dir))))])
                  (make-doc dir
                            (build-path dir (car d))
-                           (doc-path dir (caddr d) flags)
+                           (doc-path dir (cadddr d) flags)
-                           flags under-main? cat)))
+                           flags under-main? (caddr d))))
-             s
+             s)
-             (i 'doc-categories (lambda () (map (lambda (a) 'library) s))))
         (begin (fprintf (current-error-port)
                         " bad 'scribblings info: ~e from: ~e\n" s dir)
                null))))

collects/sgl/info.ss

@@ -6,4 +6,4 @@
                     "compiled"))
 (define compile-omit-paths '("examples"))
 
-(define scribblings '(("scribblings/sgl.scrbl" (multi-page))))
+(define scribblings '(("scribblings/sgl.scrbl" (multi-page) (gui-library 50))))

collects/slatex/info.ss

@@ -1,6 +1,6 @@
 #lang setup/infotab
 
-(define scribblings '(("slatex-wrap.scrbl")))
+(define scribblings '(("slatex-wrap.scrbl" () (tool-library))))
 
 ;; (define tools (list (list "slatex-lang.ss")))
 ;; (define tool-names (list "SLaTeX Language"))

collects/srfi/info.ss

@@ -1,3 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("srfi.scrbl" (multi-page))))
+(define scribblings '(("srfi.scrbl" (multi-page) (library 100))))

collects/string-constants/info.ss

@@ -1,3 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("string-constants.scrbl")))
+(define scribblings '(("string-constants.scrbl" () (gui-library))))

collects/syntax-color/info.ss

@@ -1,3 +1,3 @@
 #lang setup/infotab
 
-(define scribblings '(("syntax-color.scrbl")))
+(define scribblings '(("syntax-color.scrbl" () (gui-library))))

collects/test-box-recovery/info.ss

@@ -5,5 +5,4 @@
 (define tools (list '("tool.ss")))
 (define tool-names (list "Test Box Recovery"))
 
-(define scribblings '(("test-box-recovery.scrbl")))
+(define scribblings '(("test-box-recovery.scrbl" () (other))))
-(define doc-categories '(other))

collects/trace/scribblings/info.ss

@@ -1,4 +1,4 @@
 #lang setup/infotab
 
 (define name "Calltrace")
-(define scribblings '(("trace.scrbl" ())))
+(define scribblings '(("trace.scrbl" () (tool-library))))

collects/version/info.ss

@@ -2,3 +2,5 @@
 
 (define tools (list "tool.ss"))
 (define tool-names (list "Version"))
+
+(define scribblings '(("version.scrbl" () (tool-library))))

collects/version/version.scrbl

@@ -0,0 +1,89 @@
+#lang scribble/doc
+@(require scribble/manual
+	  scribble/bnf
+          (for-label version/patchlevel
+                     version/check
+                     version/tool
+                     scheme/base
+                     scheme/contract))
+
+@(define (selflink s) (link s (tt s)))
+
+@title{@bold{Version}: PLT Version Checking}
+
+The version collection contains several version-related pieces that
+are used by PLT Scheme. See also @scheme[version] from
+@schememodname[scheme/base].
+
+@; ----------------------------------------------------------------------
+
+@section{Installed Patch Level}
+
+@defmodule[version/patchlevel]
+
+@defthing[patchlevel exact-nonnegative-integer?]{
+
+Indicates the current installed patch level, which is normally zero,
+but may be updated by patches to DrScheme.}
+
+@; ----------------------------------------
+
+@section{Checking Available Versions}
+
+@defmodule[version/check]
+
+@defproc[(check-version) (or/c symbol? list?)]{
+
+Checks the currently available version on the PLT website
+(@selflink["http://download.plt-scheme.org"]) and returns a value that
+indicates the current state of the curent installation:
+
+  @itemize{
+
+    @item{@scheme[`ok] You're fine.}
+
+    @item{@scheme[`(ok-but ,_version)] You have a fine stable version,
+      but note that there is a newer alpha version available numbered
+      @scheme[_version].}
+
+    @item{@scheme[`(newer ,_version)] You have an old version. Please
+      upgrade to @scheme[_version].}
+
+    @item{@scheme[`(newer ,_version ,_alpha)] You have an
+      old-but-stable version, please upgrade to @scheme[_version]; you
+      may consider also the newer alpha version numbered
+      @scheme[_alpha].}
+
+    @item{@scheme[`(error ,_message)] An error occurred, and
+      @scheme[_message] is a string that indicates the error.}
+
+    @item{@scheme[`(error ,_message ,_additional-info)] An error
+       occurred; @scheme[_message] is a string that indicates the
+       error, and @scheme[_additional-info] is a string containing a
+       system error. The @scheme[_additional-info] content is always
+       parenthesizes, so @scheme[message] is a short error and
+       @scheme[(string-append message " " additional-info)] is a
+       verbose one.}
+
+  }
+
+}
+
+@; ----------------------------------------------------------------------
+
+@section{DrScheme Version Tool}
+
+@defmodule[version/tool]
+
+The @scheme[version/tool] library implements a DrScheme tool that
+
+@itemize{
+
+  @item{makes the patchlevel appears as a version @tt{p}@nonterm{N}
+  suffix in DrScheme, though the base verion reported by
+  @scheme[(version)] is not changed);}
+
+  @item{if enabled by the user, periodically checks whether a
+  new PLT Scheme distribution is available for download.}
+
+}

collects/web-server/info.ss

@@ -1,7 +1,6 @@
 #lang setup/infotab
 
-(define scribblings '(("scribblings/web-server.scrbl" (multi-page))))
+(define scribblings '(("scribblings/web-server.scrbl" (multi-page) (tool))))
-(define doc-categories '(tool))
 
 (define mzscheme-launcher-libraries '("main.ss"))
 (define mzscheme-launcher-names     '("PLT Web Server"))

collects/xml/info.ss

@@ -6,4 +6,4 @@
 (define tools '(("text-box-tool.ss")))
 (define tool-names '("Text Box"))
 
-(define scribblings '(("xml.scrbl" ())))
+(define scribblings '(("xml.scrbl" () (parsing-library))))