Add block' to scribble/text', to explicitly ask for an indentation block.

* Lists are now either blocks or splices depending on whether they
  appear inside a block or a splice (default to block).

* Adjusted the docs and a single test where this mattered.

* Change the documentation to be "text.html" and to be titled "text
  generation".

collects/scribble/text/output.rkt

@@ -1,23 +1,32 @@
-#lang scheme/base
+#lang racket/base
 
-(require scheme/promise)
+(require racket/promise)
 
 (provide output)
 
-;; Outputs some value, for the preprocessor language.
+;; Outputs some value for the `scribble/text' language:
+;; - several atomic values are printed as in `display',
+;; - promises, thunks, and boxes are indirections for the value they contain
+;;   (useful in various cases),
+;; - some "special" values are used for controlling output (eg, flushing,
+;;   prefix changes, etc),
+;; - specifically, `block's delimit indentation levels, `splice's do not,
+;; - lists (more generally, pairs) are like either one depending on the context
+;;   (same as blocks/splices when inside a `block'/`splice'), at the toplevel
+;;   they default to blocks.
 ;;
 ;; Uses global state because `output' is wrapped around each expression in a
 ;; scribble/text file so this is much more convenient than wrapping the whole
 ;; module's body in a `list' (which will be difficult with definitions etc).
 ;; The state is a pair of prefixes -- one that is the prefix for the current
-;; value (which gets accumulated to with nested lists), and the other is the
+;; value (which gets extended with nested blocks), and the other is the prefix
-;; prefix for the current "line" (which is reset after a newline).  The
+;; for the current "line" (which is reset after a newline).  The line-prefix is
-;; line-prefix is needed because a line can hold a list, which means that the
+;; needed because a line can hold a block, which means that the line-prefix
-;; line-prefix will apply for the contents of the list including newlines in
+;; will apply for the contents of the block including newlines in it.  This
-;; it.  This state is associated to a port via a hash table.  Another state
+;; state is associated with a port via a hash table.  Another state that is
-;; that is used is the port's column position, which is maintained by the
+;; used is the port's column position, which is maintained by the system (when
-;; system (when line counts are enabled) -- this is used to tell what part of a
+;; line counts are enabled) -- this is used to tell what part of a prefix is
-;; prefix is already displayed.
+;; already displayed.
 ;;
 ;; Each prefix is either an integer (for a number of spaces) or a string.  The
 ;; prefix mechanism can be disabled by using #f for the global prefix, and in
@@ -28,6 +37,8 @@
 (define (output x [p (current-output-port)])
   ;; these are the global prefix and the one that is local to the current line
   (define pfxs (port->state p))
+  ;; the current mode for lists
+  (define list=block? #t)
   ;; the low-level string output function (can change with `with-writer')
   (define write write-string)
   ;; to get the output column
@@ -98,6 +109,17 @@
                        (output-pfx col pfx lpfx)
                        ;; the spaces were already added to lpfx
                        (write x p (if m (cdar m) start)))))])))))
+  ;; blocks and splices
+  (define (output-block c)
+    (let* ([pfx (mcar pfxs)] [lpfx (mcdr pfxs)]
+           [npfx (pfx+col (pfx+ pfx lpfx))])
+      (set-mcar! pfxs npfx) (set-mcdr! pfxs 0)
+      (if (list? c)
+        (for ([c (in-list c)]) (loop c))
+        (begin (loop (car c)) (loop (cdr c))))
+      (set-mcar! pfxs pfx) (set-mcdr! pfxs lpfx)))
+  (define (output-splice c)
+    (for-each loop c))
   ;; main loop
   (define (loop x)
     (cond
@@ -107,13 +129,7 @@
       ;; one, then output the contents recursively (no need to change the
       ;; state, since we pass the values in the loop, and we'd need to restore
       ;; it afterwards anyway)
-      [(pair? x) (if (list? x)
+      [(pair? x) (if list=block? (output-block x) (output-splice x))]
-                   (let* ([pfx (mcar pfxs)] [lpfx (mcdr pfxs)]
-                          [npfx (pfx+col (pfx+ pfx lpfx))])
-                     (set-mcar! pfxs npfx) (set-mcdr! pfxs 0)
-                     (for ([x (in-list x)]) (loop x))
-                     (set-mcar! pfxs pfx) (set-mcdr! pfxs lpfx))
-                   (begin (loop (car x)) (loop (cdr x))))]
       ;; delayed values
       [(and (procedure? x) (procedure-arity-includes? x 0)) (loop (x))]
       [(promise? x) (loop (force x))]
@@ -122,7 +138,17 @@
       [(special? x)
        (let ([c (special-contents x)])
          (case (special-flag x)
-           [(splice) (for-each loop c)]
+           ;; preserve tailness & avoid `set!' for blocks/splices if possible
+           [(block) (if list=block?
+                      (output-block c)
+                      (begin (set! list=block? #t)
+                             (output-block c)
+                             (set! list=block? #f)))]
+           [(splice) (if list=block?
+                      (begin (set! list=block? #f)
+                             (output-splice c)
+                             (set! list=block? #t))
+                      (output-splice c))]
            [(flush) ; useful before `disable-prefix'
             (output-pfx (getcol) (mcar pfxs) (mcdr pfxs))]
            [(disable-prefix) ; save the previous pfxs
@@ -196,6 +222,10 @@
 
 (define-syntax define/provide-special
   (syntax-rules ()
+    [(_ (name))
+     (begin (provide name)
+            (define (name . contents)
+              (make-special 'name contents)))]
     [(_ (name x ...))
      (begin (provide name)
             (define (name x ... . contents)
@@ -204,6 +234,7 @@
      (begin (provide name)
             (define name (make-special 'name #f)))]))
 
+(define/provide-special (block))
 (define/provide-special (splice))
 (define/provide-special flush)
 (define/provide-special (disable-prefix))
@@ -215,11 +246,11 @@
 (define/provide-special (with-writer-change writer))
 
 (define make-spaces ; (efficiently)
-  (let ([t (make-hasheq)] [v (make-vector 80 #f)])
+  (let ([t (make-hasheq)] [v (make-vector 200 #f)])
     (lambda (n)
-      (or (if (< n 80) (vector-ref v n) (hash-ref t n #f))
+      (or (if (< n 200) (vector-ref v n) (hash-ref t n #f))
           (let ([spaces (make-string n #\space)])
-            (if (< n 80) (vector-set! v n spaces) (hash-set! t n spaces))
+            (if (< n 200) (vector-set! v n spaces) (hash-set! t n spaces))
             spaces)))))
 
 ;; Convenient utilities

collects/scribblings/scribble/how-to-paper.scrbl

@@ -526,13 +526,13 @@ converting @litchar{---} to an em dash or for converting @litchar{"}
 and @litchar{'} to suitable curly quotes.
 
 The decoding process for document's stream is ultimately determined by
-the @hash-lang[] line that starts the document. The @racketmodname[scribble/base],
+the @hash-lang[] line that starts the document.  The
-@racketmodname[scribble/manual], and @racketmodname[scribble/sigplan]
+@racketmodname[scribble/base], @racketmodname[scribble/manual], and
-languages all use the same @racket[decode] operation.  The
+@racketmodname[scribble/sigplan] languages all use the same
-@racketmodname[scribble/text] language, however, acts more like a
+@racket[decode] operation.  The @racketmodname[scribble/text] language,
-plain-text preprocessor and it does not perform any such decoding
+however, acts more like a plain-text genrator and preprocessor, and it
-rules. (For more on @racketmodname[scribble/text], see
+does not perform any such decoding rules.  (For more on
-@secref["preprocessor"].)
+@racketmodname[scribble/text], see @secref["text"].)
 
 @margin-note{More precisely, languages like
              @racketmodname[scribble/base] apply @racket[decode] only after
@@ -607,5 +607,5 @@ Racket, continue with @secref["reader"] and then
 @secref["generic-prose"].  Move on to @secref["internals"] when you
 need more power.
 
-If you are interested in text preprocessing, continue with
+If you are interested in text generation and preprocessing, continue
-@secref["reader"], but then skip to @secref["preprocessor"].
+with @secref["reader"], but then skip to @secref["text"].

collects/scribblings/scribble/scribble.scrbl

@@ -26,7 +26,7 @@ starting with the @filepath{scribble.scrbl} file.
 @include-section["generic.scrbl"]
 @include-section["plt.scrbl"]
 @include-section["lp.scrbl"]
-@include-section["preprocessor.scrbl"]
+@include-section["text.scrbl"]
 @include-section["internals.scrbl"]
 @include-section["running.scrbl"]
 

collects/scribblings/scribble/text.scrbl

@@ -8,27 +8,28 @@
                      ))
 @initialize-tests
 
-@title[#:tag "preprocessor"
+@title[#:tag "text"
        #:style (make-style #f (list (make-tex-addition "shaded.tex")
                                     (make-css-addition "shaded.css")))
-      ]{Text Preprocessing}
+      ]{Text Generation}
+@section-index["Preprocessor"]
 
-@defmodulelang[scribble/text]{The @racketmodname[scribble/text]
+@defmodulelang[scribble/text]{The @racketmodname[scribble/text] language
-language provides everything from @racket[racket/base] with a few
+provides everything from @racket[racket/base] with a few changes that
-changes that make it suitable as a preprocessor language:
+make it suitable as a text generation or a preprocessor language:
 
 @itemize[
 
-  @item{It uses @racket[read-syntax-inside] to read the body of the
+  @item{The language uses @racket[read-syntax-inside] to read the body
-        module, similar to @secref["docreader"].  This means that by
+        of the module, similar to @secref["docreader"].  This means that
-        default, all text is read in as Racket strings; and
+        by default, all text is read in as Racket strings; and
         @seclink["reader"]|{@-forms}| can be used to use Racket
         functions and expression escapes.}
 
-  @item{Values of expressions are printed with a custom
+  @item{Values of expressions are printed with a custom @racket[output]
-        @racket[output] function.  This function displays most values
+        function.  This function displays most values in a similar way
-        in a similar way to @racket[display], except that it is more
+        to @racket[display], except that it is more convenient for a
-        convenient for a preprocessor output.}]
+        textual output.}]
 
 }
 
@@ -39,7 +40,7 @@ changes that make it suitable as a preprocessor language:
 @; * maybe a section on additional utilities: begin/text
 
 @;--------------------------------------------------------------------
-@section{Writing Preprocessor Files}
+@section{Writing Text Files}
 
 The combination of the two features makes text in files in the
 @racket[scribble/text] language be read as strings, which get printed
@@ -380,9 +381,9 @@ number of body expressions must be fixed.
 @;--------------------------------------------------------------------
 @section{Using Printouts}
 
-Because the preprocessor language simply displays each toplevel value
+Because the text language simply displays each toplevel value as the
-as the file is run, it is possible to print text directly as part of
+file is run, it is possible to print text directly as part of the
-the output.
+output.
 
 @example|-{#lang scribble/text
            First
@@ -463,13 +464,13 @@ promises, so you can create a loop that is delayed in either form.
 @;--------------------------------------------------------------------
 @section{Indentation in Preprocessed output}
 
-An issue that can be very important in many preprocessor applications
+An issue that can be very important in many text generation applications
-is the indentation of the output.  This can be crucial in some cases,
+is the indentation of the output.  This can be crucial in some cases, if
-if you're generating code for an indentation-sensitive language (e.g.,
+you're generating code for an indentation-sensitive language (e.g.,
 Haskell, Python, or C preprocessor directives).  To get a better
-understanding of how the pieces interact, you may want to review how
+understanding of how the pieces interact, you may want to review how the
-the @seclink["reader"]|{Scribble reader}| section, but also remember
+@seclink["reader"]|{Scribble reader}| section, but also remember that
-that you can use quoted forms to see how some form is read.
+you can use quoted forms to see how some form is read.
 
 @example|-{#lang scribble/text
            @(format "~s" '@list{
@@ -479,40 +480,51 @@ that you can use quoted forms to see how some form is read.
            ---***---
            (list "a" "\n" "  " "b" "\n" "c")}-|
 
-The Scribble reader ignores indentation spaces in its body.  This is
+The Scribble reader ignores indentation spaces in its body.  This is an
-an intentional feature, since you usually do not want an expression to
+intentional feature, since you usually do not want an expression to
-depend on its position in the source.  But the question is how
+depend on its position in the source.  But the question is whether we
-@emph{can} we render some output text with proper indentation.  The
+@emph{can} render some output text with proper indentation.  The
-@racket[output] function achieves that by assigning a special meaning
+@racket[output] function achieves that by introducing @racket[block]s.
-to lists: when a newline is part of a list's contents, it causes the
+Just like a list, a @racket[block] contains a list of elements, and when
-following text to appear with indentation that corresponds to the
+one is rendered, it is done in its own indentation level.  When a
-column position at the beginning of the list.  In most cases, this
+newline is part of a @racket[block]'s contents, it causes the following
-makes the output appear ``as intended'' when lists are used for nested
+text to appear with indentation that corresponds to the column position
-pieces of text --- either from a literal @racket[list] expression, or
+at the beginning of the block.
-an expression that evaluates to a list, or when a list is passed on as
+
-a value; either as a toplevel expression, or as a nested value; either
+In addition, lists are also rendered as blocks by default, so they can
-appearing after spaces, or after other output.
+be used for the same purpose.  In most cases, this makes the output
+appear ``as intended'' where lists are used for nested pieces of text
+--- either from a literal @racket[list] expression, or an expression
+that evaluates to a list, or when a list is passed on as a value; either
+as a toplevel expression, or as a nested value; either appearing after
+spaces, or after other output.
 
 @example|-{#lang scribble/text
-           foo @list{1
+           foo @block{1
-                     2
+                      2
-                     3}
+                      3}
+           foo @list{4
+                     5
+                     6}
            ---***---
            foo 1
                2
-               3}-|
+               3
+           foo 4
+               5
+               6}-|
 
 @example|-{#lang scribble/text
-           @(define (block . text)
+           @(define (code . text)
               @list{begin
                       @text
                     end})
-           @block{first
+           @code{first
-                  second
+                 second
-                  @block{
+                 @code{
-                    third
+                   third
-                    fourth}
+                   fourth}
-                  last}
+                 last}
            ---***---
            begin
              first
@@ -697,20 +709,25 @@ appearing after spaces, or after other output.
   }-|
 
 There are, however, cases when you need more refined control over the
-output.  The @racket[scribble/text] provides a few functions for such
+output.  The @racket[scribble/text] language provides a few functions
-cases.  The @racket[splice] function is used to group together a
+for such cases in addition to @racket[block].  The @racket[splice]
-number of values but avoid introducing a new indentation context.
+function groups together a number of values but avoids introducing a new
+indentation context.  Furthermore, lists are not always rendered as
+@racket[block]s --- instead, they are rendered as @racket[splice]s when
+they are used inside one, so you essentially use @racket[splice] to
+avoid the ``indentation group'' behavior, and @racket[block] to restore
+it.
 
 @example|-{#lang scribble/text
-           @(define (block . text)
+           @(define (blah . text)
               @splice{{
-                blah(@text);
+                blah(@block{@text});
               }})
            start
              @splice{foo();
                      loop:}
-             @list{if (something) @block{one,
+             @list{if (something) @blah{one,
-                                         two}}
+                                        two}}
            end
            ---***---
            start
@@ -759,8 +776,8 @@ example, to print out CPP directives.
            }-|
 
 If there are values after a @racket[disable-prefix] value on the same
-line, they will get indented to the goal column (unless the output is
+line, they @emph{will} get indented to the goal column (unless the
-already beyond it).
+output is already beyond it).
 
 @example|-{#lang scribble/text
            @(define (thunk name . body)
@@ -922,8 +939,8 @@ property of @racket[disable-prefix] but only for a nested prefix.
 @section{Using External Files}
 
 Using additional files that contain code for your preprocessing is
-trivial: the preprocessor source is still source code in a module, so
+trivial: the source text is still source code in a module, so you can
-you can @racket[require] additional files with utility functions.
+@racket[require] additional files with utility functions.
 
 @example|-{#lang scribble/text
            @(require "itemize.rkt")
@@ -984,13 +1001,13 @@ it, it is easy to include a lot of textual content.
 
 Of course, the extreme side of this will be to put all of your content
 in a plain Racket module, using @"@"-forms for convenience.  However,
-there is no need to use the preprocessor language in this case;
+there is no need to use the text language in this case; instead, you can
-instead, you can @racket[(require scribble/text)], which will get all
+@racket[(require scribble/text)], which will get all of the bindings
-of the bindings that are available in the @racket[scribble/text]
+that are available in the @racket[scribble/text] language.  Using
-language.  Using @racket[output], switching from a preprocessed files
+@racket[output], switching from a preprocessed files to a Racket file is
-to a Racket file is very easy ---- choosing one or the other depends
+very easy ---- choosing one or the other depends on whether it is more
-on whether it is more convenient to write a text file with occasional
+convenient to write a text file with occasional Racket expressions or
-Racket expressions or the other way.
+the other way.
 
 @example|-{#lang at-exp racket/base
            (require scribble/text racket/list)
@@ -1022,11 +1039,11 @@ Racket expressions or the other way.
            }-|
 
 However, you might run into a case where it is desirable to include a
-mostly-text file from a preprocessor file.  It might be because you
+mostly-text file from a @racket[scribble/text] source file.  It might be
-prefer to split the source text to several files, or because you need
+because you prefer to split the source text to several files, or because
-to preprocess a file without even a @litchar{#lang} header (for
+you need to use a template file that cannot have a @litchar{#lang}
-example, an HTML template file that is the result of an external
+header (for example, an HTML template file that is the result of an
-editor).  For these cases, the @racket[scribble/text] language
+external editor).  In these cases, the @racket[scribble/text] language
 provides an @racket[include] form that includes a file in the
 preprocessor syntax (where the default parsing mode is text).
 
@@ -1075,12 +1092,11 @@ preprocessor syntax (where the default parsing mode is text).
            }-|
 
 (Using @racket[require] with a text file in the @racket[scribble/text]
-language will not work as intended: using the preprocessor language
+language will not work as intended: the language will display the text
-means that the text is displayed when the module is invoked, so the
+is when the module is invoked, so the required file's contents will be
-required file's contents will be printed before any of the requiring
+printed before any of the requiring module's text does.  If you find
-module's text does.  If you find yourself in such a situation, it is
+yourself in such a situation, it is better to switch to a
-better to switch to a Racket-with-@"@"-expressions file as shown
+Racket-with-@"@"-expressions file as shown above.)
-above.)
 
 @;FIXME: add more text on `restore-prefix', `set-prefix', `with-writer'
 

collects/scribblings/scribble/utils.rkt

@@ -101,7 +101,7 @@
                       (map as-flow (list spacer @expr reads-as sexpr))))
                   r))))))))
 
-;; stuff for the preprocessor examples
+;; stuff for the scribble/text examples
 
 (require racket/list (for-syntax racket/base racket/list))
 

collects/tests/scribble/main.rkt

@@ -1,9 +1,9 @@
 #lang racket/base
 
 (require tests/eli-tester
-         "reader.rkt" "preprocessor.rkt" "collect.rkt" "docs.rkt")
+         "reader.rkt" "text-lang.rkt" "collect.rkt" "docs.rkt")
 
 (test do (reader-tests)
       do (begin/collect-tests)
-      do (preprocessor-tests)
+      do (text-lang-tests)
       do (docs-tests))

collects/tests/scribble/text-lang.rkt

@@ -1,11 +1,11 @@
 #lang racket/base
 
 (require tests/eli-tester racket/runtime-path racket/port racket/sandbox
-         (prefix-in doc: (lib "scribblings/scribble/preprocessor.scrbl")))
+         (prefix-in doc: (lib "scribblings/scribble/text.scrbl")))
 
-(provide preprocessor-tests)
+(provide text-lang-tests)
 
-(define (preprocessor-tests)
+(define (text-lang-tests)
   ;; (sample-file-tests)
   (in-documentation-tests))
 
@@ -36,7 +36,7 @@
     ;; test with name indicating the source
     (define-syntax-rule (t . stuff)
       (test ;; #:failure-message
-            ;; (format "preprocessor test failure at line ~s" line)
+            ;; (format "text-lang test failure at line ~s" line)
             . stuff))
     (parameterize ([current-directory this-dir]
                    [sandbox-output o]

collects/web-server/scribblings/templates.scrbl

@@ -8,17 +8,20 @@
 
 @(define xexpr @tech[#:doc '(lib "xml/xml.scrbl")]{X-expression})
 @(define at-reader-ref @secref[#:doc '(lib "scribblings/scribble/scribble.scrbl")]{reader})
-@(define text-ref @secref[#:doc '(lib "scribblings/scribble/scribble.scrbl")]{preprocessor})
+@(define text-ref
+   @secref[#:doc '(lib "scribblings/scribble/scribble.scrbl") "text"])
 
 @title[#:tag "templates"]{Templates: Separation of View}
 
 @defmodule[web-server/templates]
 
-The @web-server provides a powerful Web template system for separating the presentation logic of a Web application
+The @web-server provides a powerful Web template system for separating
-and enabling non-programmers to contribute to Racket-based Web applications.
+the presentation logic of a Web application and enabling non-programmers
+to contribute to Racket-based Web applications.
 
-@margin-note{Although all the examples here generate HTML, the template language and the @text-ref it is based on can
+@margin-note{Although all the examples here generate HTML, the template
-             be used to generate any text-based format: C, SQL, form emails, reports, etc.} 
+  language and the @text-ref it is based on can be used to generate any
+  text-based format: C, SQL, form emails, reports, etc.}
 
 @section{Static}