finish Guide chaper on defining languages

collects/compiler/embed-unit.rkt

@@ -784,7 +784,7 @@
              [config-infos (if config?
                                (let ([a (assoc (car files) (unbox codes))])
                                  (let ([info (module-compiled-language-info (mod-code a))])
-                                   (when info
+                                   (and info
                                         (let ([get-info ((dynamic-require (vector-ref info 0) (vector-ref info 1))
                                                          (vector-ref info 2))])
                                           (get-info 'configure-runtime null)))))

collects/scribble/scheme.css

@@ -54,7 +54,7 @@
 }
 
 .ScmMeta {
-  color: #262680;
+  color: black;
 }
 
 .ScmMod {

collects/scribblings/guide/arith.rkt

@@ -17,6 +17,9 @@
   (define-values (line col pos) (port-next-location in))
   (define expr-match 
     (regexp-match 
+     ;; Match an operand followed by any number of 
+     ;; operator--operand sequences, and prohibit an
+     ;; additional operator from following immediately:
      #px"^([a-z]|[0-9]+)(?:[-+*/]([a-z]|[0-9]+))*(?![-+*/])"
      in))
 

collects/scribblings/guide/death-list-5.rkt

@@ -0,0 +1,6 @@
+#lang racket
+(list "O-Ren Ishii"
+      "Vernita Green"
+      "Elle Driver"
+      "Budd"
+      "Bill")

collects/scribblings/guide/dollar-racket.rkt

@@ -0,0 +1,7 @@
+#lang s-exp syntax/module-reader
+racket
+#:read $-read
+#:read-syntax $-read-syntax
+
+(require (prefix-in $- "dollar.rkt"))
+

collects/scribblings/guide/dollar.rkt

@@ -25,7 +25,7 @@
     (check-$-after (arith:read-syntax src in) in src)]))
 
 (define (check-$-after val in src)
-  (regexp-match #px"^\\s*" in)
+  (regexp-match #px"^\\s*" in) ; skip whitespace
   (let ([ch (peek-char in)])
     (unless (equal? ch #\$) (bad-ending ch src in))
     (read-char in))

collects/scribblings/guide/hash-languages.scrbl

@@ -0,0 +1,552 @@
+#lang scribble/doc
+@(require scribble/manual
+          scribble/eval
+          "guide-utils.ss"
+          "modfile.rkt"
+          (for-label setup/dirs
+                     syntax/strip-context
+                     syntax-color/default-lexer))
+
+@title[#:tag "hash-languages"]{Defining new @hash-lang[] Languages}
+
+When loading a module as a source program that starts
+
+@racketmod[
+@#,racket[_language]
+]
+
+the @racket[_language] determines the way that the rest of the module
+is parsed at the @tech{reader} level. The @tech{reader}-level parse
+must produce a @racket[module] form as a @tech{syntax object}. As
+always, the second sub-form after @racket[module] specifies the
+@tech{module language} that controls the meaning of the module's body
+forms. Thus, a @racket[_language] specified after @hash-lang[]
+controls both the @tech{reader}-level and @tech{expander}-level
+parsing of a module.
+
+@; ----------------------------------------
+@section{Designating a @hash-lang[] Language}
+
+The syntax of a @racket[_language] intentionally overlaps with the
+syntax of a module path as used in @racket[require] or as a
+@tech{module language}, so that names like @racketmodname[racket],
+@racketmodname[racket/base], @racketmodname[slideshow], or
+@racketmodname[scribble/manual] can be used both as @hash-lang[]
+languages and as module paths.
+
+At the same time, the syntax of @racket[_language] is far more
+restricted than a module path, because only @litchar{a}-@litchar{z},
+@litchar{A}-@litchar{Z}, @litchar{/} (not at the start or end),
+@litchar{_}, @litchar{-}, and @litchar{+} are allowed in a
+@racket[_language] name. These restrictions keep the syntax of
+@hash-lang[] as simple as possible. Keeping the syntax of @hash-lang[]
+simple, in turn, is important because the syntax is inherently
+inflexible and non-extensible; the @hash-lang[] protocol allows a
+@racket[_language] to refine and define syntax in a practically
+unconstrained way, but the @hash-lang[] protocol itself must remain
+fixed so that various different tools can ``boot'' into the extended
+world.
+
+Fortunately, the @hash-lang[] protocol provides a natural way to refer
+to languages in ways other than the rigid
+@racket[_language] syntax: by defining a @racket[_language]
+that implements its own nested protocol. We have already seen one example in
+@secref["s-exp"]: the @racketmodname[s-exp] @racket[_language] allows
+a programmer to specify a @tech{module language} using the general
+@tech{module path} syntax. Meanwhile, @racketmodname[s-exp] takes care
+of the @tech{reader}-level responsibilities of a @hash-lang[] language.
+
+Unlike @racketmodname[racket], @racketmodname[s-exp] cannot be used as a
+module path with @scheme[require]. Although the syntax of
+@racket[_language] for @hash-lang[] overlaps with the syntax of module
+paths, a @racket[_language] is not used directly as a module
+path. Instead, a @racket[_language] is suffixed with
+@racketidfont{/lang/reader} to obtain a module path, and the resulting
+module supplies @racketidfont{read} and @racketidfont{read-syntax}
+functions using a protocol that is similar to the one for
+@racketmetafont{#reader}.
+
+@guideother{@secref["hash-reader"] introduces @racketmetafont{#reader}.}
+
+A consequence of the way that a @hash-lang[] @racket[_language] is
+turned into a module path is that the language must be installed in a
+@tech{collection}, similar to the way that @filepath{racket} or
+@filepath{slideshow} are collections that are distributed with Racket.
+Again, however, there's an escape from this restriction: the
+@racketmodname[reader] language lets you specify a @tech{reader}-level
+implementation of a language using a general @tech{module path}.
+
+@; ----------------------------------------
+@section[#:tag "hash-lang reader"]{Using @racket[@#,hash-lang[] @#,racketmodname[reader]]}
+
+The @racketmodname[reader] language for @hash-lang[] is similar to
+@racketmodname[s-exp], in that it acts as a kind of meta-language.
+Whereas @racketmodname[s-exp] lets a programmer specify a @tech{module
+language} at the @tech{expander} layer of parsing,
+@racketmodname[reader] lets a programmer specify a language at the
+@tech{reader} level.
+
+The module specified after @racket[@#,hash-lang[]
+@#,racketmodname[reader]] must provide two functions:
+@racketidfont{read} and @racketidfont{read-syntax}. The protocol is
+the same as for a @racketmetafont{#reader} implementation, but for
+@hash-lang[], the @racketidfont{read} and @racketidfont{read-syntax}
+functions must produce a @scheme[module] form that is based on the
+rest of the input file for the module.
+
+The following @filepath{literal.rkt} module implements a language that
+treats its entire body as literal text and exports the text as a
+@racketidfont{data} string:
+
+@racketmodfile["literal.rkt"]
+
+The @filepath{literal.rkt} language uses @racket[strip-context] on the
+generated @racket[module] expression, because a
+@racketidfont{read-syntax} function should return a syntax obejct with
+no lexical context. Also, the @filepath{literal.rkt} language creates
+a module named @racketidfont{anything}, which is an arbitrary choice;
+the language is intended to be used in a file, and the longhand module
+name is ignored when it appears in a @racket[require]d file.
+
+The @filepath{literal.rkt} language can be used in a module
+@filepath{tuvalu.rkt}:
+
+@racketmodfile["tuvalu.rkt"]
+
+Importing @filepath{tuvalu.rkt} binds @racketidfont{data} to a
+string version of the module content:
+
+@interaction[
+(require "tuvalu.rkt")
+data
+]
+
+@; ----------------------------------------
+@section[#:tag "syntax/module-reader"]{Using @racket[@#,hash-lang[] @#,racketmodname[s-exp] @#,racketmodname[syntax/module-reader]]}
+
+Parsing a module body is usually not as trivial as in
+@filepath{literal.rkt}. A more typical module parser must iterate to
+parse multiple forms for a module body. A language is also more likely
+to extend Racket syntax---perhaps through a @tech{readtable}---instead
+of replacing Racket syntax completely.
+
+The @racketmodname[syntax/module-reader] @tech{module language}
+abstracts over common parts of a language implementation to simplify
+the creation of new languages. In its most basic form, a language
+implemented with @racketmodname[syntax/module-reader] simply specifies
+the @tech{module language} to be used for the language, in which case
+the @tech{reader} layer of the language is the same as Racket. For
+example, if @filepath{raquet-mlang.rkt} contains
+
+@racketmod[
+racket
+(provide (except-out (all-from-out racket) lambda)
+         (rename-out [lambda function]))
+]
+
+and @filepath{raquet.rkt} contains
+
+@racketmod[
+s-exp syntax/module-reader
+"raquet-mlang.rkt"
+]
+
+then
+
+@racketmod[
+reader "raquet.rkt"
+(define identity (function (x) x))
+(provide identity)
+]
+
+implements and exports the @racket[identity] function, since
+@filepath{raquet-mlang.rkt} exports @racket[lambda] as
+@racket[function].
+
+The @racketmodname[syntax/module-reader] language accepts many optional
+specifications to adjust other features of the language. For example,
+an alternate @racketidfont{read} and @racketidfont{read-syntax} for
+parsing the language can be spcified with @racket[#:read] and
+@racket[#:read-syntax], respectively. The following
+@filepath{dollar-racket.rkt} language uses @filepath{dollar.rkt} (see
+@secref["readtable"]) to build a language that is like
+@racketmodname[racket] but with a @litchar{$} escape to simple infix
+arithmetic:
+
+@racketmodfile["dollar-racket.rkt"]
+
+The @racket[require] form appears at the end of the module,
+because all of the keyword-tagged optional specifications for
+@racketmodname[syntax/module-reader] must appear before any helper
+imports or definitions.
+
+The following module uses @filepath{dollar-racket.rkt} to implement a
+@racket[cost] function using a @litchar{$} escape:
+
+@racketmodfile["store.rkt"]
+
+@; ----------------------------------------
+@section[#:tag "language-collection"]{Installing a Language}
+
+So far, we have used the @racketmodname[reader] meta-language to
+access languages like @filepath{literal.rkt} and
+@filepath{dollar-racket.rkt}. If you want to use something like
+@racket[@#,hash-lang[] literal] directly, then you must move
+@filepath{literal.rkt} into a Racket @tech{collection} named
+@filepath{literal}, and the file @filepath{literal.rkt} must be
+renamed to @filepath{reader.rkt} and placed in a @filepath{lang}
+sub-directory of the @filepath{literal} collection.
+
+To install a collection, you can create a directory either in the main
+Racket installation or in a user-specific directory. Use
+@racket[find-collects-dir] or @racket[find-user-collects-dir] from
+@racketmodname[setup/dirs] to find the directory:
+
+@interaction[
+(require setup/dirs)
+(eval:alts (find-user-collects-dir)
+           (build-path "/home/racketeer/.racket/"
+                       (version)
+                       "collects"))
+]
+
+Move @filepath{literal.rkt} to @filepath{literal/lang/reader.rkt}
+within the directory reported by @racket[find-collects-dir] or
+@racket[find-user-collects-dir]. Then, @racket[literal] can be used
+directly after @hash-lang[]:
+
+@racketmod[
+@#,racket[literal]
+Technology!
+System!
+Perfect!
+]
+
+@margin-note{See @other-manual['(lib "scribblings/raco/raco.scrbl")]
+for more information on using @exec{raco}.}
+
+You can package up a collection for others to install by using the
+@exec{raco pack} command-line tool:
+
+@commandline{raco pack --collection literal.plt literal}
+
+Then, others can install the @filepath{literal} collection using
+@exec{raco setup}:
+
+@commandline{raco setup literal.plt}
+
+@margin-note{See @other-manual['(lib "planet/planet.scrbl")] for more
+information about @|PLaneT| packages.}
+
+A better approach may be to distribute your language as a @|PLaneT|
+package. A drawback of using a @|PLaneT| package is that users must
+type @racket[@#,hash-lang[] @#,schememodname[planet]] followed by a
+@|PLaneT| path to access the language. A great advantages are that the
+@|PLaneT| package can be installed automatically, it can be versioned,
+and it co-exists more easily with other packages.
+
+@; ----------------------------------------
+@section[#:tag "language-get-info"]{Source-Handling Configuration}
+
+The Racket distribution includes a Scribble language for writing prose
+documents, where Scribble extends the normal Racket to better support
+text. Here is an example Scribble document:
+
+@verbatim[#:indent 2]|{
+#lang scribble/base
+
+@(define (get-name) "Self-Describing Document")
+
+@title[(get-name)]
+
+The title of this document is ``@(get-name).''
+}|
+
+If you put that program in DrRacket's @tech{definitions area} and
+click @onscreen{Run}, then nothing much appears to happen, because the
+@racketmodname[scribble/base] language is similar to the
+@filepath{literal.rkt} example from @secref["hash-lang reader"]: It
+just binds and exports @racketidfont{doc} as a description of a document.
+
+Simply opening a module with the language
+@racketmodname[scribble/base] in DrRacket, however, causes a
+@onscreen{Scribble HTML} button to appear. Furthermore, DrRacket knows
+how to colorize Scribble syntax by coloring green those parts of the
+document that correspond to literal text. The language name
+@racketmodname[scribble/base] is not hard-wired into
+DrRacket. Instead, the implementation of the
+@racketmodname[scribble/base] language provides button and
+syntax-coloring information in response to a query from DrRacket.
+
+For security reasons, only languages that have been specifically
+installed by a user can respond to language-information queries.  If
+you have installed the @racket[literal] language as described in
+@secref["language-collection"], then you can adjust
+@filepath{literal/lang/reader.rkt} so that DrRacket treats the content
+of a module in the @racket[literal] language as plain text instead of
+(erroneously) as Racket syntax:
+
+@racketmod[
+racket
+(require syntax/strip-context)
+
+(provide (rename-out [literal-read read]
+                     [literal-read-syntax read-syntax])
+         get-info)
+
+(define (literal-read in)
+  (syntax->datum
+   (literal-read-syntax #f in)))
+
+(define (literal-read-syntax src in)
+  (with-syntax ([str (port->string in)])
+    (strip-context
+     #'(module anything racket
+         (provide data)
+         (define data (quote str))))))
+
+(define (get-info in mod line col pos)
+  (lambda (key default)
+    (case key
+      [(color-lexer)
+       (dynamic-require 'syntax-color/default-lexer 'default-lexer)]
+      [else default])))
+]
+
+This revised @racket[literal] implementation provides a
+@racketidfont{get-info} function. The @racketidfont{get-info} function
+will be applied to the source input stream and location information,
+in case query results should depend on the content of the module after
+the language name (which is not the case for @racket[literal]). The
+result of @racketidfont{get-info} is a function of two arguments. The
+first argument is always a symbol, indicating the kind of information
+that a tool requests from the language; the second argument is the
+default result to be returned if the language does not recognize the
+query or has no information for it.
+
+After DrRacket obtains the result of @racketidfont{get-info} for a
+language, it calls the function with a @racket['color-lexer] query;
+the result should be a function that implements syntax-coloring
+parsing on an input stream. For @racket[literal], the
+@racketmodname[syntax-color/default-lexer] module provides a
+@racket[default-lexer] syntax-coloring parser that is suitable for
+plain text, so @racket[literal] loads and returns that parser in
+response to a @racket['color-lexer] query.
+
+The set of symbols that a programming tool uses for queries
+is entirely between the tool and the languages that choose to
+cooperate with it. For example, in addition to @racket['color-lexer],
+DrRacket uses a @racket['drscheme:toolbar-buttons] query to determine
+which buttons should be available in the toolbar to operate on modules
+using the language.
+
+The @racketmodname[syntax/module-reader] language lets you specify
+@racketidfont{get-info} handling through a @racket[#:info] optional
+specification. The protocol for an @racket[#:info] function is
+slightly different from the raw @racketidfont{get-info} protocol; the
+revised protocol allows @racketmodname[syntax/module-reader] the
+possibility of handling future language-information queries
+automatically.
+
+@; ----------------------------------------
+@section[#:tag "module-runtime-config"]{Module-Handling Configuration}
+
+Suppose that the file @filepath{death-list-5.rkt} contains
+
+@racketmodfile["death-list-5.rkt"]
+
+If you @scheme[require] @filepath{death-list-5.rkt} directly, then it
+prints the list in the usual Racket result format:
+
+@interaction[
+(require "death-list-5.rkt")
+]
+
+However, if @filepath{death-list-5.rkt} is required by a
+@filepath{kiddo.rkt} that is implemented with @racketmodname[scheme]
+instead of @racketmodname[racket]:
+
+@racketmodfile["kiddo.rkt"]
+
+then, if you run @filepath{kiddo.rkt} file in DrRacket or if you run it
+directly with @exec{racket}, @filepath{kiddo.rkt} causes
+@filepath{death-list-5.rkt} to print its list in traditional Scheme
+format, without the leading quote:
+
+@racketblock[
+@#,racketoutput{("O-Ren Ishii" "Vernita Green" "Elle Driver" "Budd" "Bill")}
+]
+
+The @filepath{kiddo.rkt} example illustrates how the format for
+printing a result value can depend on the main module of a program
+and the language that is used to implement it.
+
+Unlike the syntax-coloring property of a language (as described in
+@secref["language-get-info"]), the result-value format is a property of a
+@emph{module} (via its language) as opposed to a property of the
+module's @emph{source text}. That is, the run-time configuration for a
+module should be available even if the module is compiled
+to bytecode form and the source is unavailable. Due to this difference,
+language properties such as run-time configuration are not reported
+via a @racketidfont{get-info} function that exported from the language's
+parser module, but instead through a separate module whose name is
+attached to the syntax object for a parsed @racket[module] form.
+
+Going back to the @racket[literal] language (see
+@secref["language-get-info"]), we can adjust the language so that
+directly running a @racket[literal] module causes it
+to print out its string, while using a @racket[literal] module
+in a larger program simply provides @racketidfont{data}
+without printing. To make this work, we must make two changes
+to @filepath{literal/lang/reader.rkt}:
+
+@itemlist[
+
+ @item{The @racket[module] form generated by the
+       @racketidfont{read-syntax} function must import a
+       @racket[literal/show] module and call its @racketidfont{show}
+       function, which will print the given string if output has been
+       enabled.}
+
+ @item{The @racket[module] form must be annotated with a
+       @racket['language-info] syntax property, whose value points to
+       a @racketidfont{get-language-info} function exported by a
+       @racket[literal/language-info] module. The
+       @racketidfont{get-language-info} function will be responsible
+       for reporting the runtime-configuration action of the
+       language. Finally, the runtime-configuration action will turn
+       on printing in @racket[literal/show].}
+
+]
+
+The @racketidfont{get-language-info} function is not attached to the
+@racket[module] form directly; instead, the function is identified by
+a module path and a symbol, so that the function can be loaded when it
+is needed. Those two parts are combined in a vector along with a third
+part, which is an argument to supply to the
+@racketidfont{get-language-info} function (in case extra information
+needs to be propagated from the source to the module's language
+information).
+
+These changes are implemented in the following revised
+@filepath{literal/lang/reader.rkt}:
+
+@racketmod[
+racket
+(require syntax/strip-context)
+
+(provide (rename-out [literal-read read]
+                     [literal-read-syntax read-syntax])
+         get-info)
+
+(define (literal-read in)
+  (syntax->datum
+   (literal-read-syntax #f in)))
+
+(define (literal-read-syntax src in)
+  (with-syntax ([str (port->string in)])
+    (syntax-property
+     (strip-context
+      #'(module anything racket
+          (require literal/show)
+          (provide data)
+          (define data (quote str))
+          (show data)))
+     'module-language
+     '#(literal/language-info get-language-info #f))))
+
+(define (get-info in mod line col pos)
+  (lambda (key default)
+    (case key
+      [(color-lexer)
+       (dynamic-require 'syntax-color/default-lexer 'default-lexer)]
+      [else default])))
+]
+
+When a @racket[module] form with a @racket['module-language] property
+is compiled, the property value is preserved with the compiled module,
+and it is accessible via reflective functions like
+@racket[module->language-info]. When @exec{racket} or DrRacket runs a
+module, it uses @racket[module->language-info] to obtain a vector that
+contains a module name, export name, and data value. The result of the
+function applied to the data should be another function that answers
+queries, much like the @racketidfont{get-info} function in a language
+reader.
+
+For @racket[literal], @filepath{literal/language-info.rkt} is
+implemented as:
+
+@racketmod[
+racket
+
+(provide get-language-info)
+
+(define (get-language-info data)
+  (lambda (key default)
+    (case key
+      [(configure-runtime)
+       '(#(literal/runtime-config configure #f))]
+      [else default])))
+]
+
+The function returned by @racketidfont{get-language-info} answers a
+@racket['configure-runtime] query with a list of yet more vectors,
+where each vector contains a module name, an exported name, and a data
+value. This indirection through another set of vectors (as opposed to
+performing run-time configurations directly) allows the action to be
+delayed beyond the query time. For example, run-time configuration
+actions may need to collected to create a stand-alone executable, but
+the actions must be performed only when the executable is launched.
+
+For the @racket[literal] language, the run-time configuration action
+implemented in @filepath{literal/runtime-config.rkt} is to enable
+printing of strings that are sent to @racketidfont{show}:
+
+@racketmod[
+racket
+(require "show.rkt")
+
+(provide configure)
+
+(define (configure data)
+  (show-enabled #t))
+]
+
+Finally, the @filepath{literal/runtime-config.rkt} module must provide
+the @racketidfont{show-enabled} parameter and @racketidfont{show}
+function:
+
+@racketmod[
+racket
+
+(provide show show-enabled)
+
+(define show-enabled (make-parameter #f))
+
+(define (show v)
+  (when (show-enabled)
+    (display v)))
+]
+
+With all of the pieces for @racket[literal] in place, try running the
+following variant of @filepath{tuvalu.rkt} directly and through a
+@scheme[require] from another module:
+
+@racketmod[
+@#,racket[literal]
+Technology!
+System!
+Perfect!
+]
+
+Customizing the @racket[literal] language required many different
+modules, because different modules are needed to keep the different
+phases and times of different tasks separate. For example, the code
+needed to correctly color @racket[literal] text in DrRacket should not
+be required to simply run a @racket[literal] program.
+
+The @racketmodname[syntax/module-reader] language lets you specify a
+module's language information through a @racket[#:language-info]
+optional specification. The value provided through
+@racket[#:language-info] is attached to a @racket[module] form
+directly as a syntax property.

collects/scribblings/guide/html.rkt

@@ -0,0 +1,16 @@
+#lang racket
+(require racket/date)
+
+(provide (except-out (all-from-out racket)
+                     #%module-begin)
+         (rename-out [module-begin #%module-begin])
+         now)
+
+(define-syntax-rule (module-begin expr ...)
+  (#%module-begin
+   (define page `(html expr ...))
+   (provide page)))
+
+(define (now)
+  (parameterize ([date-display-format 'iso-8601])
+    (date->string (seconds->date (current-seconds)))))

collects/scribblings/guide/kiddo.rkt

@@ -0,0 +1,2 @@
+#lang scheme
+(require "death-list-5.rkt")

collects/scribblings/guide/languages.scrbl

@@ -5,8 +5,8 @@
 @title[#:tag "languages" #:style 'toc]{Creating Languages}
 
 The @tech{macro} facilities defined in the preceding chapter let a
-programmer define syntactic extensions to a language, but the
+programmer define syntactic extensions to a language, but a macro is
-expressiveness of a macro is limited in two ways:
+limited in two ways:
 
 @itemlist[
 
@@ -33,5 +33,6 @@ the @tech{reader} layer, for defining the starting point of the
 @local-table-of-contents[]
 
 @;------------------------------------------------------------------------
-@; @include-section["module-languages.scrbl"]
+@include-section["module-languages.scrbl"]
 @include-section["reader-extension.scrbl"]
+@include-section["hash-languages.scrbl"]

collects/scribblings/guide/literal.rkt

@@ -0,0 +1,16 @@
+#lang racket
+(require syntax/strip-context)
+
+(provide (rename-out [literal-read read]
+                     [literal-read-syntax read-syntax]))
+
+(define (literal-read in)
+  (syntax->datum
+   (literal-read-syntax #f in)))
+
+(define (literal-read-syntax src in)
+  (with-syntax ([str (port->string in)])
+    (strip-context
+     #'(module anything racket
+         (provide data)
+         (define data (quote str))))))

collects/scribblings/guide/modfile.rkt

@@ -0,0 +1,28 @@
+#lang racket/base
+
+(require scribble/manual
+         (for-syntax racket/base
+                     (prefix-in c: scribble/comment-reader)
+                     syntax/strip-context
+                     compiler/cm-accomplice))
+
+(provide racketmodfile)
+
+(define-syntax (racketmodfile stx)
+  (syntax-case stx ()
+    [(_ file)
+     (let ([f (path->complete-path (syntax-e #'file))])
+       (register-external-file f)
+       (with-syntax ([(content ...)
+                      (call-with-input-file* 
+                       f
+                       (lambda (in)
+                         (read-bytes 6 in)
+                         (port-count-lines! in)
+                         (let loop ()
+                           (let ([v (c:read-syntax (object-name in) in)])
+                             (if (eof-object? v) 
+                                 null 
+                                 (cons (replace-context #'file v) 
+                                       (loop)))))))])
+         #'(racketmod content ...)))]))

collects/scribblings/guide/module-languages.scrbl

@@ -0,0 +1,184 @@
+#lang scribble/doc
+@(require scribble/manual
+          scribble/eval
+          "guide-utils.ss"
+          "modfile.rkt"
+          (for-label racket/date))
+
+@title[#:tag "module-languages"]{Module Languages}
+
+When using the longhand @scheme[module] form for writing modules, the
+module path that is specified after the new module's name provides the
+initial imports for the module. Since the initial-import module
+determines even the most basic bindings that are available in a
+module's body, such as @scheme[require], the initial import can be
+called a @deftech{module language}.
+
+The most common @tech{module languages} are @racketmodname[racket] or
+@racketmodname[racket/base], but you can define your own
+@tech{module language} by defining a suitable module. For example,
+using @racket[provide] subforms like @racket[all-from-out],
+@racket[except-out], and @racket[rename-out], you can add, remove, or
+rename bindings from @racketmodname[racket] to produce a @tech{module
+language} that is a variant of @racketmodname[racket]:
+
+@guideother{@secref["module-syntax"] introduces the longhand
+@racket[module] form.}
+
+@interaction[
+(module raquet racket
+  (provide (except-out (all-from-out racket) lambda)
+           (rename-out [lambda function])))
+(module tennis 'raquet
+  (map (function (points) (case points
+                           [(0) "love"] [(1) "fifteen"]
+                           [(2) "thirty"] [(3) "forty"]))
+       (list 0 2)))
+(require 'tennis)
+]
+
+@; ----------------------------------------
+@section[#:tag "implicit-forms"]{Implicit Form Bindings}
+
+If you try to remove too much from @racketmodname[racket] in defining
+your own @tech{module language}, then the resulting module
+will no longer work right as a @tech{module language}:
+
+@interaction[
+(module just-lambda racket
+  (provide lambda))
+(module identity 'just-lambda
+  (lambda (x) x))
+]
+
+The @racket[#%module-begin] form is an implicit form that wraps the
+body of a module. It must be provided by a module that is to be used
+as @tech{module language}:
+
+@interaction[
+(module just-lambda racket
+  (provide lambda #%module-begin))
+(module identity 'just-lambda
+  (lambda (x) x))
+(require 'identity)
+]
+
+The other implicit forms provided by @racket[racket/base] are
+@racket[#%app] for function calls, @racket[#%datum] for literals, and
+@racket[#%top] for unbound identifiers:
+
+@interaction[
+(module just-lambda racket
+  (provide lambda #%module-begin
+           (code:comment @#,t{@schemeidfont{ten} needs these, too:})
+           #%app #%datum))
+(module ten 'just-lambda
+  ((lambda (x) x) 10))
+(require 'ten)
+]
+
+Implicit forms like @racket[#%app] can be used explicitly in a module,
+but they exist mainly to allow a module language to restrict or change
+the meaning of implicit uses. For example, a @racket[lambda-calculus]
+@tech{module language} might restrict functions to a single argument,
+restrict function calls to supply a single argument, restrict the
+module body to a single expression, disallow literals, and treat
+unbound identifiers as uninterpreted symbols:
+
+@interaction[
+(module lambda-calculus racket
+  (provide (rename-out [1-arg-lambda lambda]
+                       [1-arg-app #%app]
+                       [1-form-module-begin #%module-begin]
+                       [no-literals #%datum]
+                       [unbound-as-self #%top]))
+  (define-syntax-rule (1-arg-lambda (x) expr)
+    (lambda (x) expr))
+  (define-syntax-rule (1-arg-app e1 e2)
+    (#%app e1 e2))
+  (define-syntax-rule (1-form-module-begin e)
+    (#%module-begin e))
+  (define-syntax (no-literals stx)
+    (raise-syntax-error #f "no" stx))
+  (define-syntax-rule (unbound-as-self . id)
+    'id))
+(module ok 'lambda-calculus
+  ((lambda (x) (x z))
+   (lambda (y) y)))
+(require 'ok)
+(module not-ok 'lambda-calculus
+  (lambda (x y) x))
+(module not-ok 'lambda-calculus
+  (lambda (x) x)
+  (lambda (y) (y y)))
+(module not-ok 'lambda-calculus
+  (lambda (x) (x x x)))
+(module not-ok 'lambda-calculus
+  10)
+]
+
+Module languages rarely redefine @racket[#%app], @racket[#%datum], and
+@racket[#%top], but redefining @racket[#%module-begin] is more
+frequently useful. For example, when using modules to construct
+descriptions of HTML pages where a description is exported from the
+module as @racketidfont{page}, an alternate @racket[#%module-begin]
+can help eliminate @racket[provide] and quasiquoting
+boilerplate, as in @filepath{html.rkt}:
+
+@racketmodfile["html.rkt"]
+
+Using the @filepath{html.rkt} @tech{module language}, a simple web page
+can be described without having to explicitly define or export
+@racketidfont{page} and starting in @racket[quasiquote]d mode instead
+of expression mode:
+
+@interaction[
+(module lady-with-the-spinning-head "html.rkt"
+  (title "Queen of Diamonds")
+  (p "Updated: " ,(now)))
+(require 'lady-with-the-spinning-head)
+page
+]
+
+@; ----------------------------------------
+@section[#:tag "s-exp"]{Using @racket[@#,hash-lang[] @#,racketmodname[s-exp]]}
+
+Implementing a language at the level of @hash-lang[] is more complex
+than declaring a single module, because @hash-lang[] lets programmers
+control several different facets of a language. The
+@racketmodname[s-exp] language, however, acts as a kind of
+meta-language for using a @tech{module language} with the
+@hash-lang[] shorthand:
+
+@racketmod[
+s-exp _module-name
+_form ...]
+
+is the same as
+
+@racketblock[
+(module _name _module-name
+  _form ...)
+]
+
+where @racket[_name] is derived from the source file containing the
+@hash-lang[] program. The name @racketmodname[s-exp] is short for
+``@as-index{S-expression},'' which is a traditional name for
+Racket's @tech{reader}-level lexical conventions: parentheses,
+identifiers, numbers, double-quoted strings with certain backslash
+escapes, and so on.
+
+Using @racket[@#,hash-lang[] @#,racketmodname[s-exp]], the
+@racket[lady-with-the-spinning-head] example from before can be
+written more compactly as:
+
+@racketmod[
+s-exp "html.rkt"
+
+(title "Queen of Diamonds")
+(p "Updated: " ,(now))
+]
+
+The later section @secref["hash-languages"] explains how to define
+your own @hash-lang[] language, but first we explain how you can write
+@tech{reader}-level extensions to Racket.

collects/scribblings/guide/reader-extension.scrbl

@@ -5,41 +5,25 @@
           (for-label racket/match
                      syntax/readerr)
           "guide-utils.ss"
-          (for-syntax racket/base
+          "modfile.rkt")
-                      syntax/strip-context))
-
-@(define-syntax (racketmodfile stx)
-   (syntax-case stx ()
-    [(_ file)
-     (with-syntax ([(content ...)
-                    (call-with-input-file* (syntax-e #'file)
-                     (lambda (in)
-                       (read-bytes 6 in)
-                       (port-count-lines! in)
-                       (let loop ()
-                         (let ([v (read-syntax (object-name in) in)])
-                           (if (eof-object? v) 
-                               null 
-                               (cons (replace-context #'file v) 
-                                     (loop)))))))])
-        #'(racketmod content ...))]))
 
 @title[#:tag "hash-reader"]{Reader Extensions}
 
-The @tech{reader} layer of the Racket language supports a
+The @tech{reader} layer of the Racket language can be extended through
-@litchar{#reader} syntax for allowing an external processor to parse
+the @racketmetafont{#reader} form. A reader extension is implemented
-raw bytes into forms to be consumed by the @tech{expander} layer.
+as a module that is named after @racketmetafont{#rader}. The module
-The syntax of @litchar{#reader} is
+exports functions that parse raw characters into a form to be consumed
+by the @tech{expander} layer.
+
+The syntax of @racketmetafont{#reader} is
 
 @racketblock[@#,(BNF-seq @litchar{#reader} @nonterm{module-path} @nonterm{reader-specific})]
 
 where @nonterm{module-path} names a module that provides
 @racketidfont{read} and @racketidfont{read-syntax} functions. The
-@nonterm{module-path} itself is written with the reader syntax put in
+@nonterm{reader-specific} part is a sequence of characters that is
-place by its context. The @nonterm{reader-specific} part is a sequence
+parsed as determined by the @racketidfont{read} and
-of characters that is parsed as determined by the @racketidfont{read}
+@racketidfont{read-syntax} functions from @nonterm{module-path}.
-and @racketidfont{read-syntax} functions that are exported by the
-module named through @nonterm{module-path}.
 
 For example, suppose that file @filepath{five.rkt} contains
 
@@ -75,8 +59,8 @@ functions from @filepath{five.rkt} are not obliged to follow Racket
 lexical conventions and treat the continuous sequence @litchar{234567}
 as a single number. Since only the @litchar{23456} part is consumed by
 @racketidfont{read} or @racketidfont{read-syntax}, the @litchar{7}
-remains to be parsed in the usual racket way. Similarly, the reader
+remains to be parsed in the usual Racket way. Similarly, the reader
-functions from @filepath{five.rkt} are not obliged to treat spaces as
+functions from @filepath{five.rkt} are not obliged to ignore
 whitespace, and
 
 @racketmod[
@@ -96,7 +80,7 @@ racket/base
 since the first character immediately after @racket["five.rkt"] is a
 space.
 
-A @litchar{#reader} form can be used in the @tech{REPL}, too:
+A @racketmetafont{#reader} form can be used in the @tech{REPL}, too:
 
 @interaction[
 (eval:alts @#,(elem @racketmetafont{#reader}@racket["five.rkt"]@tt{abcde}) #reader"five.rkt"abcde)
@@ -108,30 +92,33 @@ A @litchar{#reader} form can be used in the @tech{REPL}, too:
 
 The difference between @racketidfont{read} and
 @racketidfont{read-syntax} is that @racketidfont{read} is meant to be
-used for data like the Racket @racket[read] function, while
+used for data while @racketidfont{read-syntax} is meant to be used to
-@racketidfont{read-syntax} is meant to be used to parse programs. More
+parse programs. More precisely, the @racketidfont{read} function will
-precisely, the @racketidfont{read} function will be used when the
+be used when the enclosing stream is being parsed by the Racket
-enclosing stream is being parsed by the Racket @racket[read], and
+@racket[read], and @racketidfont{read-syntax} is used when the
-@racketidfont{read-syntax} is used when the enclosing stream is being
+enclosing stream is being parsed by the Racket @racket[read-syntax]
-parsed by the Racket @racket[read-syntax] function. Nothing requires
+function. Nothing requires @racketidfont{read} and
-@racketidfont{read} and @racketidfont{read-syntax} to parse input in
+@racketidfont{read-syntax} to parse input in the same way, but making
-the same way, though they normally should.
+them different would confuse programmers and tools.
 
-Although the @racketidfont{read-syntax} function can return the same
+The @racketidfont{read-syntax} function can return the same kind of
-kind of value as @racketidfont{read}, it should normally return a
+value as @racketidfont{read}, but it should normally return a
 @tech{syntax object} that connects the parsed expression with source
 locations. Unlike the @filepath{five.rkt} example, the
-@racketidfont{read-syntax} function is typically implemented directly,
+@racketidfont{read-syntax} function is typically implemented directly
-and then @racketidfont{read} can use @racketidfont{read-syntax} and
+to produce @tech{syntax objects}, and then @racketidfont{read} can use
-strip away source information.
+@racketidfont{read-syntax} and strip away @tech{syntax object}
+wrappers to produce a raw result.
 
 The following @filepath{arith.rkt} module implements that reader to
 parse simple infix arithmetic expressions into Racket forms. For
 example, @litchar{1*2+3} parses into the Racket form @racket[(+ (* 1
-2) 3)]. Single-letter variables can appear in the expression. The
+2) 3)]. The supported operators are @litchar{+}, @litchar{-},
-implementation uses @racket[port-next-location] to obtain the current
+@litchar{*}, and @litchar{/}, while operands can be unsigned integers
-source location, and it uses @racket[datum->syntax] to turn raw values
+or single-letter variables. The implementation uses
-into @tech{syntax objects}.
+@racket[port-next-location] to obtain the current source location, and
+it uses @racket[datum->syntax] to turn raw values into @tech{syntax
+objects}.
 
 @racketmodfile["arith.rkt"]
 
@@ -161,13 +148,13 @@ the error message):
 
 @; ----------------------------------------------------------------------
 
-@section{Readtables}
+@section[#:tag "readtable"]{Readtables}
 
 A reader extension's ability to parse input characters in an arbitrary
 way can be powerful, but many cases of lexical extension call for a
 less general but more composable approach. In much the same way that
 the @tech{expander} level of Racket syntax can be extended through
-@tech{macros}, the @tech{reader} level of Racket syntax can be more
+@tech{macros}, the @tech{reader} level of Racket syntax can be
 composably extended through a @deftech{readtable}.
 
 The Racket reader is a recursive-descent parser, and the
@@ -208,15 +195,15 @@ character.
 The following @filepath{dollar.rkt} module defines a
 @racket[parse-dollar] function in terms of the @racketidfont{read} and
 @racketidfont{read-syntax} functions provided by @filepath{arith.rkt},
-and it puts it together with new @racketidfont{read} and
+and it puts @racket[parse-dollar] together with new @racketidfont{read} and
-@racketidfont{read-syntax} functions that install the readtable can
+@racketidfont{read-syntax} functions that install the readtable and
 chain to Racket's @racket[read] or @racket[read-syntax]:
 
 @racketmodfile["dollar.rkt"]
 
 With this reader extension, a single @racketmetafont{#reader} can be
 used at the beginning of an expression to enable multiple uses of
-@litchar{$} to switch to infix arithmetic:
+@litchar{$} that switch to infix arithmetic:
 
 @interaction[
 (eval:alts @#,(elem @racketmetafont{#reader}@racket["dollar.rkt"]@hspace[1]

collects/scribblings/guide/store.rkt

@@ -0,0 +1,8 @@
+#lang reader "dollar-racket.rkt"
+
+(provide cost)
+
+;; Cost of `n' $1 rackets with 7% sales
+;; tax and shipping-and-handling fee `h':
+(define (cost n h)
+  $n*107/100+h$)

collects/scribblings/guide/tuvalu.rkt

@@ -0,0 +1,4 @@
+#lang reader "literal.rkt"
+Technology!
+System!
+Perfect!

collects/scribblings/reference/init.scrbl

@@ -25,6 +25,8 @@ default value, and it returns @scheme['(#(racket/runtime-config
 configure #f))] if the key is @scheme['configure-runtime] or the
 default value otherwise.}
 
+@guidealso["module-runtime-config"]
+
 The vector @scheme['#(racket/language-info get-info #f)] is suitable
 for attaching to a module as its language info to get the same
 language information as the @scheme[racket/base] language.

collects/scribblings/reference/module-reflect.scrbl

@@ -309,6 +309,8 @@ import.}
 @defproc[(module-compiled-language-info [compiled-module-code compiled-module-expression?])
          (or/c #f (vector/c module-path? symbol? any/c))]{
 
+@guidealso["module-runtime-config"]
+
 Returns information intended to reflect the ``language'' of the
 module's implementation as originally attached to the syntax of the
 module's declaration though the @indexed-racket['module-language]

collects/scribblings/reference/read.scrbl

@@ -120,6 +120,8 @@ value of @racket[current-readtable]), and @litchar{#reader} forms
 (which might produce comments) are not allowed before @litchar{#lang}
 or @litchar{#!}.
 
+@guidealso["language-get-info"]
+
 When it finds a @litchar{#lang} or @litchar{#!} specification, instead
 of dispatching to a @racketidfont{read} or @racketidfont{read-syntax}
 form as @racket[read] and @racket[read-syntax] do,

collects/scribblings/reference/reader.scrbl

@@ -764,6 +764,8 @@ neither defines nor uses graph tags for other top-level forms.
 
 @section[#:tag "parse-reader"]{Reading via an Extension}
 
+@guideintro["hash-reader"]{reader extension}
+
 When the reader encounters @as-index{@litchar{#reader}}, it loads
 an external reader procedure and applies it to the current input
 stream.
@@ -814,8 +816,7 @@ sequence must not start or end with @litchar{/}. A sequence
 that the terminating whitespace (if any) is not consumed before the
 external reading procedure is called.
 
-@margin-note{The @racketmodname[syntax/module-reader] library provides a
+@guideintro["hash-languages"]{the creation languages for @hash-lang[]}
-             domain-specific language for writing language readers.}
 
 Finally, @as-index{@litchar{#!}} is a synonym for @litchar{#lang}
 followed by a space when @litchar{#!} is followed by alphanumeric
@@ -824,6 +825,9 @@ is discouraged except as needed to construct programs that conform to
 certain grammars, such as that of R@superscript{6}RS
 @cite["Sperber07"].
 
+@margin-note{The @racketmodname[syntax/module-reader] library provides a
+             domain-specific language for writing language readers.}
+
 By convention, @litchar{#lang} normally appears at the beginning of a
 file, possibly after comment forms, to specify the syntax of a module.
 
@@ -831,6 +835,8 @@ file, possibly after comment forms, to specify the syntax of a module.
 
 @defmodulelang[s-exp]
 
+@guideintro["s-exp"]{the @racketmodname[s-exp] meta-language}
+
 The @racket[s-exp] ``language'' is a kind of meta-language. It
 @racket[read]s the S-expression that follows @litchar{#lang s-exp} and
 uses it as the language of a @racket[module] form. It also reads all
@@ -855,6 +861,8 @@ is equivalent to
 
 @defmodulelang[reader]
 
+@guideintro["hash-lang reader"]{the @racketmodname[reader] meta-language}
+
 The @racket[reader] ``language'' is a kind of meta-language. It
 @racket[read]s the S-expression that follows @litchar{#lang reader}
 and uses it as a module path (relative to the module being read) that

collects/scribblings/reference/startup.scrbl

@@ -376,6 +376,8 @@ Extra arguments following the last option are available from the
 
 @section[#:tag "configure-runtime"]{Language Run-Time Configuration}
 
+@guidealso["module-runtime-config"]
+
 When a module is implemented using @hash-lang{}, the language after
 @hash-lang{} can specify configuration actions to perform when a
 module using the language is the main module of a program. The

collects/scribblings/reference/syntax.scrbl

@@ -91,8 +91,9 @@ Within such specifications,
 
 Declares a top-level module. If the
 @racket[current-module-declare-name] parameter is set, the parameter
-value is used for the module name, otherwise @racket[(#,(racket quote)
+value is used for the module name and @racket[id] is ignored,
-id)] is the name of the declared module.
+otherwise @racket[(#,(racket quote) id)] is the name of the declared
+module.
 
 @margin-note/ref{For a @racket[module]-like form for use @emph{within}
 modules and other contexts, see @racket[define-package].}
@@ -175,7 +176,7 @@ defined. No expression can refer to a @tech{top-level variable}.
 
 The evaluation of a @racket[module] form does not evaluate the
 expressions in the body of the module. Evaluation merely declares a
-module, whose full name depends both on @racket[id] and
+module, whose full name depends both on @racket[id] or
 @racket[(current-module-declare-name)].
 
 The module body is executed only when the module is explicitly

collects/syntax/scribblings/module-reader.scrbl

@@ -12,8 +12,13 @@
      (define name @scheme[#%module-begin])))
   (define-mb scheme-#%module-begin))
 
+@(define guide-doc '(lib "scribblings/guide/guide.scrbl"))
+
 @title[#:tag "module-reader"]{Module Reader}
 
+@margin-note{See also @secref[#:doc guide-doc "hash-languages"] in
+             @other-manual[guide-doc].}
+
 @defmodule[syntax/module-reader]
 
 The @schememodname[syntax/module-reader] library provides support for
@@ -188,7 +193,7 @@ identifiers used by the @scheme[reader-option]s.
  @item{@scheme[#:language-info] specifies an implementation of
        reflective information that is used by external tools to
        manipulate the module in the language @scheme[_something] in
-       its @emph{expanded}, @emph{compiled} or @emph{declared} form
+       its @emph{expanded}, @emph{compiled}, or @emph{declared} form
        (as opposed to source). For example, when Racket starts a
        program, it uses information attached to the main module to
        initialize the run-time environment.