at-exp and scribble: adjust reader to compose better with readtable extensions

The `at-exp` reader now delays picking up the current readtable until
`read`/`read-syntax` is called. Also, it uses the new 'dynamic configuration
of readers for the command and datum parts of an @-form, which delays a
decision of readtable on each part until reading the part.

Thanks to Alexander Knauth for sorting out pieces of the puzzle.

original commit: a087aea3e58560427980b28d9ccb64815edbff01

pkgs/scribble-pkgs/scribble-doc/scribblings/scribble/reader-internals.scrbl

@@ -177,33 +177,48 @@ provides direct Scribble reader functionality for advanced needs.}
          (require (for-label scribble/reader))
 
 @; *** Start reader-import section ***
+@deftogether[(
 @defproc[(read [in input-port? (current-input-port)]) any]{}
 @defproc[(read-syntax [source-name any/c (object-name in)]
                       [in input-port? (current-input-port)])
-         (or/c syntax? eof-object?)]{
+         (or/c syntax? eof-object?)]
-These procedures implement the Scribble reader.  They do so by
+)]{
-constructing a reader table based on the current one, and using that
-for reading.
-}
 
+Implements the Scribble reader using the readtable produced by
+
+@racketblock[(make-at-readtable #:command-readtable 'dynamic
+                                #:datum-readtable 'dynamic)]
+
+@history[#:changed "1.1" @elem{Changed to use @racket['dynamic] for the command and datum readtables.}]}
+
+
+@deftogether[(
 @defproc[(read-inside [in input-port? (current-input-port)]) any]{}
 @defproc[(read-syntax-inside [source-name any/c (object-name in)]
                              [in input-port? (current-input-port)]
                              [#:command-char command-char char? #\@])
-         (or/c syntax? eof-object?)]{
+         (or/c syntax? eof-object?)]
-These @racketid[-inside] variants parse as if starting inside a
+)]{
-@litchar["@{"]...@litchar["}"], and they return a (syntactic) list.
+
-The @racket[command-char] is used to customize the readtable.
+Like @racket[read] and @racket[read-syntax], but starting as if
-Useful for implementing languages that are textual by default (see
+inside a @litchar["@{"]...@litchar["}"] to return a (syntactic) list,
-@filepath{docreader.rkt} for example).
+which is useful for implementing languages that are textual by default.
+
+The given @racket[command-char] is used to customize the readtable
+used by the reader, effectively passing it along to @racket[make-at-readtable].
+
+@history[#:changed "1.1" @elem{Changed to use @racket['dynamic] for the command and datum readtables.}]
 }
 
 @defproc[(make-at-readtable
           [#:readtable readtable readtable? (current-readtable)]
           [#:command-char command-char char? #\@]
+          [#:command-readtable command-readtable (or/c readtable? 'dynamic) readtable]
           [#:datum-readtable datum-readtable
-                             (or/c readtable? boolean?
+                             (or/c readtable?
-                                              (readtable? . -> . readtable?))
+                                   boolean?
+                                   (readtable? . -> . readtable?)
+                                   'dynamic)
                              #t]
           [#:syntax-post-processor syntax-post-proc
                                    (syntax? . -> . syntax?)
@@ -220,11 +235,31 @@ resulting reader in several ways:
 
 @item{@racket[command-char] --- the character used for @tech{@"@"-forms}.}
 
-@item{@racket[datum-readtable] --- determines the readtable used for
+@item{@racket[command-readtable] --- determines the readtable that is
-  reading the datum part.  A @racket[#t] values uses the
+  extended for reading the command part of an @tech{@"@"-form}:
-  @"@"-readtable, otherwise it can be a readtable, or a
+
-  readtable-to-readtable function that will construct one from the
+  @itemlist[
-  @"@"-readtable.  The idea is that you may want to have completely
+    @item{a readtable --- extended to make @litchar{|} a delimiter
+          instead of a symbol-quoting character}
+
+    @item{@racket['dynamic] --- extends @racket[(current-readtable)]
+          at the point where a command is parsed to make @litchar{|} a
+          delimiter}
+   ]}
+
+@item{@racket[datum-readtable] --- the readtable used for
+  reading the datum part of an @tech{@"@"-form}:
+
+  @itemlist[
+    @item{@racket[#t] --- uses the constructed @"@"-readtable itself}
+    @item{a readtable --- uses the given readtable}
+    @item{a readtable-to-readtable function --- called to construct a readtable
+          from the generated @"@"-readtable}
+    @item{@racket['dynamic] --- uses @racket[(current-readtable)] at the
+          point where the datum part is parsed}
+  ]
+
+  The idea is that you may want to have completely
   different uses for the datum part, for example, introducing a
   convenient @litchar{key=val} syntax for attributes.}
 
@@ -243,7 +278,11 @@ resulting reader in several ways:
           [_else (error "@ forms must have a body")])))
   ]}
 
-]}
+]
+
+@history[#:changed "1.1" @elem{Added @racket[#:command-readtable] and
+         the @racket['dynamic] option for @racket[#:datum-readtable].}]}
+
 
 @defproc[(make-at-reader [#:syntax? syntax? #t] [#:inside? inside? #f] ...)
           procedure?]{
@@ -268,6 +307,7 @@ reading.
 Note that if @racket[syntax?] is true, the @racket[read]-like function
 is constructed by simply converting a syntax result back into a datum.}
 
+
 @defproc[(use-at-readtable ...) void?]{
 
 Passes all arguments to @racket[make-at-readtable], and installs the

pkgs/scribble-pkgs/scribble-test/tests/scribble/reader.rkt

@@ -898,6 +898,26 @@ END-OF-TESTS
 (define -@error->   (mk-error-test scr:read))
 (define -\\error->  (mk-error-test read/BS))
 
+(define (make-@+-readtable #:command-readtable [command-readtable (current-readtable)]
+                           #:datum-readtable [datum-readtable (current-readtable)])
+  (make-readtable (scr:make-at-readtable #:command-readtable command-readtable
+                                         #:datum-readtable datum-readtable)
+                  #\+ 'terminating-macro (lambda args 'PLUS)))
+(define @+-readtable (make-@+-readtable))
+(define @c+-readtable (make-@+-readtable #:command-readtable 'dynamic))
+(define @d+-readtable (make-@+-readtable #:datum-readtable 'dynamic))
+(define @cd+-readtable (make-@+-readtable #:command-readtable 'dynamic
+                                          #:datum-readtable 'dynamic))
+
+(define-syntax-rule (@+checker a b readtable)
+  (equal? (parameterize ([current-readtable readtable])
+            (read (open-input-string a)))
+          b))
+(define-syntax-rule (a . -@+> . b) (@+checker a b @+-readtable))
+(define-syntax-rule (a . -@c+> . b) (@+checker a b @c+-readtable))
+(define-syntax-rule (a . -@d+> . b) (@+checker a b @d+-readtable))
+(define-syntax-rule (a . -@cd+> . b) (@+checker a b @cd+-readtable))
+
 ;; running the tests
 (provide reader-tests)
 (module+ main (reader-tests))
@@ -932,4 +952,15 @@ END-OF-TESTS
                       (format "bad result in\n    ~a\n  results:\n    ~s != ~s"
                               (regexp-replace* #rx"\n" t "\n    ")
                               x y)
-                      (matching? x y))))))))))
+                      (matching? x y))))))))
+
+    ;; Check static versus dynamic readtable for command (dynamic when "c" in the
+    ;; name) and datum (dynamic when "d" in the name) parts:
+    (-@+> "10" 10)
+    (-@+> "(+ @+[+] +)" '(PLUS (+ +) PLUS))
+    (-@+> "@+[+]" '(+ +))
+    (-@d+> "@+[+]" '(+ PLUS))
+    (-@d+> "(+ @+[+])" '(PLUS (+ PLUS)))
+    (-@c+> "@+[+]" '(PLUS +))
+    (-@c+> "@|+|" 'PLUS)
+    (-@cd+> "@+[+]" '(PLUS PLUS))))