zo-lib: documentation and a little support for non-BC linklets

Closes #3073

pkgs/compiler-lib/compiler/private/deserialize.rkt

@@ -1,7 +1,8 @@
 #lang racket/base
 (require racket/linklet
          compiler/zo-parse
-         compiler/zo-marshal)
+         compiler/zo-marshal
+         compiler/faslable-correlated)
 
 ;; Re-implement just enough deserialization to deal with 'decl
 ;; linklets, so we can get `required`, etc.
@@ -138,24 +139,6 @@
 
 ;; ----------------------------------------
 
-(struct faslable-correlated-linklet (expr name)
-  #:prefab)
-
-(struct faslable-correlated (e source position line column span props)
-  #:prefab)
-
-(define (strip-correlated v)
-  (let strip ([v v])
-    (cond
-      [(pair? v)
-       (cons (strip (car v))
-             (strip (cdr v)))]
-      [(faslable-correlated? v)
-       (strip (faslable-correlated-e v))]
-      [else v])))
-
-;; ----------------------------------------
-
 ;; Returns (values mpi-vector requires provides phase-to-link-modules)
 (define (deserialize-requires-and-provides l)
   (define ht (linkl-bundle-table l))

pkgs/compiler-lib/info.rkt

@@ -5,7 +5,7 @@
 (define deps '(["base" #:version "8.1.0.2"]
                "scheme-lib"
                "rackunit-lib"
-               "zo-lib"))
+               ["zo-lib" #:version "1.3"]))
 
 (define implies '("zo-lib"))
 

pkgs/racket-doc/scribblings/raco/common.rkt

@@ -3,7 +3,10 @@
 
 (provide inside-doc
          guide-doc
-         reference-doc)
+         reference-doc
+
+         CS
+         BC)
 
 (define inside-doc
   '(lib "scribblings/inside/inside.scrbl"))
@@ -13,3 +16,6 @@
 
 (define reference-doc
   '(lib "scribblings/reference/reference.scrbl"))
+
+(define CS (tech #:doc guide-doc "CS"))
+(define BC (tech #:doc guide-doc "BC"))

pkgs/racket-doc/scribblings/raco/decompile.scrbl

@@ -1,11 +1,15 @@
 #lang scribble/doc
 @(require scribble/manual
           scribble/bnf
+          scribble/core
           "common.rkt"
           (for-label racket/base
+                     racket/contract/base
                      compiler/decompile
                      (only-in compiler/zo-parse linkl-directory? linkl-bundle? linkl?)
-                     compiler/zo-marshal))
+                     compiler/zo-marshal
+                     compiler/faslable-correlated
+                     racket/linklet))
 
 @title[#:tag "decompile"]{@exec{raco decompile}: Decompiling Bytecode}
 
@@ -26,7 +30,7 @@ The @exec{raco decompile} command accepts the following command-line flags:
         given file's path and an associated @filepath{.zo} file (if any)}
   @item{@Flag{n} @nonterm{n} or @DFlag{columns} @nonterm{n} --- format
         output for a display with @nonterm{n} columns}
-  @item{@DFlag{linklet} --- decompile only as far as linklets, instead
+  @item{@DFlag{linklet} --- decompile only as far as @tech[#:doc reference-doc]{linklets}, instead
         of decoding linklets to approximate Racket @racket[module] forms}
   @item{@DFlag{no-disassemble} --- show machine code as-is in a byte string,
         instead of attempting to disassemble}
@@ -58,7 +62,7 @@ module around machine-code implementations of procedures.
 
 @section{Racket BC Decompilation}
 
-Racket BC bytecode has a structure that is close enough to Racket's
+Racket @BC bytecode has a structure that is close enough to Racket's
  core language that it can more often be converted to an approximation
  of Racket code. To the degree that it can be converted back,
  many forms in the decompiled code have the same meanings as
@@ -166,7 +170,9 @@ Racket BC bytecode has a structure that is close enough to Racket's
 
 @defmodule[compiler/decompile]
 
-@defproc[(decompile [top (or/c linkl-directory? linkl-bundle? linkl?)]) any/c]{
+@defproc[(decompile [top (or/c linkl-directory? linkl-bundle? linkl?
+                               linklet? faslable-correlated-linklet?)])
+         any/c]{
 
 Consumes the result of parsing bytecode and returns an S-expression
 (as described above) that represents the compiled code.}
@@ -193,3 +199,58 @@ the marshaled bytecode.}
 @; ------------------------------------------------------------
 
 @include-section["zo-struct.scrbl"]
+
+@; ------------------------------------------------------------
+
+@section{Machine-Independent Linklets}
+
+@defmodule[compiler/faslable-correlated]
+
+@nested[#:style 'inset]{
+@elem[#:style (style #f (list (background-color-property "yellow")))]{@bold{Warning:}}
+      The @racketmodname[compiler/faslable-correlated] library exposes internals
+      of the Racket bytecode abstraction. Unlike other Racket
+      libraries, @racketmodname[compiler/faslable-correlated] is subject to
+      incompatible changes across Racket versions.}
+
+@history[#:added "1.3"]
+
+@defstruct[faslable-correlated-linklet ([expr any/c]
+                                        [name symbol?])
+           #:prefab]{
+
+A @racket[faslable-correlated-linklet] structure represents a
+@tech[#:doc reference-doc]{linklet} that has been ``compiled'' to
+machine-independent form, which just contains an S-expression
+representing the @racket[linklet] form. The S-expression is enriched
+with source-location information by wrapping some nested S-expressions
+with @racket[faslable-correlated] structures.
+
+Since @racket[faslable-correlated-linklet] is a @tech[#:doc
+reference-doc]{prefab} structure type, the contracts documented above
+for its fields are not enforced.}
+
+@defstruct[faslable-correlated ([e any/c]
+                                [source any/c]
+                                [position exact-positive-integer?]
+                                [line exact-positive-integer?]
+                                [column exact-nonnegative-integer?]
+                                [span exact-nonnegative-integer?]
+                                [props (hash/c symbol? any/c)])
+           #:prefab]{
+
+Wraps an S-expression @racket[e] to give it a @tech[#:doc
+reference-doc]{source location}. The S-expression @racket[e] may
+contain nested @racket[faslable-correlated] structures, but nesting is
+expected only within pairs.
+
+Since @racket[faslable-correlated] is a @tech[#:doc
+reference-doc]{prefab} structure type, the contracts documented above
+for its fields are not enforced.}
+
+@defproc[(strip-correlated [e any/c]) any/c]{
+
+Recurs through @racket[e] to strip away any
+@racket[faslable-correlated] structures that are reachable through
+pairs. The given @racket[e] must not contain any cycles that are
+reachable through pairs.}

pkgs/racket-doc/scribblings/raco/demod.scrbl

@@ -45,7 +45,7 @@ The @exec{raco demod} command accepts these flags:
 
  @item{@Flag{r} or @DFlag{recompile} --- (re)compiles the module to
         machine-dependent form after flattening; this mode is the
-        default except on @tech[#:doc guide-doc]{BC}, where flattening
+        default except on @BC, where flattening
         can work in terms of bytecode files.}
 
  @item{@DFlag{work} @nonterm{dir} --- uses @nonterm{dir} to cache

pkgs/racket-doc/scribblings/raco/exe.scrbl

@@ -7,7 +7,7 @@
                      compiler/embed
                      launcher/launcher))
 
-@(define (gtech s) (tech #:doc '(lib "scribblings/guide/guide.scrbl") s))
+@(define (gtech s) (tech #:doc guide-doc s))
 
 @title[#:tag "exe"]{@exec{raco exe}: Creating Stand-Alone Executables}
 

pkgs/racket-doc/scribblings/raco/setup.scrbl

@@ -2163,7 +2163,7 @@ does not match the running Racket's information, then the
 @racketmodname[setup/cross-system] module infers that Racket is being
 run in cross-installation mode.
 
-For example, if an in-place Racket @tech[#:doc guide-doc]{BC}
+For example, if an in-place Racket @BC
 installation for a different platform resides at @nonterm{cross-dir},
 then running Racket BC as
 
@@ -2177,7 +2177,7 @@ libraries need to run to perform the requested @exec{raco pkg} action
 (e.g., when installing built packages), or as long as the current
 platform's installation already includes those libraries.
 
-For Racket @tech[#:doc guide-doc]{CS}, cross compilation is more
+For Racket @CS, cross compilation is more
 complicated, because Racket CS @filepath{.zo} files are
 platform-specific:
 

pkgs/racket-doc/scribblings/raco/zo-parse.scrbl

@@ -1,8 +1,11 @@
 #lang scribble/doc
 @(require scribble/manual
+          "common.rkt"
           (for-label racket/base
                      racket/contract
                      compiler/zo-parse
+                     compiler/decompile
+                     compiler/faslable-correlated
                      racket/set))
 
 @(define-syntax-rule (defstruct+ id fields . rest)
@@ -20,12 +23,33 @@ The @racketmodname[compiler/zo-parse] module re-exports
   Parses a port (typically the result of opening a @filepath{.zo} file)
   containing bytecode.  Beware that the structure types used to
   represent the bytecode are subject to frequent changes across Racket
-  versons.
+  versions.
 
   The parsed bytecode is returned in a @racket[linkl-directory] or
   @racket[linkl-bundle] structure---the latter only for the compilation
   of a module that contains no submodules.
 
+  Beyond the linklet bundle or directory structure, the result of
+  @racket[zo-parse] contains linklets that depend on the machine for
+  which the bytecode is compiled:
+
+  @itemlist[
+
+    @item{For a machine-independent bytecode file, a linklet is
+          represented as a @racket[faslable-correlated-linklet].}
+
+    @item{For a Racket @CS bytecode file, a linklet is opaque, because
+          it is primarily machine code, but @racket[decompile] can
+          extract some information and potentially disassemble the
+          machine code.}
+
+    @item{For Racket @BC bytecode, the bytecode can be parsed into
+          structures as described below.}
+
+   ]
+
+  The rest of this section is specific to @BC bytecode.
+
   Within a linklet, the bytecode representation of an expression is closer to an
   S-expression than a traditional, flat control string.  For example, an
   @racket[if] form is represented by a @racket[branch] structure that

pkgs/racket-doc/scribblings/raco/zo-struct.scrbl

@@ -1,12 +1,15 @@
 #lang scribble/doc
 @(require scribble/manual
           scribble/core
+          "common.rkt"
           (for-label racket/base
                      racket/contract
+                     racket/linklet
                      compiler/zo-structs
                      compiler/zo-parse
                      compiler/zo-marshal
                      compiler/decompile
+                     compiler/faslable-correlated
                      racket/set))
 
 @(define-syntax-rule (defstruct+ id fields . rest)
@@ -61,7 +64,13 @@ structures that are produced by @racket[zo-parse] and consumed by
  
  For a module with submodules, the linklet directory maps submodule
  paths (as lists of symbols) to linklet bundles for the corresponding
- submodules.}
+ submodules.
+
+ An individual linklet is represented as a @racket[linkl] only if the
+ source bytecode file was for Racket @|BC|. A @CS bytecode linklet will
+ be represented by an opaque linklet (in the sense of @racket[linklet?]
+ from @racketmodname[racket/linklet]). A machine-independent linklet
+ is represented as a @racket[faslable-correlated-linklet] structure.}
 
 @defstruct+[(linkl zo)
             ([name symbol?]

pkgs/zo-lib/compiler/faslable-correlated.rkt

@@ -0,0 +1,21 @@
+#lang racket/base
+
+(provide (struct-out faslable-correlated-linklet)
+         (struct-out faslable-correlated)
+         strip-correlated)
+
+(struct faslable-correlated-linklet (expr name)
+  #:prefab)
+
+(struct faslable-correlated (e source position line column span props)
+  #:prefab)
+
+(define (strip-correlated v)
+  (let strip ([v v])
+    (cond
+      [(pair? v)
+       (cons (strip (car v))
+             (strip (cdr v)))]
+      [(faslable-correlated? v)
+       (strip (faslable-correlated-e v))]
+      [else v])))

pkgs/zo-lib/info.rkt

@@ -8,4 +8,4 @@
 
 (define pkg-authors '(mflatt))
 
-(define version "1.2")
+(define version "1.3")