extend the lexer <-> framework's color:text api

to let lexers say "call me again before you change the buffer"

also, use this in the 2d lexer

original commit: 761054890d8207d7689cce825d0fc777d15b1091

collects/framework/private/color.rkt

@@ -11,6 +11,7 @@ added get-regions
          syntax-color/token-tree
          syntax-color/paren-tree
          syntax-color/default-lexer
+         syntax-color/lexer-contract
          string-constants
          "../preferences.rkt"
          "sig.rkt"
@@ -303,7 +304,7 @@ added get-regions
             (set-lexer-state-invalid-tokens-mode! ls mode))
           (sync-invalid ls))))
     
-    (define/private (re-tokenize-move-to-next-ls start-time did-something?)
+    (define/private (re-tokenize-move-to-next-ls start-time ok-to-stop?)
       (cond
         [(null? re-tokenize-lses) 
          ;; done: return #t
@@ -317,25 +318,29 @@ added get-regions
                                    (lexer-state-end-pos ls)
                                    (λ (x) #f)))
          (port-count-lines! in)
-         (continue-re-tokenize start-time did-something? ls in
+         (continue-re-tokenize start-time ok-to-stop? ls in
                                (lexer-state-current-pos ls)
                                (lexer-state-current-lexer-mode ls))]))
     
     (define re-tokenize-lses #f)
     
-    (define/private (continue-re-tokenize start-time did-something? ls in in-start-pos lexer-mode)
+    (define/private (continue-re-tokenize start-time ok-to-stop? ls in in-start-pos lexer-mode)
       (cond
-        [(and did-something? ((+ start-time 20.0) . <= . (current-inexact-milliseconds)))
+        [(and ok-to-stop? ((+ start-time 20.0) . <= . (current-inexact-milliseconds)))
          #f]
         [else
          (define-values (_line1 _col1 pos-before) (port-next-location in))
-         (define-values (lexeme type data new-token-start new-token-end backup-delta new-lexer-mode) 
+         (define-values (lexeme type data new-token-start new-token-end backup-delta new-lexer-mode/cont) 
            (get-token in in-start-pos lexer-mode))
          (define-values (_line2 _col2 pos-after) (port-next-location in))
+         (define new-lexer-mode (if (dont-stop? new-lexer-mode/cont)
+                                    (dont-stop-val new-lexer-mode/cont)
+                                    new-lexer-mode/cont))
+         (define next-ok-to-stop? (not (dont-stop? new-lexer-mode/cont)))
          (cond
            [(eq? 'eof type) 
             (set-lexer-state-up-to-date?! ls #t)
-            (re-tokenize-move-to-next-ls start-time #t)]
+            (re-tokenize-move-to-next-ls start-time next-ok-to-stop?)]
            [else
             (unless (<= pos-before new-token-start pos-after)
               (error 'color:text<%>
@@ -369,9 +374,9 @@ added get-regions
                                (lexer-state-invalid-tokens ls))
                  (set-lexer-state-invalid-tokens-start! ls +inf.0)
                  (set-lexer-state-up-to-date?! ls #t)
-                 (re-tokenize-move-to-next-ls start-time #t)]
+                 (re-tokenize-move-to-next-ls start-time next-ok-to-stop?)]
                 [else
-                 (continue-re-tokenize start-time #t ls in in-start-pos new-lexer-mode)]))])]))
+                 (continue-re-tokenize start-time next-ok-to-stop? ls in in-start-pos new-lexer-mode)]))])]))
 
     (define/private (add-colorings type in-start-pos new-token-start new-token-end)
       (define sp (+ in-start-pos (sub1 new-token-start)))
@@ -511,7 +516,11 @@ added get-regions
                                    [else (if (lexer-state-up-to-date? (car lexer-states))
                                              (loop (cdr lexer-states))
                                              lexer-states)])))
-        (define finished? (re-tokenize-move-to-next-ls (current-inexact-milliseconds) #f))
+        (define finished? (re-tokenize-move-to-next-ls 
+                           (current-inexact-milliseconds) 
+                           ;; #f initially here ensures we do at least
+                           ;; one step of tokenization before giving up
+                           #f))
         (c-log (format "coloring stopped ~a" (if finished? "because it finished" "with more to do")))
         (when finished?
           (update-lexer-state-observers)

collects/scribblings/framework/color.scrbl

@@ -1,7 +1,6 @@
 #lang scribble/doc
 @(require scribble/manual scribble/extract)
-@(require (for-label framework))
+@(require (for-label framework scheme/gui syntax-color/lexer-contract))
-@(require (for-label scheme/gui))
 @title{Color}
 
 @definterface[color:text<%> (text:basic<%>)]{
@@ -17,7 +16,7 @@
                                                          (or/c exact-positive-integer? #f)))
                                              (-> input-port? 
                                                  exact-nonnegative-integer?
-                                                 any/c
+                                                 (not/c dont-stop?)
                                                  (values any/c 
                                                          symbol? 
                                                          (or/c symbol? #f)
@@ -57,20 +56,36 @@
           is also relative to the port's location, just like the previous value.}]
 
     When @racket[get-token] accepts an offset and mode value in addition to an
-    input port, it must also return two extra results, which are a backup
+    input port, it must also return two extra results.
-    distance and new mode. The offset given to @racket[get-token] can be added
+    The offset given to @racket[get-token] can be added
     to the position of the input port to obtain absolute coordinates within a
-    text stream. The mode argument allows @racket[get-token] to communicate
+    text stream. The extra two results are
+    @itemize[@item{a new mode; 
+    The mode argument allows @racket[get-token] to communicate
     information from earlier parsing to later.  When @racket[get-token] is
     called for the beginning on a stream, the mode argument is @racket[#f];
     thereafter, the mode returned for the previous token is provided to
-    @racket[get-token] for the next token. The mode should not be a mutable
+    @racket[get-token] for the next token. 
+    
+    If the mode result is a @racket[dont-stop] struct, then the value inside
+    the struct is considered the new mode, and the colorer is guaranteed
+    not to be interrupted until at least the next call to this tokenizing
+    function that does not return a @racket[dont-stop] struct (unless, of course,
+    it returns an eof token, in which case the new mode result is ignored).
+    This is useful, for example, when a lexer has to read ahead in the buffer
+    to decide on the tokens at this point; then that read-ahead will be 
+    inconsistent if an edit happens; returning a @racket[dont-stop]
+    struct ensures that no changes to the buffer happen.
+    
+    The mode should not be a mutable
     value; if part of the stream is re-tokenized, the mode saved from the
     immediately preceding token is given again to the @racket[get-token]
-    function. The backup distance returned by @racket[get-token] indicates the
+    function.}
+    @item{a backup distance;
+    The backup distance returned by @racket[get-token] indicates the
     maximum number of characters to back up (counting from the start of the
     token) and for re-parsing after a change to the editor within the token's
-    region.
+    region.}]
 
     The @racket[get-token] function must obey the following invariants:
     @itemize[