diff --git a/collects/framework/private/color.rkt b/collects/framework/private/color.rkt index 39e57b1c..2164b0c3 100644 --- a/collects/framework/private/color.rkt +++ b/collects/framework/private/color.rkt @@ -11,6 +11,7 @@ added get-regions syntax-color/token-tree syntax-color/paren-tree syntax-color/default-lexer + syntax-color/lexer-contract string-constants "../preferences.rkt" "sig.rkt" @@ -303,7 +304,7 @@ added get-regions (set-lexer-state-invalid-tokens-mode! ls mode)) (sync-invalid ls)))) - (define/private (re-tokenize-move-to-next-ls start-time did-something?) + (define/private (re-tokenize-move-to-next-ls start-time ok-to-stop?) (cond [(null? re-tokenize-lses) ;; done: return #t @@ -317,25 +318,29 @@ added get-regions (lexer-state-end-pos ls) (λ (x) #f))) (port-count-lines! in) - (continue-re-tokenize start-time did-something? ls in + (continue-re-tokenize start-time ok-to-stop? ls in (lexer-state-current-pos ls) (lexer-state-current-lexer-mode ls))])) (define re-tokenize-lses #f) - (define/private (continue-re-tokenize start-time did-something? ls in in-start-pos lexer-mode) + (define/private (continue-re-tokenize start-time ok-to-stop? ls in in-start-pos lexer-mode) (cond - [(and did-something? ((+ start-time 20.0) . <= . (current-inexact-milliseconds))) + [(and ok-to-stop? ((+ start-time 20.0) . <= . (current-inexact-milliseconds))) #f] [else (define-values (_line1 _col1 pos-before) (port-next-location in)) - (define-values (lexeme type data new-token-start new-token-end backup-delta new-lexer-mode) + (define-values (lexeme type data new-token-start new-token-end backup-delta new-lexer-mode/cont) (get-token in in-start-pos lexer-mode)) (define-values (_line2 _col2 pos-after) (port-next-location in)) + (define new-lexer-mode (if (dont-stop? new-lexer-mode/cont) + (dont-stop-val new-lexer-mode/cont) + new-lexer-mode/cont)) + (define next-ok-to-stop? (not (dont-stop? new-lexer-mode/cont))) (cond [(eq? 'eof type) (set-lexer-state-up-to-date?! ls #t) - (re-tokenize-move-to-next-ls start-time #t)] + (re-tokenize-move-to-next-ls start-time next-ok-to-stop?)] [else (unless (<= pos-before new-token-start pos-after) (error 'color:text<%> @@ -369,9 +374,9 @@ added get-regions (lexer-state-invalid-tokens ls)) (set-lexer-state-invalid-tokens-start! ls +inf.0) (set-lexer-state-up-to-date?! ls #t) - (re-tokenize-move-to-next-ls start-time #t)] + (re-tokenize-move-to-next-ls start-time next-ok-to-stop?)] [else - (continue-re-tokenize start-time #t ls in in-start-pos new-lexer-mode)]))])])) + (continue-re-tokenize start-time next-ok-to-stop? ls in in-start-pos new-lexer-mode)]))])])) (define/private (add-colorings type in-start-pos new-token-start new-token-end) (define sp (+ in-start-pos (sub1 new-token-start))) @@ -511,7 +516,11 @@ added get-regions [else (if (lexer-state-up-to-date? (car lexer-states)) (loop (cdr lexer-states)) lexer-states)]))) - (define finished? (re-tokenize-move-to-next-ls (current-inexact-milliseconds) #f)) + (define finished? (re-tokenize-move-to-next-ls + (current-inexact-milliseconds) + ;; #f initially here ensures we do at least + ;; one step of tokenization before giving up + #f)) (c-log (format "coloring stopped ~a" (if finished? "because it finished" "with more to do"))) (when finished? (update-lexer-state-observers) diff --git a/collects/scribblings/framework/color.scrbl b/collects/scribblings/framework/color.scrbl index 184fb0ef..ef15abaf 100644 --- a/collects/scribblings/framework/color.scrbl +++ b/collects/scribblings/framework/color.scrbl @@ -1,7 +1,6 @@ #lang scribble/doc @(require scribble/manual scribble/extract) -@(require (for-label framework)) -@(require (for-label scheme/gui)) +@(require (for-label framework scheme/gui syntax-color/lexer-contract)) @title{Color} @definterface[color:text<%> (text:basic<%>)]{ @@ -17,7 +16,7 @@ (or/c exact-positive-integer? #f))) (-> input-port? exact-nonnegative-integer? - any/c + (not/c dont-stop?) (values any/c symbol? (or/c symbol? #f) @@ -57,20 +56,36 @@ is also relative to the port's location, just like the previous value.}] When @racket[get-token] accepts an offset and mode value in addition to an - input port, it must also return two extra results, which are a backup - distance and new mode. The offset given to @racket[get-token] can be added + input port, it must also return two extra results. + The offset given to @racket[get-token] can be added to the position of the input port to obtain absolute coordinates within a - text stream. The mode argument allows @racket[get-token] to communicate + text stream. The extra two results are + @itemize[@item{a new mode; + The mode argument allows @racket[get-token] to communicate information from earlier parsing to later. When @racket[get-token] is called for the beginning on a stream, the mode argument is @racket[#f]; thereafter, the mode returned for the previous token is provided to - @racket[get-token] for the next token. The mode should not be a mutable + @racket[get-token] for the next token. + + If the mode result is a @racket[dont-stop] struct, then the value inside + the struct is considered the new mode, and the colorer is guaranteed + not to be interrupted until at least the next call to this tokenizing + function that does not return a @racket[dont-stop] struct (unless, of course, + it returns an eof token, in which case the new mode result is ignored). + This is useful, for example, when a lexer has to read ahead in the buffer + to decide on the tokens at this point; then that read-ahead will be + inconsistent if an edit happens; returning a @racket[dont-stop] + struct ensures that no changes to the buffer happen. + + The mode should not be a mutable value; if part of the stream is re-tokenized, the mode saved from the immediately preceding token is given again to the @racket[get-token] - function. The backup distance returned by @racket[get-token] indicates the + function.} + @item{a backup distance; + The backup distance returned by @racket[get-token] indicates the maximum number of characters to back up (counting from the start of the token) and for re-parsing after a change to the editor within the token's - region. + region.}] The @racket[get-token] function must obey the following invariants: @itemize[