diff --git a/collects/framework/private/color.rkt b/collects/framework/private/color.rkt index 365c48f0ea..1ade5b4ece 100644 --- a/collects/framework/private/color.rkt +++ b/collects/framework/private/color.rkt @@ -297,6 +297,14 @@ added get-regions (get-token in in-start-pos in-lexer-mode) (enable-suspend #t)))]) (unless (eq? 'eof type) + (unless (exact-nonnegative-integer? new-token-start) + (error 'color:text<%> "expected an exact nonnegative integer for the token start, got ~e" new-token-start)) + (unless (exact-nonnegative-integer? new-token-end) + (error 'color:text<%> "expected an exact nonnegative integer for the token end, got ~e" new-token-end)) + (unless (exact-nonnegative-integer? backup-delta) + (error 'color:text<%> "expected an exact nonnegative integer for the backup delta, got ~e" backup-delta)) + (unless (0 . < . (- new-token-end new-token-start)) + (error 'color:text<%> "expected the distance between the start and end position for each token to be positive, but start was ~e and end was ~e" new-token-start new-token-end)) (enable-suspend #f) #; (printf "~a at ~a to ~a\n" lexeme (+ in-start-pos (sub1 new-token-start)) (+ in-start-pos (sub1 new-token-end))) diff --git a/collects/syntax-color/module-lexer.rkt b/collects/syntax-color/module-lexer.rkt index 50d62272de..82f733a8a8 100644 --- a/collects/syntax-color/module-lexer.rkt +++ b/collects/syntax-color/module-lexer.rkt @@ -1,52 +1,65 @@ -#lang scheme/base -(require scheme/port +#lang racket/base +(require racket/port "scheme-lexer.rkt") (provide module-lexer) +;; mode = (or/c #f 'before-lang-line -- +;; 'no-lang-line +;; (cons lexer mode) +;; lexer) + +(define count 0) + (define (module-lexer in offset mode) + (set! count (+ count 1)) + (printf "~a ~s\n" count (list 'module-lexer in offset mode)) (cond - [(not mode) - ;; Starting out: look for #lang: - (let*-values ([(p) (peeking-input-port in)] - [(init) (file-position p)] - [(start-line start-col start-pos) (port-next-location p)]) - (let ([get-info (with-handlers ([exn:fail? (lambda (exn) 'fail)]) - (read-language p (lambda () #f)))] - [sync-ports (lambda () - (read-bytes (- (file-position p) init) in))]) + [(or (not mode) (eq? mode 'before-lang-line)) + (define-values (lexeme type data new-token-start new-token-end) (scheme-lexer in)) + (printf "before-lang-line lexeme ~s type ~s\n" lexeme type) + (cond + [(or (eq? type 'comment) (eq? type 'whitespace)) + (values lexeme 'other data new-token-start new-token-end 0 'before-lang-line)] + [else + ;; look for #lang: + (define p (peeking-input-port in)) + (port-count-lines! p) + (define init (file-position p)) + (define get-info (with-handlers ([exn:fail:read? values]) (read-language p (λ () 'fail)))) + (printf "get-info ~s\n" get-info) (cond - [(procedure? get-info) - ;; Produce language as first token: - (sync-ports) - (let-values ([(end-line end-col end-pos) (port-next-location in)]) - (values - "#lang" - 'other - #f - start-pos - end-pos - 0 - (or (let ([v (get-info 'color-lexer #f)]) - (and v - (if (procedure-arity-includes? v 3) - (cons v #f) - v))) - scheme-lexer)))] - [(eq? 'fail get-info) - (sync-ports) - (let*-values ([(end-line end-col end-pos) (port-next-location in)]) - (values #f 'error #f start-pos end-pos 0 scheme-lexer))] - [else - ;; Start over using the Scheme lexer - (module-lexer in offset scheme-lexer)])))] - [(pair? mode) - ;; #lang-selected language consumes and produces a mode: - (let-values ([(lexeme type data new-token-start new-token-end backup-delta new-mode) - ((car mode) in offset (cdr mode))]) - (values lexeme type data new-token-start new-token-end backup-delta (cons (car mode) new-mode)))] - [else - ;; #lang-selected language (or default) doesn't deal with modes: - (let-values ([(lexeme type data new-token-start new-token-end) - (mode in)]) - (values lexeme type data new-token-start new-token-end 0 mode))])) + [(not (procedure? get-info)) + ;(or (exn? get-info) (eq? get-info 'fail)) + (values lexeme type data new-token-start new-token-end 0 'no-lang-line)] + [(procedure? get-info) + (define end-pos (file-position p)) + (read-bytes (- end-pos init) in) ;; sync ports + ;; Produce language as first token: + (values + "#lang" + 'other + #f + 1 ;; start-pos + (+ end-pos 1) + 0 + (or (let ([v (get-info 'color-lexer #f)]) + (and v + (if (procedure-arity-includes? v 3) + (cons v #f) + v))) + scheme-lexer))])])] + [(eq? mode 'no-lang-line) + (let-values ([(lexeme type data new-token-start new-token-end) + (scheme-lexer in)]) + (values lexeme type data new-token-start new-token-end 0 'no-lang-line))] + [(pair? mode) + ;; #lang-selected language consumes and produces a mode: + (let-values ([(lexeme type data new-token-start new-token-end backup-delta new-mode) + ((car mode) in offset (cdr mode))]) + (values lexeme type data new-token-start new-token-end backup-delta (cons (car mode) new-mode)))] + [else + ;; #lang-selected language (or default) doesn't deal with modes: + (let-values ([(lexeme type data new-token-start new-token-end) + (mode in)]) + (values lexeme type data new-token-start new-token-end 0 mode))]))