suzanne.soy/2d
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix unstable/2d's lexer to handle eof better

Browse Source 
original commit: 87a8e6f677d9538001766910ba119dde8066b400
This commit is contained in:
Robby Findler 2013-02-25 10:36:46 -06:00
parent 1b75e51175
commit b49c680dd5
3 changed files with 182 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
collects/unstable/2d/private/lexer.rkt
View File

@ -162,11 +162,9 @@ todo:
;; but that works only when there are no broken
;; edges of the table that span the place I want to stop.
(define failed
(with-handlers ((exn:fail:read?
(λ (exn) exn)))
(with-handlers ((exn:fail:read? values))
(let loop ([map #f])
(define new-map
;; this might raise a read exception: what then?
(parse-2dcond-one-step peek-port (object-name peek-port) #f #f pos the-state map))
(when new-map
(loop new-map)))))
@ -177,17 +175,36 @@ todo:
;; no matter how long eol-string is, it counts for 1 position only.
(+ pos (string-length first-tok-string) 1)))
(cond
[(exn:fail:read:eof? failed)
;; in this case, the source location for the error
;; should be the beginning of the #2d token,
;; so we just turn the whole thing red in a single token
(define tok-string
(string-append
first-tok-string
(apply string
(let loop ()
(define c (read-char port))
(cond
[(eof-object? c) '()]
[else (cons c (loop))])))))
(values tok-string 'error #f
pos (+ pos (string-length tok-string))
0
#f)]
[else
(define final-tokens
(cond
[(exn:fail:read? failed)
(define error-pos (- (srcloc-position (car (exn:fail:read-srclocs failed)))
base-position)) ;; account for the newline
(when (< error-pos 0)
(error 'unstable/2d/lexer.rkt "got error-pos < 0: ~s ~s\n"
(srcloc-position (car (exn:fail:read-srclocs failed)))
base-position))
(define peek-port2 (peeking-input-port port))
(port-count-lines! peek-port2)
;; pull the newline out of peek-port2
(for ([x (in-range (string-length eol-string))]) (read-char peek-port2))
(define (pull-chars n)
(apply
string
@ -195,6 +212,13 @@ todo:
(cond
[(zero? n) '()]
[else (cons (read-char peek-port2) (loop (- n 1)))]))))
(cond
[else
;; pull the newline out of peek-port2
(for ([x (in-range (string-length eol-string))]) (read-char peek-port2))
(define before-token (list (pull-chars error-pos)
'no-color
#f
@ -217,7 +241,9 @@ todo:
#f
(+ base-position 1 error-pos)
(+ base-position 1 error-pos end-of-table-approx -1)))
(list newline-token before-token after-token)]
(if (zero? error-pos)
(list newline-token after-token)
(list newline-token before-token after-token))])]
[else
(define lhses (close-cell-graph cell-connections (length table-column-breaks) (length rows)))
@ -323,7 +349,7 @@ todo:
0
(2d-lexer-state final-tokens
#t
(2d-lexer-state-chained-state a-2d-lexer-state)))]))
(2d-lexer-state-chained-state a-2d-lexer-state)))])]))
(define (cropped-regions start end regions)
(define result-regions '())

4
collects/unstable/2d/private/read-util.rkt
View File

@ -314,9 +314,9 @@ example uses:
(cond
[(eof-object? c)
(raise-read-eof-error
"expected eof; "
"unexpected eof; "
source _line _col _pos
(and _pos (- _pos (+ current-line-start-position chars-read))))]
(and _pos (- (+ current-line-start-position chars-read) _pos)))]
[(equal? c #\return)
(cond
[(equal? #\newline (peek-char port))

9
collects/unstable/2d/private/readtable.rkt
View File

@ -31,7 +31,14 @@ example uses:
(case-lambda
[(char port)
(define-values (line col pos) (port-next-location port))
(dispatch-proc char port #f line col pos read/recursive previous-readtable)]
;; the "-2"s here are because the initial line and column
;; are supposed be at the beginning of the thing read, not
;; after the "#2" has been consumed.
(dispatch-proc char port #f line
(and col (- col 2))
(and pos (- pos 2))
read/recursive previous-readtable)]
[(char port source _line _col _pos)
(dispatch-proc char port source _line _col _pos
(λ (a b c) (read-syntax/recursive source a b c))