fix unstable/2d's lexer to handle eof better

original commit: 87a8e6f677d9538001766910ba119dde8066b400

collects/unstable/2d/private/lexer.rkt

@@ -162,11 +162,9 @@ todo:
      ;; but that works only when there are no broken
      ;; edges of the table that span the place I want to stop.
      (define failed
-       (with-handlers ((exn:fail:read? 
+       (with-handlers ((exn:fail:read? values))
-                        (λ (exn) exn)))
          (let loop ([map #f])
            (define new-map
-             ;; this might raise a read exception: what then?
              (parse-2dcond-one-step peek-port (object-name peek-port) #f #f pos the-state map))
            (when new-map
              (loop new-map)))))
@@ -177,17 +175,36 @@ todo:
              ;; no matter how long eol-string is, it counts for 1 position only.
              (+ pos (string-length first-tok-string) 1)))
      
+     (cond
+       [(exn:fail:read:eof? failed)
+        ;; in this case, the source location for the error
+        ;; should be the beginning of the #2d token, 
+        ;; so we just turn the whole thing red in a single token
+        (define tok-string 
+          (string-append
+           first-tok-string
+           (apply string
+                  (let loop ()
+                    (define c (read-char port))
+                    (cond
+                      [(eof-object? c) '()]
+                      [else (cons c (loop))])))))
+        (values tok-string 'error #f
+                pos (+ pos (string-length tok-string))
+                0
+                #f)]
+       [else
         (define final-tokens
           (cond
             [(exn:fail:read? failed)
              (define error-pos (- (srcloc-position (car (exn:fail:read-srclocs failed)))
                                   base-position)) ;; account for the newline
+             (when (< error-pos 0)
+               (error 'unstable/2d/lexer.rkt "got error-pos < 0: ~s ~s\n" 
+                      (srcloc-position (car (exn:fail:read-srclocs failed)))
+                      base-position))
              (define peek-port2 (peeking-input-port port))
              (port-count-lines! peek-port2)
-          
-          ;; pull the newline out of peek-port2
-          (for ([x (in-range (string-length eol-string))]) (read-char peek-port2))
-          
              (define (pull-chars n)
                (apply
                 string
@@ -195,6 +212,13 @@ todo:
                   (cond
                     [(zero? n) '()]
                     [else (cons (read-char peek-port2) (loop (- n 1)))]))))
+             (cond
+               
+               [else
+                
+                ;; pull the newline out of peek-port2
+                (for ([x (in-range (string-length eol-string))]) (read-char peek-port2))
+                
                 (define before-token (list (pull-chars error-pos)
                                            'no-color
                                            #f
@@ -217,7 +241,9 @@ todo:
                         #f
                         (+ base-position 1 error-pos)
                         (+ base-position 1 error-pos end-of-table-approx -1)))
-          (list newline-token before-token after-token)]
+                (if (zero? error-pos)
+                    (list newline-token after-token)
+                    (list newline-token before-token after-token))])]
             [else
              
              (define lhses (close-cell-graph cell-connections (length table-column-breaks) (length rows)))
@@ -323,7 +349,7 @@ todo:
                 0
                 (2d-lexer-state final-tokens
                                 #t
-                             (2d-lexer-state-chained-state a-2d-lexer-state)))]))
+                                (2d-lexer-state-chained-state a-2d-lexer-state)))])]))
 
 (define (cropped-regions start end regions)
   (define result-regions '())

collects/unstable/2d/private/read-util.rkt

@@ -314,9 +314,9 @@ example uses:
         (cond
           [(eof-object? c) 
            (raise-read-eof-error
-            "expected eof; "
+            "unexpected eof; "
             source _line _col _pos 
-            (and _pos (- _pos (+ current-line-start-position chars-read))))]
+            (and _pos (- (+ current-line-start-position chars-read) _pos)))]
           [(equal? c #\return)
            (cond
              [(equal? #\newline (peek-char port))

collects/unstable/2d/private/readtable.rkt

@@ -31,7 +31,14 @@ example uses:
    (case-lambda
      [(char port)
       (define-values (line col pos) (port-next-location port))
-      (dispatch-proc char port #f line col pos read/recursive previous-readtable)]
+      
+      ;; the "-2"s here are because the initial line and column
+      ;; are supposed be at the beginning of the thing read, not
+      ;; after the "#2" has been consumed.
+      (dispatch-proc char port #f line 
+                     (and col (- col 2))
+                     (and pos (- pos 2))
+                     read/recursive previous-readtable)]
      [(char port source _line _col _pos)
       (dispatch-proc char port source _line _col _pos 
                      (λ (a b c) (read-syntax/recursive source a b c))