Epsilon transitions in lexer

Ast.cs

@@ -42,6 +42,8 @@ namespace Ast {
     public override int GetHashCode() {
         return HashCode.Combine("Int", this.value);
     }
+
+    public override string ToString() => "Int";
   }
 
   public partial class Visitor<T> { public Func<string, T> String { get; set; } }
@@ -62,6 +64,8 @@ namespace Ast {
     public override int GetHashCode() {
         return HashCode.Combine("String", this.value);
     }
+
+    public override string ToString() => "String";
   }
 
 }

Lexer.cs

@@ -78,10 +78,9 @@ public static partial class Lexer {
       Rule(S.Space,   '"',                  S.StringOpen, S.String),
 
       Rule(S.Int,     C.DecimalDigitNumber, S.Int),
-      Rule(S.Int,     C.SpaceSeparator,     S.Space), // epsilon
+      Rule(S.Int,     C.SpaceSeparator,     S.Space),
       Rule(S.Int,     new[]{'.',','},       S.Decimal, S.Int),
-      Rule(S.Int,     EOF,                  S.End),   // epsilon
+      Rule(S.Decimal, C.SpaceSeparator,     S.Space),
-      Rule(S.Decimal, C.SpaceSeparator,     S.Space), // epsilon
       
       Rule(S.String,  C.LowercaseLetter,    S.String),
       Rule(S.String,  C.UppercaseLetter,    S.String),
@@ -92,8 +91,22 @@ public static partial class Lexer {
     public static Dictionary<S, List<Rule>> Dict =
       Default
         .GroupBy(r => r.oldState, r => r)
-        .ToDictionary(rs => rs.Key, rs => rs.ToList()) ;
+        .ToDefaultDictionary(
-    // TODO: upon failure, do an epsilon-transition to the whitespace state, and try again.
+          new List<Rule>(),
+          rs => rs.Key,
+          rs => rs.ToList()) ;
+
+    // This adds transitions through an implicit empty whitespace.
+    public static Dictionary<S, List<Rule>> WithEpsilonTransitions =
+      Dict.ToDefaultDictionary(
+        new List<Rule>(),
+        kv => kv.Key,
+        kv => kv.Value.Any(r => true) // r.test(" ")
+              // This is a bit of a hack, the lexer tries the rules in
+              // order so later rules with different results are masked
+              // by former rules
+              ? kv.Value.Concat(Dict[S.Space]).ToList()
+              : kv.Value);
   }
 
   public struct Lexeme {
@@ -125,7 +138,7 @@ public static partial class Lexer {
     return result;
   }
 
-  public static void ParseError(StringBuilder context, IEnumerator<GraphemeCluster> stream, S state, List<Rule> possibleNext, GraphemeCluster gc) {
+  public static Exception ParseError(StringBuilder context, IEnumerator<GraphemeCluster> stream, S state, List<Rule> possibleNext, GraphemeCluster gc) {
     var rest =
       stream
         .SingleUseEnumerable()
@@ -139,7 +152,7 @@ public static partial class Lexer {
                  .First()
                  .Match(some: (x => x.UnicodeCategory(0).ToString()),
                         none: "None (empty string)");
-    throw new Exception(
+    return new Exception(
       $"Unexpected {actual} (Unicode category {cat}) while the lexer was in state {state}: expected one of {expected}{Environment.NewLine}{context}  <--HERE  {rest}"
     );
   }
@@ -155,15 +168,13 @@ public static partial class Lexer {
     while (e.MoveNext()) {
       var c = e.Current;
       context.Append(c.str);
-      List<Rule> possibleNext;
+      var possibleNext = Rules.WithEpsilonTransitions
-      if (Rules.Dict.TryGetValue(state, out possibleNext)) {
+                           .GetOrDefault(state, new List<Rule>());
-        var rule = possibleNext.FirstOrDefault(r => r.test(c));
+      yield return
-        if (rule != null) {
+        possibleNext
-          yield return Transition(ref state, ref lexeme, c, rule);
+          .First(r => r.test(c))
-        } else {
+          .IfSome(rule => Transition(ref state, ref lexeme, c, rule))
-          ParseError(context, e, state, possibleNext, c);
+          .ElseThrow(() => ParseError(context, e, state, possibleNext, c));
-        }
-      }
     }
   }
 

LexerGenerated.cs

@@ -56,6 +56,8 @@ public static partial class Lexer {
     public override int GetHashCode() {
         return "C".GetHashCode();
     }
+
+    public override string ToString() => "End";
   }
 
   public partial class Visitor<T> { public Func<T> Space { get; set; } }
@@ -70,6 +72,8 @@ public static partial class Lexer {
     public override int GetHashCode() {
         return "C".GetHashCode();
     }
+
+    public override string ToString() => "Space";
   }
 
   public partial class Visitor<T> { public Func<T> Int { get; set; } }
@@ -84,6 +88,8 @@ public static partial class Lexer {
     public override int GetHashCode() {
         return "C".GetHashCode();
     }
+
+    public override string ToString() => "Int";
   }
 
   public partial class Visitor<T> { public Func<T> Decimal { get; set; } }
@@ -98,6 +104,8 @@ public static partial class Lexer {
     public override int GetHashCode() {
         return "C".GetHashCode();
     }
+
+    public override string ToString() => "Decimal";
   }
 
   public partial class Visitor<T> { public Func<T> String { get; set; } }
@@ -112,6 +120,8 @@ public static partial class Lexer {
     public override int GetHashCode() {
         return "C".GetHashCode();
     }
+
+    public override string ToString() => "String";
   }
 
   public partial class Visitor<T> { public Func<T> StringOpen { get; set; } }
@@ -126,6 +136,8 @@ public static partial class Lexer {
     public override int GetHashCode() {
         return "C".GetHashCode();
     }
+
+    public override string ToString() => "StringOpen";
   }
 
   public partial class Visitor<T> { public Func<T> StringClose { get; set; } }
@@ -140,6 +152,8 @@ public static partial class Lexer {
     public override int GetHashCode() {
         return "C".GetHashCode();
     }
+
+    public override string ToString() => "StringClose";
   }
 
 }

T4/Generator.cs

@@ -90,6 +90,8 @@ public static class Generator {
         o.WriteLine($"        return HashCode.Combine(\"{C}\", this.value);");
       }
       o.WriteLine($"    }}");
+      o.WriteLine("");
+      o.WriteLine($"    public override string ToString() => \"{C}\";");
       o.WriteLine($"  }}");
       o.WriteLine("");
     }

Tests/004-foo42.e

@@ -0,0 +1 @@
+"foo"42

Tests/004-foo42.o

@@ -0,0 +1 @@
+foo42

Tests/005-42foo.e

@@ -0,0 +1 @@
+42"foo"

Tests/005-42foo.o

@@ -0,0 +1 @@
+42foo

Utils/Enumerable.cs

@@ -128,6 +128,17 @@ public static class Collection {
     }
   }
 
+  public static Option<T> First<T>(this IEnumerable<T> ie, Func<T, bool> predicate) {
+    var e = ie.GetEnumerator();
+    bool found = false;
+    while (e.MoveNext()) {
+      if (predicate(e.Current)) {
+        return e.Current.Some();
+      }
+    }
+    return Option.None<T>();
+  }
+
   public static Option<T> Single<T>(this IEnumerable<T> ie) {
     var e = ie.GetEnumerator();
     if (e.MoveNext()) {
@@ -141,4 +152,41 @@ public static class Collection {
       return Option.None<T>();
     }
   }
+
+  public static Option<V> GetValue<K, V>(this Dictionary<K, V> d, K key) {
+    V result = default(V);
+    if (d.TryGetValue(key, out result)) {
+      return result.Some();
+    } else {
+      return Option.None<V>();
+    }
+  }
+
+  public static V GetOrDefault<K, V>(this Dictionary<K, V> d, K key, V defaultValue) {
+    V result = default(V);
+    if (d.TryGetValue(key, out result)) {
+      return result;
+    } else {
+      return defaultValue;
+    }
+  }
+
+  public class DefaultDictionary<TKey, TValue> : Dictionary<TKey, TValue> {
+    public readonly TValue defaultValue;
+    //public readonly Dictionary<TKey, TValue> dictionary;
+    
+    public DefaultDictionary(TValue defaultValue, Dictionary<TKey, TValue> dictionary) : base(dictionary) {
+      this.defaultValue = defaultValue;
+      //this.dictionary = dictionary;
+    }
+    
+    public TValue this[TKey key] {
+      get {
+        return this.GetOrDefault(key, defaultValue);
+      }
+    }
+  }
+
+  public static DefaultDictionary<UKey, UValue> ToDefaultDictionary<T, UKey, UValue>(this IEnumerable<T> e, UValue defaultValue, Func<T, UKey> key, Func<T, UValue> value)
+    => new DefaultDictionary<UKey, UValue>(defaultValue, e.ToDictionary(key, value));
 }

Utils/Immutable/Option.cs

@@ -1,7 +1,7 @@
 namespace Immutable {
   using System;
 
-  public interface Option<out T> {
+  public interface Option<out T> : System.Collections.Generic.IEnumerable<T> {
     U Match_<U>(Func<T, U> some, Func<U> none);
   }
 
@@ -10,18 +10,30 @@ namespace Immutable {
     public static Option<T> None<T>() => new Types.None<T>();
 
     private static class Types {
-      public class Some<T> : Option<T> {
+      public class Some<T> : Option<T>, System.Collections.IEnumerable {
         public readonly T value;
 
         public Some(T value) { this.value = value; }
 
         public U Match_<U>(Func<T, U> Some, Func<U> None) => Some(value);
+
+        public System.Collections.Generic.IEnumerator<T> GetEnumerator()
+          => value.Singleton().GetEnumerator();
+        
+        System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+          => this.GetEnumerator();
       }
 
-      public class None<T> : Option<T> {
+      public class None<T> : Option<T>, System.Collections.IEnumerable {
         public None() { }
 
         public U Match_<U>(Func<T, U> Some, Func<U> None) => None();
+
+        public System.Collections.Generic.IEnumerator<T> GetEnumerator()
+          => System.Linq.Enumerable.Empty<T>().GetEnumerator();
+        
+        System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+          => this.GetEnumerator();
       }
     }
   }
@@ -34,5 +46,20 @@ namespace Immutable {
 
     public static U Match<T, U>(this Option<T> o, Func<T, U> some, U none)
       => o.Match_(some, () => none);
+
+    public static Option<U> Map<T, U>(this Option<T> o, Func<T, U> some)
+      => o.Match_(value => some(value).Some(), () => Option.None<U>());
+
+    public static Option<U> IfSome<T, U>(this Option<T> o, Func<T, U> some)
+      => o.Map(some);
+
+    public static T Else<T>(this Option<T> o, Func<T> none)
+      => o.Match_(some => some, none);
+
+    public static Option<T> Else<T>(this Option<T> o, Func<Option<T>> none)
+      => o.Match_(value => value.Some(), none);
+
+    public static T ElseThrow<T>(this Option<T> o, Func<Exception> none)
+      => o.Match_(value => value, () => throw none());
   }
 }

Utils/Unicode.cs

@@ -12,6 +12,9 @@ public struct GraphemeCluster {
     this.str = str;
     this.codePoints = codePoints;
   }
+
+  public static implicit operator GraphemeCluster(char c)
+    => new GraphemeCluster(false, c.ToString(), c.ToString().Singleton());
 }
 
 public static class UnicodeExtensionMethods {