Lexer now uses a proper automaton.

Ast.cs

@@ -1,42 +1,79 @@
 using System;
-using Ast;
 
-public class Visitor<T> {
+// This file was generated by Generator.cs
-  public Func<int, T> Int { get; set; }
-  public Func<string, T> String { get; set; }
-}
 
 namespace Ast {
-  public interface Expr {
+
-    T Match_<T>(Visitor<T> c);
+  /* To match against an instance of Expr, write:
+     x.Match(
+       Int: value => throw new NotImplementedYetException(),,
+       String: value => throw new NotImplementedYetException(),
+     )
+  */
+  public abstract class Expr {
+    public abstract T Match_<T>(Visitor<T> c);
+    public static Expr Int(int value) => new Int(value);
+    public static Expr String(string value) => new String(value);
+    public virtual Immutable.Option<Int> AsInt() => Immutable.Option.None<Int>();
+    public virtual Immutable.Option<String> AsString() => Immutable.Option.None<String>();
+    private string GetTag() {
+      return this.Match(
+        Int: value => "Int",
+        String: value => "String"
+      );
+    }
   }
 
-  public abstract class Const<T> : Expr {
+  public partial class Visitor<T> { public Func<int, T> Int { get; set; } }
-    public readonly T value;
-    public Const(T value) { this.value = value; }
-    public abstract U Match_<U>(Visitor<U> c);
-  }
 
-  public class Int : Const<int> {
+  public sealed class Int : Expr {
-    public Int(int x) : base(x) {}
+    public readonly int value;
+    public Int(int value) { this.value = value; }
     public override T Match_<T>(Visitor<T> c) => c.Int(value);
+    public override Immutable.Option<Int> AsInt() => Immutable.Option.Some<Int>(this);
+    public override bool Equals(object other) {
+      var cast = other as Int;
+      if (Object.ReferenceEquals(cast, null)) {
+        return false;
+      } else {
+        return Equality.Field<Int, int>(this, cast, x => x.value, (x, y) => ((Object)x).Equals(y));
+      }
+    }
+    public override int GetHashCode() {
+        return HashCode.Combine("Int", this.value);
+    }
   }
 
-  public class String : Const<string> {
+  public partial class Visitor<T> { public Func<string, T> String { get; set; } }
-    public String(string x) : base(x) {}
+
+  public sealed class String : Expr {
+    public readonly string value;
+    public String(string value) { this.value = value; }
     public override T Match_<T>(Visitor<T> c) => c.String(value);
+    public override Immutable.Option<String> AsString() => Immutable.Option.Some<String>(this);
+    public override bool Equals(object other) {
+      var cast = other as String;
+      if (Object.ReferenceEquals(cast, null)) {
+        return false;
+      } else {
+        return Equality.Field<String, string>(this, cast, x => x.value, (x, y) => ((Object)x).Equals(y));
+      }
+    }
+    public override int GetHashCode() {
+        return HashCode.Combine("String", this.value);
+    }
   }
-}
 
-public static class AstExtensionMethods {
+}
+public static class ExprExtensionMethods {
   public static T Match<T>(
       this Ast.Expr e,
       Func<int, T> Int,
       Func<string, T> String
     ) {
-    return e.Match_(new Visitor<T> {
+    return e.Match_(new Ast.Visitor<T> {
       Int = Int,
       String = String
     });
   }
-}
+}

AstGenerator.cs

@@ -0,0 +1,12 @@
+// Run with: sh ./T4/Generators.sh
+
+using System.Collections.Generic;
+
+public static class AstGenerator {
+  public static void Generate() {
+    Generator.Generate("Ast.cs", "namespace Ast {", "}", "Ast.", "Expr", new Dictionary<string, string> {
+      { "Int", "int" },
+      { "String", "string" },
+    });
+  }
+}

Lexer.cs

@@ -0,0 +1,181 @@
+using System;
+using System.Text;
+using System.Collections.Generic;
+using System.Linq;
+using Immutable;
+using System.Globalization;
+using C = System.Globalization.UnicodeCategory;
+using static Global;
+
+public static partial class Lexer {
+  public class Rule {
+    public readonly S oldState;
+    public readonly string description;
+    public readonly Func<GraphemeCluster, bool> test;
+    public readonly S throughState;
+    public readonly S newState;
+
+    public Rule(S oldState, string description, Func<GraphemeCluster, bool> test, S throughState, S newState) {
+      this.oldState = oldState;
+      this.description = description;
+      this.test = test;
+      this.throughState = throughState;
+      this.newState = newState;
+    }
+  }
+
+  public sealed class EOF { }
+
+  public static class Rules {
+    private static Rule Rule(S oldState, UnicodeCategory cat, S throughState, S newState = null)
+      => new Rule(
+           oldState,
+           cat.ToString(),
+           c => c.codePoints
+                 .First()
+                 .Match(some: (x => x.UnicodeCategory(0) == cat),
+                        none: false),
+           throughState,
+           newState ?? throughState);
+
+    private static Rule Rule(S oldState, EOF eof, S throughState, S newState = null)
+      => new Rule(
+           oldState,
+           "End of file",
+           c => c.endOfFile,
+           throughState,
+           newState ?? throughState);
+
+    private static string CharDescription(char c)
+      => (c == '"') ? "'\"'" : $"\"{c.ToString()}\"";
+
+    private static Rule Rule(S oldState, char c, S throughState, S newState = null)
+      => new Rule(
+           oldState,
+           CharDescription(c),
+           x => x.codePoints
+                 .Single()
+                 .Match(some: xx => xx == c.ToString(),
+                        none: false),
+           throughState,
+           newState ?? throughState);
+
+    private static Rule Rule(S oldState, char[] cs, S throughState, S newState = null) {
+      var csl = cs.Select(x => x.ToString()).ToList();
+      return new Rule(
+        oldState,
+        ", ".Join(cs.Select(CharDescription)),
+        x => x.codePoints.Single().Match(some: csl.Contains, none: false),
+        throughState,
+        newState ?? throughState);
+    }
+
+    public static EOF EOF = new EOF();
+    public static List<Rule> Default = new List<Rule> {
+      Rule(S.Space,   C.DecimalDigitNumber, S.Int),
+      Rule(S.Space,   C.SpaceSeparator,     S.Space),
+      Rule(S.Space,   EOF,                  S.End),
+      Rule(S.Space,   '"',                  S.StringOpen, S.String),
+
+      Rule(S.Int,     C.DecimalDigitNumber, S.Int),
+      Rule(S.Int,     C.SpaceSeparator,     S.Space), // epsilon
+      Rule(S.Int,     new[]{'.',','},       S.Decimal, S.Int),
+      Rule(S.Int,     EOF,                  S.End),   // epsilon
+      Rule(S.Decimal, C.SpaceSeparator,     S.Space), // epsilon
+      
+      Rule(S.String,  C.LowercaseLetter,    S.String),
+      Rule(S.String,  C.UppercaseLetter,    S.String),
+      Rule(S.String,  C.DecimalDigitNumber, S.String),
+      Rule(S.String,  '"',                  S.StringClose, S.Space),
+    };
+
+    public static Dictionary<S, List<Rule>> Dict =
+      Default
+        .GroupBy(r => r.oldState, r => r)
+        .ToDictionary(rs => rs.Key, rs => rs.ToList()) ;
+    // TODO: upon failure, do an epsilon-transition to the whitespace state, and try again.
+  }
+
+  public struct Lexeme {
+    public readonly S state;
+    // TODO: maybe keep this as a list of grapheme clusters
+    public readonly string lexeme;
+    public Lexeme(S state, string lexeme) {
+      this.state = state;
+      this.lexeme = lexeme;
+    }
+    public override string ToString() {
+      return $"new Lexeme({state}, \"{lexeme}\")";
+    }
+  }
+
+  private static IEnumerable<Lexeme> Transition(ref S state, ref string lexeme, GraphemeCluster c, Rule rule) {
+    List<Lexeme> result = new List<Lexeme>();
+    if (rule.throughState != state) {
+      result.Add(new Lexeme(state, lexeme));
+      state = rule.throughState;
+      lexeme = "";
+    }
+    lexeme += c.str;
+    if (rule.newState != state) {
+      result.Add(new Lexeme(state, lexeme));
+      state = rule.newState;
+      lexeme = "";
+    }
+    return result;
+  }
+
+  public static void ParseError(StringBuilder context, IEnumerator<GraphemeCluster> stream, S state, List<Rule> possibleNext, GraphemeCluster gc) {
+    var rest =
+      stream
+        .SingleUseEnumerable()
+        .TakeUntil(c => c.str.StartsWith("\n"))
+        .Select(c => c.str)
+        .Aggregate(new StringBuilder(), Append);
+
+    var expected = ", ".Join(possibleNext.Select(p => p.description));
+    var actual = (gc.endOfFile ? "" : "grapheme cluster ") + gc.Description();
+    var cat = gc.codePoints
+                 .First()
+                 .Match(some: (x => x.UnicodeCategory(0).ToString()),
+                        none: "None (empty string)");
+    throw new Exception(
+      $"Unexpected {actual} (Unicode category {cat}) while the lexer was in state {state}: expected one of {expected}{Environment.NewLine}{context}  <--HERE  {rest}"
+    );
+  }
+
+  // fake Unicode category
+  private const UnicodeCategory EndOfFile = (UnicodeCategory)(-1);
+
+  public static IEnumerable<IEnumerable<Lexeme>> Lex1(string source) {
+    var context = new StringBuilder();
+    var lexeme = "";
+    var state = S.Space;
+    var e = source.TextElements().GetEnumerator();
+    while (e.MoveNext()) {
+      var c = e.Current;
+      context.Append(c.str);
+      List<Rule> possibleNext;
+      if (Rules.Dict.TryGetValue(state, out possibleNext)) {
+        var rule = possibleNext.FirstOrDefault(r => r.test(c));
+        if (rule != null) {
+          yield return Transition(ref state, ref lexeme, c, rule);
+        } else {
+          ParseError(context, e, state, possibleNext, c);
+        }
+      }
+    }
+  }
+
+  public static IEnumerable<Lexeme> Lex(string source) {
+    var first = true;
+    foreach (var x in Lex1(source).SelectMany(x => x)) {
+      if (first && "".Equals(x.lexeme)) {
+        // skip the initial empty whitespace
+      } else {
+        first = false;
+        yield return x;
+      }
+    }
+  }
+}

LexerGenerated.cs

@@ -0,0 +1,167 @@
+using System;
+
+// This file was generated by Generator.cs
+
+public static partial class Lexer {
+
+  /* To match against an instance of S, write:
+     x.Match(
+       End: () => throw new NotImplementedYetException(),,
+       Space: () => throw new NotImplementedYetException(),,
+       Int: () => throw new NotImplementedYetException(),,
+       Decimal: () => throw new NotImplementedYetException(),,
+       String: () => throw new NotImplementedYetException(),,
+       StringOpen: () => throw new NotImplementedYetException(),,
+       StringClose: () => throw new NotImplementedYetException(),
+     )
+  */
+  public abstract class S {
+    public abstract T Match_<T>(Visitor<T> c);
+    public static S End = new End();
+    public static S Space = new Space();
+    public static S Int = new Int();
+    public static S Decimal = new Decimal();
+    public static S String = new String();
+    public static S StringOpen = new StringOpen();
+    public static S StringClose = new StringClose();
+    public virtual Immutable.Option<End> AsEnd() => Immutable.Option.None<End>();
+    public virtual Immutable.Option<Space> AsSpace() => Immutable.Option.None<Space>();
+    public virtual Immutable.Option<Int> AsInt() => Immutable.Option.None<Int>();
+    public virtual Immutable.Option<Decimal> AsDecimal() => Immutable.Option.None<Decimal>();
+    public virtual Immutable.Option<String> AsString() => Immutable.Option.None<String>();
+    public virtual Immutable.Option<StringOpen> AsStringOpen() => Immutable.Option.None<StringOpen>();
+    public virtual Immutable.Option<StringClose> AsStringClose() => Immutable.Option.None<StringClose>();
+    private string GetTag() {
+      return this.Match(
+        End: () => "End",
+        Space: () => "Space",
+        Int: () => "Int",
+        Decimal: () => "Decimal",
+        String: () => "String",
+        StringOpen: () => "StringOpen",
+        StringClose: () => "StringClose"
+      );
+    }
+  }
+
+  public partial class Visitor<T> { public Func<T> End { get; set; } }
+
+  public sealed class End : S {
+    public End() { }
+    public override T Match_<T>(Visitor<T> c) => c.End();
+    public override Immutable.Option<End> AsEnd() => Immutable.Option.Some<End>(this);
+    public override bool Equals(object other) {
+        return (other is End);
+    }
+    public override int GetHashCode() {
+        return "C".GetHashCode();
+    }
+  }
+
+  public partial class Visitor<T> { public Func<T> Space { get; set; } }
+
+  public sealed class Space : S {
+    public Space() { }
+    public override T Match_<T>(Visitor<T> c) => c.Space();
+    public override Immutable.Option<Space> AsSpace() => Immutable.Option.Some<Space>(this);
+    public override bool Equals(object other) {
+        return (other is Space);
+    }
+    public override int GetHashCode() {
+        return "C".GetHashCode();
+    }
+  }
+
+  public partial class Visitor<T> { public Func<T> Int { get; set; } }
+
+  public sealed class Int : S {
+    public Int() { }
+    public override T Match_<T>(Visitor<T> c) => c.Int();
+    public override Immutable.Option<Int> AsInt() => Immutable.Option.Some<Int>(this);
+    public override bool Equals(object other) {
+        return (other is Int);
+    }
+    public override int GetHashCode() {
+        return "C".GetHashCode();
+    }
+  }
+
+  public partial class Visitor<T> { public Func<T> Decimal { get; set; } }
+
+  public sealed class Decimal : S {
+    public Decimal() { }
+    public override T Match_<T>(Visitor<T> c) => c.Decimal();
+    public override Immutable.Option<Decimal> AsDecimal() => Immutable.Option.Some<Decimal>(this);
+    public override bool Equals(object other) {
+        return (other is Decimal);
+    }
+    public override int GetHashCode() {
+        return "C".GetHashCode();
+    }
+  }
+
+  public partial class Visitor<T> { public Func<T> String { get; set; } }
+
+  public sealed class String : S {
+    public String() { }
+    public override T Match_<T>(Visitor<T> c) => c.String();
+    public override Immutable.Option<String> AsString() => Immutable.Option.Some<String>(this);
+    public override bool Equals(object other) {
+        return (other is String);
+    }
+    public override int GetHashCode() {
+        return "C".GetHashCode();
+    }
+  }
+
+  public partial class Visitor<T> { public Func<T> StringOpen { get; set; } }
+
+  public sealed class StringOpen : S {
+    public StringOpen() { }
+    public override T Match_<T>(Visitor<T> c) => c.StringOpen();
+    public override Immutable.Option<StringOpen> AsStringOpen() => Immutable.Option.Some<StringOpen>(this);
+    public override bool Equals(object other) {
+        return (other is StringOpen);
+    }
+    public override int GetHashCode() {
+        return "C".GetHashCode();
+    }
+  }
+
+  public partial class Visitor<T> { public Func<T> StringClose { get; set; } }
+
+  public sealed class StringClose : S {
+    public StringClose() { }
+    public override T Match_<T>(Visitor<T> c) => c.StringClose();
+    public override Immutable.Option<StringClose> AsStringClose() => Immutable.Option.Some<StringClose>(this);
+    public override bool Equals(object other) {
+        return (other is StringClose);
+    }
+    public override int GetHashCode() {
+        return "C".GetHashCode();
+    }
+  }
+
+}
+public static class SExtensionMethods {
+  public static T Match<T>(
+      this Lexer.S e,
+      Func<T> End,
+      Func<T> Space,
+      Func<T> Int,
+      Func<T> Decimal,
+      Func<T> String,
+      Func<T> StringOpen,
+      Func<T> StringClose
+    ) {
+    return e.Match_(new Lexer.Visitor<T> {
+      End = End,
+      Space = Space,
+      Int = Int,
+      Decimal = Decimal,
+      String = String,
+      StringOpen = StringOpen,
+      StringClose = StringClose
+    });
+  }
+}

LexerGenerator.cs

@@ -0,0 +1,17 @@
+// Run with: sh ./T4/Generators.sh
+
+using System.Collections.Generic;
+
+public static class LexerGenerator {
+  public static void Generate() {
+    Generator.Generate("LexerGenerated.cs", "public static partial class Lexer {", "}", "Lexer.", "S", new Dictionary<string, string> {
+      { "End", null },
+      { "Space", null },
+      { "Int", null },
+      { "Decimal", null },
+      { "String", null },
+      { "StringOpen", null },
+      { "StringClose", null },
+    });
+  }
+}

Parser.cs

@@ -3,137 +3,21 @@ using System.Text;
 using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
-using C = System.Globalization.UnicodeCategory;
+using S = Lexer.S;
 using static Global;
 
 public static class Parser {
-  public enum S {
-    End,
-    Space,
-    Int,
-    Decimal,
-    String,
-  }
-
-  public struct Lexeme {
-    public readonly S state;
-    // TODO: maybe keep this as a list of grapheme clusters
-    public readonly string lexeme;
-    public Lexeme(S state, string lexeme) {
-      this.state = state;
-      this.lexeme = lexeme;
-    }
-    public override string ToString() {
-      return $"new Lexeme({state}, \"{lexeme}\")";
-    }
-  }
-
-  // Transition
-  private static IEnumerable<Lexeme> T(ref S state, ref string lexeme, GraphemeCluster c, S newState) {
-    if (newState != state) {
-      var toReturn = new Lexeme(state, lexeme);
-      state = newState;
-      lexeme = "";
-      lexeme += c.str;
-      return toReturn.Singleton();
-    } else {
-      lexeme += c.str;
-      return Enumerable.Empty<Lexeme>();
-    }
-  }
-
-  public static void ParseError(StringBuilder context, IEnumerator<GraphemeCluster> stream) {
-    var rest =
-      stream
-        .SingleUseEnumerable()
-        .TakeUntil(c => c.str.StartsWith("\n"))
-        .Select(c => c.str)
-        .Aggregate(new StringBuilder(), Append);
-
-    throw new Exception(
-      $"Cannot parse this:{Environment.NewLine}{context}__HERE:__{rest}"
-    );
-  }
-
-  // fake Unicode category
-  private const UnicodeCategory EndOfFile = (UnicodeCategory)(-1);
-
-  public static IEnumerable<IEnumerable<Lexeme>> Lex1(string source) {
-    var context = new StringBuilder();
-    var lexeme = "";
-    var state = S.Space;
-    var e = source.TextElements().GetEnumerator();
-    while (e.MoveNext()) {
-      var c = e.Current;
-      context.Append(c.str);
-      var charCategory =
-        c.endOfFile
-          ? EndOfFile
-          : Char.GetUnicodeCategory(c.codePoints.First(), 0);
-      switch (state) {
-        case S.Space:
-        {
-          switch (charCategory) {
-            case C.DecimalDigitNumber:
-              yield return T(ref state, ref lexeme, c, S.Int);
-              break;
-            case C.SpaceSeparator:
-              yield return T(ref state, ref lexeme, c, S.Space);
-              break;
-            case EndOfFile:
-              yield return T(ref state, ref lexeme, c, S.End);
-              break;
-            default:
-              ParseError(context, e);
-              break;
-          }
-        }
-        break;
-
-        case S.Int:
-        {
-          switch (charCategory) {
-            case C.DecimalDigitNumber:
-              yield return T(ref state, ref lexeme, c, S.Int);
-              break;
-            case C.SpaceSeparator:
-              yield return T(ref state, ref lexeme, c, S.Space);
-              break;
-            case EndOfFile:
-              yield return T(ref state, ref lexeme, c, S.End);
-              break;
-            default:
-              ParseError(context, e);
-              break;
-          }
-        }
-        break;
-      }
-    }
-  }
-
-  public static IEnumerable<Lexeme> Lex(string source) {
-    var first = true;
-    foreach (var x in Lex1(source).SelectMany(x => x)) {
-      if (first && "".Equals(x.lexeme)) {
-        // skip the initial empty whitespace
-      } else {
-        first = false;
-        yield return x;
-      }
-    }
-  }
-
   public static Ast.Expr Parse(string source) {
-    foreach (var lexeme in Lex(source)) {
+    foreach (var lexeme in Lexer.Lex(source)) {
-      switch (lexeme.state) {
+      return lexeme.state.Match(
-        case S.Int:
+        Int: () => Ast.Expr.Int(Int32.Parse(lexeme.lexeme)),
-          return new Ast.Int(Int32.Parse(lexeme.lexeme));
+        String: () => Ast.Expr.String(lexeme.lexeme),
-        case S.String:
+        Space: () => throw new NotImplementedException(), // ignore
-          return new Ast.String(lexeme.lexeme);
+        End: () => throw new NotImplementedException(),
-        default:
+        Decimal: () => throw new NotImplementedException(),
-          throw new NotImplementedException();
+        StringOpen: () => throw new NotImplementedException(),
-      }
+        StringClose: () => throw new NotImplementedException()
+      );
     }
     throw new Exception("empty file, rm this when consuming the whole stream of lexemes.");
   }

T4/Generator.cs

@@ -0,0 +1,120 @@
+// Run with: sh ./T4/Generators.sh
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public static class Generator {
+  public static void WriteVariant(this System.IO.StreamWriter o, string header, string footer, string qualifier, string name, Dictionary<string, string> variant) {
+    o.WriteLine("// This file was generated by Generator.cs");
+    o.WriteLine("");
+
+    o.WriteLine($"{header}");
+    o.WriteLine("");
+
+    o.WriteLine($"  /* To match against an instance of {name}, write:");
+    o.WriteLine($"     x.Match(");
+    o.WriteLine(String.Join(",\n", variant.Select(@case => 
+                $"       {@case.Key}: {@case.Value == null ? "()" : "value"} => throw new NotImplementedYetException(),")));
+    o.WriteLine($"     )");
+    o.WriteLine($"  */");
+
+    o.WriteLine($"  public abstract class {name} {{");
+    o.WriteLine($"    public abstract T Match_<T>(Visitor<T> c);");
+    foreach (var @case in variant) {
+      var C = @case.Key;
+      var Ty = @case.Value;
+      o.WriteLine($"    public static {name} {C}{Ty == null ? $" = new {C}()" : $"({Ty} value) => new {C}(value)"};");
+    }
+
+    foreach (var @case in variant) {
+      var C = @case.Key;
+      var Ty = @case.Value;
+      o.WriteLine($"    public virtual Immutable.Option<{C}> As{C}() => Immutable.Option.None<{C}>();");
+    }
+
+    o.WriteLine($"    private string GetTag() {{");
+    o.WriteLine($"      return this.Match(");
+    o.WriteLine(String.Join(",\n", variant.Select(@case =>
+                $"        {@case.Key}: {@case.Value == null ? "()" : "value"} => \"{@case.Key}\"")));
+    o.WriteLine($"      );");
+    o.WriteLine($"    }}");
+
+/*
+    public abstract override bool Equals(object other) {
+      if (object.ReferenceEquals(other, null) || !this.GetType().Equals(other.GetType())) {
+        return false;
+      } else {
+        var cast = (S)other;
+        return    String.Equal(this.GetTag(), other.GetTag)
+               && 
+    }
+*/
+
+    o.WriteLine($"  }}");
+    o.WriteLine("");
+
+    foreach (var @case in variant) {
+      var C = @case.Key;
+      var Ty = @case.Value;
+
+      o.WriteLine(
+        $"  public partial class Visitor<T> {{"
+        + $" public Func<{Ty == null ? "" : $"{Ty}, "}T> {C} {{ get; set; }} "
+        + @"}");
+      o.WriteLine("");
+
+      o.WriteLine($"  public sealed class {C} : {name} {{");
+      if (Ty != null) {
+        o.WriteLine($"    public readonly {Ty} value;");
+      }
+      o.WriteLine($"    public {C}({Ty == null ? "" : $"{Ty} value"}) {{ {Ty == null ? "" : $"this.value = value; "}}}");
+      o.WriteLine($"    public override T Match_<T>(Visitor<T> c) => c.{C}({Ty == null ? "" : "value"});");
+      o.WriteLine($"    public override Immutable.Option<{C}> As{C}() => Immutable.Option.Some<{C}>(this);");
+      o.WriteLine($"    public override bool Equals(object other) {{");
+      if (Ty == null) {
+        o.WriteLine($"        return (other is {C});");
+      } else {
+        o.WriteLine($"      var cast = other as {C};");
+        o.WriteLine($"      if (Object.ReferenceEquals(cast, null)) {{");
+        o.WriteLine($"        return false;");
+        o.WriteLine($"      }} else {{");
+        o.WriteLine($"        return Equality.Field<{C}, {Ty}>(this, cast, x => x.value, (x, y) => ((Object)x).Equals(y));");
+        o.WriteLine($"      }}");
+      }
+      o.WriteLine($"    }}");
+      o.WriteLine($"    public override int GetHashCode() {{");
+      if (Ty == null) {
+        o.WriteLine($"        return \"C\".GetHashCode();");
+      } else {
+        o.WriteLine($"        return HashCode.Combine(\"{C}\", this.value);");
+      }
+      o.WriteLine($"    }}");
+      o.WriteLine($"  }}");
+      o.WriteLine("");
+    }
+
+    o.WriteLine($"}}");
+
+    o.WriteLine($"public static class {name}ExtensionMethods {{");
+    o.WriteLine($"  public static T Match<T>(");
+    o.WriteLine($"      this {qualifier}{name} e,");
+    o.WriteLine(String.Join(",\n", variant.Select(c =>
+                $"      Func<{c.Value == null ? "" : $"{c.Value}, "}T> {c.Key}")));
+    o.WriteLine($"    ) {{");
+    o.WriteLine($"    return e.Match_(new {qualifier}Visitor<T> {{");
+    o.WriteLine(String.Join(",\n", variant.Select(c =>
+                $"      {c.Key} = {c.Key}")));
+    o.WriteLine($"    }});");
+    o.WriteLine($"  }}");
+    o.WriteLine($"{footer}");
+  }
+
+  public static void Generate(string outputFile, string header, string footer, string qualifier, string name, Dictionary<string, string> variant) {
+    using (var o = new System.IO.StreamWriter(outputFile)) {
+      o.WriteLine("using System;");
+      o.WriteLine("");
+      o.WriteVariant(header, footer, qualifier, name, variant);
+    }
+  }
+}

T4/Generators.cs

@@ -0,0 +1,8 @@
+// Run with: sh ./T4/Generators.sh
+
+public static class Generators {
+  public static void Generate() {
+    AstGenerator.Generate();
+    LexerGenerator.Generate();
+  }
+}

T4/Generators.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# Run with: sh ./T4/Generators.sh
+
+set -e
+set -x
+
+mcs -out:T4.exe T4/Generator.cs T4/Generators.cs LexerGenerator.cs AstGenerator.cs T4/T4.compileTime && mono T4.exe

T4/T4.compileTime

@@ -0,0 +1,14 @@
+// Run with: sh ./T4/Generators.sh
+
+// Workaround to start the compile-time code generation
+// repl.it lacks T4, if the generated code is out of sync it's not
+// possible to build main.exe, and there can't be a fallback Main
+// method with a .cs extension (the C# compiler complains there are
+// two implementations of Main, and repl.it's command-line lacks
+// the option to specify which class to use).
+
+public static class MainClass {
+  public static void Main() {
+    Generators.Generate();
+  }
+}

Tests/003-foo.e

@@ -0,0 +1 @@
+"foo"

Tests/003-foo.o

@@ -0,0 +1 @@
+foo

Utils/Enumerable.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Linq;
 using System.Collections.Generic;
+using Immutable;
 
 public static class Collection {
   public static void ForEach<T>(this IEnumerable<T> x, Action<T> f)
@@ -45,9 +46,12 @@ public static class Collection {
   public static IEnumerable<Item<T>> Indexed<T>(this IEnumerable<T> e) {
     long i         = 0L;
     bool first     = true;
-    T    prevX     = default(T); // Dummy
+
+    // These default(…) are written below before being read
+    T    prevX     = default(T);
     long prevI     = default(long);
     bool prevFirst = default(bool);
+
     foreach (var x in e) {
       if (!first) {
         yield return new Item<T>(prevX, prevI, prevFirst, false);
@@ -73,7 +77,7 @@ public static class Collection {
     }
     public bool MoveNext() {
       this.peeked = false;
-      this.previous = default(T);
+      this.previous = default(T); // guarded by peeked
       return this.e.MoveNext();
     }
     public bool Peek() {
@@ -91,7 +95,7 @@ public static class Collection {
     public Peekable(IEnumerable<T> e) {
       this.e = e.GetEnumerator();
       this.peeked = false;
-      this.previous = default(T);
+      this.previous = default(T); // guarded by peeked
     }
   }
 
@@ -111,4 +115,30 @@ public static class Collection {
   public static IEnumerable<T> Singleton<T>(this T x) {
     yield return x;
   }
+
+  public static string Join(this string separator, IEnumerable<string> strings)
+    => String.Join(separator, strings);
+
+  public static Option<T> First<T>(this IEnumerable<T> ie) {
+    var e = ie.GetEnumerator();
+    if (e.MoveNext()) {
+      return e.Current.Some();
+    } else {
+      return Option.None<T>();
+    }
+  }
+
+  public static Option<T> Single<T>(this IEnumerable<T> ie) {
+    var e = ie.GetEnumerator();
+    if (e.MoveNext()) {
+      var value = e.Current;
+      if (e.MoveNext()) {
+        return Option.None<T>();
+      } else {
+        return value.Some();
+      }
+    } else {
+      return Option.None<T>();
+    }
+  }
 }

Utils/Equality.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Linq;
+
+public static class Equality {
+  public static bool Test<T>(T a, T b, Func<T, T, bool> compare) {
+    if (Object.ReferenceEquals(a, null)) {
+      return Object.ReferenceEquals(b, null);
+    } else {
+      return compare(a, b);
+    }
+  }
+
+
+  public static bool Field<T, U>(T a, T b, Func<T, U> getField, Func<U, U, bool> compareField) {
+    if (Object.ReferenceEquals(a, null)) {
+      return Object.ReferenceEquals(b, null);
+    } else {
+      return compareField(getField(a), getField(b));
+    }
+  }
+}

Utils/Global.cs

@@ -8,4 +8,6 @@ public static class Global {
 
   public static System.Text.StringBuilder Append(System.Text.StringBuilder b, string s)
     => b.Append(s);
+
+  public static Immutable.Option<T> None<T>() => Immutable.Option.None<T>();
 }

Utils/Immutable/Option.cs

@@ -1,9 +1,8 @@
-/*
 namespace Immutable {
   using System;
 
   public interface Option<out T> {
-    U Match_<U>(Func<T, U> Some, Func<U> None);
+    U Match_<U>(Func<T, U> some, Func<U> none);
   }
 
   public static class Option {
@@ -29,8 +28,11 @@ namespace Immutable {
 
   public static class OptionExtensionMethods {
     public static Option<T> Some<T>(this T value) => Option.Some<T>(value);
-    public static U Match<T, U>(this Option<T> o, Func<T, U> Some, Func<U> None)
+    
-      => o.Match_(Some, None);
+    public static U Match<T, U>(this Option<T> o, Func<T, U> some, Func<U> none)
- }
+      => o.Match_(some, none);
-}
+
-*/
+    public static U Match<T, U>(this Option<T> o, Func<T, U> some, U none)
+      => o.Match_(some, () => none);
+  }
+}

Utils/Unicode.cs

@@ -67,6 +67,7 @@ public static class UnicodeExtensionMethods {
           if (te2.Length == 2 && first2 >= sp && first2 <= cancelTag) {
             te += te2;
             if (first2 == cancelTag) {
+              alreadyMoved = false;
               break;
             }
           } else {
@@ -86,4 +87,17 @@ public static class UnicodeExtensionMethods {
       Enumerable.Empty<string>()
     );
   }
+
+  public static UnicodeCategory UnicodeCategory(this string s, int startIndex)
+    => Char.GetUnicodeCategory(s, startIndex);
+
+  public static string Description(this GraphemeCluster gc) {
+    if (gc.endOfFile) {
+      return "end of file";
+    } else if (gc.str == "\"") {
+      return "'\"'";
+    } else {
+      return $"\"{gc.str}\"";
+    }
+  }
 }

main.cs

@@ -22,7 +22,7 @@ public static class MainClass {
     var sourcePath = tests.Combine(source);
     var expected = sourcePath.DropExtension().Combine(Ext(".o"));
     
-    Console.Write($"Running test {source} ({toolchainName}) ");
+    Console.Write($"\x1b[KRunning test {source} ({toolchainName}) ");
     
     destPath.DirName().Create();
 
@@ -34,7 +34,7 @@ public static class MainClass {
       Console.WriteLine("\x1b[1;31mFail\x1b[m");
       throw new Exception($"Test failed {source}: expected {expectedStr} but got {actualStr}.");
     } else {
-      Console.WriteLine("\x1b[1;32mOK\x1b[m");
+      Console.Write("\x1b[1;32mOK\x1b[m\r");
     }
   }
 
@@ -46,14 +46,20 @@ public static class MainClass {
       .Cons(" js ", Compilers.JS.Compile, Exe("node"))
       .Cons("eval", Evaluator.Evaluate,   Exe("cat"));
 
+    var total = 0;
     foreach (var t in Dir("Tests/").GetFiles("*.e", SearchOption.AllDirectories)) {
       foreach (var compiler in compilers) {
         RunTest(compiler.Item1, compiler.Item2, compiler.Item3, t);
+        total++;
       }
     }
+    Console.WriteLine($"\x1b[K{total} tests run.");
   }
 
   public static void Main (string[] args) {
+    // Refresh code generated at compile-time
+    Generators.Generate();
+
     if (args.Length != 1) {
       Console.WriteLine("Usage: mono main.exe path/to/file.e");
       Console.WriteLine("");