Debugging the parser

DefaultGrammar.cs

@@ -11,10 +11,10 @@ public static class DefaultGrammar {
       .WithOperator("equality", NonAssociative, "int|terminal|additive|multiplicative", S.Eq, "int|terminal|additive|multiplicative")
       .WithOperator("int",      NonAssociative, S.Int)
       .WithOperator("additive", LeftAssociative, "int|terminal|multiplicative", S.Plus, "int|terminal|multiplicative")
-//      .WithOperator("multiplicative", LeftAssociative, "int|terminal", S.Times, "int|terminal")
+      .WithOperator("multiplicative", LeftAssociative, "int|terminal", S.Times, "int|terminal")
       .WithOperator("terminal", NonAssociative, S.Ident)
       // This is the root set of operators
       .WithOperator("program",  NonAssociative,
       // "bool" // TODO: this needs aliases
-      "equality|terminal", S.And, "equality|terminal");
+      "terminal", S.And, "terminal");// TODO: re-add equality|
 }

Lexer.cs

@@ -213,22 +213,36 @@ public static partial class Lexer {
 
   [F]
   private partial class SkipInitialEmptyWhitespace {
+    public IImmutableEnumerator<Lexeme> F(
+      IImmutableEnumerator<Lexeme> lx
+    )
+    => lx.FirstAndRest().Match(
+      Some: hdtl =>
+        // skip the initial empty whitespace
+          "".Equals(hdtl.Item1.lexeme)
+        ? hdtl.Item2
+        : hdtl.Item1.ImSingleton().Concat(hdtl.Item2),
+      None: Empty<Lexeme>());
+  }
+
+  // TODO: move this to a .Filter() extension method.
+  [F]
+  private partial class DiscardWhitespace {
     public IImmutableEnumerator<Lexeme> F(
       IImmutableEnumerator<Lexeme> lx
     )
     => lx.FirstAndRest().Match<Tuple<Lexeme, IImmutableEnumerator<Lexeme>>, IImmutableEnumerator<Lexeme>>(
       Some: hdtl =>
         // skip the initial empty whitespace
- string.Equals(
+          hdtl.Item1.state.Equals(S.Space)
- "",
- hdtl.Item1.lexeme)
         ? hdtl.Item2
-            : hdtl.Item1.ImSingleton().Concat(hdtl.Item2),
+        : hdtl.Item1.ImSingleton().Concat(hdtl.Item2.Lazy(DiscardWhitespace.Eq)),
       None: Empty<Lexeme>());
   }
 
   public static IImmutableEnumerator<Lexeme> Lex(string source)
     => Lex1(source)
          .Flatten()
-         .Lazy(SkipInitialEmptyWhitespace.Eq);
+         //.Lazy(SkipInitialEmptyWhitespace.Eq)
+         .Lazy(DiscardWhitespace.Eq);
 }

MixFix.cs

@@ -21,9 +21,20 @@ public static partial class MixFix {
     public static Grammar1 Or(params Grammar1[] xs)
       => Or(xs.ToImmutableList());
 
+    public static Grammar1 Empty
+      = new Grammar1.Cases.Sequence(Enumerable.Empty<Grammar1>());
+      
+    public static Grammar1 Impossible
+      = new Grammar1.Cases.Or(Enumerable.Empty<Grammar1>());
+
+    // TODO: inline the OR, detect impossible cases (Or of 0)
     public static Grammar1 Sequence(IEnumerable<Grammar1> xs) {
       var filteredXs = xs.Where(x => !x.IsEmpty);
-      if (filteredXs.Count() == 1) {
+      if (filteredXs.Any(x => x.IsImpossible)) {
+        return Impossible;
+      } else if (filteredXs.Count() == 0) {
+        return Empty;
+      } else if (filteredXs.Count() == 1) {
         return filteredXs.Single().ElseThrow(() => new Exception("TODO: use an either to prove that this is safe."));
       } else {
         return new Grammar1.Cases.Sequence(filteredXs);
@@ -31,8 +42,17 @@ public static partial class MixFix {
     }
 
     public static Grammar1 Or(IEnumerable<Grammar1> xs) {
-      var filteredXs = xs.Where(x => !x.IsEmpty);
+      var filteredXsNoEmpty = 
-      if (filteredXs.Count() == 1) {
+        xs.Where(x => !x.IsImpossible)
+          .Where(x => !x.IsEmpty);
+      var filteredXs =
+        (   xs.Any(x => x.IsEmpty)
+         && !filteredXsNoEmpty.Any(x => x.AllowsEmpty))
+        ? Empty.Cons(filteredXsNoEmpty)
+        : filteredXsNoEmpty;
+      if (filteredXs.All(x => x.IsImpossible)) {
+        return new Grammar1.Cases.Or(Enumerable.Empty<Grammar1>());
+      } else if (filteredXs.Count() == 1) {
         return filteredXs.Single().ElseThrow(() => new Exception("TODO: use an either to prove that this is safe."));
       } else {
         return new Grammar1.Cases.Or(filteredXs);
@@ -42,20 +62,43 @@ public static partial class MixFix {
     public static Grammar1 RepeatOnePlus(Grammar1 g)
       => g.IsEmpty
         ? Grammar1.Empty
+        : g.IsImpossible
+        ? Grammar1.Impossible
         : new Grammar1.Cases.RepeatOnePlus(g);
 
-    public static Grammar1 Empty = new Grammar1.Cases.Or(Enumerable.Empty<Grammar1>());
-
     public bool IsEmpty {
       get => this.Match(
-        Or: l => l.Count() == 0,
+        Or: l => l.All(g => g.IsEmpty),
-        Sequence: l => l.Count() == 0,
+        Sequence: l => l.All(g => g.IsEmpty),
         RepeatOnePlus: g => g.IsEmpty,
         Terminal: t => false,
         Rule: r => false
       );
     }
 
+    // TODO: cache this!
+    public bool AllowsEmpty {
+      get => this.Match(
+        Or: l => l.Any(g => g.AllowsEmpty),
+        Sequence: l => l.All(g => g.IsEmpty),
+        // This one should not be true, if it is
+        // then the precedence graph may be ill-formed?
+        RepeatOnePlus: g => g.AllowsEmpty,
+        Terminal: t => false,
+        Rule: r => false
+      );
+    }
+
+    public bool IsImpossible {
+      get => this.Match(
+        Or: l => l.All(g => g.IsImpossible),
+        Sequence: l => l.Any(g => g.IsImpossible),
+        RepeatOnePlus: g => g.IsImpossible,
+        Terminal: t => false,
+        Rule: r => false
+      );
+    }
+
     public static Grammar1 operator |(Grammar1 a, Grammar1 b)
       => Or(a, b);
 
@@ -65,8 +108,8 @@ public static partial class MixFix {
     public static implicit operator Grammar1((Grammar1 a, Grammar1 b, Grammar1 c) gs)
       => Sequence(gs.a, gs.b, gs.c);
 
-    public static bool operator true(Grammar1 g) => !g.IsEmpty;
+    public static bool operator true(Grammar1 g) => !g.IsImpossible;
-    public static bool operator false(Grammar1 g) => g.IsEmpty;
+    public static bool operator false(Grammar1 g) => g.IsImpossible;
 
     public Grammar1 this[string multiplicity] {
       get {
@@ -85,20 +128,55 @@ public static partial class MixFix {
     private string Paren(bool paren, string s)
       => paren ? $"({s})" : s;
 
-    string CustomToString()
+    string CustomToString() =>
-      => this.Match<string>(
+        this.IsEmpty ? "Empty"
-        Or: l => l.Count() == 0
+      : this.IsImpossible ? "Impossible"
-          ? "Or(Empty)"
+      : this.Match<string>(
-          : Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(" | ")),
+        Or: l =>
-        Sequence: l => l.Count() == 0
+          Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(" | ")),
-          ? "Sequence(Empty)"
+        Sequence: l =>
-          : Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(",")),
+          Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(", ")),
-        RepeatOnePlus: g => $"({g.Str()})+",
+        RepeatOnePlus: g => $"{g.Str()}+",
         Terminal: t => t.Str(),
         Rule: r => r
       );
   }
 
+  public partial class Grammar2 {
+    public bool IsEmpty {
+      get => this.Match(
+        Or: l => l.All(g => g.IsEmpty),
+        Sequence: l => l.All(g => g.IsEmpty),
+        RepeatOnePlus: g => g.IsEmpty,
+        Terminal: t => false
+      );
+    }
+
+    public bool IsImpossible {
+      get => this.Match(
+        Or: l => l.All(g => g.IsImpossible),
+        Sequence: l => l.Any(g => g.IsImpossible),
+        RepeatOnePlus: g => g.IsImpossible,
+        Terminal: t => false
+      );
+    }
+
+    private string Paren(bool paren, string s)
+      => paren ? $"({s})" : s;
+
+    string CustomToString() =>
+        this.IsEmpty ? "Empty"
+      : this.IsImpossible ? "Impossible"
+      : this.Match<string>(
+        Or: l =>
+          Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(" | ")),
+        Sequence: l =>
+          Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(", ")),
+        RepeatOnePlus: g => $"{g.Str()}+",
+        Terminal: t => t.Str()
+      );
+  }
+
   public partial class Operator {
     private string CustomToString()
     => $"Operator(\"{precedenceGroup}\", {fixity}, {parts.Select(x => x.Match(Hole: h => h.Select(g => g.ToString()).JoinWith("|"), Name: n => $"\"{n}\"")).JoinWith(", ")})";
@@ -222,7 +300,7 @@ public static partial class MixFix {
     = new PrecedenceDAG(EmptyDAGNode);
 
   public static Whole Add<Whole>(this ILens<DAGNode, Whole> node, Operator @operator) {
-    return @operator.fixity.Match(
+    return @operator.Cons(@operator.fixity.Match(
       Closed:
         () => node.Closed(),
       Prefix:
@@ -235,7 +313,7 @@ public static partial class MixFix {
         () => node.InfixRightAssociative(),
       InfixLeftAssociative:
         () => node.InfixLeftAssociative()
-    ).Cons(@operator);
+    ));
   }
 
   public static void CheckHole(PrecedenceDAG precedenceDAG, Operator @operator, string name, Option<Hole> existing, Option<Hole> @new) {
@@ -344,15 +422,27 @@ public static partial class MixFix {
     var infixl = node.infixLeftAssociative.ToGrammar1();
     var infixr = node.infixRightAssociative.ToGrammar1();
 
+
+    //Log("closed.IsImpossible:"+closed.IsImpossible);
+
+
+
+
+
     // TODO: BUG: only include these parts if there are
-    // any operators with that fixity.
+    // any operators with that fixity, with the code below
+    // they are empty.
+
+
+
+
     return
-        closed
+        (closed ? closed : Grammar1.Impossible)
-      | (nonAssoc ? (lsucc, nonAssoc, rsucc) : Grammar1.Empty)
+      | (nonAssoc ? (lsucc, nonAssoc, rsucc) : Grammar1.Impossible)
+      // TODO: post-processsing of the rightassoc list.
+      | ((prefix || infixr) ? ((prefix || (lsucc, infixr))["+"], rsucc) : Grammar1.Impossible)
       // TODO: post-processsing of the leftassoc list.
-      | ((prefix || infixr) ? ((prefix | (lsucc, infixr))["+"], rsucc) : Grammar1.Empty)
+      | ((postfix || infixl) ? (lsucc, (postfix || (infixl, rsucc))["+"]) : Grammar1.Impossible);
-      // TODO: post-processsing of the leftassoc list.
-      | ((postfix || infixl) ? (lsucc, (postfix | (infixl, rsucc))["+"]) : Grammar1.Empty);
   }
 
   public static EquatableDictionary<string, Grammar1> ToGrammar1(this PrecedenceDAG precedenceDAG)
@@ -363,15 +453,28 @@ public static partial class MixFix {
   private static Grammar2 Recur(Func<Grammar1, EquatableDictionary<string, Grammar1>, Grammar2> recur, Grammar1 grammar1, EquatableDictionary<string, Grammar1> labeled)
     => grammar1.Match<Grammar2>(
       // TODO: throw exception if lookup fails
-      Rule:          r => recur(labeled[r], labeled),
+      Rule:          r => {
+        Grammar1 lr = null;
+        try {
+          lr = labeled[r];
+        } catch (Exception e) {
+          throw new ParserExtensionException($"Internal error: could not find node {r} in labeled grammar. It only contains labels for: {labeled.Select(kvp => kvp.Key.ToString()).JoinWith(", ")}.");
+        }
+        return recur(labeled[r], labeled);
+      },
       Terminal:      t => Grammar2.Terminal(t),
       Sequence:      l => Grammar2.Sequence(l.Select(g => recur(g, labeled))),
       Or:            l => Grammar2.Or(l.Select(g => recur(g, labeled))),
       RepeatOnePlus: g => Grammar2.RepeatOnePlus(recur(g, labeled))
     );
 
-  public static Grammar2 ToGrammar2(this EquatableDictionary<string, Grammar1> labeled)
+  public static Grammar2 ToGrammar2(this EquatableDictionary<string, Grammar1> labeled) {
-    => Func.YMemoize<Grammar1, EquatableDictionary<string, Grammar1>, Grammar2>(Recur)(Grammar1.Rule("program"), labeled);
+    foreach (var kvp in labeled) {
+      Log($"{kvp.Key} -> {kvp.Value.ToString()}");
+    }
+    Log("");
+    return Func.YMemoize<Grammar1, EquatableDictionary<string, Grammar1>, Grammar2>(Recur)(Grammar1.Rule("program"), labeled);
+  }
 
   public static Grammar2 ToGrammar2(this PrecedenceDAG precedenceDAG)
     => precedenceDAG.ToGrammar1().ToGrammar2();

Parser.cs

@@ -25,26 +25,34 @@ public static partial class Parser {
       .FirstAndRest()
       .Match(
         None: () =>
-          throw new Exception("EOF, what to do?"),
+          //throw new Exception("EOF, what to do?"),
+          None<ValueTuple<IImmutableEnumerator<Lexeme>, AstNode>>(),
         Some: firstRest => {
-          // Case("IImmutableEnumerable<AstNode>", "Operator"))
           var first = firstRest.Item1;
           var rest = firstRest.Item2;
+          Log(first.lexeme);
+          Log(grammar.ToString());
+          Log(grammar.Match(
+            RepeatOnePlus: _ => "RepeatOnePlus",
+            Or: _ => "Or",
+            Sequence: _ => "Sequence",
+            Terminal: t => "Terminal:"+t.ToString()));
           return grammar.Match(
             RepeatOnePlus: g =>
               rest.FoldMapWhileSome(restI => Parse3(restI, g))
                 .If<IImmutableEnumerator<Lexeme>, IEnumerable<AstNode>>((restN, nodes) => nodes.Count() > 1)
                 .IfSome((restN, nodes) => (restN, AstNode.Operator(nodes))),
-                //.IfSome(rest1 =>
-                  // TODO: remove IfSome above (useless) && aggregate
-                //  WhileSome(rest1, restI => Parse3(restI, g))),
             // TODO: to check for ambiguous parses, we can use
-            // .SingleArg(…) instead of .FirstArg(…).
+            // .Single(…) instead of .First(…).
             Or: l =>
               l.First(g => Parse3(rest, g)),
-            Sequence: l =>
+            Sequence: l => {
-              l.BindFoldMap(rest, (restI, g) => Parse3(restI, g))
+              return l.BindFoldMap(rest, (restI, g) => Parse3(restI, g))
-               .IfSome((restN, nodes) => (restN, AstNode.Operator(nodes))),
+               .IfSome((restN, nodes) => {
+                 Log($"{nodes.Count()}/{l.Count()}");
+                 return (restN, AstNode.Operator(nodes));
+               });
+            },
             Terminal: t =>
               first.state.Equals(t)
               ? (rest,
@@ -60,7 +68,7 @@ public static partial class Parser {
   public static Option<ValueTuple<IImmutableEnumerator<Lexeme>, AstNode>> Parse2(string source) {
     Grammar2 grammar =
       DefaultGrammar.DefaultPrecedenceDAG.ToGrammar2();
-    Log(grammar.Str());
+    //Log(grammar.Str());
 
     var P = Func.YMemoize<
               IImmutableEnumerator<Lexeme>,
@@ -73,6 +81,11 @@ public static partial class Parser {
   }
 
   public static Ast.Expr Parse(string source) {
+    Log("");
+    Log("" + Parse2(source).ToString());
+    Log("");
+    Environment.Exit(0);
+
     return Lexer.Lex(source)
       .SelectMany(lexeme =>
         lexeme.state.Match(

Tests/001-42.e

@@ -1 +1 @@
-42
+true && false

Utils/Enumerable.cs

@@ -8,14 +8,20 @@ public static class Collection {
   public static void ForEach<T>(this IEnumerable<T> x, Action<T> f)
     => x.ToImmutableList().ForEach(f);
 
-  public static ImmutableList<T> Cons<T>(this ImmutableList<T> l, T x)
+  public static ImmutableList<T> Cons<T>(this T x, ImmutableList<T> l)
     => l.Add(x);
 
-  public static ImmutableList<Tuple<T,U>> Cons<T,U>(this ImmutableList<Tuple<T,U>> l, T x, U y)
+  public static IEnumerable<T> Cons<T>(this T x, IEnumerable<T> l)
-    => l.Cons(Tuple.Create(x,y));
+    => x.Singleton().Concat(l);
 
-  public static ImmutableList<Tuple<T,U,V>> Cons<T,U,V>(this ImmutableList<Tuple<T,U,V>> l, T x, U y, V z)
+  public static IEnumerable<T> Concat<T>(this IEnumerable<T> l, T x)
-    => l.Cons(Tuple.Create(x,y,z));
+    => l.Concat(x.Singleton());
+
+  public static ImmutableList<Tuple<T,U>> Add<T,U>(this ImmutableList<Tuple<T,U>> l, T x, U y)
+    => l.Add(Tuple.Create(x,y));
+
+  public static ImmutableList<Tuple<T,U,V>> Add<T,U,V>(this ImmutableList<Tuple<T,U,V>> l, T x, U y, V z)
+    => l.Add(Tuple.Create(x,y,z));
 
   public static void Deconstruct<A, B>(this Tuple<A, B> t, out A a, out B b) {
     a = t.Item1;
@@ -241,7 +247,7 @@ public static class Collection {
     foreach (var x in e) {
       var newAcc = f(acc, x);
       if (newAcc.IsNone) {
-        break;
+        return Option.None<A>();
       } else {
         acc = newAcc.ElseThrow(new Exception("impossible"));
       }

Utils/Immutable/Option.cs

@@ -27,6 +27,9 @@ namespace Immutable {
         
         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
           => this.GetEnumerator();
+
+        public override string ToString()
+          => $"Some({value.ToString()})";
       }
 
       public class None<T> : Option<T>, System.Collections.IEnumerable {
@@ -42,6 +45,9 @@ namespace Immutable {
         
         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
           => this.GetEnumerator();
+
+        public override string ToString()
+          => $"None";
       }
     }
   }

Utils/Lens.cs

@@ -10,8 +10,8 @@ public static class LensExtensionMethods {
   public static Whole Update<Hole, Whole>(this ILens<Hole, Whole> lens, Hole newHole)
     => lens.Update(oldHole => newHole);
 
-  public static Whole Cons<T, Whole>(this ILens<ImmutableList<T>, Whole> lens, T value)
+  public static Whole Cons<T, Whole>(this T value, ILens<ImmutableList<T>, Whole> lens)
-    => lens.Update(oldHole => oldHole.Cons(value));
+    => lens.Update(oldHole => value.Cons(oldHole));
 
   public static ILens<string, Whole> ChainLens<Whole>(this string hole, System.Func<string, Whole> wrap) => new LeafLens<string, Whole>(wrap: wrap, oldHole: hole);
 

Utils/ToString.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Linq;
+using System.Collections.Generic;
 using System.Collections.Immutable;
 
 public interface IString {
@@ -25,6 +26,12 @@ public static class ToStringImplementations {
   public static string Str<T>(this ImmutableHashSet<string> h)
     => $"ImmutableHashSet({h.Select(x => x.Str<string>()).JoinWith(", ")})";
 
+  public static string Str<T>(this IEnumerable<MixFix.Grammar2> e)
+    => $"IEnumerabl({e.Select(x => x.Str<MixFix.Grammar2>()).JoinWith(", ")})";
+
+  public static string Str<T>(this IEnumerable<Ast.AstNode> e)
+    => $"IEnumerab({e.Select(x => x.Str<Ast.AstNode>()).JoinWith(", ")})";
+
   public static string Str<Grammar>(this ImmutableDictionary<string,Grammar> h)
     => $"ImmutableDictionary(\n{h.Select(x => $"  {x.Key.Str<string>()}:{x.Value.Str<Grammar>()}").JoinWith(",\n")}\n)";
 

main.cs

@@ -47,7 +47,7 @@ public static class MainClass {
         Console.WriteLine($"\x1b[1;33m{source}: expected {expectedStr} but got {actualStr}.\x1b[m\n");
         return false;
       } else {
-        Console.Write("\x1b[1;32mOK\x1b[m\r");
+        Console.Write("\x1b[1;32mOK\x1b[m"); // \r at the end for quiet
         return true;
       }
     }
@@ -58,13 +58,13 @@ public static class MainClass {
     // first-class functions by using repeated .Add()
     // See https://repl.it/@suzannesoy/WarlikeWorstTraining#main.cs
     var compilers = ImmutableList<Tuple<string, Compiler, Exe>>.Empty
-      .Cons(" js ", Compilers.JS.Compile, Exe("node"))
+      .Add(" js ", Compilers.JS.Compile, Exe("node"))
-      .Cons("eval", Evaluator.Evaluate,   Exe("cat"));
+      .Add("eval", Evaluator.Evaluate,   Exe("cat"));
 
     var total = 0;
     var passed = 0;
     var failed = 0;
-    foreach (var t in Dir("Tests/").GetFiles("*.e", SearchOption.AllDirectories)) {
+    foreach (var t in Dir("Tests/").GetFiles("*.e", SearchOption.AllDirectories).OrderBy(f => f.ToString())) {
       foreach (var compiler in compilers) {
         if (RunTest(compiler.Item1, compiler.Item2, compiler.Item3, t)) {
           passed++;