Keep annotations in Parser

AstGenerator.cs

@@ -12,6 +12,10 @@ public static class AstGenerator {
         Variant("Expr",
           Case("int", "Int"),
           Case("string", "String")),
+        Variant("ParserResult",
+          Case("MixFix.Annotation", "Annotated"),
+          Case("Lexer.Lexeme", "Terminal"),
+          Case("IEnumerable<ParserResult>", "Productions")),
         Variant("AstNode",
           Case("Expr", "Terminal"),
           Case("IEnumerable<AstNode>", "Operator"))));

DefaultGrammar.cs

@@ -7,14 +7,14 @@ using static MixFix.Associativity;
 public static class DefaultGrammar {
   public static PrecedenceDAG DefaultPrecedenceDAG
     = EmptyPrecedenceDAG
-      /*.WithOperator("bool",     NonAssociative, "equality|terminal", S.And, "equality|terminal")
+      .WithOperator("bool",     NonAssociative, "equality|terminal", S.And, "equality|terminal")
       .WithOperator("equality", NonAssociative, "int|terminal|additive|multiplicative", S.Eq, "int|terminal|additive|multiplicative")
       .WithOperator("int",      NonAssociative, S.Int)
       .WithOperator("additive", LeftAssociative, "int|terminal|multiplicative", S.Plus, "int|terminal|multiplicative")
-      .WithOperator("multiplicative", LeftAssociative, "int|terminal", S.Times, "int|terminal")*/
+      .WithOperator("multiplicative", LeftAssociative, "int|terminal", S.Times, "int|terminal")
       .WithOperator("terminal", NonAssociative, S.Ident)
       // This is the root set of operators
       .WithOperator("program",  NonAssociative,
       // "bool" // TODO: this needs aliases
-      "terminal", S.And, "terminal");// TODO: re-add equality|
+      "equality|terminal", S.And, "equality|terminal");
 }

Lexer.cs

@@ -232,11 +232,12 @@ public static partial class Lexer {
       IImmutableEnumerator<Lexeme> lx
     )
     => lx.FirstAndRest().Match<Tuple<Lexeme, IImmutableEnumerator<Lexeme>>, IImmutableEnumerator<Lexeme>>(
-      Some: hdtl =>
+      Some: hdtl => {
-        // skip the initial empty whitespace
+        var rest = hdtl.Item2.Lazy(DiscardWhitespace.Eq);
-          hdtl.Item1.state.Equals(S.Space)
+        return hdtl.Item1.state.Equals(S.Space)
-        ? hdtl.Item2
+          ? rest
-        : hdtl.Item1.ImSingleton().Concat(hdtl.Item2.Lazy(DiscardWhitespace.Eq)),
+          : hdtl.Item1.ImSingleton().Concat(rest);
+      },
       None: Empty<Lexeme>());
   }
 

MixFix.cs

@@ -155,7 +155,8 @@ public static partial class MixFix {
         Or: l => l.All(g => g.IsEmpty),
         Sequence: l => l.All(g => g.IsEmpty),
         RepeatOnePlus: g => g.IsEmpty,
-        Terminal: t => false
+        Terminal: t => false,
+        Annotated: a => a.Item2.IsEmpty
       );
     }
 
@@ -164,7 +165,8 @@ public static partial class MixFix {
         Or: l => l.All(g => g.IsImpossible),
         Sequence: l => l.Any(g => g.IsImpossible),
         RepeatOnePlus: g => g.IsImpossible,
-        Terminal: t => false
+        Terminal: t => false,
+        Annotated: a => a.Item2.IsImpossible
       );
     }
 
@@ -182,7 +184,8 @@ public static partial class MixFix {
           ? "EmptySequence"
           : Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(", ")),
         RepeatOnePlus: g => $"{g.Str()}+",
-        Terminal: t => t.Str()
+        Terminal: t => t.Str(),
+        Annotated: a => $"Annotated({a.Item1.Str()}, {a.Item2.Str()})"
       );
   }
 
@@ -414,16 +417,27 @@ public static partial class MixFix {
       Hole: precedenceGroups => precedenceGroups.ToGrammar1());
 
   public static Grammar1 ToGrammar1(this Operator @operator)
-    => Grammar1.Sequence(
+    => Grammar1.Annotated(
-      @operator.internalParts.Select(part => part.ToGrammar1()));
+         (Annotation.Operator(@operator),
+          Grammar1.Sequence(
+            @operator.internalParts.Select(
+              part => part.ToGrammar1()))));
 
   public static Grammar1 ToGrammar1(this IEnumerable<Operator> operators)
     => Grammar1.Or(
       operators.Select(@operator => @operator.ToGrammar1()));
 
   public static Grammar1 ToGrammar1(this DAGNode node) {
-    var lsucc = node.leftmostHole_.ToGrammar1();
+    Func<Associativity, Grammar1, Grammar1> SamePrecedence = (a, g)
-    var rsucc = node.rightmostHole_.ToGrammar1();
+      => Grammar1.Annotated((Annotation.SamePrecedence(a), g));
+    Func<Grammar1, Grammar1> R = g => SamePrecedence(Associativity.RightAssociative, g);
+    Func<Grammar1, Grammar1> L = g => SamePrecedence(Associativity.LeftAssociative, g);
+    Func<Grammar1, Grammar1> N = g => SamePrecedence(Associativity.NonAssociative, g);
+    Func<Grammar1, Grammar1> H = g => Grammar1.Annotated((Annotation.Hole, g));
+    var Impossible = Grammar1.Impossible;
+
+    var lsucc = H(node.leftmostHole_.ToGrammar1());
+    var rsucc = H(node.rightmostHole_.ToGrammar1());
     var closed = node.closed.ToGrammar1();
     var nonAssoc = node.infixNonAssociative.ToGrammar1();
     var prefix = node.prefix.ToGrammar1();
@@ -431,25 +445,13 @@ public static partial class MixFix {
     var infixl = node.infixLeftAssociative.ToGrammar1();
     var infixr = node.infixRightAssociative.ToGrammar1();
 
-
-
-
-
-
-    // TODO: BUG: only include these parts if there are
-    // any operators with that fixity, with the code below
-    // they are empty.
-
-
-    Func<string, Grammar1, Grammar1> Annotated = (s, g) => Grammar1.Annotated((s, g));
-
     return
-        Annotated("closed", closed ? closed : Grammar1.Impossible)
+        // TODO: we can normally remove the ?: checks, as the constructors for grammars
-      | Annotated("nonAssoc", (nonAssoc ? (lsucc, nonAssoc, rsucc) : Grammar1.Impossible))
+        // now coalesce Impossible cases in the correct way.
-      // TODO: post-processsing of the rightassoc list.
+        (closed ? N(closed) : Impossible)
-      | Annotated("prefix||infixr", ((prefix || infixr) ? ((prefix || (lsucc, infixr))["+"], rsucc) : Grammar1.Impossible))
+      | (nonAssoc ? N( (lsucc, nonAssoc, rsucc) ) : Impossible)
-      // TODO: post-processsing of the leftassoc list.
+      | ((prefix || infixr) ? R( ((prefix | (lsucc, infixr))["+"], rsucc) ) : Impossible)
-      | Annotated("postfix||infixl", ((postfix || infixl) ? (lsucc, (postfix || (infixl, rsucc))["+"]) : Grammar1.Impossible));
+      | ((postfix || infixl) ? L( (lsucc, (postfix || (infixl, rsucc))["+"]) ) : Impossible);
   }
 
   public static EquatableDictionary<string, Grammar1> ToGrammar1(this PrecedenceDAG precedenceDAG)
@@ -469,7 +471,7 @@ public static partial class MixFix {
         }
         return recur(labeled[r], labeled);
       },
-      Annotated:     a => recur(a.Item2, labeled),
+      Annotated:     a => Grammar2.Annotated((a.Item1, recur(a.Item2, labeled))),
       Terminal:      t => Grammar2.Terminal(t),
       Sequence:      l => Grammar2.Sequence(l.Select(g => recur(g, labeled))),
       Or:            l => Grammar2.Or(l.Select(g => recur(g, labeled))),

MixFixGenerator.cs

@@ -20,14 +20,20 @@ public static class ParserGenerator {
           Case("IEnumerable<Grammar1>", "Or"),
           Case("IEnumerable<Grammar1>", "Sequence"),
           Case("S",                     "Terminal"),
-          Case("ValueTuple<string, Grammar1>", "Annotated"),
+          Case("ValueTuple<Annotation, Grammar1>", "Annotated"),
           Case("string",                "Rule")),
 
+        Variant("Annotation",
+          Case("MixFix.Operator", "Operator"),
+          Case("Associativity", "SamePrecedence"),
+          Case("Hole")),
+
         Variant("Grammar2",
           Case("Grammar2",              "RepeatOnePlus"),
           Case("IEnumerable<Grammar2>", "Or"),
           Case("IEnumerable<Grammar2>", "Sequence"),
-          Case("S",                     "Terminal")),
+          Case("S",                     "Terminal"),
+          Case("ValueTuple<Annotation, Grammar2>", "Annotated")),
 
         Variant("Fixity",
           Case("Closed"),

Parser.cs

@@ -11,61 +11,44 @@ using Grammar2 = MixFix.Grammar2;
 using static Global;
 
 public static partial class Parser {
-  public static Option<ValueTuple<IImmutableEnumerator<Lexeme>, AstNode>> Parse3(
+  public static Option<ValueTuple<IImmutableEnumerator<Lexeme>, ParserResult>> Parse3(
     Func<IImmutableEnumerator<Lexeme>,
          Grammar2,
          //Option<IImmutableEnumerator<Lexeme>>
-         Option<ValueTuple<IImmutableEnumerator<Lexeme>, AstNode>>
+         Option<ValueTuple<IImmutableEnumerator<Lexeme>, ParserResult>>
          >
       Parse3,
     IImmutableEnumerator<Lexeme> tokens,
     Grammar2 grammar
-    ) =>
+    )
-    tokens
+    => grammar.Match(
-      .FirstAndRest()
+        RepeatOnePlus: g =>
-      .Match(
+          tokens.FoldMapWhileSome(restI => Parse3(restI, g))
-        None: () =>
+            .If((restN, nodes) => nodes.Count() > 1)
-          //throw new Exception("EOF, what to do?"),
+            .IfSome((restN, nodes) => (restN, ParserResult.Productions(nodes))),
-          None<ValueTuple<IImmutableEnumerator<Lexeme>, AstNode>>(),
+        // TODO: to check for ambiguous parses, we can use
-        Some: firstRest => {
+        // .Single(…) instead of .First(…).
-          var first = firstRest.Item1;
+        Or: l =>
-          var rest = firstRest.Item2;
+          l.First(g => Parse3(tokens, g)),
-          Log(first.lexeme);
+        Sequence: l =>
-          Log(grammar.ToString());
+          l.BindFoldMap(tokens, (restI, g) => Parse3(restI, g))
-          Log(grammar.Match(
+            .IfSome((restN, nodes) =>
-            RepeatOnePlus: _ => "RepeatOnePlus",
+              Log($"{nodes.Count()}/{l.Count()}", () =>
-            Or: _ => "Or",
+                (restN, ParserResult.Productions(nodes)))),
-            Sequence: _ => "Sequence",
+        Terminal: t =>
-            Terminal: t => "Terminal:"+t.ToString()));
+        // TODO: move the FirstAndRest here!
-          return grammar.Match(
+          tokens
-            RepeatOnePlus: g =>
+            .FirstAndRest()
-              rest.FoldMapWhileSome(restI => Parse3(restI, g))
+            // When EOF is reached, the parser can't accept this derivation.
-                .If<IImmutableEnumerator<Lexeme>, IEnumerable<AstNode>>((restN, nodes) => nodes.Count() > 1)
+            .If((first, rest) => first.state.Equals(t))
-                .IfSome((restN, nodes) => (restN, AstNode.Operator(nodes))),
+            .IfSome((first, rest) => (rest, ParserResult.Terminal(first))),
-            // TODO: to check for ambiguous parses, we can use
+        Annotated: a =>
-            // .Single(…) instead of .First(…).
+          // TODO: use the annotation to give some shape to these lists
-            Or: l =>
+          Parse3(tokens, a.Item2));
-              l.First(g => Parse3(rest, g)),
+      // TODO: at the top-level, check that the lexemes
-            Sequence: l => {
+      // are empty if the parser won't accept anything else.
-              return l.BindFoldMap(rest, (restI, g) => Parse3(restI, g))
-               .IfSome((restN, nodes) => {
-                 Log($"{nodes.Count()}/{l.Count()}");
-                 return (restN, AstNode.Operator(nodes));
-               });
-            },
-            Terminal: t =>
-              first.state.Equals(t)
-              ? (rest,
-                 AstNode.Terminal(/* TODO: */ Expr.String(rest.ToString())))
-                .Some()
-              : None<ValueTuple<IImmutableEnumerator<Lexeme>, AstNode>>()
-          );
-          // TODO: at the top-level, check that the lexemes
-          // are empty if the parser won't accept anything else.
-        }
-      );
 
-  public static Option<ValueTuple<IImmutableEnumerator<Lexeme>, AstNode>> Parse2(string source) {
+  public static Option<ValueTuple<IImmutableEnumerator<Lexeme>, ParserResult>> Parse2(string source) {
     Grammar2 grammar =
       DefaultGrammar.DefaultPrecedenceDAG.ToGrammar2();
     //Log(grammar.Str());
@@ -73,7 +56,7 @@ public static partial class Parser {
     var P = Func.YMemoize<
               IImmutableEnumerator<Lexeme>,
               Grammar2,
-              Option<ValueTuple<IImmutableEnumerator<Lexeme>, AstNode>>>(
+              Option<ValueTuple<IImmutableEnumerator<Lexeme>, ParserResult>>>(
       Parse3
     );
 

Utils/Global.cs

@@ -28,6 +28,8 @@ public static class Global {
 
   public static T To<T>(this T x) => x;
 
+  public static Option<T> If<T>(this bool cond, T x) => x.If(_ => cond);
+
   public static A FoldWhileSome<A>(A init, Func<A, Option<A>> f)
     => Collection.FoldWhileSome(init, f);
 

Utils/Immutable/Option.cs

@@ -76,6 +76,12 @@ namespace Immutable {
     public static Option<ValueTuple<T1, T2>> If<T1, T2>(this ValueTuple<T1, T2> value, Func<T1, T2, bool> predicate)
       => predicate(value.Item1, value.Item2) ? value.Some() : Option.None<ValueTuple<T1, T2>>();
 
+    public static Option<Tuple<T1, T2>> If<T1, T2>(this Tuple<T1, T2> value, Func<T1, T2, bool> predicate)
+      => predicate(value.Item1, value.Item2) ? value.Some() : Option.None<Tuple<T1, T2>>();
+
+    public static Option<Tuple<T1, T2>> If<T1, T2>(this Option<Tuple<T1, T2>> value, Func<T1, T2, bool> predicate)
+      => value.Bind(val => val.If(predicate));
+
     public static Option<U> IfSome<T1, T2, U>(this Option<Tuple<T1, T2>> o, Func<T1, T2, U> some)
       => o.Map(o1o2 => some(o1o2.Item1, o1o2.Item2));
 

Utils/ToString.cs

@@ -32,6 +32,9 @@ public static class ToStringImplementations {
   public static string Str<T>(this IEnumerable<Ast.AstNode> e)
     => $"IEnumerab({e.Select(x => x.Str<Ast.AstNode>()).JoinWith(", ")})";
 
+  public static string Str<T>(this IEnumerable<Ast.ParserResult> e)
+    => $"IEnumerab({e.Select(x => x.Str<Ast.ParserResult>()).JoinWith(", ")})";
+
   public static string Str<Grammar>(this ImmutableDictionary<string,Grammar> h)
     => $"ImmutableDictionary(\n{h.Select(x => $"  {x.Key.Str<string>()}:{x.Value.Str<Grammar>()}").JoinWith(",\n")}\n)";