From 592a758fb1e9af381b4ae5a5a31d6d17dfd4acd9 Mon Sep 17 00:00:00 2001 From: Suzanne Soy Date: Mon, 31 Aug 2020 18:38:52 +0000 Subject: [PATCH] Keep annotations in Parser --- AstGenerator.cs | 4 ++ DefaultGrammar.cs | 6 +-- Lexer.cs | 11 +++--- MixFix.cs | 54 +++++++++++++------------- MixFixGenerator.cs | 10 ++++- Parser.cs | 79 +++++++++++++++------------------------ Utils/Global.cs | 2 + Utils/Immutable/Option.cs | 6 +++ Utils/ToString.cs | 3 ++ 9 files changed, 91 insertions(+), 84 deletions(-) diff --git a/AstGenerator.cs b/AstGenerator.cs index 1154fa6..916e3d9 100644 --- a/AstGenerator.cs +++ b/AstGenerator.cs @@ -12,6 +12,10 @@ public static class AstGenerator { Variant("Expr", Case("int", "Int"), Case("string", "String")), + Variant("ParserResult", + Case("MixFix.Annotation", "Annotated"), + Case("Lexer.Lexeme", "Terminal"), + Case("IEnumerable", "Productions")), Variant("AstNode", Case("Expr", "Terminal"), Case("IEnumerable", "Operator")))); diff --git a/DefaultGrammar.cs b/DefaultGrammar.cs index 9410fa3..ff13e35 100644 --- a/DefaultGrammar.cs +++ b/DefaultGrammar.cs @@ -7,14 +7,14 @@ using static MixFix.Associativity; public static class DefaultGrammar { public static PrecedenceDAG DefaultPrecedenceDAG = EmptyPrecedenceDAG - /*.WithOperator("bool", NonAssociative, "equality|terminal", S.And, "equality|terminal") + .WithOperator("bool", NonAssociative, "equality|terminal", S.And, "equality|terminal") .WithOperator("equality", NonAssociative, "int|terminal|additive|multiplicative", S.Eq, "int|terminal|additive|multiplicative") .WithOperator("int", NonAssociative, S.Int) .WithOperator("additive", LeftAssociative, "int|terminal|multiplicative", S.Plus, "int|terminal|multiplicative") - .WithOperator("multiplicative", LeftAssociative, "int|terminal", S.Times, "int|terminal")*/ + .WithOperator("multiplicative", LeftAssociative, "int|terminal", S.Times, "int|terminal") .WithOperator("terminal", NonAssociative, S.Ident) // This is the root set of operators .WithOperator("program", NonAssociative, // "bool" // TODO: this needs aliases - "terminal", S.And, "terminal");// TODO: re-add equality| + "equality|terminal", S.And, "equality|terminal"); } \ No newline at end of file diff --git a/Lexer.cs b/Lexer.cs index 64703bb..7a5d9c4 100644 --- a/Lexer.cs +++ b/Lexer.cs @@ -232,11 +232,12 @@ public static partial class Lexer { IImmutableEnumerator lx ) => lx.FirstAndRest().Match>, IImmutableEnumerator>( - Some: hdtl => - // skip the initial empty whitespace - hdtl.Item1.state.Equals(S.Space) - ? hdtl.Item2 - : hdtl.Item1.ImSingleton().Concat(hdtl.Item2.Lazy(DiscardWhitespace.Eq)), + Some: hdtl => { + var rest = hdtl.Item2.Lazy(DiscardWhitespace.Eq); + return hdtl.Item1.state.Equals(S.Space) + ? rest + : hdtl.Item1.ImSingleton().Concat(rest); + }, None: Empty()); } diff --git a/MixFix.cs b/MixFix.cs index 4b5af8e..45b3fce 100644 --- a/MixFix.cs +++ b/MixFix.cs @@ -155,7 +155,8 @@ public static partial class MixFix { Or: l => l.All(g => g.IsEmpty), Sequence: l => l.All(g => g.IsEmpty), RepeatOnePlus: g => g.IsEmpty, - Terminal: t => false + Terminal: t => false, + Annotated: a => a.Item2.IsEmpty ); } @@ -164,7 +165,8 @@ public static partial class MixFix { Or: l => l.All(g => g.IsImpossible), Sequence: l => l.Any(g => g.IsImpossible), RepeatOnePlus: g => g.IsImpossible, - Terminal: t => false + Terminal: t => false, + Annotated: a => a.Item2.IsImpossible ); } @@ -182,7 +184,8 @@ public static partial class MixFix { ? "EmptySequence" : Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(", ")), RepeatOnePlus: g => $"{g.Str()}+", - Terminal: t => t.Str() + Terminal: t => t.Str(), + Annotated: a => $"Annotated({a.Item1.Str()}, {a.Item2.Str()})" ); } @@ -414,16 +417,27 @@ public static partial class MixFix { Hole: precedenceGroups => precedenceGroups.ToGrammar1()); public static Grammar1 ToGrammar1(this Operator @operator) - => Grammar1.Sequence( - @operator.internalParts.Select(part => part.ToGrammar1())); + => Grammar1.Annotated( + (Annotation.Operator(@operator), + Grammar1.Sequence( + @operator.internalParts.Select( + part => part.ToGrammar1())))); public static Grammar1 ToGrammar1(this IEnumerable operators) => Grammar1.Or( operators.Select(@operator => @operator.ToGrammar1())); public static Grammar1 ToGrammar1(this DAGNode node) { - var lsucc = node.leftmostHole_.ToGrammar1(); - var rsucc = node.rightmostHole_.ToGrammar1(); + Func SamePrecedence = (a, g) + => Grammar1.Annotated((Annotation.SamePrecedence(a), g)); + Func R = g => SamePrecedence(Associativity.RightAssociative, g); + Func L = g => SamePrecedence(Associativity.LeftAssociative, g); + Func N = g => SamePrecedence(Associativity.NonAssociative, g); + Func H = g => Grammar1.Annotated((Annotation.Hole, g)); + var Impossible = Grammar1.Impossible; + + var lsucc = H(node.leftmostHole_.ToGrammar1()); + var rsucc = H(node.rightmostHole_.ToGrammar1()); var closed = node.closed.ToGrammar1(); var nonAssoc = node.infixNonAssociative.ToGrammar1(); var prefix = node.prefix.ToGrammar1(); @@ -431,25 +445,13 @@ public static partial class MixFix { var infixl = node.infixLeftAssociative.ToGrammar1(); var infixr = node.infixRightAssociative.ToGrammar1(); - - - - - - // TODO: BUG: only include these parts if there are - // any operators with that fixity, with the code below - // they are empty. - - - Func Annotated = (s, g) => Grammar1.Annotated((s, g)); - return - Annotated("closed", closed ? closed : Grammar1.Impossible) - | Annotated("nonAssoc", (nonAssoc ? (lsucc, nonAssoc, rsucc) : Grammar1.Impossible)) - // TODO: post-processsing of the rightassoc list. - | Annotated("prefix||infixr", ((prefix || infixr) ? ((prefix || (lsucc, infixr))["+"], rsucc) : Grammar1.Impossible)) - // TODO: post-processsing of the leftassoc list. - | Annotated("postfix||infixl", ((postfix || infixl) ? (lsucc, (postfix || (infixl, rsucc))["+"]) : Grammar1.Impossible)); + // TODO: we can normally remove the ?: checks, as the constructors for grammars + // now coalesce Impossible cases in the correct way. + (closed ? N(closed) : Impossible) + | (nonAssoc ? N( (lsucc, nonAssoc, rsucc) ) : Impossible) + | ((prefix || infixr) ? R( ((prefix | (lsucc, infixr))["+"], rsucc) ) : Impossible) + | ((postfix || infixl) ? L( (lsucc, (postfix || (infixl, rsucc))["+"]) ) : Impossible); } public static EquatableDictionary ToGrammar1(this PrecedenceDAG precedenceDAG) @@ -469,7 +471,7 @@ public static partial class MixFix { } return recur(labeled[r], labeled); }, - Annotated: a => recur(a.Item2, labeled), + Annotated: a => Grammar2.Annotated((a.Item1, recur(a.Item2, labeled))), Terminal: t => Grammar2.Terminal(t), Sequence: l => Grammar2.Sequence(l.Select(g => recur(g, labeled))), Or: l => Grammar2.Or(l.Select(g => recur(g, labeled))), diff --git a/MixFixGenerator.cs b/MixFixGenerator.cs index a3e5da4..21163d6 100644 --- a/MixFixGenerator.cs +++ b/MixFixGenerator.cs @@ -20,14 +20,20 @@ public static class ParserGenerator { Case("IEnumerable", "Or"), Case("IEnumerable", "Sequence"), Case("S", "Terminal"), - Case("ValueTuple", "Annotated"), + Case("ValueTuple", "Annotated"), Case("string", "Rule")), + Variant("Annotation", + Case("MixFix.Operator", "Operator"), + Case("Associativity", "SamePrecedence"), + Case("Hole")), + Variant("Grammar2", Case("Grammar2", "RepeatOnePlus"), Case("IEnumerable", "Or"), Case("IEnumerable", "Sequence"), - Case("S", "Terminal")), + Case("S", "Terminal"), + Case("ValueTuple", "Annotated")), Variant("Fixity", Case("Closed"), diff --git a/Parser.cs b/Parser.cs index 8fd37da..0863b18 100644 --- a/Parser.cs +++ b/Parser.cs @@ -11,61 +11,44 @@ using Grammar2 = MixFix.Grammar2; using static Global; public static partial class Parser { - public static Option, AstNode>> Parse3( + public static Option, ParserResult>> Parse3( Func, Grammar2, //Option> - Option, AstNode>> + Option, ParserResult>> > Parse3, IImmutableEnumerator tokens, Grammar2 grammar - ) => - tokens - .FirstAndRest() - .Match( - None: () => - //throw new Exception("EOF, what to do?"), - None, AstNode>>(), - Some: firstRest => { - var first = firstRest.Item1; - var rest = firstRest.Item2; - Log(first.lexeme); - Log(grammar.ToString()); - Log(grammar.Match( - RepeatOnePlus: _ => "RepeatOnePlus", - Or: _ => "Or", - Sequence: _ => "Sequence", - Terminal: t => "Terminal:"+t.ToString())); - return grammar.Match( - RepeatOnePlus: g => - rest.FoldMapWhileSome(restI => Parse3(restI, g)) - .If, IEnumerable>((restN, nodes) => nodes.Count() > 1) - .IfSome((restN, nodes) => (restN, AstNode.Operator(nodes))), - // TODO: to check for ambiguous parses, we can use - // .Single(…) instead of .First(…). - Or: l => - l.First(g => Parse3(rest, g)), - Sequence: l => { - return l.BindFoldMap(rest, (restI, g) => Parse3(restI, g)) - .IfSome((restN, nodes) => { - Log($"{nodes.Count()}/{l.Count()}"); - return (restN, AstNode.Operator(nodes)); - }); - }, - Terminal: t => - first.state.Equals(t) - ? (rest, - AstNode.Terminal(/* TODO: */ Expr.String(rest.ToString()))) - .Some() - : None, AstNode>>() - ); - // TODO: at the top-level, check that the lexemes - // are empty if the parser won't accept anything else. - } - ); + ) + => grammar.Match( + RepeatOnePlus: g => + tokens.FoldMapWhileSome(restI => Parse3(restI, g)) + .If((restN, nodes) => nodes.Count() > 1) + .IfSome((restN, nodes) => (restN, ParserResult.Productions(nodes))), + // TODO: to check for ambiguous parses, we can use + // .Single(…) instead of .First(…). + Or: l => + l.First(g => Parse3(tokens, g)), + Sequence: l => + l.BindFoldMap(tokens, (restI, g) => Parse3(restI, g)) + .IfSome((restN, nodes) => + Log($"{nodes.Count()}/{l.Count()}", () => + (restN, ParserResult.Productions(nodes)))), + Terminal: t => + // TODO: move the FirstAndRest here! + tokens + .FirstAndRest() + // When EOF is reached, the parser can't accept this derivation. + .If((first, rest) => first.state.Equals(t)) + .IfSome((first, rest) => (rest, ParserResult.Terminal(first))), + Annotated: a => + // TODO: use the annotation to give some shape to these lists + Parse3(tokens, a.Item2)); + // TODO: at the top-level, check that the lexemes + // are empty if the parser won't accept anything else. - public static Option, AstNode>> Parse2(string source) { + public static Option, ParserResult>> Parse2(string source) { Grammar2 grammar = DefaultGrammar.DefaultPrecedenceDAG.ToGrammar2(); //Log(grammar.Str()); @@ -73,7 +56,7 @@ public static partial class Parser { var P = Func.YMemoize< IImmutableEnumerator, Grammar2, - Option, AstNode>>>( + Option, ParserResult>>>( Parse3 ); diff --git a/Utils/Global.cs b/Utils/Global.cs index b68e80c..16de047 100644 --- a/Utils/Global.cs +++ b/Utils/Global.cs @@ -28,6 +28,8 @@ public static class Global { public static T To(this T x) => x; + public static Option If(this bool cond, T x) => x.If(_ => cond); + public static A FoldWhileSome(A init, Func> f) => Collection.FoldWhileSome(init, f); diff --git a/Utils/Immutable/Option.cs b/Utils/Immutable/Option.cs index 2ab3d38..f53ca82 100644 --- a/Utils/Immutable/Option.cs +++ b/Utils/Immutable/Option.cs @@ -76,6 +76,12 @@ namespace Immutable { public static Option> If(this ValueTuple value, Func predicate) => predicate(value.Item1, value.Item2) ? value.Some() : Option.None>(); + public static Option> If(this Tuple value, Func predicate) + => predicate(value.Item1, value.Item2) ? value.Some() : Option.None>(); + + public static Option> If(this Option> value, Func predicate) + => value.Bind(val => val.If(predicate)); + public static Option IfSome(this Option> o, Func some) => o.Map(o1o2 => some(o1o2.Item1, o1o2.Item2)); diff --git a/Utils/ToString.cs b/Utils/ToString.cs index 80de4e4..739fa7f 100644 --- a/Utils/ToString.cs +++ b/Utils/ToString.cs @@ -32,6 +32,9 @@ public static class ToStringImplementations { public static string Str(this IEnumerable e) => $"IEnumerab({e.Select(x => x.Str()).JoinWith(", ")})"; + public static string Str(this IEnumerable e) + => $"IEnumerab({e.Select(x => x.Str()).JoinWith(", ")})"; + public static string Str(this ImmutableDictionary h) => $"ImmutableDictionary(\n{h.Select(x => $" {x.Key.Str()}:{x.Value.Str()}").JoinWith(",\n")}\n)";