From 74c3abcccc1acdf1b2e71efba87ddf57e5639a0f Mon Sep 17 00:00:00 2001 From: Suzanne Soy Date: Mon, 31 Aug 2020 13:36:14 +0000 Subject: [PATCH] Debugging the parser --- DefaultGrammar.cs | 4 +- Lexer.cs | 26 ++++-- MixFix.cs | 161 +++++++++++++++++++++++++++++++------- Parser.cs | 33 +++++--- Tests/001-42.e | 2 +- Utils/Enumerable.cs | 18 +++-- Utils/Immutable/Option.cs | 6 ++ Utils/Lens.cs | 4 +- Utils/ToString.cs | 7 ++ main.cs | 10 +-- 10 files changed, 210 insertions(+), 61 deletions(-) diff --git a/DefaultGrammar.cs b/DefaultGrammar.cs index b62e221..6533427 100644 --- a/DefaultGrammar.cs +++ b/DefaultGrammar.cs @@ -11,10 +11,10 @@ public static class DefaultGrammar { .WithOperator("equality", NonAssociative, "int|terminal|additive|multiplicative", S.Eq, "int|terminal|additive|multiplicative") .WithOperator("int", NonAssociative, S.Int) .WithOperator("additive", LeftAssociative, "int|terminal|multiplicative", S.Plus, "int|terminal|multiplicative") -// .WithOperator("multiplicative", LeftAssociative, "int|terminal", S.Times, "int|terminal") + .WithOperator("multiplicative", LeftAssociative, "int|terminal", S.Times, "int|terminal") .WithOperator("terminal", NonAssociative, S.Ident) // This is the root set of operators .WithOperator("program", NonAssociative, // "bool" // TODO: this needs aliases - "equality|terminal", S.And, "equality|terminal"); + "terminal", S.And, "terminal");// TODO: re-add equality| } \ No newline at end of file diff --git a/Lexer.cs b/Lexer.cs index 9f04d78..64703bb 100644 --- a/Lexer.cs +++ b/Lexer.cs @@ -213,22 +213,36 @@ public static partial class Lexer { [F] private partial class SkipInitialEmptyWhitespace { + public IImmutableEnumerator F( + IImmutableEnumerator lx + ) + => lx.FirstAndRest().Match( + Some: hdtl => + // skip the initial empty whitespace + "".Equals(hdtl.Item1.lexeme) + ? hdtl.Item2 + : hdtl.Item1.ImSingleton().Concat(hdtl.Item2), + None: Empty()); + } + + // TODO: move this to a .Filter() extension method. + [F] + private partial class DiscardWhitespace { public IImmutableEnumerator F( IImmutableEnumerator lx ) => lx.FirstAndRest().Match>, IImmutableEnumerator>( Some: hdtl => // skip the initial empty whitespace - string.Equals( - "", - hdtl.Item1.lexeme) - ? hdtl.Item2 - : hdtl.Item1.ImSingleton().Concat(hdtl.Item2), + hdtl.Item1.state.Equals(S.Space) + ? hdtl.Item2 + : hdtl.Item1.ImSingleton().Concat(hdtl.Item2.Lazy(DiscardWhitespace.Eq)), None: Empty()); } public static IImmutableEnumerator Lex(string source) => Lex1(source) .Flatten() - .Lazy(SkipInitialEmptyWhitespace.Eq); + //.Lazy(SkipInitialEmptyWhitespace.Eq) + .Lazy(DiscardWhitespace.Eq); } \ No newline at end of file diff --git a/MixFix.cs b/MixFix.cs index 1846935..d471d7d 100644 --- a/MixFix.cs +++ b/MixFix.cs @@ -21,9 +21,20 @@ public static partial class MixFix { public static Grammar1 Or(params Grammar1[] xs) => Or(xs.ToImmutableList()); + public static Grammar1 Empty + = new Grammar1.Cases.Sequence(Enumerable.Empty()); + + public static Grammar1 Impossible + = new Grammar1.Cases.Or(Enumerable.Empty()); + + // TODO: inline the OR, detect impossible cases (Or of 0) public static Grammar1 Sequence(IEnumerable xs) { var filteredXs = xs.Where(x => !x.IsEmpty); - if (filteredXs.Count() == 1) { + if (filteredXs.Any(x => x.IsImpossible)) { + return Impossible; + } else if (filteredXs.Count() == 0) { + return Empty; + } else if (filteredXs.Count() == 1) { return filteredXs.Single().ElseThrow(() => new Exception("TODO: use an either to prove that this is safe.")); } else { return new Grammar1.Cases.Sequence(filteredXs); @@ -31,8 +42,17 @@ public static partial class MixFix { } public static Grammar1 Or(IEnumerable xs) { - var filteredXs = xs.Where(x => !x.IsEmpty); - if (filteredXs.Count() == 1) { + var filteredXsNoEmpty = + xs.Where(x => !x.IsImpossible) + .Where(x => !x.IsEmpty); + var filteredXs = + ( xs.Any(x => x.IsEmpty) + && !filteredXsNoEmpty.Any(x => x.AllowsEmpty)) + ? Empty.Cons(filteredXsNoEmpty) + : filteredXsNoEmpty; + if (filteredXs.All(x => x.IsImpossible)) { + return new Grammar1.Cases.Or(Enumerable.Empty()); + } else if (filteredXs.Count() == 1) { return filteredXs.Single().ElseThrow(() => new Exception("TODO: use an either to prove that this is safe.")); } else { return new Grammar1.Cases.Or(filteredXs); @@ -42,20 +62,43 @@ public static partial class MixFix { public static Grammar1 RepeatOnePlus(Grammar1 g) => g.IsEmpty ? Grammar1.Empty + : g.IsImpossible + ? Grammar1.Impossible : new Grammar1.Cases.RepeatOnePlus(g); - public static Grammar1 Empty = new Grammar1.Cases.Or(Enumerable.Empty()); - public bool IsEmpty { get => this.Match( - Or: l => l.Count() == 0, - Sequence: l => l.Count() == 0, + Or: l => l.All(g => g.IsEmpty), + Sequence: l => l.All(g => g.IsEmpty), RepeatOnePlus: g => g.IsEmpty, Terminal: t => false, Rule: r => false ); } + // TODO: cache this! + public bool AllowsEmpty { + get => this.Match( + Or: l => l.Any(g => g.AllowsEmpty), + Sequence: l => l.All(g => g.IsEmpty), + // This one should not be true, if it is + // then the precedence graph may be ill-formed? + RepeatOnePlus: g => g.AllowsEmpty, + Terminal: t => false, + Rule: r => false + ); + } + + public bool IsImpossible { + get => this.Match( + Or: l => l.All(g => g.IsImpossible), + Sequence: l => l.Any(g => g.IsImpossible), + RepeatOnePlus: g => g.IsImpossible, + Terminal: t => false, + Rule: r => false + ); + } + public static Grammar1 operator |(Grammar1 a, Grammar1 b) => Or(a, b); @@ -65,8 +108,8 @@ public static partial class MixFix { public static implicit operator Grammar1((Grammar1 a, Grammar1 b, Grammar1 c) gs) => Sequence(gs.a, gs.b, gs.c); - public static bool operator true(Grammar1 g) => !g.IsEmpty; - public static bool operator false(Grammar1 g) => g.IsEmpty; + public static bool operator true(Grammar1 g) => !g.IsImpossible; + public static bool operator false(Grammar1 g) => g.IsImpossible; public Grammar1 this[string multiplicity] { get { @@ -85,20 +128,55 @@ public static partial class MixFix { private string Paren(bool paren, string s) => paren ? $"({s})" : s; - string CustomToString() - => this.Match( - Or: l => l.Count() == 0 - ? "Or(Empty)" - : Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(" | ")), - Sequence: l => l.Count() == 0 - ? "Sequence(Empty)" - : Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(",")), - RepeatOnePlus: g => $"({g.Str()})+", + string CustomToString() => + this.IsEmpty ? "Empty" + : this.IsImpossible ? "Impossible" + : this.Match( + Or: l => + Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(" | ")), + Sequence: l => + Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(", ")), + RepeatOnePlus: g => $"{g.Str()}+", Terminal: t => t.Str(), Rule: r => r ); } + public partial class Grammar2 { + public bool IsEmpty { + get => this.Match( + Or: l => l.All(g => g.IsEmpty), + Sequence: l => l.All(g => g.IsEmpty), + RepeatOnePlus: g => g.IsEmpty, + Terminal: t => false + ); + } + + public bool IsImpossible { + get => this.Match( + Or: l => l.All(g => g.IsImpossible), + Sequence: l => l.Any(g => g.IsImpossible), + RepeatOnePlus: g => g.IsImpossible, + Terminal: t => false + ); + } + + private string Paren(bool paren, string s) + => paren ? $"({s})" : s; + + string CustomToString() => + this.IsEmpty ? "Empty" + : this.IsImpossible ? "Impossible" + : this.Match( + Or: l => + Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(" | ")), + Sequence: l => + Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(", ")), + RepeatOnePlus: g => $"{g.Str()}+", + Terminal: t => t.Str() + ); + } + public partial class Operator { private string CustomToString() => $"Operator(\"{precedenceGroup}\", {fixity}, {parts.Select(x => x.Match(Hole: h => h.Select(g => g.ToString()).JoinWith("|"), Name: n => $"\"{n}\"")).JoinWith(", ")})"; @@ -222,7 +300,7 @@ public static partial class MixFix { = new PrecedenceDAG(EmptyDAGNode); public static Whole Add(this ILens node, Operator @operator) { - return @operator.fixity.Match( + return @operator.Cons(@operator.fixity.Match( Closed: () => node.Closed(), Prefix: @@ -235,7 +313,7 @@ public static partial class MixFix { () => node.InfixRightAssociative(), InfixLeftAssociative: () => node.InfixLeftAssociative() - ).Cons(@operator); + )); } public static void CheckHole(PrecedenceDAG precedenceDAG, Operator @operator, string name, Option existing, Option @new) { @@ -344,15 +422,27 @@ public static partial class MixFix { var infixl = node.infixLeftAssociative.ToGrammar1(); var infixr = node.infixRightAssociative.ToGrammar1(); + + //Log("closed.IsImpossible:"+closed.IsImpossible); + + + + + // TODO: BUG: only include these parts if there are - // any operators with that fixity. + // any operators with that fixity, with the code below + // they are empty. + + + + return - closed - | (nonAssoc ? (lsucc, nonAssoc, rsucc) : Grammar1.Empty) + (closed ? closed : Grammar1.Impossible) + | (nonAssoc ? (lsucc, nonAssoc, rsucc) : Grammar1.Impossible) + // TODO: post-processsing of the rightassoc list. + | ((prefix || infixr) ? ((prefix || (lsucc, infixr))["+"], rsucc) : Grammar1.Impossible) // TODO: post-processsing of the leftassoc list. - | ((prefix || infixr) ? ((prefix | (lsucc, infixr))["+"], rsucc) : Grammar1.Empty) - // TODO: post-processsing of the leftassoc list. - | ((postfix || infixl) ? (lsucc, (postfix | (infixl, rsucc))["+"]) : Grammar1.Empty); + | ((postfix || infixl) ? (lsucc, (postfix || (infixl, rsucc))["+"]) : Grammar1.Impossible); } public static EquatableDictionary ToGrammar1(this PrecedenceDAG precedenceDAG) @@ -363,15 +453,28 @@ public static partial class MixFix { private static Grammar2 Recur(Func, Grammar2> recur, Grammar1 grammar1, EquatableDictionary labeled) => grammar1.Match( // TODO: throw exception if lookup fails - Rule: r => recur(labeled[r], labeled), + Rule: r => { + Grammar1 lr = null; + try { + lr = labeled[r]; + } catch (Exception e) { + throw new ParserExtensionException($"Internal error: could not find node {r} in labeled grammar. It only contains labels for: {labeled.Select(kvp => kvp.Key.ToString()).JoinWith(", ")}."); + } + return recur(labeled[r], labeled); + }, Terminal: t => Grammar2.Terminal(t), Sequence: l => Grammar2.Sequence(l.Select(g => recur(g, labeled))), Or: l => Grammar2.Or(l.Select(g => recur(g, labeled))), RepeatOnePlus: g => Grammar2.RepeatOnePlus(recur(g, labeled)) ); - public static Grammar2 ToGrammar2(this EquatableDictionary labeled) - => Func.YMemoize, Grammar2>(Recur)(Grammar1.Rule("program"), labeled); + public static Grammar2 ToGrammar2(this EquatableDictionary labeled) { + foreach (var kvp in labeled) { + Log($"{kvp.Key} -> {kvp.Value.ToString()}"); + } + Log(""); + return Func.YMemoize, Grammar2>(Recur)(Grammar1.Rule("program"), labeled); + } public static Grammar2 ToGrammar2(this PrecedenceDAG precedenceDAG) => precedenceDAG.ToGrammar1().ToGrammar2(); diff --git a/Parser.cs b/Parser.cs index cc363ea..8fd37da 100644 --- a/Parser.cs +++ b/Parser.cs @@ -25,26 +25,34 @@ public static partial class Parser { .FirstAndRest() .Match( None: () => - throw new Exception("EOF, what to do?"), + //throw new Exception("EOF, what to do?"), + None, AstNode>>(), Some: firstRest => { - // Case("IImmutableEnumerable", "Operator")) var first = firstRest.Item1; var rest = firstRest.Item2; + Log(first.lexeme); + Log(grammar.ToString()); + Log(grammar.Match( + RepeatOnePlus: _ => "RepeatOnePlus", + Or: _ => "Or", + Sequence: _ => "Sequence", + Terminal: t => "Terminal:"+t.ToString())); return grammar.Match( RepeatOnePlus: g => rest.FoldMapWhileSome(restI => Parse3(restI, g)) .If, IEnumerable>((restN, nodes) => nodes.Count() > 1) .IfSome((restN, nodes) => (restN, AstNode.Operator(nodes))), - //.IfSome(rest1 => - // TODO: remove IfSome above (useless) && aggregate - // WhileSome(rest1, restI => Parse3(restI, g))), // TODO: to check for ambiguous parses, we can use - // .SingleArg(…) instead of .FirstArg(…). + // .Single(…) instead of .First(…). Or: l => l.First(g => Parse3(rest, g)), - Sequence: l => - l.BindFoldMap(rest, (restI, g) => Parse3(restI, g)) - .IfSome((restN, nodes) => (restN, AstNode.Operator(nodes))), + Sequence: l => { + return l.BindFoldMap(rest, (restI, g) => Parse3(restI, g)) + .IfSome((restN, nodes) => { + Log($"{nodes.Count()}/{l.Count()}"); + return (restN, AstNode.Operator(nodes)); + }); + }, Terminal: t => first.state.Equals(t) ? (rest, @@ -60,7 +68,7 @@ public static partial class Parser { public static Option, AstNode>> Parse2(string source) { Grammar2 grammar = DefaultGrammar.DefaultPrecedenceDAG.ToGrammar2(); - Log(grammar.Str()); + //Log(grammar.Str()); var P = Func.YMemoize< IImmutableEnumerator, @@ -73,6 +81,11 @@ public static partial class Parser { } public static Ast.Expr Parse(string source) { + Log(""); + Log("" + Parse2(source).ToString()); + Log(""); + Environment.Exit(0); + return Lexer.Lex(source) .SelectMany(lexeme => lexeme.state.Match( diff --git a/Tests/001-42.e b/Tests/001-42.e index f70d7bb..5aa6034 100644 --- a/Tests/001-42.e +++ b/Tests/001-42.e @@ -1 +1 @@ -42 \ No newline at end of file +true && false \ No newline at end of file diff --git a/Utils/Enumerable.cs b/Utils/Enumerable.cs index b30c652..7a71b69 100644 --- a/Utils/Enumerable.cs +++ b/Utils/Enumerable.cs @@ -8,14 +8,20 @@ public static class Collection { public static void ForEach(this IEnumerable x, Action f) => x.ToImmutableList().ForEach(f); - public static ImmutableList Cons(this ImmutableList l, T x) + public static ImmutableList Cons(this T x, ImmutableList l) => l.Add(x); - public static ImmutableList> Cons(this ImmutableList> l, T x, U y) - => l.Cons(Tuple.Create(x,y)); + public static IEnumerable Cons(this T x, IEnumerable l) + => x.Singleton().Concat(l); - public static ImmutableList> Cons(this ImmutableList> l, T x, U y, V z) - => l.Cons(Tuple.Create(x,y,z)); + public static IEnumerable Concat(this IEnumerable l, T x) + => l.Concat(x.Singleton()); + + public static ImmutableList> Add(this ImmutableList> l, T x, U y) + => l.Add(Tuple.Create(x,y)); + + public static ImmutableList> Add(this ImmutableList> l, T x, U y, V z) + => l.Add(Tuple.Create(x,y,z)); public static void Deconstruct(this Tuple t, out A a, out B b) { a = t.Item1; @@ -241,7 +247,7 @@ public static class Collection { foreach (var x in e) { var newAcc = f(acc, x); if (newAcc.IsNone) { - break; + return Option.None(); } else { acc = newAcc.ElseThrow(new Exception("impossible")); } diff --git a/Utils/Immutable/Option.cs b/Utils/Immutable/Option.cs index d0b9b8b..2ab3d38 100644 --- a/Utils/Immutable/Option.cs +++ b/Utils/Immutable/Option.cs @@ -27,6 +27,9 @@ namespace Immutable { System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => this.GetEnumerator(); + + public override string ToString() + => $"Some({value.ToString()})"; } public class None : Option, System.Collections.IEnumerable { @@ -42,6 +45,9 @@ namespace Immutable { System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => this.GetEnumerator(); + + public override string ToString() + => $"None"; } } } diff --git a/Utils/Lens.cs b/Utils/Lens.cs index fb759f7..b70b125 100644 --- a/Utils/Lens.cs +++ b/Utils/Lens.cs @@ -10,8 +10,8 @@ public static class LensExtensionMethods { public static Whole Update(this ILens lens, Hole newHole) => lens.Update(oldHole => newHole); - public static Whole Cons(this ILens, Whole> lens, T value) - => lens.Update(oldHole => oldHole.Cons(value)); + public static Whole Cons(this T value, ILens, Whole> lens) + => lens.Update(oldHole => value.Cons(oldHole)); public static ILens ChainLens(this string hole, System.Func wrap) => new LeafLens(wrap: wrap, oldHole: hole); diff --git a/Utils/ToString.cs b/Utils/ToString.cs index 2fac8b1..80de4e4 100644 --- a/Utils/ToString.cs +++ b/Utils/ToString.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Collections.Generic; using System.Collections.Immutable; public interface IString { @@ -25,6 +26,12 @@ public static class ToStringImplementations { public static string Str(this ImmutableHashSet h) => $"ImmutableHashSet({h.Select(x => x.Str()).JoinWith(", ")})"; + public static string Str(this IEnumerable e) + => $"IEnumerabl({e.Select(x => x.Str()).JoinWith(", ")})"; + + public static string Str(this IEnumerable e) + => $"IEnumerab({e.Select(x => x.Str()).JoinWith(", ")})"; + public static string Str(this ImmutableDictionary h) => $"ImmutableDictionary(\n{h.Select(x => $" {x.Key.Str()}:{x.Value.Str()}").JoinWith(",\n")}\n)"; diff --git a/main.cs b/main.cs index d2829d2..58e701b 100644 --- a/main.cs +++ b/main.cs @@ -22,7 +22,7 @@ public static class MainClass { var destPath = tests_results.Combine(source); var sourcePath = tests.Combine(source); var expected = sourcePath.DropExtension().Combine(Ext(".o")); - + Console.Write($"\x1b[KRunning test {source} ({toolchainName}) "); destPath.DirName().Create(); @@ -47,7 +47,7 @@ public static class MainClass { Console.WriteLine($"\x1b[1;33m{source}: expected {expectedStr} but got {actualStr}.\x1b[m\n"); return false; } else { - Console.Write("\x1b[1;32mOK\x1b[m\r"); + Console.Write("\x1b[1;32mOK\x1b[m"); // \r at the end for quiet return true; } } @@ -58,13 +58,13 @@ public static class MainClass { // first-class functions by using repeated .Add() // See https://repl.it/@suzannesoy/WarlikeWorstTraining#main.cs var compilers = ImmutableList>.Empty - .Cons(" js ", Compilers.JS.Compile, Exe("node")) - .Cons("eval", Evaluator.Evaluate, Exe("cat")); + .Add(" js ", Compilers.JS.Compile, Exe("node")) + .Add("eval", Evaluator.Evaluate, Exe("cat")); var total = 0; var passed = 0; var failed = 0; - foreach (var t in Dir("Tests/").GetFiles("*.e", SearchOption.AllDirectories)) { + foreach (var t in Dir("Tests/").GetFiles("*.e", SearchOption.AllDirectories).OrderBy(f => f.ToString())) { foreach (var compiler in compilers) { if (RunTest(compiler.Item1, compiler.Item2, compiler.Item3, t)) { passed++;