suzanne.soy/envlang-csharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Debugging the parser

Browse Source
This commit is contained in:
Suzanne Soy 2020-08-31 13:36:14 +00:00
parent 829bff6c2b
commit 74c3abcccc
10 changed files with 210 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
DefaultGrammar.cs
View File

@ -11,10 +11,10 @@ public static class DefaultGrammar {
.WithOperator("equality", NonAssociative, "int|terminal|additive|multiplicative", S.Eq, "int|terminal|additive|multiplicative")
.WithOperator("int", NonAssociative, S.Int)
.WithOperator("additive", LeftAssociative, "int|terminal|multiplicative", S.Plus, "int|terminal|multiplicative")
// .WithOperator("multiplicative", LeftAssociative, "int|terminal", S.Times, "int|terminal")
.WithOperator("multiplicative", LeftAssociative, "int|terminal", S.Times, "int|terminal")
.WithOperator("terminal", NonAssociative, S.Ident)
// This is the root set of operators
.WithOperator("program", NonAssociative,
// "bool" // TODO: this needs aliases
"equality|terminal", S.And, "equality|terminal");
"terminal", S.And, "terminal");// TODO: re-add equality|
}

26
Lexer.cs
View File

@ -213,22 +213,36 @@ public static partial class Lexer {
[F]
private partial class SkipInitialEmptyWhitespace {
public IImmutableEnumerator<Lexeme> F(
IImmutableEnumerator<Lexeme> lx
)
=> lx.FirstAndRest().Match(
Some: hdtl =>
// skip the initial empty whitespace
"".Equals(hdtl.Item1.lexeme)
? hdtl.Item2
: hdtl.Item1.ImSingleton().Concat(hdtl.Item2),
None: Empty<Lexeme>());
}
// TODO: move this to a .Filter() extension method.
[F]
private partial class DiscardWhitespace {
public IImmutableEnumerator<Lexeme> F(
IImmutableEnumerator<Lexeme> lx
)
=> lx.FirstAndRest().Match<Tuple<Lexeme, IImmutableEnumerator<Lexeme>>, IImmutableEnumerator<Lexeme>>(
Some: hdtl =>
// skip the initial empty whitespace
string.Equals(
"",
hdtl.Item1.lexeme)
? hdtl.Item2
: hdtl.Item1.ImSingleton().Concat(hdtl.Item2),
hdtl.Item1.state.Equals(S.Space)
? hdtl.Item2
: hdtl.Item1.ImSingleton().Concat(hdtl.Item2.Lazy(DiscardWhitespace.Eq)),
None: Empty<Lexeme>());
}
public static IImmutableEnumerator<Lexeme> Lex(string source)
=> Lex1(source)
.Flatten()
.Lazy(SkipInitialEmptyWhitespace.Eq);
//.Lazy(SkipInitialEmptyWhitespace.Eq)
.Lazy(DiscardWhitespace.Eq);
}

161
MixFix.cs
View File

@ -21,9 +21,20 @@ public static partial class MixFix {
public static Grammar1 Or(params Grammar1[] xs)
=> Or(xs.ToImmutableList());
public static Grammar1 Empty
= new Grammar1.Cases.Sequence(Enumerable.Empty<Grammar1>());
public static Grammar1 Impossible
= new Grammar1.Cases.Or(Enumerable.Empty<Grammar1>());
// TODO: inline the OR, detect impossible cases (Or of 0)
public static Grammar1 Sequence(IEnumerable<Grammar1> xs) {
var filteredXs = xs.Where(x => !x.IsEmpty);
if (filteredXs.Count() == 1) {
if (filteredXs.Any(x => x.IsImpossible)) {
return Impossible;
} else if (filteredXs.Count() == 0) {
return Empty;
} else if (filteredXs.Count() == 1) {
return filteredXs.Single().ElseThrow(() => new Exception("TODO: use an either to prove that this is safe."));
} else {
return new Grammar1.Cases.Sequence(filteredXs);
@ -31,8 +42,17 @@ public static partial class MixFix {
}
public static Grammar1 Or(IEnumerable<Grammar1> xs) {
var filteredXs = xs.Where(x => !x.IsEmpty);
if (filteredXs.Count() == 1) {
var filteredXsNoEmpty =
xs.Where(x => !x.IsImpossible)
.Where(x => !x.IsEmpty);
var filteredXs =
( xs.Any(x => x.IsEmpty)
&& !filteredXsNoEmpty.Any(x => x.AllowsEmpty))
? Empty.Cons(filteredXsNoEmpty)
: filteredXsNoEmpty;
if (filteredXs.All(x => x.IsImpossible)) {
return new Grammar1.Cases.Or(Enumerable.Empty<Grammar1>());
} else if (filteredXs.Count() == 1) {
return filteredXs.Single().ElseThrow(() => new Exception("TODO: use an either to prove that this is safe."));
} else {
return new Grammar1.Cases.Or(filteredXs);
@ -42,20 +62,43 @@ public static partial class MixFix {
public static Grammar1 RepeatOnePlus(Grammar1 g)
=> g.IsEmpty
? Grammar1.Empty
: g.IsImpossible
? Grammar1.Impossible
: new Grammar1.Cases.RepeatOnePlus(g);
public static Grammar1 Empty = new Grammar1.Cases.Or(Enumerable.Empty<Grammar1>());
public bool IsEmpty {
get => this.Match(
Or: l => l.Count() == 0,
Sequence: l => l.Count() == 0,
Or: l => l.All(g => g.IsEmpty),
Sequence: l => l.All(g => g.IsEmpty),
RepeatOnePlus: g => g.IsEmpty,
Terminal: t => false,
Rule: r => false
);
}
// TODO: cache this!
public bool AllowsEmpty {
get => this.Match(
Or: l => l.Any(g => g.AllowsEmpty),
Sequence: l => l.All(g => g.IsEmpty),
// This one should not be true, if it is
// then the precedence graph may be ill-formed?
RepeatOnePlus: g => g.AllowsEmpty,
Terminal: t => false,
Rule: r => false
);
}
public bool IsImpossible {
get => this.Match(
Or: l => l.All(g => g.IsImpossible),
Sequence: l => l.Any(g => g.IsImpossible),
RepeatOnePlus: g => g.IsImpossible,
Terminal: t => false,
Rule: r => false
);
}
public static Grammar1 operator |(Grammar1 a, Grammar1 b)
=> Or(a, b);
@ -65,8 +108,8 @@ public static partial class MixFix {
public static implicit operator Grammar1((Grammar1 a, Grammar1 b, Grammar1 c) gs)
=> Sequence(gs.a, gs.b, gs.c);
public static bool operator true(Grammar1 g) => !g.IsEmpty;
public static bool operator false(Grammar1 g) => g.IsEmpty;
public static bool operator true(Grammar1 g) => !g.IsImpossible;
public static bool operator false(Grammar1 g) => g.IsImpossible;
public Grammar1 this[string multiplicity] {
get {
@ -85,20 +128,55 @@ public static partial class MixFix {
private string Paren(bool paren, string s)
=> paren ? $"({s})" : s;
string CustomToString()
=> this.Match<string>(
Or: l => l.Count() == 0
? "Or(Empty)"
: Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(" | ")),
Sequence: l => l.Count() == 0
? "Sequence(Empty)"
: Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(",")),
RepeatOnePlus: g => $"({g.Str()})+",
string CustomToString() =>
this.IsEmpty ? "Empty"
: this.IsImpossible ? "Impossible"
: this.Match<string>(
Or: l =>
Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(" | ")),
Sequence: l =>
Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(", ")),
RepeatOnePlus: g => $"{g.Str()}+",
Terminal: t => t.Str(),
Rule: r => r
);
}
public partial class Grammar2 {
public bool IsEmpty {
get => this.Match(
Or: l => l.All(g => g.IsEmpty),
Sequence: l => l.All(g => g.IsEmpty),
RepeatOnePlus: g => g.IsEmpty,
Terminal: t => false
);
}
public bool IsImpossible {
get => this.Match(
Or: l => l.All(g => g.IsImpossible),
Sequence: l => l.Any(g => g.IsImpossible),
RepeatOnePlus: g => g.IsImpossible,
Terminal: t => false
);
}
private string Paren(bool paren, string s)
=> paren ? $"({s})" : s;
string CustomToString() =>
this.IsEmpty ? "Empty"
: this.IsImpossible ? "Impossible"
: this.Match<string>(
Or: l =>
Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(" | ")),
Sequence: l =>
Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(", ")),
RepeatOnePlus: g => $"{g.Str()}+",
Terminal: t => t.Str()
);
}
public partial class Operator {
private string CustomToString()
=> $"Operator(\"{precedenceGroup}\", {fixity}, {parts.Select(x => x.Match(Hole: h => h.Select(g => g.ToString()).JoinWith("|"), Name: n => $"\"{n}\"")).JoinWith(", ")})";
@ -222,7 +300,7 @@ public static partial class MixFix {
= new PrecedenceDAG(EmptyDAGNode);
public static Whole Add<Whole>(this ILens<DAGNode, Whole> node, Operator @operator) {
return @operator.fixity.Match(
return @operator.Cons(@operator.fixity.Match(
Closed:
() => node.Closed(),
Prefix:
@ -235,7 +313,7 @@ public static partial class MixFix {
() => node.InfixRightAssociative(),
InfixLeftAssociative:
() => node.InfixLeftAssociative()
).Cons(@operator);
));
}
public static void CheckHole(PrecedenceDAG precedenceDAG, Operator @operator, string name, Option<Hole> existing, Option<Hole> @new) {
@ -344,15 +422,27 @@ public static partial class MixFix {
var infixl = node.infixLeftAssociative.ToGrammar1();
var infixr = node.infixRightAssociative.ToGrammar1();
//Log("closed.IsImpossible:"+closed.IsImpossible);
// TODO: BUG: only include these parts if there are
// any operators with that fixity.
// any operators with that fixity, with the code below
// they are empty.
return
closed
| (nonAssoc ? (lsucc, nonAssoc, rsucc) : Grammar1.Empty)
(closed ? closed : Grammar1.Impossible)
| (nonAssoc ? (lsucc, nonAssoc, rsucc) : Grammar1.Impossible)
// TODO: post-processsing of the rightassoc list.
| ((prefix || infixr) ? ((prefix || (lsucc, infixr))["+"], rsucc) : Grammar1.Impossible)
// TODO: post-processsing of the leftassoc list.
| ((prefix || infixr) ? ((prefix | (lsucc, infixr))["+"], rsucc) : Grammar1.Empty)
// TODO: post-processsing of the leftassoc list.
| ((postfix || infixl) ? (lsucc, (postfix | (infixl, rsucc))["+"]) : Grammar1.Empty);
| ((postfix || infixl) ? (lsucc, (postfix || (infixl, rsucc))["+"]) : Grammar1.Impossible);
}
public static EquatableDictionary<string, Grammar1> ToGrammar1(this PrecedenceDAG precedenceDAG)
@ -363,15 +453,28 @@ public static partial class MixFix {
private static Grammar2 Recur(Func<Grammar1, EquatableDictionary<string, Grammar1>, Grammar2> recur, Grammar1 grammar1, EquatableDictionary<string, Grammar1> labeled)
=> grammar1.Match<Grammar2>(
// TODO: throw exception if lookup fails
Rule: r => recur(labeled[r], labeled),
Rule: r => {
Grammar1 lr = null;
try {
lr = labeled[r];
} catch (Exception e) {
throw new ParserExtensionException($"Internal error: could not find node {r} in labeled grammar. It only contains labels for: {labeled.Select(kvp => kvp.Key.ToString()).JoinWith(", ")}.");
}
return recur(labeled[r], labeled);
},
Terminal: t => Grammar2.Terminal(t),
Sequence: l => Grammar2.Sequence(l.Select(g => recur(g, labeled))),
Or: l => Grammar2.Or(l.Select(g => recur(g, labeled))),
RepeatOnePlus: g => Grammar2.RepeatOnePlus(recur(g, labeled))
);
public static Grammar2 ToGrammar2(this EquatableDictionary<string, Grammar1> labeled)
=> Func.YMemoize<Grammar1, EquatableDictionary<string, Grammar1>, Grammar2>(Recur)(Grammar1.Rule("program"), labeled);
public static Grammar2 ToGrammar2(this EquatableDictionary<string, Grammar1> labeled) {
foreach (var kvp in labeled) {
Log($"{kvp.Key} -> {kvp.Value.ToString()}");
}
Log("");
return Func.YMemoize<Grammar1, EquatableDictionary<string, Grammar1>, Grammar2>(Recur)(Grammar1.Rule("program"), labeled);
}
public static Grammar2 ToGrammar2(this PrecedenceDAG precedenceDAG)
=> precedenceDAG.ToGrammar1().ToGrammar2();

33
Parser.cs
View File

@ -25,26 +25,34 @@ public static partial class Parser {
.FirstAndRest()
.Match(
None: () =>
throw new Exception("EOF, what to do?"),
//throw new Exception("EOF, what to do?"),
None<ValueTuple<IImmutableEnumerator<Lexeme>, AstNode>>(),
Some: firstRest => {
// Case("IImmutableEnumerable<AstNode>", "Operator"))
var first = firstRest.Item1;
var rest = firstRest.Item2;
Log(first.lexeme);
Log(grammar.ToString());
Log(grammar.Match(
RepeatOnePlus: _ => "RepeatOnePlus",
Or: _ => "Or",
Sequence: _ => "Sequence",
Terminal: t => "Terminal:"+t.ToString()));
return grammar.Match(
RepeatOnePlus: g =>
rest.FoldMapWhileSome(restI => Parse3(restI, g))
.If<IImmutableEnumerator<Lexeme>, IEnumerable<AstNode>>((restN, nodes) => nodes.Count() > 1)
.IfSome((restN, nodes) => (restN, AstNode.Operator(nodes))),
//.IfSome(rest1 =>
// TODO: remove IfSome above (useless) && aggregate
// WhileSome(rest1, restI => Parse3(restI, g))),
// TODO: to check for ambiguous parses, we can use
// .SingleArg(…) instead of .FirstArg(…).
// .Single(…) instead of .First(…).
Or: l =>
l.First(g => Parse3(rest, g)),
Sequence: l =>
l.BindFoldMap(rest, (restI, g) => Parse3(restI, g))
.IfSome((restN, nodes) => (restN, AstNode.Operator(nodes))),
Sequence: l => {
return l.BindFoldMap(rest, (restI, g) => Parse3(restI, g))
.IfSome((restN, nodes) => {
Log($"{nodes.Count()}/{l.Count()}");
return (restN, AstNode.Operator(nodes));
});
},
Terminal: t =>
first.state.Equals(t)
? (rest,
@ -60,7 +68,7 @@ public static partial class Parser {
public static Option<ValueTuple<IImmutableEnumerator<Lexeme>, AstNode>> Parse2(string source) {
Grammar2 grammar =
DefaultGrammar.DefaultPrecedenceDAG.ToGrammar2();
Log(grammar.Str());
//Log(grammar.Str());
var P = Func.YMemoize<
IImmutableEnumerator<Lexeme>,
@ -73,6 +81,11 @@ public static partial class Parser {
}
public static Ast.Expr Parse(string source) {
Log("");
Log("" + Parse2(source).ToString());
Log("");
Environment.Exit(0);
return Lexer.Lex(source)
.SelectMany(lexeme =>
lexeme.state.Match(

2
Tests/001-42.e
View File

@ -1 +1 @@
42
true && false

18
Utils/Enumerable.cs
View File

@ -8,14 +8,20 @@ public static class Collection {
public static void ForEach<T>(this IEnumerable<T> x, Action<T> f)
=> x.ToImmutableList().ForEach(f);
public static ImmutableList<T> Cons<T>(this ImmutableList<T> l, T x)
public static ImmutableList<T> Cons<T>(this T x, ImmutableList<T> l)
=> l.Add(x);
public static ImmutableList<Tuple<T,U>> Cons<T,U>(this ImmutableList<Tuple<T,U>> l, T x, U y)
=> l.Cons(Tuple.Create(x,y));
public static IEnumerable<T> Cons<T>(this T x, IEnumerable<T> l)
=> x.Singleton().Concat(l);
public static ImmutableList<Tuple<T,U,V>> Cons<T,U,V>(this ImmutableList<Tuple<T,U,V>> l, T x, U y, V z)
=> l.Cons(Tuple.Create(x,y,z));
public static IEnumerable<T> Concat<T>(this IEnumerable<T> l, T x)
=> l.Concat(x.Singleton());
public static ImmutableList<Tuple<T,U>> Add<T,U>(this ImmutableList<Tuple<T,U>> l, T x, U y)
=> l.Add(Tuple.Create(x,y));
public static ImmutableList<Tuple<T,U,V>> Add<T,U,V>(this ImmutableList<Tuple<T,U,V>> l, T x, U y, V z)
=> l.Add(Tuple.Create(x,y,z));
public static void Deconstruct<A, B>(this Tuple<A, B> t, out A a, out B b) {
a = t.Item1;
@ -241,7 +247,7 @@ public static class Collection {
foreach (var x in e) {
var newAcc = f(acc, x);
if (newAcc.IsNone) {
break;
return Option.None<A>();
} else {
acc = newAcc.ElseThrow(new Exception("impossible"));
}

6
Utils/Immutable/Option.cs
View File

@ -27,6 +27,9 @@ namespace Immutable {
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
=> this.GetEnumerator();
public override string ToString()
=> $"Some({value.ToString()})";
}
public class None<T> : Option<T>, System.Collections.IEnumerable {
@ -42,6 +45,9 @@ namespace Immutable {
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
=> this.GetEnumerator();
public override string ToString()
=> $"None";
}
}
}

4
Utils/Lens.cs
View File

@ -10,8 +10,8 @@ public static class LensExtensionMethods {
public static Whole Update<Hole, Whole>(this ILens<Hole, Whole> lens, Hole newHole)
=> lens.Update(oldHole => newHole);
public static Whole Cons<T, Whole>(this ILens<ImmutableList<T>, Whole> lens, T value)
=> lens.Update(oldHole => oldHole.Cons(value));
public static Whole Cons<T, Whole>(this T value, ILens<ImmutableList<T>, Whole> lens)
=> lens.Update(oldHole => value.Cons(oldHole));
public static ILens<string, Whole> ChainLens<Whole>(this string hole, System.Func<string, Whole> wrap) => new LeafLens<string, Whole>(wrap: wrap, oldHole: hole);

7
Utils/ToString.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Linq;
using System.Collections.Generic;
using System.Collections.Immutable;
public interface IString {
@ -25,6 +26,12 @@ public static class ToStringImplementations {
public static string Str<T>(this ImmutableHashSet<string> h)
=> $"ImmutableHashSet({h.Select(x => x.Str<string>()).JoinWith(", ")})";
public static string Str<T>(this IEnumerable<MixFix.Grammar2> e)
=> $"IEnumerabl({e.Select(x => x.Str<MixFix.Grammar2>()).JoinWith(", ")})";
public static string Str<T>(this IEnumerable<Ast.AstNode> e)
=> $"IEnumerab({e.Select(x => x.Str<Ast.AstNode>()).JoinWith(", ")})";
public static string Str<Grammar>(this ImmutableDictionary<string,Grammar> h)
=> $"ImmutableDictionary(\n{h.Select(x => $" {x.Key.Str<string>()}:{x.Value.Str<Grammar>()}").JoinWith(",\n")}\n)";

10
main.cs
View File

@ -22,7 +22,7 @@ public static class MainClass {
var destPath = tests_results.Combine(source);
var sourcePath = tests.Combine(source);
var expected = sourcePath.DropExtension().Combine(Ext(".o"));
Console.Write($"\x1b[KRunning test {source} ({toolchainName}) ");
destPath.DirName().Create();
@ -47,7 +47,7 @@ public static class MainClass {
Console.WriteLine($"\x1b[1;33m{source}: expected {expectedStr} but got {actualStr}.\x1b[m\n");
return false;
} else {
Console.Write("\x1b[1;32mOK\x1b[m\r");
Console.Write("\x1b[1;32mOK\x1b[m"); // \r at the end for quiet
return true;
}
}
@ -58,13 +58,13 @@ public static class MainClass {
// first-class functions by using repeated .Add()
// See https://repl.it/@suzannesoy/WarlikeWorstTraining#main.cs
var compilers = ImmutableList<Tuple<string, Compiler, Exe>>.Empty
.Cons(" js ", Compilers.JS.Compile, Exe("node"))
.Cons("eval", Evaluator.Evaluate, Exe("cat"));
.Add(" js ", Compilers.JS.Compile, Exe("node"))
.Add("eval", Evaluator.Evaluate, Exe("cat"));
var total = 0;
var passed = 0;
var failed = 0;
foreach (var t in Dir("Tests/").GetFiles("*.e", SearchOption.AllDirectories)) {
foreach (var t in Dir("Tests/").GetFiles("*.e", SearchOption.AllDirectories).OrderBy(f => f.ToString())) {
foreach (var compiler in compilers) {
if (RunTest(compiler.Item1, compiler.Item2, compiler.Item3, t)) {
passed++;