suzanne.soy/envlang-csharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Keep annotations in Parser

Browse Source
This commit is contained in:
Suzanne Soy 2020-08-31 18:38:52 +00:00
parent 89c9908957
commit 592a758fb1
9 changed files with 91 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
AstGenerator.cs
View File

@ -12,6 +12,10 @@ public static class AstGenerator {
Variant("Expr",
Case("int", "Int"),
Case("string", "String")),
Variant("ParserResult",
Case("MixFix.Annotation", "Annotated"),
Case("Lexer.Lexeme", "Terminal"),
Case("IEnumerable<ParserResult>", "Productions")),
Variant("AstNode",
Case("Expr", "Terminal"),
Case("IEnumerable<AstNode>", "Operator"))));

6
DefaultGrammar.cs
View File

@ -7,14 +7,14 @@ using static MixFix.Associativity;
public static class DefaultGrammar {
public static PrecedenceDAG DefaultPrecedenceDAG
= EmptyPrecedenceDAG
/*.WithOperator("bool", NonAssociative, "equality|terminal", S.And, "equality|terminal")
.WithOperator("bool", NonAssociative, "equality|terminal", S.And, "equality|terminal")
.WithOperator("equality", NonAssociative, "int|terminal|additive|multiplicative", S.Eq, "int|terminal|additive|multiplicative")
.WithOperator("int", NonAssociative, S.Int)
.WithOperator("additive", LeftAssociative, "int|terminal|multiplicative", S.Plus, "int|terminal|multiplicative")
.WithOperator("multiplicative", LeftAssociative, "int|terminal", S.Times, "int|terminal")*/
.WithOperator("multiplicative", LeftAssociative, "int|terminal", S.Times, "int|terminal")
.WithOperator("terminal", NonAssociative, S.Ident)
// This is the root set of operators
.WithOperator("program", NonAssociative,
// "bool" // TODO: this needs aliases
"terminal", S.And, "terminal");// TODO: re-add equality|
"equality|terminal", S.And, "equality|terminal");
}

11
Lexer.cs
View File

@ -232,11 +232,12 @@ public static partial class Lexer {
IImmutableEnumerator<Lexeme> lx
)
=> lx.FirstAndRest().Match<Tuple<Lexeme, IImmutableEnumerator<Lexeme>>, IImmutableEnumerator<Lexeme>>(
Some: hdtl =>
// skip the initial empty whitespace
hdtl.Item1.state.Equals(S.Space)
? hdtl.Item2
: hdtl.Item1.ImSingleton().Concat(hdtl.Item2.Lazy(DiscardWhitespace.Eq)),
Some: hdtl => {
var rest = hdtl.Item2.Lazy(DiscardWhitespace.Eq);
return hdtl.Item1.state.Equals(S.Space)
? rest
: hdtl.Item1.ImSingleton().Concat(rest);
},
None: Empty<Lexeme>());
}

54
MixFix.cs
View File

@ -155,7 +155,8 @@ public static partial class MixFix {
Or: l => l.All(g => g.IsEmpty),
Sequence: l => l.All(g => g.IsEmpty),
RepeatOnePlus: g => g.IsEmpty,
Terminal: t => false
Terminal: t => false,
Annotated: a => a.Item2.IsEmpty
);
}
@ -164,7 +165,8 @@ public static partial class MixFix {
Or: l => l.All(g => g.IsImpossible),
Sequence: l => l.Any(g => g.IsImpossible),
RepeatOnePlus: g => g.IsImpossible,
Terminal: t => false
Terminal: t => false,
Annotated: a => a.Item2.IsImpossible
);
}
@ -182,7 +184,8 @@ public static partial class MixFix {
? "EmptySequence"
: Paren(l.Count() != 1, l.Select(x => x.Str()).JoinWith(", ")),
RepeatOnePlus: g => $"{g.Str()}+",
Terminal: t => t.Str()
Terminal: t => t.Str(),
Annotated: a => $"Annotated({a.Item1.Str()}, {a.Item2.Str()})"
);
}
@ -414,16 +417,27 @@ public static partial class MixFix {
Hole: precedenceGroups => precedenceGroups.ToGrammar1());
public static Grammar1 ToGrammar1(this Operator @operator)
=> Grammar1.Sequence(
@operator.internalParts.Select(part => part.ToGrammar1()));
=> Grammar1.Annotated(
(Annotation.Operator(@operator),
Grammar1.Sequence(
@operator.internalParts.Select(
part => part.ToGrammar1()))));
public static Grammar1 ToGrammar1(this IEnumerable<Operator> operators)
=> Grammar1.Or(
operators.Select(@operator => @operator.ToGrammar1()));
public static Grammar1 ToGrammar1(this DAGNode node) {
var lsucc = node.leftmostHole_.ToGrammar1();
var rsucc = node.rightmostHole_.ToGrammar1();
Func<Associativity, Grammar1, Grammar1> SamePrecedence = (a, g)
=> Grammar1.Annotated((Annotation.SamePrecedence(a), g));
Func<Grammar1, Grammar1> R = g => SamePrecedence(Associativity.RightAssociative, g);
Func<Grammar1, Grammar1> L = g => SamePrecedence(Associativity.LeftAssociative, g);
Func<Grammar1, Grammar1> N = g => SamePrecedence(Associativity.NonAssociative, g);
Func<Grammar1, Grammar1> H = g => Grammar1.Annotated((Annotation.Hole, g));
var Impossible = Grammar1.Impossible;
var lsucc = H(node.leftmostHole_.ToGrammar1());
var rsucc = H(node.rightmostHole_.ToGrammar1());
var closed = node.closed.ToGrammar1();
var nonAssoc = node.infixNonAssociative.ToGrammar1();
var prefix = node.prefix.ToGrammar1();
@ -431,25 +445,13 @@ public static partial class MixFix {
var infixl = node.infixLeftAssociative.ToGrammar1();
var infixr = node.infixRightAssociative.ToGrammar1();
// TODO: BUG: only include these parts if there are
// any operators with that fixity, with the code below
// they are empty.
Func<string, Grammar1, Grammar1> Annotated = (s, g) => Grammar1.Annotated((s, g));
return
Annotated("closed", closed ? closed : Grammar1.Impossible)
| Annotated("nonAssoc", (nonAssoc ? (lsucc, nonAssoc, rsucc) : Grammar1.Impossible))
// TODO: post-processsing of the rightassoc list.
| Annotated("prefix||infixr", ((prefix || infixr) ? ((prefix || (lsucc, infixr))["+"], rsucc) : Grammar1.Impossible))
// TODO: post-processsing of the leftassoc list.
| Annotated("postfix||infixl", ((postfix || infixl) ? (lsucc, (postfix || (infixl, rsucc))["+"]) : Grammar1.Impossible));
// TODO: we can normally remove the ?: checks, as the constructors for grammars
// now coalesce Impossible cases in the correct way.
(closed ? N(closed) : Impossible)
| (nonAssoc ? N( (lsucc, nonAssoc, rsucc) ) : Impossible)
| ((prefix || infixr) ? R( ((prefix | (lsucc, infixr))["+"], rsucc) ) : Impossible)
| ((postfix || infixl) ? L( (lsucc, (postfix || (infixl, rsucc))["+"]) ) : Impossible);
}
public static EquatableDictionary<string, Grammar1> ToGrammar1(this PrecedenceDAG precedenceDAG)
@ -469,7 +471,7 @@ public static partial class MixFix {
}
return recur(labeled[r], labeled);
},
Annotated: a => recur(a.Item2, labeled),
Annotated: a => Grammar2.Annotated((a.Item1, recur(a.Item2, labeled))),
Terminal: t => Grammar2.Terminal(t),
Sequence: l => Grammar2.Sequence(l.Select(g => recur(g, labeled))),
Or: l => Grammar2.Or(l.Select(g => recur(g, labeled))),

10
MixFixGenerator.cs
View File

@ -20,14 +20,20 @@ public static class ParserGenerator {
Case("IEnumerable<Grammar1>", "Or"),
Case("IEnumerable<Grammar1>", "Sequence"),
Case("S", "Terminal"),
Case("ValueTuple<string, Grammar1>", "Annotated"),
Case("ValueTuple<Annotation, Grammar1>", "Annotated"),
Case("string", "Rule")),
Variant("Annotation",
Case("MixFix.Operator", "Operator"),
Case("Associativity", "SamePrecedence"),
Case("Hole")),
Variant("Grammar2",
Case("Grammar2", "RepeatOnePlus"),
Case("IEnumerable<Grammar2>", "Or"),
Case("IEnumerable<Grammar2>", "Sequence"),
Case("S", "Terminal")),
Case("S", "Terminal"),
Case("ValueTuple<Annotation, Grammar2>", "Annotated")),
Variant("Fixity",
Case("Closed"),

79
Parser.cs
View File

@ -11,61 +11,44 @@ using Grammar2 = MixFix.Grammar2;
using static Global;
public static partial class Parser {
public static Option<ValueTuple<IImmutableEnumerator<Lexeme>, AstNode>> Parse3(
public static Option<ValueTuple<IImmutableEnumerator<Lexeme>, ParserResult>> Parse3(
Func<IImmutableEnumerator<Lexeme>,
Grammar2,
//Option<IImmutableEnumerator<Lexeme>>
Option<ValueTuple<IImmutableEnumerator<Lexeme>, AstNode>>
Option<ValueTuple<IImmutableEnumerator<Lexeme>, ParserResult>>
>
Parse3,
IImmutableEnumerator<Lexeme> tokens,
Grammar2 grammar
) =>
tokens
.FirstAndRest()
.Match(
None: () =>
//throw new Exception("EOF, what to do?"),
None<ValueTuple<IImmutableEnumerator<Lexeme>, AstNode>>(),
Some: firstRest => {
var first = firstRest.Item1;
var rest = firstRest.Item2;
Log(first.lexeme);
Log(grammar.ToString());
Log(grammar.Match(
RepeatOnePlus: _ => "RepeatOnePlus",
Or: _ => "Or",
Sequence: _ => "Sequence",
Terminal: t => "Terminal:"+t.ToString()));
return grammar.Match(
RepeatOnePlus: g =>
rest.FoldMapWhileSome(restI => Parse3(restI, g))
.If<IImmutableEnumerator<Lexeme>, IEnumerable<AstNode>>((restN, nodes) => nodes.Count() > 1)
.IfSome((restN, nodes) => (restN, AstNode.Operator(nodes))),
// TODO: to check for ambiguous parses, we can use
// .Single(…) instead of .First(…).
Or: l =>
l.First(g => Parse3(rest, g)),
Sequence: l => {
return l.BindFoldMap(rest, (restI, g) => Parse3(restI, g))
.IfSome((restN, nodes) => {
Log($"{nodes.Count()}/{l.Count()}");
return (restN, AstNode.Operator(nodes));
});
},
Terminal: t =>
first.state.Equals(t)
? (rest,
AstNode.Terminal(/* TODO: */ Expr.String(rest.ToString())))
.Some()
: None<ValueTuple<IImmutableEnumerator<Lexeme>, AstNode>>()
);
// TODO: at the top-level, check that the lexemes
// are empty if the parser won't accept anything else.
}
);
)
=> grammar.Match(
RepeatOnePlus: g =>
tokens.FoldMapWhileSome(restI => Parse3(restI, g))
.If((restN, nodes) => nodes.Count() > 1)
.IfSome((restN, nodes) => (restN, ParserResult.Productions(nodes))),
// TODO: to check for ambiguous parses, we can use
// .Single(…) instead of .First(…).
Or: l =>
l.First(g => Parse3(tokens, g)),
Sequence: l =>
l.BindFoldMap(tokens, (restI, g) => Parse3(restI, g))
.IfSome((restN, nodes) =>
Log($"{nodes.Count()}/{l.Count()}", () =>
(restN, ParserResult.Productions(nodes)))),
Terminal: t =>
// TODO: move the FirstAndRest here!
tokens
.FirstAndRest()
// When EOF is reached, the parser can't accept this derivation.
.If((first, rest) => first.state.Equals(t))
.IfSome((first, rest) => (rest, ParserResult.Terminal(first))),
Annotated: a =>
// TODO: use the annotation to give some shape to these lists
Parse3(tokens, a.Item2));
// TODO: at the top-level, check that the lexemes
// are empty if the parser won't accept anything else.
public static Option<ValueTuple<IImmutableEnumerator<Lexeme>, AstNode>> Parse2(string source) {
public static Option<ValueTuple<IImmutableEnumerator<Lexeme>, ParserResult>> Parse2(string source) {
Grammar2 grammar =
DefaultGrammar.DefaultPrecedenceDAG.ToGrammar2();
//Log(grammar.Str());
@ -73,7 +56,7 @@ public static partial class Parser {
var P = Func.YMemoize<
IImmutableEnumerator<Lexeme>,
Grammar2,
Option<ValueTuple<IImmutableEnumerator<Lexeme>, AstNode>>>(
Option<ValueTuple<IImmutableEnumerator<Lexeme>, ParserResult>>>(
Parse3
);

2
Utils/Global.cs
View File

@ -28,6 +28,8 @@ public static class Global {
public static T To<T>(this T x) => x;
public static Option<T> If<T>(this bool cond, T x) => x.If(_ => cond);
public static A FoldWhileSome<A>(A init, Func<A, Option<A>> f)
=> Collection.FoldWhileSome(init, f);

6
Utils/Immutable/Option.cs
View File

@ -76,6 +76,12 @@ namespace Immutable {
public static Option<ValueTuple<T1, T2>> If<T1, T2>(this ValueTuple<T1, T2> value, Func<T1, T2, bool> predicate)
=> predicate(value.Item1, value.Item2) ? value.Some() : Option.None<ValueTuple<T1, T2>>();
public static Option<Tuple<T1, T2>> If<T1, T2>(this Tuple<T1, T2> value, Func<T1, T2, bool> predicate)
=> predicate(value.Item1, value.Item2) ? value.Some() : Option.None<Tuple<T1, T2>>();
public static Option<Tuple<T1, T2>> If<T1, T2>(this Option<Tuple<T1, T2>> value, Func<T1, T2, bool> predicate)
=> value.Bind(val => val.If(predicate));
public static Option<U> IfSome<T1, T2, U>(this Option<Tuple<T1, T2>> o, Func<T1, T2, U> some)
=> o.Map(o1o2 => some(o1o2.Item1, o1o2.Item2));

3
Utils/ToString.cs
View File

@ -32,6 +32,9 @@ public static class ToStringImplementations {
public static string Str<T>(this IEnumerable<Ast.AstNode> e)
=> $"IEnumerab({e.Select(x => x.Str<Ast.AstNode>()).JoinWith(", ")})";
public static string Str<T>(this IEnumerable<Ast.ParserResult> e)
=> $"IEnumerab({e.Select(x => x.Str<Ast.ParserResult>()).JoinWith(", ")})";
public static string Str<Grammar>(this ImmutableDictionary<string,Grammar> h)
=> $"ImmutableDictionary(\n{h.Select(x => $" {x.Key.Str<string>()}:{x.Value.Str<Grammar>()}").JoinWith(",\n")}\n)";