suzanne.soy/envlang-csharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Epsilon transitions in lexer

Browse Source
This commit is contained in:
Suzanne Soy 2020-08-16 02:57:20 +00:00
parent ff459fb1fd
commit c534c2c6ab
11 changed files with 132 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Ast.cs
View File

@ -42,6 +42,8 @@ namespace Ast {
public override int GetHashCode() {
return HashCode.Combine("Int", this.value);
}
public override string ToString() => "Int";
}
public partial class Visitor<T> { public Func<string, T> String { get; set; } }
@ -62,6 +64,8 @@ namespace Ast {
public override int GetHashCode() {
return HashCode.Combine("String", this.value);
}
public override string ToString() => "String";
}
}

43
Lexer.cs
View File

@ -78,10 +78,9 @@ public static partial class Lexer {
Rule(S.Space, '"', S.StringOpen, S.String),
Rule(S.Int, C.DecimalDigitNumber, S.Int),
Rule(S.Int, C.SpaceSeparator, S.Space), // epsilon
Rule(S.Int, C.SpaceSeparator, S.Space),
Rule(S.Int, new[]{'.',','}, S.Decimal, S.Int),
Rule(S.Int, EOF, S.End), // epsilon
Rule(S.Decimal, C.SpaceSeparator, S.Space), // epsilon
Rule(S.Decimal, C.SpaceSeparator, S.Space),
Rule(S.String, C.LowercaseLetter, S.String),
Rule(S.String, C.UppercaseLetter, S.String),
@ -92,8 +91,22 @@ public static partial class Lexer {
public static Dictionary<S, List<Rule>> Dict =
Default
.GroupBy(r => r.oldState, r => r)
.ToDictionary(rs => rs.Key, rs => rs.ToList()) ;
// TODO: upon failure, do an epsilon-transition to the whitespace state, and try again.
.ToDefaultDictionary(
new List<Rule>(),
rs => rs.Key,
rs => rs.ToList()) ;
// This adds transitions through an implicit empty whitespace.
public static Dictionary<S, List<Rule>> WithEpsilonTransitions =
Dict.ToDefaultDictionary(
new List<Rule>(),
kv => kv.Key,
kv => kv.Value.Any(r => true) // r.test(" ")
// This is a bit of a hack, the lexer tries the rules in
// order so later rules with different results are masked
// by former rules
? kv.Value.Concat(Dict[S.Space]).ToList()
: kv.Value);
}
public struct Lexeme {
@ -125,7 +138,7 @@ public static partial class Lexer {
return result;
}
public static void ParseError(StringBuilder context, IEnumerator<GraphemeCluster> stream, S state, List<Rule> possibleNext, GraphemeCluster gc) {
public static Exception ParseError(StringBuilder context, IEnumerator<GraphemeCluster> stream, S state, List<Rule> possibleNext, GraphemeCluster gc) {
var rest =
stream
.SingleUseEnumerable()
@ -139,7 +152,7 @@ public static partial class Lexer {
.First()
.Match(some: (x => x.UnicodeCategory(0).ToString()),
none: "None (empty string)");
throw new Exception(
return new Exception(
$"Unexpected {actual} (Unicode category {cat}) while the lexer was in state {state}: expected one of {expected}{Environment.NewLine}{context} <--HERE {rest}"
);
}
@ -155,15 +168,13 @@ public static partial class Lexer {
while (e.MoveNext()) {
var c = e.Current;
context.Append(c.str);
List<Rule> possibleNext;
if (Rules.Dict.TryGetValue(state, out possibleNext)) {
var rule = possibleNext.FirstOrDefault(r => r.test(c));
if (rule != null) {
yield return Transition(ref state, ref lexeme, c, rule);
} else {
ParseError(context, e, state, possibleNext, c);
}
}
var possibleNext = Rules.WithEpsilonTransitions
.GetOrDefault(state, new List<Rule>());
yield return
possibleNext
.First(r => r.test(c))
.IfSome(rule => Transition(ref state, ref lexeme, c, rule))
.ElseThrow(() => ParseError(context, e, state, possibleNext, c));
}
}

14
LexerGenerated.cs
View File

@ -56,6 +56,8 @@ public static partial class Lexer {
public override int GetHashCode() {
return "C".GetHashCode();
}
public override string ToString() => "End";
}
public partial class Visitor<T> { public Func<T> Space { get; set; } }
@ -70,6 +72,8 @@ public static partial class Lexer {
public override int GetHashCode() {
return "C".GetHashCode();
}
public override string ToString() => "Space";
}
public partial class Visitor<T> { public Func<T> Int { get; set; } }
@ -84,6 +88,8 @@ public static partial class Lexer {
public override int GetHashCode() {
return "C".GetHashCode();
}
public override string ToString() => "Int";
}
public partial class Visitor<T> { public Func<T> Decimal { get; set; } }
@ -98,6 +104,8 @@ public static partial class Lexer {
public override int GetHashCode() {
return "C".GetHashCode();
}
public override string ToString() => "Decimal";
}
public partial class Visitor<T> { public Func<T> String { get; set; } }
@ -112,6 +120,8 @@ public static partial class Lexer {
public override int GetHashCode() {
return "C".GetHashCode();
}
public override string ToString() => "String";
}
public partial class Visitor<T> { public Func<T> StringOpen { get; set; } }
@ -126,6 +136,8 @@ public static partial class Lexer {
public override int GetHashCode() {
return "C".GetHashCode();
}
public override string ToString() => "StringOpen";
}
public partial class Visitor<T> { public Func<T> StringClose { get; set; } }
@ -140,6 +152,8 @@ public static partial class Lexer {
public override int GetHashCode() {
return "C".GetHashCode();
}
public override string ToString() => "StringClose";
}
}

2
T4/Generator.cs
View File

@ -90,6 +90,8 @@ public static class Generator {
o.WriteLine($" return HashCode.Combine(\"{C}\", this.value);");
}
o.WriteLine($" }}");
o.WriteLine("");
o.WriteLine($" public override string ToString() => \"{C}\";");
o.WriteLine($" }}");
o.WriteLine("");
}

1
Tests/004-foo42.e Normal file
View File

@ -0,0 +1 @@
"foo"42

1
Tests/004-foo42.o Normal file
View File

@ -0,0 +1 @@
foo42

1
Tests/005-42foo.e Normal file
View File

@ -0,0 +1 @@
42"foo"

1
Tests/005-42foo.o Normal file
View File

@ -0,0 +1 @@
42foo

48
Utils/Enumerable.cs
View File

@ -128,6 +128,17 @@ public static class Collection {
}
}
public static Option<T> First<T>(this IEnumerable<T> ie, Func<T, bool> predicate) {
var e = ie.GetEnumerator();
bool found = false;
while (e.MoveNext()) {
if (predicate(e.Current)) {
return e.Current.Some();
}
}
return Option.None<T>();
}
public static Option<T> Single<T>(this IEnumerable<T> ie) {
var e = ie.GetEnumerator();
if (e.MoveNext()) {
@ -141,4 +152,41 @@ public static class Collection {
return Option.None<T>();
}
}
public static Option<V> GetValue<K, V>(this Dictionary<K, V> d, K key) {
V result = default(V);
if (d.TryGetValue(key, out result)) {
return result.Some();
} else {
return Option.None<V>();
}
}
public static V GetOrDefault<K, V>(this Dictionary<K, V> d, K key, V defaultValue) {
V result = default(V);
if (d.TryGetValue(key, out result)) {
return result;
} else {
return defaultValue;
}
}
public class DefaultDictionary<TKey, TValue> : Dictionary<TKey, TValue> {
public readonly TValue defaultValue;
//public readonly Dictionary<TKey, TValue> dictionary;
public DefaultDictionary(TValue defaultValue, Dictionary<TKey, TValue> dictionary) : base(dictionary) {
this.defaultValue = defaultValue;
//this.dictionary = dictionary;
}
public TValue this[TKey key] {
get {
return this.GetOrDefault(key, defaultValue);
}
}
}
public static DefaultDictionary<UKey, UValue> ToDefaultDictionary<T, UKey, UValue>(this IEnumerable<T> e, UValue defaultValue, Func<T, UKey> key, Func<T, UValue> value)
=> new DefaultDictionary<UKey, UValue>(defaultValue, e.ToDictionary(key, value));
}

33
Utils/Immutable/Option.cs
View File

@ -1,7 +1,7 @@
namespace Immutable {
using System;
public interface Option<out T> {
public interface Option<out T> : System.Collections.Generic.IEnumerable<T> {
U Match_<U>(Func<T, U> some, Func<U> none);
}
@ -10,18 +10,30 @@ namespace Immutable {
public static Option<T> None<T>() => new Types.None<T>();
private static class Types {
public class Some<T> : Option<T> {
public class Some<T> : Option<T>, System.Collections.IEnumerable {
public readonly T value;
public Some(T value) { this.value = value; }
public U Match_<U>(Func<T, U> Some, Func<U> None) => Some(value);
public System.Collections.Generic.IEnumerator<T> GetEnumerator()
=> value.Singleton().GetEnumerator();
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
=> this.GetEnumerator();
}
public class None<T> : Option<T> {
public class None<T> : Option<T>, System.Collections.IEnumerable {
public None() { }
public U Match_<U>(Func<T, U> Some, Func<U> None) => None();
public System.Collections.Generic.IEnumerator<T> GetEnumerator()
=> System.Linq.Enumerable.Empty<T>().GetEnumerator();
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
=> this.GetEnumerator();
}
}
}
@ -34,5 +46,20 @@ namespace Immutable {
public static U Match<T, U>(this Option<T> o, Func<T, U> some, U none)
=> o.Match_(some, () => none);
public static Option<U> Map<T, U>(this Option<T> o, Func<T, U> some)
=> o.Match_(value => some(value).Some(), () => Option.None<U>());
public static Option<U> IfSome<T, U>(this Option<T> o, Func<T, U> some)
=> o.Map(some);
public static T Else<T>(this Option<T> o, Func<T> none)
=> o.Match_(some => some, none);
public static Option<T> Else<T>(this Option<T> o, Func<Option<T>> none)
=> o.Match_(value => value.Some(), none);
public static T ElseThrow<T>(this Option<T> o, Func<Exception> none)
=> o.Match_(value => value, () => throw none());
}
}

3
Utils/Unicode.cs
View File

@ -12,6 +12,9 @@ public struct GraphemeCluster {
this.str = str;
this.codePoints = codePoints;
}
public static implicit operator GraphemeCluster(char c)
=> new GraphemeCluster(false, c.ToString(), c.ToString().Singleton());
}
public static class UnicodeExtensionMethods {