Skip to content

Commit 4bff7bf

Browse files
wiredwizwiredwiz
committed
Partial redesign work on integrating the custom token factory to generate syntax tokens directly.
1 parent 77c8908 commit 4bff7bf

File tree

17 files changed

+203
-147
lines changed

17 files changed

+203
-147
lines changed

‎Code Grapher/Properties/AssemblyInfo.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,5 +32,5 @@
3232
// You can specify all the values or you can default the Build and Revision Numbers
3333
// by using the '*' as shown below:
3434
// [assembly: AssemblyVersion("1.0.*")]
35-
[assembly: AssemblyVersion("1.2.21215.2")]
36-
[assembly: AssemblyFileVersion("1.2.21215.2")]
35+
[assembly: AssemblyVersion("1.2.22221.0")]
36+
[assembly: AssemblyFileVersion("1.2.22221.0")]

‎Common/Grammar/SyntaxToken.cs‎

Lines changed: 90 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,10 @@
3434
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
3535
#endregion
3636

37+
using System;
38+
3739
using Antlr4.Runtime;
40+
using Antlr4.Runtime.Misc;
3841

3942
using Org.Edgerunner.ANTLR4.Tools.Common.Extensions;
4043

@@ -44,55 +47,79 @@ namespace Org.Edgerunner.ANTLR4.Tools.Common.Grammar
4447
/// Struct that represents a lightweight view model for IToken instances
4548
/// </summary>
4649
/// <seealso cref="Antlr4.Runtime.IToken"/>
47-
public struct SyntaxToken
50+
public class SyntaxToken : CommonToken
4851
{
52+
private string _DisplayText;
53+
private string _TypeNameUpperCase;
54+
private int? _EndingLineNumber;
55+
private int? _EndingColumnPosition;
56+
private Place? _EndPlace;
57+
58+
private string _TypeName;
59+
60+
/// <summary>
61+
/// Initializes a new instance of the <see cref="SyntaxToken"/> class.
62+
/// </summary>
63+
/// <param name="source">The source.</param>
64+
/// <param name="type">The type.</param>
65+
/// <param name="channel">The channel.</param>
66+
/// <param name="start">The start.</param>
67+
/// <param name="stop">The stop.</param>
68+
public SyntaxToken([NotNull] Tuple<ITokenSource, ICharStream> source, int type, int channel, int start, int stop)
69+
: base(source, type, channel, start, stop)
70+
{
71+
ColumnPosition = charPositionInLine + 1;
72+
Length = stop - start + 1;
73+
74+
//TypeName = parserToken.Type > -1 ? lexer.Vocabulary.GetDisplayName(parserToken.Type) : string.Empty;
75+
//TypeNameUpperCase = TypeName.ToUpperInvariant();
76+
}
77+
4978
/// <summary>
50-
/// Initializes a new instance of the <see cref="SyntaxToken"/> struct.
79+
/// Initializes a new instance of the <see cref="SyntaxToken"/> class.
5180
/// </summary>
52-
/// <param name="lexer">The lexer.</param>
53-
/// <param name="parserToken">The token.</param>
54-
public SyntaxToken(Lexer lexer, IToken parserToken)
81+
/// <param name="type">The type.</param>
82+
/// <param name="text">The text.</param>
83+
public SyntaxToken(int type, string text)
84+
: base(type, text)
5585
{
56-
ActualParserToken = parserToken;
57-
Text = FormatTokenText(parserToken);
58-
Type = parserToken.Type > -1 ? lexer.Vocabulary.GetDisplayName(parserToken.Type) : string.Empty;
59-
TypeUpperCase = Type.ToUpperInvariant();
60-
LineNumber = parserToken.Line;
61-
ColumnPosition = parserToken.Column + 1;
62-
ChannelId = parserToken.Channel;
63-
Length = parserToken.StopIndex - parserToken.StartIndex + 1;
64-
StartPosition = parserToken.StartIndex;
65-
StopPosition = parserToken.StopIndex;
66-
67-
var spot = parserToken.GetEndPlace();
68-
EndingLineNumber = spot.Line;
69-
EndingColumnPosition = spot.Position + 1;
7086
}
7187

7288
/// <summary>
73-
/// Gets the token text.
89+
/// Initializes a new instance of the <see cref="SyntaxToken"/> class.
7490
/// </summary>
75-
/// <value>The token text.</value>
76-
public string Text { get; }
91+
/// <param name="type">The type.</param>
92+
public SyntaxToken(int type)
93+
: base(type)
94+
{
95+
}
7796

7897
/// <summary>
79-
/// Gets the token type.
98+
/// Gets the token display text.
8099
/// </summary>
81-
/// <value>The token type.</value>
82-
public string Type { get; }
100+
/// <value>The token display text.</value>
101+
public string DisplayText => _DisplayText ?? (_DisplayText = FormatTokenText(Text));
83102

84103
/// <summary>
85-
/// Gets the upper case type.
104+
/// Gets or sets the token type.
86105
/// </summary>
87-
/// <value>The upper case type.</value>
88-
/// <remarks>Exists solely to avoid repeated upper casing of the Type property.</remarks>
89-
public string TypeUpperCase { get; }
106+
/// <value>The token type name.</value>
107+
public string TypeName
108+
{
109+
get => _TypeName;
110+
set
111+
{
112+
_TypeNameUpperCase = null;
113+
_TypeName = value;
114+
}
115+
}
90116

91117
/// <summary>
92-
/// Gets the line number where the token occurs.
118+
/// Gets the upper case type name.
93119
/// </summary>
94-
/// <value>The line number.</value>
95-
public int LineNumber { get; }
120+
/// <value>The upper case type name.</value>
121+
/// <remarks>Exists solely to avoid repeated upper casing of the TypeName property.</remarks>
122+
public string TypeNameUpperCase => _TypeNameUpperCase ?? (_TypeNameUpperCase = TypeName.ToUpperInvariant());
96123

97124
/// <summary>
98125
/// Gets the column position of the token within the source line.
@@ -110,48 +137,50 @@ public SyntaxToken(Lexer lexer, IToken parserToken)
110137
/// Gets the line number for the end of token.
111138
/// </summary>
112139
/// <value>The ending line number.</value>
113-
public int EndingLineNumber { get; }
140+
public int EndingLineNumber
141+
{
142+
get
143+
{
144+
if (!_EndingLineNumber.HasValue)
145+
{
146+
if (!_EndPlace.HasValue)
147+
_EndPlace = this.GetEndPlace();
148+
_EndingLineNumber = _EndPlace.Value.Line;
149+
}
150+
151+
return _EndingLineNumber.Value;
152+
}
153+
}
114154

115155
/// <summary>
116156
/// Gets the column position for the end of the token.
117157
/// </summary>
118158
/// <value>The ending column position.</value>
119-
public int EndingColumnPosition { get; }
120-
121-
/// <summary>
122-
/// Gets the start position of the token within the source.
123-
/// </summary>
124-
/// <value>The start position.</value>
125-
public int StartPosition { get; }
126-
127-
/// <summary>
128-
/// Gets the stop position of the token within the source.
129-
/// </summary>
130-
/// <value>The stop position.</value>
131-
public int StopPosition { get; }
132-
133-
/// <summary>
134-
/// Gets the channel id for the token.
135-
/// </summary>
136-
/// <value>The channel id.</value>
137-
public int ChannelId { get; }
138-
139-
/// <summary>
140-
/// Gets the actual parser token.
141-
/// </summary>
142-
/// <value>The actual parser token.</value>
143-
public IToken ActualParserToken { get; }
159+
public int EndingColumnPosition
160+
{
161+
get
162+
{
163+
if (!_EndingColumnPosition.HasValue)
164+
{
165+
if (!_EndPlace.HasValue)
166+
_EndPlace = this.GetEndPlace();
167+
_EndingColumnPosition = _EndPlace.Value.Position + 1;
168+
}
169+
170+
return _EndingColumnPosition.Value;
171+
}
172+
}
144173

145-
private static string FormatTokenText(IToken token)
174+
private static string FormatTokenText(string text)
146175
{
147-
switch (token.Text)
176+
switch (text)
148177
{
149178
case "\r": return "\\r";
150179
case "\n": return "\\n";
151180
case "\t": return "\\t";
152181
}
153182

154-
return token.Text;
183+
return text;
155184
}
156185
}
157186
}

‎Common/Grammar/SyntaxTokenFactory.cs‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
using System;
3838

3939
using Antlr4.Runtime;
40+
using Antlr4.Runtime.Misc;
4041

4142
// ReSharper disable TooManyArguments
4243
namespace Org.Edgerunner.ANTLR4.Tools.Common.Grammar
@@ -48,16 +49,29 @@ namespace Org.Edgerunner.ANTLR4.Tools.Common.Grammar
4849
/// <seealso cref="Antlr4.Runtime.ITokenFactory" />
4950
public class SyntaxTokenFactory : ITokenFactory
5051
{
52+
/// <summary>
53+
/// The default static instance factory.
54+
/// </summary>
55+
public static readonly ITokenFactory Instance = new SyntaxTokenFactory();
56+
5157
IToken ITokenFactory.Create(Tuple<ITokenSource, ICharStream> source, int type, string text, int channel, int start, int stop, int line, int charPositionInLine)
5258
{
53-
throw new NotImplementedException();
59+
return Create(source, type, text, channel, start, stop, line, charPositionInLine);
5460
}
5561

5662
IToken ITokenFactory.Create(int type, string text)
5763
{
58-
throw new NotImplementedException();
64+
return Create(type, text);
5965
}
6066

67+
/// <summary>
68+
/// Creates the specified type.
69+
/// </summary>
70+
/// <param name="type">The type.</param>
71+
/// <param name="text">The text.</param>
72+
/// <returns>A new SyntaxToken.</returns>
73+
public virtual SyntaxToken Create(int type, string text) => new SyntaxToken(type, text);
74+
6175
/// <summary>
6276
/// Creates the specified source.
6377
/// </summary>
@@ -72,7 +86,13 @@ IToken ITokenFactory.Create(int type, string text)
7286
/// <returns>A new SyntaxToken instance.</returns>
7387
public SyntaxToken Create(Tuple<ITokenSource, ICharStream> source, int type, string text, int channel, int start, int stop, int line, int charPositionInLine)
7488
{
75-
throw new NotImplementedException();
89+
SyntaxToken token =
90+
new SyntaxToken(source, type, channel, start, stop) { Line = line, Column = charPositionInLine };
91+
if (text != null)
92+
token.Text = text;
93+
else if (source.Item2 != null)
94+
token.Text = source.Item2.GetText(Interval.Of(start, stop));
95+
return token;
7696
}
7797
}
7898
}

‎Common/Properties/AssemblyInfo.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,5 +32,5 @@
3232
// You can specify all the values or you can default the Build and Revision Numbers
3333
// by using the '*' as shown below:
3434
// [assembly: AssemblyVersion("1.0.*")]
35-
[assembly: AssemblyVersion("1.2.21215.2")]
36-
[assembly: AssemblyFileVersion("1.2.21215.2")]
35+
[assembly: AssemblyVersion("1.2.22221.0")]
36+
[assembly: AssemblyFileVersion("1.2.22221.0")]

‎Grun/Program.cs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -323,13 +323,13 @@ private static void HighlightSyntaxInConsole(int lineOffset, Analyzer analyzer,
323323

324324
private static void ColorToken(SyntaxToken token, int lineOffset, ISyntaxHighlightingGuide guide)
325325
{
326-
if (token.ChannelId != 0)
326+
if (token.Channel != 0)
327327
return;
328328

329-
if (token.Text == "<EOF>")
329+
if (token.DisplayText == "<EOF>")
330330
return;
331331

332-
var startLine = token.LineNumber + lineOffset;
332+
var startLine = token.Line + lineOffset;
333333
var endLine = token.EndingLineNumber + lineOffset;
334334

335335
if (startLine < 0)
@@ -344,7 +344,7 @@ private static void ColorToken(SyntaxToken token, int lineOffset, ISyntaxHighlig
344344
for (int col = token.ColumnPosition; col < token.EndingColumnPosition + 1; col++)
345345
{
346346
Console.SetCursorPosition(col - 1, ln);
347-
Console.Write(token.Text[index++]);
347+
Console.Write(token.DisplayText[index++]);
348348
}
349349
}
350350

@@ -425,7 +425,7 @@ private static void LoadApplicationSettings()
425425

426426
private static int MakeTokenKey(SyntaxToken token)
427427
{
428-
return $"{token.LineNumber}-{token.ColumnPosition}-{token.Text}".GetHashCode();
428+
return $"{token.Line}-{token.ColumnPosition}-{token.DisplayText}".GetHashCode();
429429
}
430430

431431
#endregion

‎Grun/SyntaxHighlighting/ConsoleWrapperColor.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,16 +108,16 @@ private static void HighlightToken(SyntaxToken token, ISyntaxHighlightingGuide g
108108

109109
private static void ColorToken(SyntaxToken token, int lineOffset, ISyntaxHighlightingGuide guide)
110110
{
111-
if (token.ChannelId != 0)
111+
if (token.Channel != 0)
112112
return;
113113

114-
if (token.Text == "<EOF>")
114+
if (token.DisplayText == "<EOF>")
115115
return;
116116

117117
var defaultForeGround = Console.ForegroundColor;
118118
var defaultBackground = Console.BackgroundColor;
119119

120-
var startLine = token.LineNumber + lineOffset;
120+
var startLine = token.Line + lineOffset;
121121
var endLine = token.EndingLineNumber + lineOffset;
122122

123123
if (startLine < 0)
@@ -133,7 +133,7 @@ private static void ColorToken(SyntaxToken token, int lineOffset, ISyntaxHighlig
133133
for (int col = token.ColumnPosition; col < token.EndingColumnPosition + 1; col++)
134134
{
135135
Console.SetCursorPosition(col - 1, ln);
136-
Console.Write(token.Text[index++]);
136+
Console.Write(token.DisplayText[index++]);
137137
}
138138
}
139139

‎Grun/SyntaxHighlighting/HighlightingTokenCache.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public HighlightingTokenCache()
5151

5252
public bool IsKnown(SyntaxToken token)
5353
{
54-
if (_KnownTokens.TryGetValue(token.LineNumber, out var lineCache))
54+
if (_KnownTokens.TryGetValue(token.Line, out var lineCache))
5555
if (lineCache.TryGetValue(token.ColumnPosition, out var hash))
5656
if (hash == token.GetHashCode())
5757
return true;
@@ -61,10 +61,10 @@ public bool IsKnown(SyntaxToken token)
6161

6262
public void RegisterToken(SyntaxToken token)
6363
{
64-
if (_KnownTokens.TryGetValue(token.LineNumber, out var lineCache))
64+
if (_KnownTokens.TryGetValue(token.Line, out var lineCache))
6565
lineCache[token.ColumnPosition] = token.GetHashCode();
6666
else
67-
_KnownTokens[token.LineNumber] = new Dictionary<int, int> { { token.ColumnPosition, token.GetHashCode() } };
67+
_KnownTokens[token.Line] = new Dictionary<int, int> { { token.ColumnPosition, token.GetHashCode() } };
6868
}
6969

7070
public void FlushTokensForLine(int lineNumber)

‎GunWin/Editor/SyntaxHighlighting/EditorSyntaxHighlighter.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ public void ColorizeTokens(FastColoredTextBox editor, IStyleRegistry registry, I
8686
{
8787
foreach (var token in tokens)
8888
{
89-
var startingPlace = new Place(token.ActualParserToken.Column, token.ActualParserToken.Line - 1);
89+
var startingPlace = new Place(token.Column, token.Line - 1);
9090
var stoppingPlace = new Place(token.EndingColumnPosition, token.EndingLineNumber - 1);
9191
var tokenRange = editor.GetRange(startingPlace, stoppingPlace);
9292
tokenRange.ClearStyle(StyleIndex.All);

0 commit comments

Comments
 (0)