Skip to content

Commit dcf70ca

Browse files
smailletsmaillet
authored
Refactored and corrected bugs in Kaleidoscope samples (#46)
* Moved more common code into Kaleidoscope.Runtime so that the individual chapters are focused on the the distinct aspect of the tutorial chapter instead of on gerenal * Fixed grammar parsing of assignment expressions * Refactored grammar to allow parser to reject incorrect/invalid user operator overloads of built-in operators. * Converted operator tables and support to use the token type integral value instead of characters or strings for simpler use and ensures the code generator's aren't using token chars or strings that aren't available in the language. * Fixed AntlrUtilities source interval and text extraction. * Simplified lexer by moving to use Semantic predicates on the Lexemes for fueature controlled keywords. * Unified name generation for user defined binary and unary operatos so all should be using a consistent formatting from one place. * Added ScopeStack<T> for a cleaner, simpler (and actually functionally correct) variable scoping. * Fixed Repl loop parsing to properly handle blank/empty lines
1 parent ae98a7e commit dcf70ca

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+1974
-1543
lines changed

‎Directory.Build.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@
6161
<IsTestProject>$(MSBuildProjectName.Contains('Test'))</IsTestProject>
6262
</PropertyGroup>
6363
<ItemGroup Condition="'$(NoCommonAnalyzers)'!='true' and '$(IsTestProject)' != 'true' and '$(SourceLinkEnabled)' != 'false'">
64-
<PackageReference Include="SourceLink.Create.CommandLine" Version="2.6.0" PrivateAssets="All" />
64+
<PackageReference Include="SourceLink.Create.CommandLine" Version="2.6.1" PrivateAssets="All" />
6565
</ItemGroup>
6666
<ItemGroup Condition="'$(NoCommonAnalyzers)'!='true'">
6767
<PackageReference Include="Microsoft.AnalyzerPowerPack" Version="1.1.0" PrivateAssets="All" />

‎Samples/Kaleidoscope/Chapter2/CodeGenerator.cs‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,11 @@
33
// </copyright>
44

55
using System;
6+
using Antlr4.Runtime;
67
using Antlr4.Runtime.Misc;
8+
using Antlr4.Runtime.Tree;
79
using Kaleidoscope.Grammar;
10+
using Kaleidoscope.Runtime;
811

912
using static Kaleidoscope.Grammar.KaleidoscopeParser;
1013

@@ -13,34 +16,45 @@ namespace Kaleidoscope
1316
/// <summary>Static extension methods to perform LLVM IR Code generation from the Kaledoscope AST</summary>
1417
internal sealed class CodeGenerator
1518
: KaleidoscopeBaseVisitor<int>
19+
, IDisposable
20+
, IKaleidoscopeCodeGenerator<int>
1621
{
17-
public CodeGenerator( LanguageLevel level )
22+
public CodeGenerator( DynamicRuntimeState globalState )
1823
{
19-
ParserStack = new ReplParserStack( level );
24+
RuntimeState = globalState;
2025
}
2126

22-
public ReplParserStack ParserStack { get; }
27+
public void Dispose( )
28+
{
29+
}
30+
31+
public int Generate( Parser parser, IParseTree tree, DiagnosticRepresentations additionalDiagnostics )
32+
{
33+
return Visit( tree );
34+
}
2335

2436
public override int VisitBinaryPrototype( [NotNull] BinaryPrototypeContext context )
2537
{
26-
if( !ParserStack.GlobalState.TryAddOperator( context.Op, OperatorKind.InfixLeftAssociative, context.Precedence ) )
38+
if( !RuntimeState.TryAddOperator( context.OpToken, OperatorKind.InfixLeftAssociative, context.Precedence ) )
2739
{
28-
throw new ArgumentException( "Cannot replace built-in operators", nameof( context ) );
40+
throw new CodeGeneratorException( "Cannot replace built-in operators" );
2941
}
3042

3143
return 0;
3244
}
3345

3446
public override int VisitUnaryPrototype( [NotNull] UnaryPrototypeContext context )
3547
{
36-
if( !ParserStack.GlobalState.TryAddOperator( context.Op, OperatorKind.PreFix, 0 ) )
48+
if( !RuntimeState.TryAddOperator( context.OpToken, OperatorKind.PreFix, 0 ) )
3749
{
38-
throw new ArgumentException( "Cannot replace built-in operators", nameof( context ) );
50+
throw new CodeGeneratorException( "Cannot replace built-in operators" );
3951
}
4052

4153
return 0;
4254
}
4355

4456
protected override int DefaultResult => 0;
57+
58+
private readonly DynamicRuntimeState RuntimeState;
4559
}
4660
}

‎Samples/Kaleidoscope/Chapter2/Program.cs‎

Lines changed: 18 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@
44

55
using System;
66
using System.Diagnostics.CodeAnalysis;
7+
using Antlr4.Runtime.Tree;
78
using Kaleidoscope.Grammar;
9+
using Kaleidoscope.Runtime;
810

911
[assembly: SuppressMessage( "StyleCop.CSharp.DocumentationRules", "SA1652:Enable XML documentation output", Justification = "Sample application" )]
1012

@@ -20,57 +22,26 @@ public static void Main( string[ ] args )
2022
// Using language level that includes the complete set
2123
// of language features to allow exploring and verifying
2224
// the parser support for the whole language.
23-
var generator = new CodeGenerator( LanguageLevel.MutableVariables );
24-
RunReplLoop( generator );
25-
}
26-
27-
/// <summary>Runs the REPL loop for the language</summary>
28-
/// <param name="generator">Generator for generating code</param>
29-
/// <remarks>
30-
/// Since ANTLR doesn't have an "interactive" input stream, this sort of fakes
31-
/// it by using the <see cref="ReplLoopExtensions.ReadStatements(System.IO.TextReader)"/>
32-
/// extension to provide an enumeration of lines that may be partial statements read in.
33-
/// This is consistent with the behavior of the official LLVM C++ version and allows
34-
/// for full use of ANTLR4 instead of wrting a parser by hand.
35-
/// </remarks>
36-
private static void RunReplLoop( CodeGenerator generator )
37-
{
38-
Console.WriteLine( "LLVM Kaleidoscope Syntax Viewer - {0}", generator.ParserStack.LanguageLevel );
39-
Console.Write( "Ready>" );
40-
foreach( var (Txt, IsPartial) in Console.In.ReadStatements( ) )
25+
var parser = new ReplParserStack( LanguageLevel.MutableVariables );
26+
using( var generator = new CodeGenerator( parser.GlobalState ) )
4127
{
42-
if( !IsPartial )
43-
{
44-
var parseTree = generator.ParserStack.ReplParse( Txt );
45-
if( parseTree != null )
46-
{
47-
// pass the tree through simplistic generator to track user defined ops so that
48-
// subsequent references parse successfully. No code generation in this version.
49-
generator.Visit( parseTree );
50-
51-
// This departs a tad from the official C++ version for this "chapter"
52-
// by printing generating a representation of the complete parse tree
53-
// as opposed to the official version's simplistic "parsed an XYZ"
54-
// message.
28+
Console.WriteLine( "LLVM Kaleidoscope Syntax Viewer - {0}", parser.LanguageLevel );
5529

56-
// This provides much more detailed information about the actual parse
57-
// to help in diagnosing issues. Whenever, adding functionality to
58-
// the grammar itself it is useful to come back to this to verify what
59-
// the parser is actually producing for a given input.
60-
#if NET47
61-
// For desktop, generate a DGML from the parse tree. This is useful when modifying
62-
// or debugging the gramar in general as you can open the DGML in VS
63-
// and as each new tree is parsed VS can auto update the visual graph
64-
string path = System.IO.Path.GetFullPath( "parsetree.dgml" );
65-
generator.ParserStack.GenerateDgml( parseTree, path );
66-
Console.WriteLine( "Generated {0}", path );
67-
#endif
68-
Console.WriteLine( "Parsed:\n{0}", generator.ParserStack.GenerateXmlTree( parseTree ) );
69-
}
70-
}
30+
var replLoop = new ReplLoop<int>( generator, parser, DiagnosticRepresentations.Xml | DiagnosticRepresentations.Dgml );
31+
replLoop.ReadyStateChanged += ( s, e ) => Console.Write( e.PartialParse ? ">" : "Ready>" );
32+
replLoop.GeneratedResultAvailable += OnGeneratedResultAvailable;
7133

72-
Console.Write( IsPartial ? ">" : "Ready>" );
34+
replLoop.Run( );
7335
}
7436
}
37+
38+
private static void OnGeneratedResultAvailable( object sender, GeneratedResultAvailableArgs<int> e )
39+
{
40+
var docListener = new XDocumentListener( e.Recognizer );
41+
ParseTreeWalker.Default.Walk( docListener, e.ParseTree );
42+
Console.WriteLine( "Parsed:" );
43+
docListener.Document.Save( Console.Out );
44+
Console.WriteLine( );
45+
}
7546
}
7647
}

‎Samples/Kaleidoscope/Chapter3/CodeGenerator.cs‎

Lines changed: 31 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@
55
using System;
66
using System.Collections.Generic;
77
using System.Linq;
8+
using Antlr4.Runtime;
89
using Antlr4.Runtime.Misc;
910
using Antlr4.Runtime.Tree;
1011
using Kaleidoscope.Grammar;
12+
using Kaleidoscope.Runtime;
1113
using Llvm.NET;
1214
using Llvm.NET.Instructions;
1315
using Llvm.NET.Values;
@@ -20,31 +22,32 @@ namespace Kaleidoscope
2022
internal sealed class CodeGenerator
2123
: KaleidoscopeBaseVisitor<Value>
2224
, IDisposable
25+
, IKaleidoscopeCodeGenerator<Value>
2326
{
24-
public CodeGenerator( LanguageLevel level )
27+
public CodeGenerator( DynamicRuntimeState globalState )
2528
{
29+
RuntimeState = globalState;
2630
Context = new Context( );
2731
Module = Context.CreateBitcodeModule( "Kaleidoscope" );
2832
InstructionBuilder = new InstructionBuilder( Context );
2933
NamedValues = new Dictionary<string, Value>( );
30-
ParserStack = new ReplParserStack( level );
3134
}
3235

33-
public ReplParserStack ParserStack { get; }
34-
35-
public Context Context { get; }
36-
37-
public BitcodeModule Module { get; }
38-
39-
public InstructionBuilder InstructionBuilder { get; }
40-
41-
public IDictionary<string, Value> NamedValues { get; }
42-
4336
public void Dispose( )
4437
{
4538
Context.Dispose( );
4639
}
4740

41+
public Value Generate( Parser parser, IParseTree tree, DiagnosticRepresentations additionalDiagnostics )
42+
{
43+
if( parser.NumberOfSyntaxErrors > 0 )
44+
{
45+
return null;
46+
}
47+
48+
return Visit( tree );
49+
}
50+
4851
public override Value VisitParenExpression( [NotNull] ParenExpressionContext context )
4952
{
5053
return context.Expression.Accept( this );
@@ -97,7 +100,8 @@ public override Value VisitFunctionDefinition( [NotNull] FunctionDefinitionConte
97100

98101
public override Value VisitTopLevelExpression( [NotNull] TopLevelExpressionContext context )
99102
{
100-
var function = GetOrDeclareFunction( new Prototype( $"anon_expr_{AnonNameIndex++}" ) );
103+
var proto = new Prototype( $"anon_expr_{AnonNameIndex++}" );
104+
var function = GetOrDeclareFunction( proto );
101105

102106
return DefineFunction( function, context.expression() );
103107
}
@@ -117,45 +121,45 @@ public override Value VisitExpression( [NotNull] ExpressionContext context )
117121

118122
protected override Value DefaultResult => null;
119123

120-
private Value EmitBinaryOperator( Value lhs, OpsymbolContext opSymbol, IParseTree rightTree )
124+
private Value EmitBinaryOperator( Value lhs, BinaryopContext op, IParseTree rightTree )
121125
{
122126
var rhs = rightTree.Accept( this );
123127
if( lhs == null || rhs == null )
124128
{
125129
return null;
126130
}
127131

128-
switch( opSymbol.Op )
132+
switch( op.Token.Type )
129133
{
130-
case '<':
134+
case LEFTANGLE:
131135
{
132136
var tmp = InstructionBuilder.Compare( RealPredicate.UnorderedOrLessThan, lhs, rhs )
133137
.RegisterName( "cmptmp" );
134138
return InstructionBuilder.UIToFPCast( tmp, InstructionBuilder.Context.DoubleType )
135139
.RegisterName( "booltmp" );
136140
}
137141

138-
case '^':
142+
case CARET:
139143
{
140144
var pow = GetOrDeclareFunction( new Prototype( "llvm.pow.f64", "value", "power" ) );
141145
return InstructionBuilder.Call( pow, lhs, rhs )
142146
.RegisterName( "powtmp" );
143147
}
144148

145-
case '+':
149+
case PLUS:
146150
return InstructionBuilder.FAdd( lhs, rhs ).RegisterName( "addtmp" );
147151

148-
case '-':
152+
case MINUS:
149153
return InstructionBuilder.FSub( lhs, rhs ).RegisterName( "subtmp" );
150154

151-
case '*':
155+
case ASTERISK:
152156
return InstructionBuilder.FMul( lhs, rhs ).RegisterName( "multmp" );
153157

154-
case '/':
158+
case SLASH:
155159
return InstructionBuilder.FDiv( lhs, rhs ).RegisterName( "divtmp" );
156160

157161
default:
158-
throw new CodeGeneratorException( $"Invalid binary operator {opSymbol.Op}" );
162+
throw new CodeGeneratorException( $"Invalid binary operator {op.Token.Text}" );
159163
}
160164
}
161165

@@ -215,6 +219,11 @@ private Function DefineFunction( Function function, ExpressionContext body )
215219
return function;
216220
}
217221

222+
private readonly DynamicRuntimeState RuntimeState;
218223
private static int AnonNameIndex;
224+
private readonly Context Context;
225+
private BitcodeModule Module;
226+
private readonly InstructionBuilder InstructionBuilder;
227+
private readonly IDictionary<string, Value> NamedValues;
219228
}
220229
}

‎Samples/Kaleidoscope/Chapter3/Program.cs‎

Lines changed: 31 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,14 @@
44

55
using System;
66
using System.Diagnostics.CodeAnalysis;
7+
using System.IO;
78
using Kaleidoscope.Grammar;
8-
using Llvm.NET;
9+
using Kaleidoscope.Runtime;
910
using Llvm.NET.Values;
1011

12+
using static Kaleidoscope.Runtime.Utilities;
13+
using static Llvm.NET.StaticState;
14+
1115
[assembly: SuppressMessage( "StyleCop.CSharp.DocumentationRules", "SA1652:Enable XML documentation output", Justification = "Sample application" )]
1216

1317
namespace Kaleidoscope
@@ -19,48 +23,44 @@ public static class Program
1923
[SuppressMessage( "Redundancies in Symbol Declarations", "RECS0154:Parameter is never used", Justification = "Standard required signature" )]
2024
public static void Main( string[ ] args )
2125
{
22-
Utilities.WaitForDebugger( );
26+
WaitForDebugger( );
2327

24-
using( StaticState.InitializeLLVM( ) )
28+
using( InitializeLLVM( ) )
2529
{
26-
StaticState.RegisterNative( );
27-
using( var generator = new CodeGenerator( LanguageLevel.SimpleExpressions ) )
30+
RegisterNative( );
31+
var parser = new ReplParserStack( LanguageLevel.SimpleExpressions );
32+
using( var generator = new CodeGenerator( parser.GlobalState ) )
2833
{
29-
RunReplLoop( generator );
30-
Console.WriteLine( generator.Module.WriteToString( ) );
34+
Console.WriteLine( "LLVM Kaleidoscope Interpreter - {0}", parser.LanguageLevel );
35+
36+
var replLoop = new ReplLoop<Value>( generator, parser );
37+
replLoop.ReadyStateChanged += ( s, e ) => Console.Write( e.PartialParse ? ">" : "Ready>" );
38+
replLoop.GeneratedResultAvailable += OnGeneratedResultAvailable;
39+
40+
replLoop.Run( );
3141
}
3242
}
3343
}
3444

35-
/// <summary>Runs the REPL loop for the language</summary>
36-
/// <param name="generator">Generator for generating code</param>
37-
/// <remarks>
38-
/// Since ANTLR doesn't have an "interactive" input stream, this sort of fakes
39-
/// it by using the <see cref="ReplLoopExtensions.ReadStatements(System.IO.TextReader)"/>
40-
/// extension to provide an enumeration of lines that may be partial statements read in.
41-
/// This is consistent with the behavior of the official LLVM C++ version and allows
42-
/// for full use of ANTLR4 instead of wrting a parser by hand.
43-
/// </remarks>
44-
private static void RunReplLoop( CodeGenerator generator )
45+
private static void OnGeneratedResultAvailable( object sender, GeneratedResultAvailableArgs<Value> e )
4546
{
46-
Console.WriteLine( "LLVM Kaleidoscope Generator - {0}", generator.ParserStack.LanguageLevel );
47-
Console.Write( "Ready>" );
48-
foreach( var (Txt, IsPartial) in Console.In.ReadStatements( ) )
47+
var source = ( ReplLoop<Value> )sender;
48+
49+
switch( e.Result )
4950
{
50-
if( !IsPartial )
51+
case ConstantFP result:
52+
Console.WriteLine( "Evaluated to {0}", result.Value );
53+
break;
54+
55+
case Function function:
56+
if( source.AdditionalDiagnostics.HasFlag( DiagnosticRepresentations.LlvmIR ) )
5157
{
52-
var parseTree = generator.ParserStack.ReplParse( Txt );
53-
if( parseTree != null )
54-
{
55-
Value value = generator.Visit( parseTree );
56-
if( value != null )
57-
{
58-
Console.WriteLine( "Parsed {0}", value );
59-
}
60-
}
58+
function.ParentModule.WriteToTextFile( Path.ChangeExtension( GetSafeFileName( function.Name ), "ll" ), out string ignoredMsg );
6159
}
6260

63-
Console.Write( IsPartial ? ">" : "Ready>" );
61+
Console.WriteLine( "Defined function: {0}", function.Name );
62+
Console.WriteLine( function );
63+
break;
6464
}
6565
}
6666
}

0 commit comments

Comments
 (0)