Skip to content

Commit ae98a7e

Browse files
authored
Full OrcJit for Kaleidoscope and Llvm.NET Fixes #43,#44 (#45)
* Fixes #43,#44 Full OrcJit for Kaleidoscope * Kaleidoscope now correctly runs the mandelbrot example! * Completely re-wrote Value handle interning/cache to use a tracking map that handles RAUW and deletions from various transforms. * Updated Kaleidosopce samples to use OrcJit * Fixed KaleidoscopeJIT and generator to handle re-defining a function by tracking the jitmodule handles for a function and removing the module for the previous definition before adding the new one. * Added support for global symbol resolving to delegates so that JITed code can call back to the host in a controlled fashion. * Fixed bug where user defined operator expressions were re-generating the operand expressions. * Added DGML generator for the Kaleidoscope parser to make visualizing the parse tree a LOT easier. * Refactored common support and JIT for Kaledoscope to eliminate most of the duplicated code files. * refactored Kaleidoscope grammar to split out the individual context rules * re-worked dynamic operator precedence to fix incorrect evaluations * reorganized the various codegnerator.cs files to make doing a diff between tutorial chapters easier to see what's new for each chapter. * Fixed bug in for loop code generation where condition was inverted so it basically guaranteed only a single loop iteration always. * Fixed bug in OrcJit where the symbolresolver delegate was at the mercy of the garbage collector, It is now contained in a WrappedNativeCallback that is mapped per module. * Fixed module dispose bug found by running the unit tests and sample test app * * simplified dispose for BitcodeModule as it doesn't really ned finalization. (context will destory any modules not explicitly disposed) * removed temporary per module value clean up now that ValueCache tracks destroy and RAUW it isn't needed and was causing a new manged wrapper for a module it was in the process of disposing
1 parent b8b7339 commit ae98a7e

File tree

102 files changed

+2697
-2096
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

102 files changed

+2697
-2096
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,3 +150,4 @@ Tools/
150150
*.bc
151151
*.o
152152
*.s
153+
*.dgml

BuildVersion.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<BuildVersionData
22
BuildMajor = "5"
33
BuildMinor = "0"
4-
BuildPatch = "0"
4+
BuildPatch = "1"
55
PreReleaseName = "alpha"
66
/>

Directory.Build.props

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@
6565
</ItemGroup>
6666
<ItemGroup Condition="'$(NoCommonAnalyzers)'!='true'">
6767
<PackageReference Include="Microsoft.AnalyzerPowerPack" Version="1.1.0" PrivateAssets="All" />
68-
<PackageReference Include="RefactoringEssentials" Version="5.4.0" PrivateAssets="All" />
68+
<PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="All" />
6969
<PackageReference Include="StyleCop.Analyzers" Version="1.0.2" PrivateAssets="All" />
7070
<AdditionalFiles Include="$(MSBuildThisFileDirectory)stylecop.json" Link="stylecop.json" />
7171
</ItemGroup>

Samples/CodeGenWithDebugInfo/Program.cs

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -168,26 +168,27 @@ public static void Main( string[ ] args )
168168
{// force a GC to verify callback delegate for diagnostics is still valid, this is for test only and wouldn't
169169
// normally be done in production code.
170170
GC.Collect( GC.MaxGeneration );
171-
var modForOpt = module.Clone( );
172-
173-
// NOTE:
174-
// The ordering of passes can matter depending on the pass and passes may be added more than once
175-
// the caller has full control of ordering, this is just a sample of effectively randomly picked
176-
// passes and not necessarily a reflection of any particular use case.
177-
var pm = new ModulePassManager( )
178-
.AddAlwaysInlinerPass( )
179-
.AddAggressiveDCEPass( )
180-
.AddArgumentPromotionPass( )
181-
.AddBasicAliasAnalysisPass( )
182-
.AddBitTrackingDCEPass( )
183-
.AddCFGSimplificationPass( )
184-
.AddConstantMergePass( )
185-
.AddConstantPropagationPass( )
186-
.AddFunctionInliningPass( )
187-
.AddGlobalOptimizerPass( )
188-
.AddInstructionCombiningPass( );
189-
190-
pm.Run( modForOpt );
171+
using( var modForOpt = module.Clone( ) )
172+
{
173+
// NOTE:
174+
// The ordering of passes can matter depending on the pass and passes may be added more than once
175+
// the caller has full control of ordering, this is just a sample of effectively randomly picked
176+
// passes and not necessarily a reflection of any particular use case.
177+
var pm = new ModulePassManager( )
178+
.AddAlwaysInlinerPass( )
179+
.AddAggressiveDCEPass( )
180+
.AddArgumentPromotionPass( )
181+
.AddBasicAliasAnalysisPass( )
182+
.AddBitTrackingDCEPass( )
183+
.AddCFGSimplificationPass( )
184+
.AddConstantMergePass( )
185+
.AddConstantPropagationPass( )
186+
.AddFunctionInliningPass( )
187+
.AddGlobalOptimizerPass( )
188+
.AddInstructionCombiningPass( );
189+
190+
pm.Run( modForOpt );
191+
}
191192
}
192193

193194
// Module is good, so generate the output files

Samples/Kaleidoscope/Chapter2/Chapter2.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
</PropertyGroup>
99
<ItemGroup>
1010
<ProjectReference Include="..\Kaleidoscope.Parser\Kaleidoscope.Grammar.csproj" />
11+
<ProjectReference Include="..\Kaleidoscope.Runtime\Kaleidoscope.Runtime.csproj" />
1112
</ItemGroup>
1213

1314
</Project>

Samples/Kaleidoscope/Chapter2/CodeGenerator.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ public CodeGenerator( LanguageLevel level )
2323

2424
public override int VisitBinaryPrototype( [NotNull] BinaryPrototypeContext context )
2525
{
26-
if( !ParserStack.Parser.TryAddOperator( context.Op, OperatorKind.InfixLeftAssociative, context.Precedence ) )
26+
if( !ParserStack.GlobalState.TryAddOperator( context.Op, OperatorKind.InfixLeftAssociative, context.Precedence ) )
2727
{
2828
throw new ArgumentException( "Cannot replace built-in operators", nameof( context ) );
2929
}
@@ -33,7 +33,7 @@ public override int VisitBinaryPrototype( [NotNull] BinaryPrototypeContext conte
3333

3434
public override int VisitUnaryPrototype( [NotNull] UnaryPrototypeContext context )
3535
{
36-
if( !ParserStack.Parser.TryAddOperator( context.Op, OperatorKind.PreFix, 0 ) )
36+
if( !ParserStack.GlobalState.TryAddOperator( context.Op, OperatorKind.PreFix, 0 ) )
3737
{
3838
throw new ArgumentException( "Cannot replace built-in operators", nameof( context ) );
3939
}

Samples/Kaleidoscope/Chapter2/Program.cs

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
using System;
66
using System.Diagnostics.CodeAnalysis;
7-
using Antlr4.Runtime.Tree;
87
using Kaleidoscope.Grammar;
98

109
[assembly: SuppressMessage( "StyleCop.CSharp.DocumentationRules", "SA1652:Enable XML documentation output", Justification = "Sample application" )]
@@ -50,15 +49,23 @@ private static void RunReplLoop( CodeGenerator generator )
5049
generator.Visit( parseTree );
5150

5251
// This departs a tad from the official C++ version for this "chapter"
53-
// by printing out an XML representation of the complete parse tree
54-
// as opposed to the official version's "parsed an XYZ" message.
52+
// by printing generating a representation of the complete parse tree
53+
// as opposed to the official version's simplistic "parsed an XYZ"
54+
// message.
55+
5556
// This provides much more detailed information about the actual parse
5657
// to help in diagnosing issues. Whenever, adding functionality to
5758
// the grammar itself it is useful to come back to this to verify what
5859
// the parser is actually producing for a given input.
59-
var docListener = new XDocumentListener( generator.ParserStack.Parser );
60-
ParseTreeWalker.Default.Walk( docListener, parseTree );
61-
Console.WriteLine( "Parsed:\n{0}", docListener.Document.ToString( ) );
60+
#if NET47
61+
// For desktop, generate a DGML from the parse tree. This is useful when modifying
62+
// or debugging the gramar in general as you can open the DGML in VS
63+
// and as each new tree is parsed VS can auto update the visual graph
64+
string path = System.IO.Path.GetFullPath( "parsetree.dgml" );
65+
generator.ParserStack.GenerateDgml( parseTree, path );
66+
Console.WriteLine( "Generated {0}", path );
67+
#endif
68+
Console.WriteLine( "Parsed:\n{0}", generator.ParserStack.GenerateXmlTree( parseTree ) );
6269
}
6370
}
6471

Samples/Kaleidoscope/Chapter3/Chapter3.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
<ItemGroup>
1717
<ProjectReference Include="..\Kaleidoscope.Parser\Kaleidoscope.Grammar.csproj" />
18+
<ProjectReference Include="..\Kaleidoscope.Runtime\Kaleidoscope.Runtime.csproj" />
1819
</ItemGroup>
1920

2021
</Project>

Samples/Kaleidoscope/Chapter3/CodeGenerator.cs

Lines changed: 53 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using System.Collections.Generic;
77
using System.Linq;
88
using Antlr4.Runtime.Misc;
9+
using Antlr4.Runtime.Tree;
910
using Kaleidoscope.Grammar;
1011
using Llvm.NET;
1112
using Llvm.NET.Instructions;
@@ -64,22 +65,67 @@ public override Value VisitVariableExpression( [NotNull] VariableExpressionConte
6465
string varName = context.Name;
6566
if( !NamedValues.TryGetValue( varName, out Value value ) )
6667
{
67-
throw new ArgumentException( "Unknown variable name", nameof( context ) );
68+
throw new CodeGeneratorException( $"Unknown variable name: {context}" );
6869
}
6970

7071
return value;
7172
}
7273

73-
public override Value VisitBinaryOpExpression( [NotNull] BinaryOpExpressionContext context )
74+
public override Value VisitFunctionCallExpression( [NotNull] FunctionCallExpressionContext context )
75+
{
76+
var function = GetFunction( context.CaleeName );
77+
if( function == null )
78+
{
79+
throw new CodeGeneratorException( $"function '{context.CaleeName}' is unknown" );
80+
}
81+
82+
var args = context.Args.Select( ctx => ctx.Accept( this ) ).ToArray( );
83+
return InstructionBuilder.Call( function, args ).RegisterName( "calltmp" );
84+
}
85+
86+
public override Value VisitFunctionPrototype( [NotNull] FunctionPrototypeContext context )
87+
{
88+
return GetOrDeclareFunction( new Prototype( context ) );
89+
}
90+
91+
public override Value VisitFunctionDefinition( [NotNull] FunctionDefinitionContext context )
92+
{
93+
return DefineFunction( ( Function )context.Signature.Accept( this )
94+
, context.BodyExpression
95+
);
96+
}
97+
98+
public override Value VisitTopLevelExpression( [NotNull] TopLevelExpressionContext context )
99+
{
100+
var function = GetOrDeclareFunction( new Prototype( $"anon_expr_{AnonNameIndex++}" ) );
101+
102+
return DefineFunction( function, context.expression() );
103+
}
104+
105+
public override Value VisitExpression( [NotNull] ExpressionContext context )
106+
{
107+
// Expression: PrimaryExpression (op expression)*
108+
// the sub-expressions are in evaluation order
109+
var lhs = context.primaryExpression( ).Accept( this );
110+
foreach( var (op, rhs) in context.OperatorExpressions )
111+
{
112+
lhs = EmitBinaryOperator( lhs, op, rhs );
113+
}
114+
115+
return lhs;
116+
}
117+
118+
protected override Value DefaultResult => null;
119+
120+
private Value EmitBinaryOperator( Value lhs, OpsymbolContext opSymbol, IParseTree rightTree )
74121
{
75-
var lhs = context.Lhs.Accept( this );
76-
var rhs = context.Rhs.Accept( this );
122+
var rhs = rightTree.Accept( this );
77123
if( lhs == null || rhs == null )
78124
{
79125
return null;
80126
}
81127

82-
switch( context.Op )
128+
switch( opSymbol.Op )
83129
{
84130
case '<':
85131
{
@@ -109,43 +155,10 @@ public override Value VisitBinaryOpExpression( [NotNull] BinaryOpExpressionConte
109155
return InstructionBuilder.FDiv( lhs, rhs ).RegisterName( "divtmp" );
110156

111157
default:
112-
throw new ArgumentException( $"Invalid binary operator {context.Op}", nameof( context ) );
113-
}
114-
}
115-
116-
public override Value VisitFunctionCallExpression( [NotNull] FunctionCallExpressionContext context )
117-
{
118-
var function = GetFunction( context.CaleeName );
119-
if( function == null )
120-
{
121-
throw new ArgumentException( $"Unknown function reference {context.CaleeName}", nameof( context ) );
158+
throw new CodeGeneratorException( $"Invalid binary operator {opSymbol.Op}" );
122159
}
123-
124-
var args = context.Args.Select( ctx => ctx.Accept( this ) ).ToArray( );
125-
return InstructionBuilder.Call( function, args ).RegisterName( "calltmp" );
126160
}
127161

128-
public override Value VisitFunctionPrototype( [NotNull] FunctionPrototypeContext context )
129-
{
130-
return GetOrDeclareFunction( new Prototype( context ) );
131-
}
132-
133-
public override Value VisitFunctionDefinition( [NotNull] FunctionDefinitionContext context )
134-
{
135-
return DefineFunction( ( Function )context.Signature.Accept( this )
136-
, context.BodyExpression
137-
);
138-
}
139-
140-
public override Value VisitTopLevelExpression( [NotNull] TopLevelExpressionContext context )
141-
{
142-
var function = GetOrDeclareFunction( new Prototype( $"anon_expr_{AnonNameIndex++}" ) );
143-
144-
return DefineFunction( function, context.expression() );
145-
}
146-
147-
protected override Value DefaultResult => null;
148-
149162
private Function GetFunction( string name )
150163
{
151164
return Module.GetFunction( name );
@@ -178,7 +191,7 @@ private Function DefineFunction( Function function, ExpressionContext body )
178191
{
179192
if( !function.IsDeclaration )
180193
{
181-
throw new ArgumentException( $"Function {function.Name} cannot be redefined", nameof( function ) );
194+
throw new CodeGeneratorException( $"Function {function.Name} cannot be redefined in the same module" );
182195
}
183196

184197
var basicBlock = function.AppendBasicBlock( "entry" );

Samples/Kaleidoscope/Chapter3/Utilities.cs

Lines changed: 0 additions & 40 deletions
This file was deleted.

0 commit comments

Comments
 (0)