Kaleidoscope tutorial chapter 4 updates (#55)

Author: smaillet

Samples/Kaleidoscope/Chapter3/CodeGenerator.cs

@@ -83,7 +83,7 @@ public override Value VisitVariableExpression( [NotNull] VariableExpressionConte
 
         public override Value VisitFunctionCallExpression( [NotNull] FunctionCallExpressionContext context )
         {
-            var function = GetFunction( context.CaleeName );
+            var function = FindCallTarget( context.CaleeName );
             if( function == null )
             {
                 throw new CodeGeneratorException( $"function '{context.CaleeName}' is unknown" );
@@ -185,7 +185,7 @@ private Value EmitBinaryOperator( Value lhs, BinaryopContext op, IParseTree righ
         }
         // </EmitBinaryOperator>
 
-        private Function GetFunction( string name )
+        private Function FindCallTarget( string name )
         {
             return Module.GetFunction( name );
         }

Samples/Kaleidoscope/Chapter3/Program.cs

@@ -47,6 +47,7 @@ public static void Main( string[ ] args )
                     var replLoop = new ReplLoop<Value>( generator, parser );
                     replLoop.ReadyStateChanged += ( s, e ) => Console.Write( e.PartialParse ? ">" : "Ready>" );
                     replLoop.GeneratedResultAvailable += OnGeneratedResultAvailable;
+                    replLoop.CodeGenerationError += OnGeneratorError;
 
                     replLoop.Run( );
                     Console.WriteLine( generator.GeneratedModule.WriteToString( ) );
@@ -55,6 +56,22 @@ public static void Main( string[ ] args )
             }
         }
 
+        // <ErrorHandling>
+        private static void OnGeneratorError( object sender, CodeGenerationExceptionArgs e )
+        {
+            var color = Console.ForegroundColor;
+            Console.ForegroundColor = ConsoleColor.Red;
+            try
+            {
+                Console.Error.WriteLine( e.Exception.Message );
+            }
+            finally
+            {
+                Console.ForegroundColor = color;
+            }
+        }
+        // </ErrorHandling>
+
         // <ResultProcessing>
         private static void OnGeneratedResultAvailable( object sender, GeneratedResultAvailableArgs<Value> e )
         {

Samples/Kaleidoscope/Chapter4/CodeGenerator.cs

@@ -34,9 +34,9 @@ public CodeGenerator( DynamicRuntimeState globalState )
         {
             RuntimeState = globalState;
             Context = new Context( );
+            JIT = new KaleidoscopeJIT( );
             InitializeModuleAndPassManager( );
             InstructionBuilder = new InstructionBuilder( Context );
-            JIT = new KaleidoscopeJIT( );
             FunctionPrototypes = new PrototypeCollection( );
             FunctionModuleMap = new Dictionary<string, IJitModuleHandle>( );
             NamedValues = new Dictionary<string, Value>( );
@@ -84,7 +84,7 @@ public override Value VisitVariableExpression( [NotNull] VariableExpressionConte
 
         public override Value VisitFunctionCallExpression( [NotNull] FunctionCallExpressionContext context )
         {
-            var function = GetFunction( context.CaleeName );
+            var function = FindCallTarget( context.CaleeName );
             if( function == null )
             {
                 throw new CodeGeneratorException( $"function '{context.CaleeName}' is unknown" );
@@ -104,13 +104,16 @@ public override Value VisitFunctionPrototype( [NotNull] FunctionPrototypeContext
             return GetOrDeclareFunction( new Prototype( context ) );
         }
 
+        // <VisitFunctionDefinition>
         public override Value VisitFunctionDefinition( [NotNull] FunctionDefinitionContext context )
         {
             return DefineFunction( ( Function )context.Signature.Accept( this )
                                  , context.BodyExpression
                                  ).Function;
         }
+        // </VisitFunctionDefinition>
 
+        // <VisitTopLevelExpression>
         public override Value VisitTopLevelExpression( [NotNull] TopLevelExpressionContext context )
         {
             var proto = new Prototype( $"anon_expr_{AnonNameIndex++}" );
@@ -120,9 +123,11 @@ public override Value VisitTopLevelExpression( [NotNull] TopLevelExpressionConte
 
             var nativeFunc = JIT.GetDelegateForFunction<AnonExpressionFunc>( proto.Identifier.Name );
             var retVal = Context.CreateConstant( nativeFunc( ) );
+            FunctionModuleMap.Remove( function.Name );
             JIT.RemoveModule( jitHandle );
             return retVal;
         }
+        // </VisitTopLevelExpression>
 
         public override Value VisitExpression( [NotNull] ExpressionContext context )
         {
@@ -181,19 +186,25 @@ private Value EmitBinaryOperator( Value lhs, BinaryopContext op, IParseTree righ
             }
         }
 
+        // <InitializeModuleAndPassManager>
         private void InitializeModuleAndPassManager( )
         {
             Module = Context.CreateBitcodeModule( );
+            Module.Layout = JIT.TargetMachine.TargetData;
             FunctionPassManager = new FunctionPassManager( Module );
             FunctionPassManager.AddInstructionCombiningPass( )
                                .AddReassociatePass( )
                                .AddGVNPass( )
                                .AddCFGSimplificationPass( )
                                .Initialize( );
         }
+        // </InitializeModuleAndPassManager>
 
-        private Function GetFunction( string name )
+        // <FindCallTarget>
+        private Function FindCallTarget( string name )
         {
+            // lookup the prototype for the function to get the signature
+            // and create a declaration in this module
             if( FunctionPrototypes.TryGetValue( name, out var signature ) )
             {
                 return GetOrDeclareFunction( signature );
@@ -207,7 +218,9 @@ private Function GetFunction( string name )
 
             return null;
         }
+        // </FindCallTarget>
 
+        // <GetOrDeclareFunction>
         private Function GetOrDeclareFunction( Prototype prototype, bool isAnonymous = false )
         {
             var function = Module.GetFunction( prototype.Identifier.Name );
@@ -235,7 +248,9 @@ private Function GetOrDeclareFunction( Prototype prototype, bool isAnonymous = f
 
             return retVal;
         }
+        // </GetOrDeclareFunction>
 
+        // <DefineFunction>
         private (Function Function, IJitModuleHandle JitHandle) DefineFunction( Function function, ExpressionContext body )
         {
             if( !function.IsDeclaration )
@@ -283,6 +298,7 @@ private Function GetOrDeclareFunction( Prototype prototype, bool isAnonymous = f
             InitializeModuleAndPassManager( );
             return (function, jitHandle);
         }
+        // </DefineFunction>
 
         // <PrivateMembers>
         private readonly DynamicRuntimeState RuntimeState;

Samples/Kaleidoscope/Chapter4/Program.cs

@@ -14,6 +14,8 @@
 
 [assembly: SuppressMessage( "StyleCop.CSharp.DocumentationRules", "SA1652:Enable XML documentation output", Justification = "Sample application" )]
 
+#pragma warning disable SA1512, SA1513, SA1515 // single line comments used to tag regions for extraction into docs
+
 namespace Kaleidoscope
 {
     public static class Program
@@ -35,6 +37,8 @@ public static void Main( string[ ] args )
             using( InitializeLLVM( ) )
             {
                 RegisterNative( );
+
+                // <generatorloop>
                 var parser = new ReplParserStack( LanguageLevel.SimpleExpressions );
                 using( var generator = new CodeGenerator( parser.GlobalState ) )
                 {
@@ -43,12 +47,31 @@ public static void Main( string[ ] args )
                     var replLoop = new ReplLoop<Value>( generator, parser );
                     replLoop.ReadyStateChanged += ( s, e ) => Console.Write( e.PartialParse ? ">" : "Ready>" );
                     replLoop.GeneratedResultAvailable += OnGeneratedResultAvailable;
+                    replLoop.CodeGenerationError += OnGeneratorError;
 
                     replLoop.Run( );
                 }
+                // </generatorloop>
+            }
+        }
+
+        // <ErrorHandling>
+        private static void OnGeneratorError( object sender, CodeGenerationExceptionArgs e )
+        {
+            var color = Console.ForegroundColor;
+            Console.ForegroundColor = ConsoleColor.Red;
+            try
+            {
+                Console.Error.WriteLine( e.Exception.Message );
+            }
+            finally
+            {
+                Console.ForegroundColor = color;
             }
         }
+        // </ErrorHandling>
 
+        // <ResultProcessing>
         private static void OnGeneratedResultAvailable( object sender, GeneratedResultAvailableArgs<Value> e )
         {
             var source = ( ReplLoop<Value> )sender;
@@ -69,5 +92,6 @@ private static void OnGeneratedResultAvailable( object sender, GeneratedResultAv
                 break;
             }
         }
+        // </ResultProcessing>
     }
 }

Samples/Kaleidoscope/Kaleidoscope.Runtime/CodeGenerationExceptionArgs.cs

@@ -0,0 +1,19 @@
+// <copyright file="ReplLoop.cs" company=".NET Foundation">
+// Copyright (c) .NET Foundation. All rights reserved.
+// </copyright>
+
+using System;
+
+namespace Kaleidoscope.Runtime
+{
+    public class CodeGenerationExceptionArgs
+        : EventArgs
+    {
+        public CodeGenerationExceptionArgs( CodeGeneratorException exception )
+        {
+            Exception = exception;
+        }
+
+        public CodeGeneratorException Exception { get; }
+    }
+}

Samples/Kaleidoscope/Kaleidoscope.Runtime/ReplLoop.cs

@@ -42,6 +42,8 @@ public ReplLoop( IKaleidoscopeCodeGenerator<TResult> generator
 
         public event EventHandler<ReadyStateChangedArgs> ReadyStateChanged = (s,e)=> { };
 
+        public event EventHandler<CodeGenerationExceptionArgs> CodeGenerationError = (s,e) => { };
+
         public bool Ready { get; private set; } = true;
 
         public DiagnosticRepresentations AdditionalDiagnostics { get; }
@@ -63,8 +65,15 @@ public void Run( )
                 if( !IsPartial )
                 {
                     var (parseTree, recognizer) = Parser.Parse( Txt, AdditionalDiagnostics );
-                    TResult value = Generator.Generate( recognizer, parseTree, AdditionalDiagnostics );
-                    GeneratedResultAvailable( this, new GeneratedResultAvailableArgs<TResult>( value, recognizer, parseTree ) );
+                    try
+                    {
+                        TResult value = Generator.Generate( recognizer, parseTree, AdditionalDiagnostics );
+                        GeneratedResultAvailable( this, new GeneratedResultAvailableArgs<TResult>( value, recognizer, parseTree ) );
+                    }
+                    catch(CodeGeneratorException ex)
+                    {
+                        CodeGenerationError( this, new CodeGenerationExceptionArgs( ex ) );
+                    }
                 }
 
                 /*

docfx/articles/Samples/Kaleidoscope-ch2.md

@@ -1,14 +1,13 @@
-# Implementing the parser
+# 2. Kaleidoscope: Implementing the parser
 The Lllvm.NET version of Kaleidoscope leverages ANTLR4 to parse the language into a parse tree.
 The chapter 2 sample doesn't actually generate any code. Instead it focuses on the general structure
 of the samples and parsing of the language. The sample for this chapter enables all language features
 to allow exploring the language and how it is parsed to help better understand the rest of the chapters
-better. It is hoped that users of this library find this helpful as understanding the language grammar
+better. It is hoped that users of this library find this helpful. Understanding the language grammar
 from reading the LVM tutorials and source was a difficult task since it isn't formally defined in one
 place. (There are some EBNF like comments in the code but it is spread around without much real discussion
 of the language the tutorials guide you to implement)
 
-
 ## Formal Grammar
 ### Lexer symbols
 
@@ -78,10 +77,10 @@ for the language. Subsequent chapters will introduce the meaning and use of each
 
 #### Language Feature Defined Keywords
 Chapters 5-7 each introduce new language features that introduce new keywords into the language. In order to
-maintain a single grammar for all chapters the lexer uses a technique of ANTLR4 called semantic predicates.
-These are basically boolean expressions that determine if a given rule should be applied. These are applied
-to the rules for the feature specific keywords. Thus, at runtime, if a given feature is disabled then the
-keyword is not recognized.
+maintain a single grammar for all chapters the lexer uses a technique of ANTLR4 called [Semantic Predicates](https://github.com/antlr/antlr4/blob/master/doc/predicates.md).
+These are basically boolean expressions that determine if a given rule should be applied while parsing the
+input language. These are applied to the rules for the feature specific keywords. Thus, at runtime, if a given
+feature is disabled then the keyword is not recognized.
 
 ```antlr
 IF:     {FeatureControlFlow}? 'if';
@@ -266,9 +265,78 @@ for the call `foo(1, 2, 3);` is:
 
 ![Parse Tree](parsetree-func-call.svg)
 
-###### Other expressions
-The other expressions are either simple tokens like `Identifier` and `Number` or more complex expressions that are
-covered in detail in later chapters.
+###### VarInExpression
+```antlr
+VAR initializer (COMMA initializer)* IN expression[0]
+```
+The VarInExpression rule provides variable declaration, with optional initialization. The scope of the
+variables is that of the expression on the right of the `in` keyword. The `var ... in ...` expression is
+in many ways like a declaration of an inline function. The variables declared are scoped to the internal
+implementation of the function. Once the function produces the return value the variables no longer exist.
+
+###### ConditionalExpression
+```antlr
+IF expression[0] THEN expression[0] ELSE expression[0]
+```
+Conditional expressions use the very common and familiar if-then-else syntax and semantics with one noteable
+unique quality. In Kaleidoscope every language construct is an expression, there are no statements. Expressions
+all produce a value. So the result of the conditional expression is the result of the sub expression selected
+based on the condition. The condition value is computed and if the result == 0.0 (false) the `else` expression
+is used to produce the final result. Otherwise, the `then` expression is executed to produce the result. Thus,
+the actual semantics are more like the ternary operator found C and other languages:
+```C
+condition ? thenExpression : elseExpression`
+```
+
+Example:
+```Kaleidoscope
+def fib(x)
+  if x < 3 then
+    1
+  else
+    fib(x-1)+fib(x-2);
+```
+##### ForInExpression
+The ForInExpression provides support for classic for loop constructs. In particular it provides a variable scope for a loop
+value, a condition to test when to exit the loop and an optional step value for incrementing the loop value (default is 1.0).
+
+```Kaleidoscope
+extern putchard(char);
+def printstar(n)
+  for i = 1, i < n, 1.0 in
+    putchard(42);  # ascii 42 = '*'
+
+# print 100 '*' characters
+printstar(100);
+```
+Note: That there are no statements in Kaleidoscope, everything is an expression and has a value. putchard() implicitly returns a
+value as does printstar(). (e.g. there is no void return ALL functions implictly return a floating point value, even if it is always 0.0)
+For loops with mutable values support in the language may provide a result that isn't always 0.0, for example:
+
+```Kaleidoscope
+# Define ':' for sequencing: as a low-precedence operator that ignores operands
+# and just returns the RHS.
+def binary : 1 (x y) y;
+
+# Recursive fib, we could do this before.
+def fib(x)
+  if (x < 3) then
+    1
+  else
+    fib(x-1)+fib(x-2);
+
+# Iterative fib.
+def fibi(x)
+  var a = 1, b = 1, c in
+  (for i = 3, i < x in
+     c = a + b :
+     a = b :
+     b = c) :
+  b;
+
+# Call it.
+fibi(10);
+```
 
 #### Parse Tree
 The Llvm.NET implementation of Kaleidoscope doesn't use an AST per se. Instead it use the parse tree generated