sf.net # 994: Bad handling of macro argument evaluation

- reset file pointers to previous context after evaluating arguments
- in __fb_query_symbol__, __fb_eval__, __fb_arg_extract__, __fb_iif__,
- and after releasing lexer context
- add enum LEX_TKCTX_CONTEXT to specify the kind of lexer context
- LEX_TKCTX_CONTEXT_INIT    for initializing, first time only
- LEX_TKCTX_CONTEXT_INCLUDE for include file
- LEX_TKCTX_CONTEXT_EVAL    for preprocessor evaluation

Author: jayrm

changelog.txt

@@ -24,7 +24,7 @@ Version 1.10.1
 - rtlib: emscripten: CURDIR and EXEPATH always returned ""
 - emscripten/fbc: -sASYNCIFY wasn't passed to emcc at link-time as needed by recent Emscripten
 - sf.net #993: ignore (parse) single line comments for macros invoked with optional parentheses instead of passing as an argument to macro
-- sf.net #994: Bad handling of macro argument evaluation - reset file pointers after evaluating arguments in __fb_query_symbol__, __fb_eval__, __fb_arg_extract__, __fb_iif__
+- sf.net #994: Bad handling of macro argument evaluation - reset file pointers to previous context after evaluating arguments in __fb_query_symbol__, __fb_eval__, __fb_arg_extract__, __fb_iif__, and releasing lexer context
 
 
 Version 1.10.0

src/compiler/fb.bas

@@ -496,7 +496,7 @@ sub fbInit _
 	hashInit( @env.inconcehash, FB_INITINCFILES, FALSE )
 
 	stackNew( @parser.stmt.stk, FB_INITSTMTSTACKNODES, len( FB_CMPSTMTSTK ), FALSE )
-	lexInit( FALSE, FALSE )
+	lexInit( LEX_TKCTX_CONTEXT_INIT )
 	parserInit( )
 	rtlInit( )
 
@@ -1661,7 +1661,7 @@ sub fbIncludeFile(byval filename as zstring ptr, byval isonce as integer)
 	'' parse
 	lexPushCtx( )
 
-	lexInit( TRUE, FALSE )
+	lexInit( LEX_TKCTX_CONTEXT_INCLUDE )
 
 	cProgram()
 

src/compiler/lex.bas

@@ -60,10 +60,15 @@ sub lexPopCtx( )
 		DWstrAllocate( lex.ctx->deftextw, 0 )
 	end if
 
-	'' restore file pointer position if current context is different from previous
-	if( (lex.ctx-1)->physfilepos > 0 ) then
-		if( (lex.ctx-1)->physfilepos <> lex.ctx->physfilepos ) then
-			seek #env.inf.num, lex.ctx->physfilepos
+
+	'' if it's an include file, don't bother restoring file position because
+	'' we are going to be closing the file anyway
+	if( lex.ctx->kind <> LEX_TKCTX_CONTEXT_INCLUDE ) then
+		'' restore file pointer position if current context is different from previous
+		if( (lex.ctx-1)->physfilepos > 0 ) then
+			if( (lex.ctx-1)->physfilepos <> lex.ctx->physfilepos ) then
+				seek #env.inf.num, (lex.ctx-1)->physfilepos
+			end if
 		end if
 	end if
 
@@ -75,14 +80,16 @@ end sub
 '':::::
 sub lexInit _
 	( _
-		byval isinclude as integer, _
-		byval is_fb_eval as integer _
+		byval ctx_kind as LEX_TKCTX_CONTEXT _
 	)
 
 	dim as integer i
 	dim as FBTOKEN ptr n
 
-	if( (env.includerec = 0) and (is_fb_eval = FALSE) ) then
+	'' !!!TODO!!! - determine if env.includerec check can be removed
+
+	'' first time? make sure lex.ctx points to something
+	if( (env.includerec = 0) and (ctx_kind = LEX_TKCTX_CONTEXT_INIT) ) then
 		lex.ctx = @lex.ctxTB(0)
 	end if
 
@@ -108,12 +115,15 @@ sub lexInit _
 	lex.ctx->lahdchar1 = UINVALID
 	lex.ctx->lahdchar2 = UINVALID
 
-	lex.ctx->is_fb_eval = is_fb_eval
+	lex.ctx->kind = ctx_kind
 
-	if( is_fb_eval ) then
+	'' preprocessor evaluation?
+	if( ctx_kind = LEX_TKCTX_CONTEXT_EVAL ) then
 		lex.ctx->linenum = (lex.ctx-1)->linenum
 		lex.ctx->reclevel = (lex.ctx-1)->reclevel
 		lex.ctx->currmacro = (lex.ctx-1)->currmacro
+
+	'' else it is an include file or first time initialization
 	else
 		lex.ctx->linenum = 1
 		lex.ctx->reclevel = 0
@@ -127,7 +137,7 @@ sub lexInit _
 	lex.ctx->deflen = 0
 
 	if( env.inf.format = FBFILE_FORMAT_ASCII ) then
-		lex.ctx->buffptr = iif( is_fb_eval, @lex.ctx->buff, NULL )
+		lex.ctx->buffptr = iif( ctx_kind = LEX_TKCTX_CONTEXT_EVAL, @lex.ctx->buff, NULL )
 		lex.ctx->defptr = NULL
 		DZstrAllocate( lex.ctx->deftext, 0 )
 	else
@@ -136,26 +146,30 @@ sub lexInit _
 		DWstrAllocate( lex.ctx->deftextw, 0 )
 	end if
 
-	''
-	if( is_fb_eval ) then
+	'' preprocessor evaluation?
+	if( ctx_kind = LEX_TKCTX_CONTEXT_EVAL ) then
 		lex.ctx->filepos = (lex.ctx-1)->filepos
 		lex.ctx->lastfilepos = (lex.ctx-1)->lastfilepos
 		lex.ctx->physfilepos = (lex.ctx-1)->physfilepos
+
+	'' else it is an include file or first time initialization
 	else
 		lex.ctx->filepos = 0
 		lex.ctx->lastfilepos = 0
 		lex.ctx->physfilepos = 0
 	end if
 
 	'' only if it's not on an inc file
-	if( (env.includerec = 0) or (is_fb_eval = TRUE) ) then
+	'' !!!TODO!!! - determine if env.includerec check can be removed
+	'' if( ctx_kind <> LEX_TKCTX_CONTEXT_INCLUDE ) then
+	if( (env.includerec = 0) or (ctx_kind = LEX_TKCTX_CONTEXT_EVAL) ) then
 		DZstrAllocate( lex.ctx->currline, 0 )
 		lex.insidemacro = FALSE
 	end if
 
 	lex.ctx->after_space = FALSE
 
-	if( (isinclude = FALSE) and (is_fb_eval = FALSE ) ) then
+	if( ctx_kind = LEX_TKCTX_CONTEXT_INIT ) then
 		ppInit( )
 	end if
 
@@ -2445,7 +2459,7 @@ function lexPeekCurrentLine _
 	'' get file contents around current token
 	old_p = seek( env.inf.num )
 
-	if( lex.ctx->is_fb_eval ) then
+	if( lex.ctx->kind = LEX_TKCTX_CONTEXT_EVAL ) then
 		p = (lex.ctx-1)->lastfilepos - 512
 	else
 		p = lex.ctx->lastfilepos - 512

src/compiler/lex.bi

@@ -77,6 +77,12 @@ const FB_LEX_MAXK   = 3
 
 const LEX_MAXBUFFCHARS = 8192
 
+enum LEX_TKCTX_CONTEXT
+	LEX_TKCTX_CONTEXT_INIT = 0                  '' initializing, first time only
+	LEX_TKCTX_CONTEXT_INCLUDE = 1               '' include file
+	LEX_TKCTX_CONTEXT_EVAL = 2                  '' preprocessor evaluation
+end enum
+
 type LEX_TKCTX
 	tokenTB(0 to FB_LEX_MAXK) as FBTOKEN
 	k               as integer                  '' look ahead cnt (1..MAXK)
@@ -95,7 +101,7 @@ type LEX_TKCTX
 	currmacro       as FBSYMBOL ptr             '' used to check macro recursion
 
 	kwdns           as FBSYMBOL ptr             '' used by the PP
-	is_fb_eval      as integer                  '' TRUE if inside an FB_EVAL
+	kind            as LEX_TKCTX_CONTEXT        '' the kind of lexer context
 
 	'' last #define's text
 	deflen          as integer
@@ -159,8 +165,7 @@ end type
 
 declare sub lexInit _
 	( _
-		byval isinclude as integer, _
-		byval is_fb_eval as integer _
+		byval ctx_kind as LEX_TKCTX_CONTEXT _
 	)
 
 declare sub lexEnd _

src/compiler/symb-define.bas

@@ -388,8 +388,8 @@ private function hMacro_EvalZ( byval arg as zstring ptr, byval errnum as integer
 		'' - text to expand is to be loaded in LEX.CTX->DEFTEXT[W]
 		'' - use the parser to build an AST for the literal result
 
-		lexPushCtx()
-		lexInit( FALSE, TRUE )
+		lexPushCtx( )
+		lexInit( LEX_TKCTX_CONTEXT_EVAL )
 
 		'' prevent cExpression from writing to .pp.bas file
 		lex.ctx->reclevel += 1
@@ -428,7 +428,7 @@ private function hMacro_EvalZ( byval arg as zstring ptr, byval errnum as integer
 
 		lex.ctx->reclevel -= 1
 
-		lexPopCtx()
+		lexPopCtx( )
 
 		if( errmsg <> FB_ERRMSG_OK ) then
 			errReportEx( errmsg, *arg )
@@ -464,8 +464,8 @@ private function hMacro_EvalW( byval arg as wstring ptr, byval errnum as integer
 		'' - text to expand is to be loaded in LEX.CTX->DEFTEXT[W]
 		'' - use the parser to build an AST for the literal result
 
-		lexPushCtx()
-		lexInit( FALSE, TRUE )
+		lexPushCtx( )
+		lexInit( LEX_TKCTX_CONTEXT_EVAL )
 
 		'' prevent cExpression from writing to .pp.bas file
 		lex.ctx->reclevel += 1
@@ -504,7 +504,7 @@ private function hMacro_EvalW( byval arg as wstring ptr, byval errnum as integer
 
 		lex.ctx->reclevel -= 1
 
-		lexPopCtx()
+		lexPopCtx( )
 
 		if( errmsg <> FB_ERRMSG_OK ) then
 			errReportEx( errmsg, *arg )
@@ -1058,8 +1058,8 @@ private function hDefQuerySymZ_cb( byval argtb as LEXPP_ARGTB ptr, byval errnum
 			var errmsg = FB_ERRMSG_OK
 
 			'' create a lightweight context push
-			lexPushCtx()
-			lexInit( FALSE, TRUE )
+			lexPushCtx( )
+			lexInit( LEX_TKCTX_CONTEXT_EVAL )
 
 			'' prevent cExpression from writing to .pp.bas file
 			lex.ctx->reclevel += 1
@@ -1153,7 +1153,10 @@ private function hDefQuerySymZ_cb( byval argtb as LEXPP_ARGTB ptr, byval errnum
 					sym = NULL
 
 					'' reset the current lexer context and refresh the text to parse.
-					lexInit( FALSE, TRUE )
+					'' !!!TODO!!! - probably more efficient with some kind of 'lexReinit()' function
+					lexPopCtx( )
+					lexPushCtx( )
+					lexInit( LEX_TKCTX_CONTEXT_EVAL )
 
 					hArgInsertArgA( sexpr )
 					hArgAppendLFCHAR()
@@ -1219,7 +1222,7 @@ private function hDefQuerySymZ_cb( byval argtb as LEXPP_ARGTB ptr, byval errnum
 
 			lex.ctx->reclevel -= 1
 
-			lexPopCtx()
+			lexPopCtx( )
 
 		else
 			'' NUMARG isn't a number