[Monodevelop-patches-list] r489 - trunk/MonoDevelop/src/Libraries/SharpRefactory/src/Lexer

[commit-watcher at mono-cvs.ximian.com]

Author: benm
Date: 2004-01-15 15:49:35 -0500 (Thu, 15 Jan 2004)
New Revision: 489

Modified:
   trunk/MonoDevelop/src/Libraries/SharpRefactory/src/Lexer/Lexer.cs
Log:
ok, make the tokenizer sane. this is still a hack, we should really do it right, eg without a list, but at least this makes things *reasonable*

Modified: trunk/MonoDevelop/src/Libraries/SharpRefactory/src/Lexer/Lexer.cs
===================================================================
--- trunk/MonoDevelop/src/Libraries/SharpRefactory/src/Lexer/Lexer.cs	2004-01-15 06:40:33 UTC (rev 488)
+++ trunk/MonoDevelop/src/Libraries/SharpRefactory/src/Lexer/Lexer.cs	2004-01-15 20:49:35 UTC (rev 489)
@@ -13,20 +13,16 @@
 using System.Text;
 using ICSharpCode.SharpRefactory.Parser;
 
-namespace ICSharpCode.SharpRefactory.Parser
-{
-	public class Token
-	{
-		public int kind;
+namespace ICSharpCode.SharpRefactory.Parser {
+	public struct Token {
+		public readonly int kind;
 		
-		public int col;
-		public int line;
+		public readonly int col;
+		public readonly int line;
 		
-		public object literalValue = null;
-		public string val;
-		public Token  next;
-//		public ArrayList specials;
-		
+		public readonly object literalValue;
+		public readonly string val;
+
 		public Point EndLocation {
 			get {
 				return new Point(col + val.Length, line);
@@ -38,28 +34,22 @@
 			}
 		}
 		
-		public Token()
-		{
-		}
-		
 		public Token(int kind)
 		{
-			this.kind = kind;
+			this.kind = kind;
+			this.col  = 0;
+			this.line = 0;
+			this.val  = null;
+			this.literalValue = null;
 		}
 		
-//		public Token(Tokens kind, int col, int line)
-//		{
-//			this.kind = kind;
-//			this.col  = col;
-//			this.line = line;
-//		}
-		
 		public Token(int kind, int col, int line, string val)
 		{
 			this.kind = kind;
 			this.col  = col;
 			this.line = line;
-			this.val  = val;
+			this.val  = val;
+			this.literalValue = null;
 		}
 		
 		public Token(int kind, int col, int line, string val, object literalValue)
@@ -70,7 +60,9 @@
 			this.val          = val;
 			this.literalValue = literalValue;
 		}
-	}
+	}
+	
+
 	
 	public class Lexer
 	{
@@ -80,13 +72,20 @@
 		int col  = 1;
 		int line = 1;
 		
-		Errors errors   = new Errors();
+		Errors errors   = new Errors();
+		class TokenLink {
+			public Token t;
+			public TokenLink next;
+			
+			public TokenLink (Token t) { this.t = t; }
+		}
 		
 //		SpecialTracker specialTracker = new SpecialTracker();
 		
-		Token lastToken = null;
-		Token curToken  = null;
-		Token peekToken = null;
+		TokenLink lastToken = null;
+		TokenLink curToken  = null;
+		TokenLink peekToken = null;
+		TokenLink reuse;
 		
 		public Errors Errors {
 			get {
@@ -96,13 +95,13 @@
 		
 		public Token Token {
 			get {
-				return lastToken;
+				return lastToken.t;
 			}
 		}
 		
 		public Token LookAhead {
 			get {
-				return curToken;
+				return curToken.t;
 			}
 		}
 		
@@ -113,50 +112,48 @@
 		
 		public Token Peek()
 		{
-			if (peekToken.next == null) {
-				peekToken.next = Next();
-//				peekToken.next.specials = this.specialTracker.RetrieveSpecials();
-			}
+			if (peekToken.next == null) {
+				if (reuse != null) {
+					reuse.t = Next ();
+					peekToken.next = reuse;
+					reuse = null;
+				} else
+					peekToken.next = new TokenLink (Next());
+			}
+				
 			peekToken = peekToken.next;
-			return peekToken;
+			return peekToken.t;
 		}
 		
 		public Token NextToken()
 		{
 			if (curToken == null) {
-				curToken = Next();
-//				curToken.specials = this.specialTracker.RetrieveSpecials();
-				return curToken;
+				curToken = new TokenLink (Next());
+				return curToken.t;
 			}
+			
+			reuse = lastToken;
+			// make sure we dont keep a chain around
+			if (reuse != null)
+				reuse.next = null;
 			
-//			if (lastToken != null && lastToken.specials != null) {
-//				curToken.specials.InsertRange(0, lastToken.specials);
-//			}
-			
 			lastToken = curToken;
 			
-			if (curToken.next == null) {
-				curToken.next = Next();
-//				curToken.next.specials = this.specialTracker.RetrieveSpecials();
+			if (curToken.next == null) {
+				
+				if (reuse != null) {
+					reuse.t = Next ();
+					curToken.next = reuse;
+					reuse = null;
+				} else
+					curToken.next = new TokenLink (Next());
 			}
 			
 			curToken  = curToken.next;
-			return curToken;
+			return curToken.t;
 		}
+
 		
-//		public ArrayList RetrieveSpecials()
-//		{
-//			if (lastToken == null) {
-//				return this.specialTracker.RetrieveSpecials();
-//			}
-//			
-//			Debug.Assert(lastToken.specials != null);
-//			
-//			ArrayList tmp = lastToken.specials;
-//			lastToken.specials = null;
-//			return tmp;
-//		}
-		
 		static string[] keywordStrings = {
 			"abstract",
 			"as",
@@ -320,7 +317,7 @@
 				Token token = ReadOperator(ch);
 				
 				// try error recovery :)
-				if (token == null) {
+				if (token.kind == -1) {
 					return Next();
 				}
 				return token;
@@ -889,7 +886,7 @@
 					return new Token(Tokens.OpenCurlyBrace, x, y, "{");
 				default:
 					--col;
-					return null;
+					return new Token (-1);
 			}
 		}