Mirror
/
icsharpcode-ILSpy
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

clean up the mini C# lexer a bit

pull/904/head
Daniel Grunwald 8 years ago
parent
ebab7d5de0
commit
e8b3ce99dc
1 changed files with 63 additions and 48 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 111
      ILSpy/Languages/CSharpLexer.cs

111
ILSpy/Languages/CSharpLexer.cs
View File

@ -101,35 +101,11 @@ namespace ICSharpCode.ILSpy
protected Literal curToken = null;
protected Literal peekToken = null;
string[] specialCommentTags = null;
protected StringBuilder sb = new StringBuilder();
// used for the original value of strings (with escape sequences).
protected StringBuilder originalValue = new StringBuilder();
public bool SkipAllComments { get; set; }
public bool EvaluateConditionalCompilation { get; set; }
public virtual IDictionary<string, object> ConditionalCompilationSymbols {
get { throw new NotSupportedException(); }
}
protected static IEnumerable<string> GetSymbols(string symbols)
{
if (!string.IsNullOrEmpty(symbols)) {
foreach (string symbol in symbols.Split(';', ' ', '\t')) {
string s = symbol.Trim();
if (s.Length == 0)
continue;
yield return s;
}
}
}
public virtual void SetConditionalCompilationSymbols(string symbols)
{
throw new NotSupportedException();
}
protected int Line {
get {
return line;
@ -367,8 +343,6 @@ namespace ICSharpCode.ILSpy
internal sealed class Lexer : AbstractLexer
{
bool isAtLineBegin = true;
public Lexer(TextReader reader) : base(reader)
{
}
@ -376,10 +350,6 @@ namespace ICSharpCode.ILSpy
protected override Literal Next()
{
char ch;
if (Line == 1 && Col == 1) {
isAtLineBegin = true;
}
while (true) {
int nextChar = ReaderRead();
if (nextChar == -1)
@ -394,18 +364,36 @@ namespace ICSharpCode.ILSpy
case '\r':
case '\n':
HandleLineEnd((char)nextChar);
isAtLineBegin = true;
continue;
case '"':
token = ReadString();
isAtLineBegin = false;
break;
case '\'':
token = ReadChar();
isAtLineBegin = false;
break;
default:
isAtLineBegin = false; // non-ws chars are handled here
case '@':
int next = ReaderRead();
if (next == -1) {
Error(Line, Col, String.Format("EOF after @"));
continue;
} else {
int x = Col - 1;
int y = Line;
ch = (char)next;
if (ch == '"') {
token = ReadVerbatimString();
} else if (Char.IsLetterOrDigit(ch) || ch == '_') {
bool canBeKeyword;
string s = ReadIdent(ch, out canBeKeyword);
return new Literal(null, null, LiteralFormat.None);
} else {
HandleLineEnd(ch);
Error(y, x, String.Format("Unexpected char in Lexer.Next() : {0}", ch));
continue;
}
}
break;
default: // non-ws chars are handled here
ch = (char)nextChar;
if (Char.IsLetter(ch) || ch == '_' || ch == '\\') {
int x = Col - 1; // Col was incremented above, but we want the start of the identifier
@ -441,16 +429,26 @@ namespace ICSharpCode.ILSpy
while (true) {
if (ch == '\\') {
peek = ReaderPeek();
if (peek != 'u' && peek != 'U') {
Error(Line, Col, "Identifiers can only contain unicode escape sequences");
}
canBeKeyword = false;
string surrogatePair;
ReadEscapeSequence(out ch, out surrogatePair);
if (surrogatePair != null) {
if (!char.IsLetterOrDigit(surrogatePair, 0)) {
Error(Line, Col, "Unicode escape sequences in identifiers cannot be used to represent characters that are invalid in identifiers");
}
for (int i = 0; i < surrogatePair.Length - 1; i++) {
if (curPos < MAX_IDENTIFIER_LENGTH) {
identBuffer[curPos++] = surrogatePair[i];
}
}
ch = surrogatePair[surrogatePair.Length - 1];
} else {
if (!IsIdentifierPart(ch)) {
Error(Line, Col, "Unicode escape sequences in identifiers cannot be used to represent characters that are invalid in identifiers");
}
}
}
@ -459,6 +457,7 @@ namespace ICSharpCode.ILSpy
// prevents \ from being added
identBuffer[curPos++] = ch;
} else {
Error(Line, Col, String.Format("Identifier too long"));
while (IsIdentifierPart(ReaderPeek())) {
ReaderRead();
}
@ -507,6 +506,7 @@ namespace ICSharpCode.ILSpy
}
if (sb.Length == 0) {
sb.Append('0'); // dummy value to prevent exception
Error(y, x, "Invalid hexadecimal integer literal");
}
ishex = true;
prefix = "0x";
@ -527,6 +527,9 @@ namespace ICSharpCode.ILSpy
peek = '.';
} else {
isdouble = true; // double is default
if (ishex) {
Error(y, x, "No hexadecimal floating point values allowed");
}
sb.Append('.');
while (Char.IsDigit((char)ReaderPeek())) { // read decimal digits beyond the dot
@ -592,6 +595,7 @@ namespace ICSharpCode.ILSpy
if (float.TryParse(digit, NumberStyles.Any, CultureInfo.InvariantCulture, out num)) {
return new Literal(stringValue, num, LiteralFormat.DecimalNumber);
} else {
Error(y, x, String.Format("Can't parse float {0}", digit));
return new Literal(stringValue, 0f, LiteralFormat.DecimalNumber);
}
}
@ -600,6 +604,7 @@ namespace ICSharpCode.ILSpy
if (decimal.TryParse(digit, NumberStyles.Any, CultureInfo.InvariantCulture, out num)) {
return new Literal(stringValue, num, LiteralFormat.DecimalNumber);
} else {
Error(y, x, String.Format("Can't parse decimal {0}", digit));
return new Literal(stringValue, 0m, LiteralFormat.DecimalNumber);
}
}
@ -608,7 +613,7 @@ namespace ICSharpCode.ILSpy
if (double.TryParse(digit, NumberStyles.Any, CultureInfo.InvariantCulture, out num)) {
return new Literal(stringValue, num, LiteralFormat.DecimalNumber);
} else {
Error(y, x, String.Format("Can't parse double {0}", digit));
return new Literal(stringValue, 0d, LiteralFormat.DecimalNumber);
}
}
@ -617,12 +622,12 @@ namespace ICSharpCode.ILSpy
ulong result;
if (ishex) {
if (!ulong.TryParse(digit, NumberStyles.HexNumber, null, out result)) {
Error(y, x, String.Format("Can't parse hexadecimal constant {0}", digit));
return new Literal(stringValue.ToString(), 0, LiteralFormat.HexadecimalNumber);
}
} else {
if (!ulong.TryParse(digit, NumberStyles.Integer, null, out result)) {
Error(y, x, String.Format("Can't parse integral constant {0}", digit));
return new Literal(stringValue.ToString(), 0, LiteralFormat.DecimalNumber);
}
}
@ -645,7 +650,7 @@ namespace ICSharpCode.ILSpy
if (ulong.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number, CultureInfo.InvariantCulture, out num)) {
token = new Literal(stringValue, num, literalFormat);
} else {
Error(y, x, String.Format("Can't parse unsigned long {0}", digit));
token = new Literal(stringValue, 0UL, literalFormat);
}
} else {
@ -653,7 +658,7 @@ namespace ICSharpCode.ILSpy
if (long.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number, CultureInfo.InvariantCulture, out num)) {
token = new Literal(stringValue, num, literalFormat);
} else {
Error(y, x, String.Format("Can't parse long {0}", digit));
token = new Literal(stringValue, 0L, literalFormat);
}
}
@ -663,7 +668,7 @@ namespace ICSharpCode.ILSpy
if (uint.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number, CultureInfo.InvariantCulture, out num)) {
token = new Literal(stringValue, num, literalFormat);
} else {
Error(y, x, String.Format("Can't parse unsigned int {0}", digit));
token = new Literal(stringValue, (uint)0, literalFormat);
}
} else {
@ -671,7 +676,7 @@ namespace ICSharpCode.ILSpy
if (int.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number, CultureInfo.InvariantCulture, out num)) {
token = new Literal(stringValue, num, literalFormat);
} else {
Error(y, x, String.Format("Can't parse int {0}", digit));
token = new Literal(stringValue, 0, literalFormat);
}
}
@ -711,7 +716,7 @@ namespace ICSharpCode.ILSpy
}
} else if (HandleLineEnd(ch)) {
// call HandleLineEnd to ensure line numbers are still correct after the error
Error(y, x, "No new line is allowed inside a string literal");
break;
} else {
originalValue.Append(ch);
@ -720,7 +725,7 @@ namespace ICSharpCode.ILSpy
}
if (!doneNormally) {
Error(y, x, "End of file reached inside string literal");
}
return new Literal(originalValue.ToString(), sb.ToString(), LiteralFormat.StringLiteral);
@ -753,7 +758,7 @@ namespace ICSharpCode.ILSpy
}
if (nextChar == -1) {
Error(Line, Col, "End of file reached inside verbatim string literal");
}
return new Literal(originalValue.ToString(), sb.ToString(), LiteralFormat.VerbatimStringLiteral);
@ -777,6 +782,7 @@ namespace ICSharpCode.ILSpy
int nextChar = ReaderRead();
if (nextChar == -1) {
Error(Line, Col, "End of file reached inside escape sequence");
ch = '\0';
return String.Empty;
}
@ -825,6 +831,9 @@ namespace ICSharpCode.ILSpy
number = GetHexNumber(c);
escapeSequenceBuffer[curPos++] = c;
if (number < 0) {
Error(Line, Col - 1, String.Format("Invalid char in literal : {0}", c));
}
for (int i = 0; i < 3; ++i) {
if (IsHex((char)ReaderPeek())) {
c = (char)ReaderRead();
@ -847,6 +856,7 @@ namespace ICSharpCode.ILSpy
escapeSequenceBuffer[curPos++] = c;
number = 16 * number + idx;
} else {
Error(Line, Col - 1, String.Format("Invalid char in literal : {0}", (char)ReaderPeek()));
break;
}
}
@ -858,6 +868,7 @@ namespace ICSharpCode.ILSpy
}
break;
default:
Error(Line, Col, String.Format("Unexpected escape sequence : {0}", c));
ch = '\0';
break;
}
@ -879,16 +890,20 @@ namespace ICSharpCode.ILSpy
string surrogatePair;
escapeSequence = ReadEscapeSequence(out chValue, out surrogatePair);
if (surrogatePair != null) {
Error(y, x, "The unicode character must be represented by a surrogate pair and does not fit into a System.Char");
}
}
unchecked {
if ((char)ReaderRead() != '\'') {
Error(y, x, "Char not terminated");
}
}
return new Literal("'" + ch + escapeSequence + "'", chValue, LiteralFormat.CharLiteral);
}
void Error(int y, int x, string message)
{
}
}
}
Write Preview
Loading…
Cancel
Save