一个简单词法分析器的实现代码

一个简单的java词法分析器的实现（c语言版）

#include <stdio.h>
#include <string.h>
#define BUFSIZE 256

static char keyWords[][13]={"abstract", "boolean", "break", "byte", "case", "catch", "char", 
       "class", "const", "continue", "default", "do", "double", "else", "extends", "false", 
       "final", "finally", "float", "for", "goto", "if", "implements", "import", "instanceof", 
       "int", "interface", "long", "native", "new", "null", "package", "private", "protected", 
       "public", "return", "short", "static", "super", "switch", "synchronized", "this", 
       "throw", "throws", "transient", "true", "try", "void", "volatile", "while"};      /*关键字 50个*/ 
FILE *fin;                                     /*要处理的java文件*/ 
FILE *fout;                                    /*保存处理结果的文件*/   
char filePath[100];				               /*要处理的java文件路径*/ 
char savePath[100] = "javascan_result.txt";			   /*保存处理结果文件的路径*/ 
char inBuf[2][BUFSIZE];                        /*输入缓冲区*/
char scanBuf[BUFSIZE];                         /*扫描缓冲区*/ 
int bufNum = 0;								   /*输入缓冲区编号*/
long totalWords = 0;						   /*总单词数量*/
long lineWords = 0;							   /*单行单词个数*/
long lineCount = 0;							   /*记录行数*/

void dealWord();                               /*词法分析*/ 
void clearNote();							   /*过滤注释及多余空格*/
void getWords(int state);					   /*解析获取单词*/
void getWordKind(char str[]);				   /*获取单词类型*/
int getCharKind(char c);					   /*判断字符类型*/
int isOper(char c);						       /*判断是否为可消除空格的字符*/
int isWord(char c);							   /*判断是否为字母*/
int isKeyWord(char str[]);					   /*判断是否为关键字*/
int isSignWord(char str[]);					   /*判断是否为标识符*/
int isNumber(char c);						   /*判断是否为数字*/
int isInteger(char str[]);					   /*判断是否为整型*/
int isFloat(char str[]);					   /*判断是否为浮点型*/

int main()
{
	/*printf("Please set the path of the Java File that you want to analyse:");*/
	printf("请设置要分析的Java程序文件的路径：");
	scanf("%s", filePath);
	if((fin = fopen(filePath, "r")) == NULL)
	{
		printf("Can't open this file!/n");
		return 0;
	}
	if((fout = fopen(savePath, "w")) == NULL)
	{
		printf("Can't write to this file!/n");
		return 0;
	}
	fprintf(fout, "Java词法分析器 Powered By Hamvorinf /n------------------------------------------------------/n");
	dealWord();
	fprintf(fout, "------------------------------------------------------/n分析结束！/n");
	fclose(fin);
	fclose(fout);
	system("pause");
	return 0;
}

/*******************词法分析*******************/ 
void dealWord()
{
	char c;
	int i, j, k;
	int count = 0;
	int chgLine = 0;			/*换行标志*/

	/*循环读取字符*/
	while(1)
	{
		c = fgetc(fin);
		if(c == EOF)
		{
			break;
		}
		/*如果是换行符*/
		if(c == '/n' || count == BUFSIZE - 2)
		{
			/*如果长度超出，则放入对半互补区*/
			if(count == BUFSIZE - 2)
			{
				inBuf[bufNum][count] = c;
				for(i = count; i > 0; i--)
				{
					if(isOper(inBuf[bufNum][i]) == 1)
					{
						for(j = 0, k = i+1; k <= count; k++, j++)
						{
							inBuf[1-bufNum][j] = inBuf[bufNum][k];
						}
						inBuf[1-bufNum][j] = '/0';
						inBuf[bufNum][i+1] = '/0';
					}
				}
			}
			else
			{
				inBuf[bufNum][count] = '/0';
			}
			
			/*若一行读取完毕，将换行标志置为1*/
			if(c == '/n')
			{
				chgLine = 1;
				lineCount++;
			}

			clearNote(inBuf[bufNum]);	/*过滤注释和多余空格、制表符*/

			/*分析处理*/
			if(strlen(inBuf[bufNum]) > 0)
			{
				/*将数据从输入缓冲区拷贝如扫描缓冲区*/
				strcpy(scanBuf, inBuf[bufNum]);

				/*对扫描缓冲区进行词语分析处理*/
				getWords(0);
			}

			/*如果长度超出，启用对半互补区*/
			if(count == BUFSIZE - 2)
			{
				bufNum = 1 - bufNum;
			}

			/*一行处理完毕输出当行单词数*/
			if(chgLine == 1)
			{
				printf("第%-4ld行共有 %-4ld个单词./n", lineCount, lineWords);
				chgLine = 0;
				lineWords = 0;
			}

			count = 0;
		}
		else
		{
            inBuf[bufNum][count++] = c;
        }
	}
	printf("共处理了 %-4ld行的 %-4ld个单词/n", lineCount, totalWords);
	printf("处理结果已经保存到 %s 下！/n", savePath);
}

/*******************过滤注释及多余空格*******************/ 
void clearNote()
{
	int i, j, k;
	int noteCount = 0;
	int flag = 0;			/*是否是字符串*/
	char note[100];

	/*去除注释*/
	for(i = 0; inBuf[bufNum][i] != '/0'; i++)
	{
		if(inBuf[bufNum][i] == '"')
		{
			flag = 1 - flag;
			continue;
		}
		if(inBuf[bufNum][i] == '/' && flag == 0)
		{
			/*如果是“//”注释*/
			if(inBuf[bufNum][i+1] == '/')
			{
				for(j = i; inBuf[bufNum][j] != '/0'; j++)
				{
					note[noteCount++] = inBuf[bufNum][j];
				}
				note[noteCount] = '/0';
				noteCount = 0;
				lineWords++;
				fprintf(fout, "%4ld.%-4ld  【%s】     【注释】---【0x101】/n",lineCount, lineWords, note);
				inBuf[bufNum][i] = '/0';
				break;
			}
			
			if(inBuf[bufNum][i+1] == '*')
			{
				note[noteCount++] = '/';
				note[noteCount++] = '*';
				for(j = i+2; inBuf[bufNum][j] != '/0'; j++)
				{
					note[noteCount++] = inBuf[bufNum][j];
					if(inBuf[bufNum][j] == '*' && inBuf[bufNum][j+1] == '/')
					{
						j += 2;
						note[noteCount++] = inBuf[bufNum][j];
						note[noteCount] = '/0';
						noteCount = 0;
						lineWords++;
						fprintf(fout, "%4ld.%-4ld  【%s】     【注释】---【0x101】/n",lineCount, lineWords, note);
						break;
					}
				}
				for(; inBuf[bufNum][j] != '/0'; j++, i++)
				{
					inBuf[bufNum][i] = inBuf[bufNum][j];
				}
				if(inBuf[bufNum][j] == '/0')
				{
					inBuf[bufNum][i] = '/0';
				}
			}
		}
	}

	/*去除多余空格*/
	for(i = 0, flag = 0; inBuf[bufNum][i] != '/0'; i++)
	{
		if(inBuf[bufNum][i] == '"')
		{
			flag = 1 - flag;
			continue;
		}
		if(inBuf[bufNum][i] == ' ' && flag == 0)
		{
			for(j = i+1; inBuf[bufNum][j] != '/0' && inBuf[bufNum][j] == ' '; j++)
			{
			}
			if(inBuf[bufNum][j] == '/0')
			{
				inBuf[bufNum][i] = '/0';
				break;
			}
			if(inBuf[bufNum][j] != '/0' && ((isOper(inBuf[bufNum][j]) == 1) || (i > 0 && isOper(inBuf[bufNum][i-1]) == 1)))
			{
				for(k = i; inBuf[bufNum][j] != '/0'; j++, k++)
				{
					inBuf[bufNum][k] = inBuf[bufNum][j];
				}
				inBuf[bufNum][k] = '/0';
				i--;
			}
		}
	}

	/*去除多余制表符*/
	for(i = 0, flag = 0; inBuf[bufNum][i] != '/0'; i++)
	{
		if(inBuf[bufNum][i] == '/t')
		{
            for( j = i; inBuf[bufNum][j] != '/0'; j++)
			{
				inBuf[bufNum][j] = inBuf[bufNum][j+1];
			}
			i = -1;
		}
	}
}



/*******************判断是否为可消除空格的字符/操作符*******************/
int isOper(char c)
{
	if((c > 'z' || (c < 'a' && c > 'Z') || (c < 'A' && c > '9') || (c < '0')) && c != '_' && c != '$')
	{
		return 1;
	}
	return 0;
}


/*******************判断是否为字母*******************/
int isWord(char c)
{
	if((c <= 'z' && c >= 'a') || (c <= 'Z' && c >= 'A'))
	{
		return 1;
	}
	return 0;
}

/*******************判断是否为数字*******************/
int isNumber(char c)
{
	if(c <= '9' && c >= '0')
	{
		return 1;
	}
	return 0;
}

/*******************判断是否为整型*******************/
int isInteger(char str[])
{
	int i;
	if(str[0] == '-' || isNumber(str[0]) == 1)
	{
		for(i = 0; i < strlen(str); i++)
		{
			if(str[i] == '.')
			{
				return 0;
			}
			if((str[i] == 'x' || str[i] == 'X') && (((str[0] == '-' || str[0] == '+') && (str[1] != '0' || i > 2)) || (str[0] != '-' && str[0] != '+' && (str[0] != '0' || i > 1))))
			{
				return 0;
			}
			if((i < strlen(str) -1) && isNumber(str[i]) == 0 && str[i] != 'x' && str[i] != 'X')
			{
				if(strlen(str) > 2 && strnicmp("0x", str, 2) == 0 || strnicmp("-0x", str, 3) == 0)
				{
					if(str[i] >= 'A' && str[i] <= 'F')
					{
						continue;
					}
				}
				return 0;
			}
			if((i == strlen(str) - 1) && isNumber(str[i]) ==0 && str[i] != 'L')
			{
				if(strlen(str) > 2 && strnicmp("0x", str, 2) == 0 || strnicmp("-0x", str, 3) == 0)
				{
					if(str[i] >= 'A' && str[i] <= 'F')
					{
						continue;
					}
				}
				return 0;
			}
		}
		return 1;	
	}
	return 0;
}
/*******************判断是否为浮点型*******************/
int isFloat(char str[])
{
	int i;
	int flag = 0;
	if(str[0] == '-' || isNumber(str[0]) == 1)
	{
		for(i = 0; i < strlen(str); i++)
		{
			if(str[i] == '.')
			{
				if(flag == 0)
				{
					flag = 1;
					continue;
				}
				else
				{
					return 0;
				}
			}
			if((str[i] == 'x' || str[i] == 'X') && (((str[0] == '-' || str[0] == '+') && (str[1] != '0' || i > 2)) || (str[0] != '-' && str[0] != '+' && (str[0] != '0' || i > 1))))
			{
				return 0;
			}
			if(isNumber(str[i]) == 0 && str[i] != 'x' && str[i] != 'X')
			{
				if(strlen(str) > 2 && strnicmp("0x", str, 2) == 0 || strnicmp("-0x", str, 3) == 0)
				{
					if(str[i] >= 'A' && str[i] <= 'F')
					{
						continue;
					}
				}
				return 0;
			}
		}
		return flag;	
	}
	return 0;
}
/*******************判断字符类型*******************/
int getCharKind(char c)
{
	/*是字母*/
	if(isWord(c) == 1)
	{
		return 1;
	}
	/*是数字*/
	if(isNumber(c) == 1)
	{
		return 2;
	}
	/*是$或_*/
	if(c == '$' || c == '_')
	{
		return 3;
	}
	/*是转义字符*/
	if(c == '//')
	{
		return 4;
	}
	/*是等号*/
	/*if(c == '=')
	{
		return 5;
	}*/
	return 0;
}
/*******************判断是否为关键字*******************/
int isKeyWord(char str[])
{
	int i;
	for(i = 0; i < 50; i++)
	{
		if(strcmp(str, keyWords[i]) == 0)
		{
			return 1;
		}
	}
	return 0;
}
/*******************判断是否为标识符*******************/
int isSignWord(char str[])
{
	int i;
	if(str[0] == '$' || str[0] == '_' || isWord(str[0]) == 1)
	{
		for(i = 0; str[i] != '/0'; i++)
		{
			if(isOper(str[i]) == 1)
			{
				return 0;
			}
		}
		return 1;
	}
	return 0;
}
/*******************获取单词类型*******************/
void getWordKind(char str[])
{
	int i, j, k;
	int flag = 0;

	/*判断是否为关键字或标识符*/
	if(isKeyWord(str) == 1)
	{
		if(strcmp(str,"true") == 0 || strcmp(str,"false") == 0)
		{
			fprintf(fout, "%4ld.%-4ld  【%s】     【布尔型】---【0x105】/n",lineCount, lineWords, str);
		}
		else
		{
			fprintf(fout, "%4ld.%-4ld  【%s】     【关键字】---【0x103】/n",lineCount, lineWords, str);
		}
	}
	else if(isSignWord(str) == 1)
	{
		fprintf(fout, "%4ld.%-4ld  【%s】     【标识符】---【0x104】/n",lineCount, lineWords, str);
	}
	else if(isInteger(str) == 1)
	{
		fprintf(fout, "%4ld.%-4ld  【%s】     【整型】---【0x107】/n",lineCount, lineWords, str);
	}
	else if(isFloat(str) == 1)
	{
		fprintf(fout, "%4ld.%-4ld  【%s】     【浮点型】---【0x108】/n",lineCount, lineWords, str);
	}
	else if(str[0] == '/'' && str[strlen(str)-1] == '/'')
	{
		fprintf(fout, "%4ld.%-4ld  【%s】     【字符型】---【0x106】/n",lineCount, lineWords, str);
	}
	else if(str[0] == '"' && str[strlen(str)-1] == '"')
	{
		fprintf(fout, "%4ld.%-4ld  【%s】     【字符串】---【0x109】/n",lineCount, lineWords, str);
	}
	else if(isOper(str[0]) == 1 && str[0] != '"' && str[0] != '/'')
	{
		if(strcmp(str, "<") == 0 || strcmp(str, ">") == 0 || strcmp(str, "<=") == 0 || strcmp(str, ">=") == 0)
		{
			fprintf(fout, "%4ld.%-4ld  【%s】     【< > <= >=】---【0x118】/n",lineCount, lineWords, str);	
		}
		else if(strcmp(str, "<<") == 0 || strcmp(str, ">>") == 0 || strcmp(str, ">>>") == 0 || strcmp(str, "<<<") == 0)
		{
			fprintf(fout, "%4ld.%-4ld  【%s】     【<< >> <<< >>>】---【0x119】/n",lineCount, lineWords, str);	
		}
		else if(strchr(str, '=') != NULL)
		{
			if(strcmp(str, "==") == 0 || strcmp(str, "!=") == 0)
			{
				fprintf(fout, "%4ld.%-4ld  【%s】     【== !=】---【0x117】/n",lineCount, lineWords, str);	
			}
			else
			{
				/*fprintf(fout, "%4ld.%-4ld  【%s】     【= += -= *= /= %%= &= ^= |= >>= <<= >>>= <<<=】---【0x110】/n",lineCount, lineWords, str);*/
				fprintf(fout, "%4ld.%-4ld  【%s】     【特殊符号】---【0x110】/n",lineCount, lineWords, str);
			}
		}
		else if(strcmp(str, "||") == 0)
		{
			fprintf(fout, "%4ld.%-4ld  【%s】     【||】---【0x112】/n",lineCount, lineWords, str);
		}
		else if(strcmp(str, "&&") == 0)
		{
			fprintf(fout, "%4ld.%-4ld  【%s】     【&&】---【0x113】/n",lineCount, lineWords, str);
		}
		else if(strcmp(str, "++") == 0 || strcmp(str, "--") == 0 || strcmp(str, "!") == 0 || strcmp(str, "~") == 0)
		{
			fprintf(fout, "%4ld.%-4ld  【%s】     【++ -- +(正) -(负) ! ~】---【0x11c】/n",lineCount, lineWords, str);
		}
		else if(strlen(str) == 1)
		{
			switch(str[0])
			{
				case '?': case ':': fprintf(fout, "%4ld.%-4ld  【%s】     【? :】---【0x111】/n",lineCount, lineWords, str); break;
				case ' ': fprintf(fout, "%4ld.%-4ld  【%s】     【空格】---【0x102】/n",lineCount, lineWords, str); break;
				case '{':case '}': fprintf(fout, "%4ld.%-4ld  【%s】     【{}】---【0x121】/n",lineCount, lineWords, str); break;
				case '[':case ']':case '(':case ')':case '.': fprintf(fout, "%4ld.%-4ld  【%s】     【[]().】---【0x11d】/n",lineCount, lineWords, str); break;
				case ',': fprintf(fout, "%4ld.%-4ld  【%s】     【,】---【0x120】/n",lineCount, lineWords, str); break;
				case ';': fprintf(fout, "%4ld.%-4ld  【%s】     【;】---【0x122】/n",lineCount, lineWords, str); break;
				case '+':case '-': fprintf(fout, "%4ld.%-4ld  【%s】     【+ -】---【0x11a】/n",lineCount, lineWords, str); break;
				case '*':case '/':case '%': fprintf(fout, "%4ld.%-4ld  【%s】     【* / %%】---【0x11b】/n",lineCount, lineWords, str); break;
				case '|': fprintf(fout, "%4ld.%-4ld  【%s】     【|】---【0x114】/n",lineCount, lineWords, str); break;
				case '^': fprintf(fout, "%4ld.%-4ld  【%s】     【^】---【0x115】/n",lineCount, lineWords, str); break;
				case '&': fprintf(fout, "%4ld.%-4ld  【%s】     【&】---【0x116】/n",lineCount, lineWords, str); break;
				default: fprintf(fout, "%4ld.%-4ld  【%s】     【其他符号】---【0x999】/n",lineCount, lineWords, str); break;
			}
		}
	}
	else
	{
		fprintf(fout, "%4ld.%-4ld  【%s】     【错误的单词】---【0x100】/n",lineCount, lineWords, str);
	}
}
/*******************从扫描缓冲区解析获取单词********************
state说明：
0表示初始，10表示只含有字母，20表示含有数字，30表示含有$或_，40表示含有其他， 50表示读取完一个单词
41表示字符串，42表示字符
****************************************************************/
void getWords(int state)
{
	char word[100];
	int charCount = 0;
	int finish = 0;
	int i, j, k;  
	for(i = 0; scanBuf[i] != '/0'; i++)
	{
		switch(state/10)
		{
			case 0:
				switch(getCharKind(scanBuf[i]))
				{
					case 1:
						word[charCount++] = scanBuf[i];
						state = 10;
						break;
					case 2:
						word[charCount++] = scanBuf[i];
						state = 20;
						break;
					case 3:
						word[charCount++] = scanBuf[i];
						state = 30;
						break;
					case 0:
						word[charCount++] = scanBuf[i];
						switch(scanBuf[i])
						{
							case '"':
								state = 41;
								break;
							case '/'':
								state = 42;
								break;
							case '(': case ')': case '{': case '}': case '[': case ']': case ';': case ',': case '.':
								state = 50;
								word[charCount] = '/0';
								finish = 1;
								break;
							case '=':
								state = 43;
								break;
							default:
								state = 40;
								break;
						}
						break;
					default: word[charCount++] = scanBuf[i]; break;
				}
				break;
			case 1:
				switch(getCharKind(scanBuf[i]))
				{
					case 1:
						word[charCount++] = scanBuf[i];
						state = 10;
						break;
					case 2:
						word[charCount++] = scanBuf[i];
						state = 20;
						break;
					case 3:
						word[charCount++] = scanBuf[i];
						state = 30;
						break;
					case 0:
						word[charCount] = '/0';
						i--;
						finish = 1;
						state = 50;
						break;
					default: word[charCount++] = scanBuf[i];break;
				}
				break;
			case 2:
				switch(getCharKind(scanBuf[i]))
				{
					case 1:
						word[charCount++] = scanBuf[i];
						state = 20;
						break;
					case 2:
						word[charCount++] = scanBuf[i];
						state = 20;
						break;
					case 3:
						word[charCount++] = scanBuf[i];
						state = 30;
						break;
					case 0:
						if(scanBuf[i] == '.')
						{
							word[charCount++] = scanBuf[i];
							state = 20;
							break;
						}
						word[charCount] = '/0';
						i--;
						finish = 1;
						state = 50;
						break;
					default: word[charCount++] = scanBuf[i];break;
				}
				break;
			case 3:
				switch(getCharKind(scanBuf[i]))
				{
					case 1:
						word[charCount++] = scanBuf[i];
						state = 30;
						break;
					case 2:
						word[charCount++] = scanBuf[i];
						state = 30;
						break;
					case 3:
						word[charCount++] = scanBuf[i];
						state = 30;
						break;
					case 0:
						word[charCount] = '/0';
						i--;
						finish = 1;
						state = 50;
						break;
					default: word[charCount++] = scanBuf[i];break;
				}
				break;
			case 4:
				switch(state)
				{
					case 40:
						switch(getCharKind(scanBuf[i]))
						{
							case 1:
								word[charCount] = '/0';
								i--;
								finish = 1;
								state = 50;
								break;
							case 2:
								word[charCount] = '/0';
								i--;
								finish = 1;
								state = 50;
								break;
							case 3:
								word[charCount] = '/0';
								i--;
								finish = 1;
								state = 50;
								break;
							case 0:
								word[charCount++] = scanBuf[i];
								state = 40;
								break;
							default: word[charCount++] = scanBuf[i];break;
						}
						break;
					case 41:
						word[charCount++] = scanBuf[i];
						if(scanBuf[i] == '"')
						{
							if(getCharKind(scanBuf[i-1]) == 4)
							{
							}
							else
							{
								word[charCount] = '/0';
								finish = 1;
								state = 50;
							}
						}
						break;
					case 42:
						word[charCount++] = scanBuf[i];
						if(scanBuf[i] == '/'')
						{
							word[charCount] = '/0';
							finish = 1;
							state = 50;
						}
						break;
					case 43:
						if(scanBuf[i] == '=')
						{
							word[charCount++] = scanBuf[i];
							state = 43;
						}
						else
						{
							word[charCount] = '/0';
							finish = 1;
							i--;
							state = 50;
						}
						break;
					default: word[charCount++] = scanBuf[i];break;
				}
				break;
			case 5:
				finish = 0;
				state = 0;
				charCount = 0;
				i--;
				lineWords++;
				totalWords++;
				getWordKind(word);
				break;
			default:break;
		}
		if(scanBuf[i+1] == '/0')
		{
			word[charCount] = '/0';
			lineWords++;
			totalWords++;
			getWordKind(word);
		}
	}
}