Lexical Analyzer in C and C++

Here you will get the program to implement lexical analyzer in C and C++.

The compiler is responsible for converting high-level language into machine language. There are several phases involved in this and lexical analysis is the first phase.

A lexical analyzer reads the characters from the source code and converts them into tokens.

Different tokens or lexemes are:

• Keywords
• Identifiers
• Operators
• Constants

Take the below example.

c = a + b;

After lexical analysis, a symbol table is generated as given below.

Token	Type
c	identifier
=	operator
a	identifier
+	operator
b	identifier
;	separator

Now below I have given the implementation of a very simple lexical analyzer that reads source code from a file and then generates tokens.

Program for Lexical Analyzer in C

#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include<ctype.h>

int isKeyword(char buffer[]){
	char keywords[32][10] = {"auto","break","case","char","const","continue","default",
							"do","double","else","enum","extern","float","for","goto",
							"if","int","long","register","return","short","signed",
							"sizeof","static","struct","switch","typedef","union",
							"unsigned","void","volatile","while"};
	int i, flag = 0;
	
	for(i = 0; i < 32; ++i){
		if(strcmp(keywords[i], buffer) == 0){
			flag = 1;
			break;
		}
	}
	
	return flag;
}

int main(){
	char ch, buffer[15], operators[] = "+-*/%=";
	FILE *fp;
	int i,j=0;
	
	fp = fopen("program.txt","r");
	
	if(fp == NULL){
		printf("error while opening the file\n");
		exit(0);
	}
	
	while((ch = fgetc(fp)) != EOF){
   		for(i = 0; i < 6; ++i){
   			if(ch == operators[i])
   				printf("%c is operator\n", ch);
   		}
   		
   		if(isalnum(ch)){
   			buffer[j++] = ch;
   		}
   		else if((ch == ' ' || ch == '\n') && (j != 0)){
   				buffer[j] = '\0';
   				j = 0;
   				   				
   				if(isKeyword(buffer) == 1)
   					printf("%s is keyword\n", buffer);
   				else
   					printf("%s is indentifier\n", buffer);
   		}
   		
	}
	
	fclose(fp);
	
	return 0;
}

Program for Lexical Analyzer in C++

#include<iostream>
#include<fstream>
#include<stdlib.h>
#include<string.h>
#include<ctype.h>

using namespace std;

int isKeyword(char buffer[]){
	char keywords[32][10] = {"auto","break","case","char","const","continue","default",
							"do","double","else","enum","extern","float","for","goto",
							"if","int","long","register","return","short","signed",
							"sizeof","static","struct","switch","typedef","union",
							"unsigned","void","volatile","while"};
	int i, flag = 0;
	
	for(i = 0; i < 32; ++i){
		if(strcmp(keywords[i], buffer) == 0){
			flag = 1;
			break;
		}
	}
	
	return flag;
}

int main(){
	char ch, buffer[15], operators[] = "+-*/%=";
	ifstream fin("program.txt");
	int i,j=0;
	
	if(!fin.is_open()){
		cout<<"error while opening the file\n";
		exit(0);
	}
	
	while(!fin.eof()){
   		ch = fin.get();
   		
		for(i = 0; i < 6; ++i){
   			if(ch == operators[i])
   				cout<<ch<<" is operator\n";
   		}
   		
   		if(isalnum(ch)){
   			buffer[j++] = ch;
   		}
   		else if((ch == ' ' || ch == '\n') && (j != 0)){
   				buffer[j] = '\0';
   				j = 0;
   				   				
   				if(isKeyword(buffer) == 1)
   					cout<<buffer<<" is keyword\n";
   				else
   					cout<<buffer<<" is indentifier\n";
   		}
   		
	}
	
	fin.close();
	
	return 0;
}

Output

The source code present in the file is shown in the above image.

The below video explains how a lexical analyzer works.

Comment below if you have any queries regarding the above program for lexical analyzer in C and C++.