#include <stdlib.h>#include <stdio.h>#include <string.h>#include <ctype.h>#include "../global/macros.h"#include "tokenizer.h"#define TOKENIZER_DEBUG 0/************************************* * Private functions. *************************************//* Make sure a character is allowed in an identifier token */int char_allowed_in_ids(char ch) { return (isalnum(ch) || ch == '_' || ch == '$');}/* Get a character from anywhere in the buffer */char get_char(SwTokenBuffer *buffer, int i) { return (i > buffer->len) ? '\0' : buffer->stream[i];}/* Peek at a future character in the buffer */char peek_char(SwTokenBuffer *buffer, int add) { return get_char(buffer, buffer->pos+add);}/* Return the current buffer character */char current_char(SwTokenBuffer *buffer) { return get_char(buffer, buffer->pos);}/* Free the memory allocated for the value of a token. Values are only allocated for string literals / unknown identifiers */void free_token_value(SwTokenBuffer *buffer) { if(buffer->token->value != NULL) buffer->token->value = (free(buffer->token->value), NULL);}/* Free the memory in the current buffer token. */void free_token(SwTokenBuffer *buffer) { if(buffer->token != NULL) { free_token_value(buffer); buffer->token = (free(buffer->token), NULL); }}/* Allocate and return a token. If no value is needed for the token, as is the case for most tokens, then NULL will be supplied. The 'val' is useful for numbers, string literals, and identifiers. */int make_token(SwTokenBuffer *buffer, SwSymbol tk, char *val) { buffer->token->symbol = tk; buffer->token->value = val; return 1;}/* This is for when we find a character and are looking to see if the next character is the same. If the next character is the same, then we have found the 'xx' token, otherwise we stick with the original guess of having found the 'x' token. */int make_token_cond(SwTokenBuffer *buffer, int cond, SwSymbol x, SwSymbol xx){ return make_token(buffer, (SwSymbol)(cond == 1 ? x : xx), NULL);}/* Allocate the space for the value of a token. */char *alloc_token_value(SwTokenBuffer *buffer, int started_at, int ended_at) { char *literal; size_t len; len = ended_at - started_at; if( (literal = (char *)malloc(sizeof(char) * (len + 1))) == NULL) FATAL_ERROR("Could not allocate memory for string literal."); /* Make the string. Copies the memory at the right start location by using pointer arithmetic. */ literal = (char *)memcpy(literal, buffer->stream + started_at, len); literal[len] = '\0'; return literal;}/* See if we can match the next characters in the buffer against the chars in chs. */int match_next_chars(SwTokenBuffer *buffer, int len, char *chs) { int i = buffer->pos, z = 0; char ch = get_char(buffer, i); /* See if we can match the chars in chs to the chars in buffer */ for(; i < (buffer->pos + len) \ && char_allowed_in_ids(ch); i++, z++, ch = get_char(buffer, i)) { if(ch != chs[z]) return 0; } /* So we've matched those characters, but that's not enough. If there's a character after the last matched one that can be used in an identifier then this test fails. */ if(char_allowed_in_ids(peek_char(buffer, 1))) return 0; /* Change the pos pointer so that in the future the characters we just scanned are ignored */ buffer->pos = i-1; return 1;}/* Make a token for a string literal. This will work for single/double quotes and Python-like triple quoting. */int make_token_string_literal(SwTokenBuffer *buffer, const char q) { int i, started_at = 0, ended_at = 0; int is_triple = 0; char ch; /* start from the character *after* this quotation mark */ started_at = buffer->pos + 1; /* We could be dealing with a triple-quotes literal or an empty string. */ if(q == peek_char(buffer, 1)) { buffer->pos++; /* yes, this is triple-quotes */ if(q == peek_char(buffer, 1)) { is_triple = 1; buffer->pos++; /* no, this is an empty string */ } else { ended_at = buffer->pos + 1; goto SwTokenMakeLiteral; } } /* Get the current char and set up the loop */ i = started_at; ch = get_char(buffer, i); /* Go over the characters in the buffer and try to find the end of this string literal. */ for(; ch != '\0'; i++, ch = get_char(buffer, i)) { if(ch == q) { /* ignore this quote because the previous character negated it with a backslash */ if(get_char(buffer, i-1) == '\\') continue; /* If this is a triple, than this might be an everyday quote inside it which is allowed, otherwise it could be the ending one. Let's make sure. */ else { /* Closing of triple quotes OR closing of single quotes. */ if(!is_triple || (is_triple && get_char(buffer, i+1) == q \ && get_char(buffer, i+2) == q)) { ended_at = i; break; } } } } /* change the pos to the end of the string */ buffer->pos = is_triple ? (i + 2) : i; /* We now have the start and end position of the string. Let's allocate space for it and then make the token. */ SwTokenMakeLiteral: return make_token(buffer, SW_TOKEN_STRING_LITERAL, \ alloc_token_value(buffer, started_at, ended_at));}/* Create a token for a text-only identifier. The identifier can have a lot in it but there are restrictions. This can find ints/floats, or normal identifiers. Note: we don't worry about numbers being negative. The parser can figure this out :) */int make_token_identifier(SwTokenBuffer *buffer) { int started_at = buffer->pos, is_float = 0; char *val = NULL; char ch = current_char(buffer); SwSymbol tk = SW_TOKEN_UNKNOWN; /* number, could be a float, integer, etc. */ if(isdigit(ch)) { for(; isdigit(ch) || ch == '.'; buffer->pos++, \ ch = current_char(buffer)) { /* Check to see if we are dealing with a floating-point number, if so make sure it doesn't have more than one decimal place in it. */ if(ch == '.') { if(is_float) { buffer->pos--; /* Rewind this so that we capture the period again later. */ break; } is_float = 1; } } tk = is_float ? SW_TOKEN_FLOAT : SW_TOKEN_INTEGER; val = alloc_token_value(buffer, started_at, buffer->pos); /* identifier of some sort, could be a built-in construct */ } else if(char_allowed_in_ids(ch)) { /* Let's start with the first characters of some identifiers that we recognize. If at the end we don't match a known identifier *then* we will do a malloc. Otherwise, a malloc is just a waste because the memory will be free'd anyway. So, is this more or less efficient than the macro-based SWITCH_STR? It definitely looks uglier... */ switch(ch) { case 'b': /* break */ if(match_next_chars(buffer, 5, "break")) tk = SW_TOKEN_BREAK; break; case 'c': /* const, continue */ if(match_next_chars(buffer, 5, "const")) tk = SW_TOKEN_CONST; else if(match_next_chars(buffer, 7, "continue")) tk = SW_TOKEN_CONT; break; case 'd': /* dict, do */ if(match_next_chars(buffer, 4, "dict")) tk = SW_TOKEN_TYPE_DICT; else if(match_next_chars(buffer, 2, "do")) tk = SW_TOKEN_DO; break; case 'f': /* for, false, float */ if(match_next_chars(buffer, 3, "for")) tk = SW_TOKEN_FOR; else if(match_next_chars(buffer, 5, "false")) tk = SW_TOKEN_BOOL_FALSE; else if(match_next_chars(buffer, 5, "float")) tk = SW_TOKEN_TYPE_FLOAT; break; case 'i': /* in, int, is */ if(peek_char(buffer, 1) == 'n') { if(match_next_chars(buffer, 3, "int")) tk = SW_TOKEN_TYPE_INT; else tk = SW_TOKEN_IN; } else { if(match_next_chars(buffer, 2, "is")) tk = SW_TOKEN_IS; } break; case 'l': /* list */ if(match_next_chars(buffer, 4, "list")) tk = SW_TOKEN_TYPE_LIST; break; case 'm': /* my */ if(match_next_chars(buffer, 2, "my")) tk = SW_TOKEN_MY; break; case 'r': /* return */ if(match_next_chars(buffer, 6, "return")) tk = SW_TOKEN_RETURN; break; case 't': /* true */ if(match_next_chars(buffer, 4, "true")) tk = SW_TOKEN_BOOL_TRUE; break; case 'w': /* while */ if(match_next_chars(buffer, 5, "while")) tk = SW_TOKEN_WHILE; break; } /* allocate the value if necessary */ if(tk == SW_TOKEN_UNKNOWN) { tk = SW_TOKEN_IDENTIFIER; /* go find the last valid character in this token */ for(; char_allowed_in_ids(current_char(buffer)); buffer->pos++); val = alloc_token_value(buffer, started_at, buffer->pos); } /* unknown character, todo: error or act as comment? */ } else PARSE_ERROR("Unknown character encountered."); /* make the token */ SwTokenMakeIdentifier: return make_token(buffer, tk, val);}/* Return a single token from the buffer. Note: the buffer and the current character position need to be initialized and maintained *outside* of this function. To work this function, use a while/for loop checking that this function doesn't return NULL. At the end of loop increment 'pos'!! */int get_token(SwTokenBuffer *buffer) { char ch, next_ch; SwSymbol tk; /* It could be that the token we're passing in was already used, and had a value component to it that wasn't free'd. If that's the case then we will free up that memory. */ free_token_value(buffer); /* Ignore all white space. */ for(; isspace(current_char(buffer)); buffer->pos++); /* Whoop, at the end of the string, don't need to check for out of bounds because current_char already does that. */ ch = current_char(buffer); if(ch == '\0') return 0; next_ch = peek_char(buffer, 1); /* We will do cases for every one-character token, then the default will be looked at as a possible identifier. The one-character tokens are used so that we can peek ahead to the next character(s) and see if we should collect more characters for the token on stay with the original single-character token. */ switch(ch) { /* The following tokens have only a single character and no other tokens start with the same character, so we can handle these in the same way. */ case ':': tk = SW_TOKEN_COLON; goto SwTokenSimpleCreate; case ';': tk = SW_TOKEN_SEMICOLON; goto SwTokenSimpleCreate; case ',': tk = SW_TOKEN_COMMA; goto SwTokenSimpleCreate; case '_': tk = SW_TOKEN_JOIN; goto SwTokenSimpleCreate; case '/': tk = SW_TOKEN_DIVIDE; goto SwTokenSimpleCreate; case '+': tk = SW_TOKEN_ADD; goto SwTokenSimpleCreate; case '(': tk = SW_TOKEN_PAREN_OPEN; goto SwTokenSimpleCreate; case ')': tk = SW_TOKEN_PAREN_CLOSE; goto SwTokenSimpleCreate; case '[': tk = SW_TOKEN_BRACKET_OPEN; goto SwTokenSimpleCreate; case ']': tk = SW_TOKEN_BRACKET_CLOSE; goto SwTokenSimpleCreate; case '{': tk = SW_TOKEN_BRACE_OPEN; goto SwTokenSimpleCreate; case '}': tk = SW_TOKEN_BRACE_CLOSE; goto SwTokenSimpleCreate; case '~': tk = SW_TOKEN_BIT_COMPLEMENT; goto SwTokenSimpleCreate; case '%': tk = SW_TOKEN_MODULO; goto SwTokenSimpleCreate; SwTokenSimpleCreate: /* goto's come here */ return make_token(buffer, tk, NULL); /* Make tokens given that the next character. The choice for these is between two possible tokens where there are only two tokens that start with the same character. */ case '!': return make_token_cond(buffer, next_ch != '=', SW_TOKEN_NOT, \ (buffer->pos++, SW_TOKEN_NOT_EQUALS)); case '-': return make_token_cond(buffer, next_ch != '>', SW_TOKEN_SUBTRACT,\ (buffer->pos++, SW_TOKEN_RELATE)); case '*': return make_token_cond(buffer, ch != next_ch, SW_TOKEN_MULTIPLY, \ (buffer->pos++, SW_TOKEN_EXPONENT)); case '|': return make_token_cond(buffer, ch != next_ch, SW_TOKEN_BIT_OR, \ (buffer->pos++, SW_TOKEN_LOGICAL_OR)); case '&': return make_token_cond(buffer, ch != next_ch, SW_TOKEN_BIT_AND, \ (buffer->pos++, SW_TOKEN_LOGICAL_AND)); case '^': return make_token_cond(buffer, ch != next_ch, SW_TOKEN_BIT_XOR, \ (buffer->pos++, SW_TOKEN_LOGICAL_XOR)); /* Make tokens given the next character, where there are three possible tokens that start with the same character. */ case '=': return next_ch == '>' ? make_token(buffer, (buffer->pos++, SW_TOKEN_PROCEDURE), NULL): make_token_cond(buffer, ch != next_ch, SW_TOKEN_SET, \ (buffer->pos++, SW_TOKEN_EQUALS)); case '<': return next_ch == '=' ? make_token(buffer, (buffer->pos++, SW_TOKEN_LT_EQUAL), NULL) : make_token_cond(buffer, ch != next_ch, SW_TOKEN_LT, \ (buffer->pos++, SW_TOKEN_BIT_LSHIFT)); case '>': return next_ch == '=' ? make_token(buffer, (buffer->pos++, SW_TOKEN_GT_EQUAL), NULL) : make_token_cond(buffer, ch != next_ch, SW_TOKEN_GT, \ (buffer->pos++, SW_TOKEN_BIT_RSHIFT)); /* Special case for the ellipsis signifying more arguments. */ case '.': if(next_ch != '.') return make_token(buffer, SW_TOKEN_PERIOD, NULL); else if(next_ch == '.' && peek_char(buffer, 2) == '.') { buffer->pos += 2; return make_token(buffer, SW_TOKEN_MORE_ARGS, NULL); } else { return make_token(buffer, SW_TOKEN_RANGE, NULL); } /* Special cases, when we find a ' or " then we will try to find the string literal and we will consider that to be a token instead of tokenizing the actual ' or ". */ case '\'': case '"': return make_token_string_literal(buffer, ch); /* We will accept all other characters from here. We can assume that they will somehow be identifiers. In the parsing stage, if they are misplaced then the program will report an error. */ default: return make_token_identifier(buffer); } return 0;}/************************************* * Interface functions. *************************************//* Get the next token from the token buffer */SwToken *next_token(SwTokenBuffer *buffer) { if(get_token(buffer)) { buffer->pos++; return buffer->token; } free_token(buffer); return NULL;}/* Get the current token in the buffer */SwToken *current_token(SwTokenBuffer *buffer) { return buffer->token;}/* Initialize the tokenizer, allocate space for a token. */SwTokenBuffer *init_tokenizer(char *stream) { SwTokenBuffer *buffer;#if TOKENIZER_DEBUG == 1 printf("Tokenizer initialized.\n"); printf("buffer malloc: size: %d\n", sizeof(SwTokenBuffer)); printf("token malloc: size: %d\n", sizeof(SwToken));#endif if((buffer = (SwTokenBuffer *)malloc(sizeof(SwTokenBuffer))) == NULL) FATAL_ERROR("Could not allocate space for the token buffer."); /* allocate space for the token on the heap */ if((buffer->token = (SwToken *)malloc(sizeof(SwToken))) == NULL) FATAL_ERROR("Could not allocate space for the token."); /* default values */ buffer->token->symbol = SW_TOKEN_UNKNOWN; buffer->token->value = NULL; buffer->stream = stream; buffer->len = strlen(stream); buffer->pos = 0; return buffer;}/* Disable the tokenizer, deallocate any left over strings in the token value (this is also done in get_token) and then free the token from memory. */void disable_tokenizer(SwTokenBuffer *buffer) { if(buffer != NULL) { free_token(buffer); buffer = (free(buffer), NULL); }#if TOKENIZER_DEBUG == 1 printf("Tokenizer disabled.\n");#endif}