#include <stdlib.h>#include <stdio.h>#include <string.h>#include <ctype.h>#include "../global/macros.h"#include "tokenizer.h"/** * Tokens! *//* These are the core types, advanced types such as lists/dicts contain these core types (among other things). */const TOKEN SW_TOKEN_STRING_LITERAL = 1; /* using ', ", """, or ''' */const TOKEN SW_TOKEN_INTEGER = 2; /* 0-9+ */const TOKEN SW_TOKEN_FLOAT = 3; /* 0-9.(0-9)* */const TOKEN SW_TOKEN_BOOL_TRUE = 4; /* true */const TOKEN SW_TOKEN_BOOL_FALSE = 5; /* false */const TOKEN SW_TOKEN_CONST = 6; /* const *//* Symbols/Identifiers */const TOKEN SW_TOKEN_PROCEDURE = 10; /* => */const TOKEN SW_TOKEN_IDENTIFIER = 11; /* variables, functions, etc. */const TOKEN SW_TOKEN_RELATE = 12; /* -> */const TOKEN SW_TOKEN_DO = 13; /* do: ... ; */const TOKEN SW_TOKEN_WHILE = 14; /* while ...: ... ; */const TOKEN SW_TOKEN_FOR = 15; /* for ...: ... ; */const TOKEN SW_TOKEN_RETURN = 16; /* return ... */const TOKEN SW_TOKEN_CONT = 17; /* continue */const TOKEN SW_TOKEN_BREAK = 18; /* break *//* General symbols that can be ambiguous */const TOKEN SW_TOKEN_COLON = 20; /* : */const TOKEN SW_TOKEN_SEMICOLON = 21; /* ; */const TOKEN SW_TOKEN_PERIOD = 22; /* . */const TOKEN SW_TOKEN_COMMA = 23; /* , */const TOKEN SW_TOKEN_SET = 24; /* = */const TOKEN SW_TOKEN_JOIN = 25; /* _ */const TOKEN SW_TOKEN_MORE_ARGS = 26; /* ... */const TOKEN SW_TOKEN_RANGE = 27; /* .. *//* Operators */const TOKEN SW_TOKEN_NOT = 30; /* ! */const TOKEN SW_TOKEN_IS = 31; /* is */const TOKEN SW_TOKEN_EQUALS = 32; /* == */const TOKEN SW_TOKEN_IN = 33; /* in */const TOKEN SW_TOKEN_NOT_EQUALS = 34; /* != */const TOKEN SW_TOKEN_LOGICAL_AND = 35; /* && */const TOKEN SW_TOKEN_LOGICAL_OR = 36; /* || */const TOKEN SW_TOKEN_LOGICAL_XOR = 37; /* ^^ */const TOKEN SW_TOKEN_LT = 38; /* < */const TOKEN SW_TOKEN_GT = 39; /* > */const TOKEN SW_TOKEN_LT_EQUAL = 40; /* <= */const TOKEN SW_TOKEN_GT_EQUAL = 41; /* >= */const TOKEN SW_TOKEN_ADD = 43; /* + */const TOKEN SW_TOKEN_SUBTRACT = 44; /* - */const TOKEN SW_TOKEN_DIVIDE = 45; /* / */const TOKEN SW_TOKEN_MULTIPLY = 46; /* * */const TOKEN SW_TOKEN_EXPONENT = 47; /* ** */const TOKEN SW_TOKEN_SUBSTITUTE = 48; /* % *//* Parens/brackets/braces (would be cool if we could have chevrons as well) */const TOKEN SW_TOKEN_PAREN_OPEN = 50; /* ( */const TOKEN SW_TOKEN_PAREN_CLOSE = 51; /* ) */const TOKEN SW_TOKEN_BRACKET_OPEN = 52; /* [ */const TOKEN SW_TOKEN_BRACKET_CLOSE = 53; /* ] */const TOKEN SW_TOKEN_BRACE_OPEN = 54; /* { */const TOKEN SW_TOKEN_BRACE_CLOSE = 55; /* } *//* Bitwise operators */const TOKEN SW_TOKEN_BIT_OR = 60; /* | */const TOKEN SW_TOKEN_BIT_XOR = 61; /* ^ */const TOKEN SW_TOKEN_BIT_AND = 62; /* & */const TOKEN SW_TOKEN_BIT_LSHIFT = 64; /* << */const TOKEN SW_TOKEN_BIT_RSHIFT = 65; /* >> */const TOKEN SW_TOKEN_BIT_COMPLEMENT = 66; /* ~ *//* Type casts */const TOKEN SW_TOKEN_TYPE_DICT = 70; /* dict */const TOKEN SW_TOKEN_TYPE_LIST = 71; /* list */const TOKEN SW_TOKEN_TYPE_INT = 72; /* int */const TOKEN SW_TOKEN_TYPE_FLOAT = 73; /* float *//** * Functions! *//* Make sure a character is allowed in an identifier token */int char_allowed_in_identifier(char ch) { return (isalnum(ch) || ch == '_' || ch == '$');}/* Peek at the next character in the buffer */char peek_next_token(const char *buffer, const int *pos) { return buffer[(*pos)+1];}/* Allocate and return a token. If no value is needed for the token, as is the case for most tokens, then NULL will be supplied. The 'val' is useful for numbers, string literals, and identifiers. */SwToken *make_token(TOKEN tk, char *val) { SwToken *token; if( (token = (SwToken *)malloc(sizeof(SwToken))) == NULL) FATAL_ERROR("Could not allocate space for token."); token->tk = tk; token->value = val; return token;}/* This is for when we find a character and are looking to see if the next character is the same. If the next character is the same, then we have found the 'xx' token, otherwise we stick with the original guess of having found the 'x' token. */SwToken *make_token_cond(int cond, TOKEN x, TOKEN xx) { return make_token((TOKEN)(cond == 1 ? x : xx), NULL);}/* Allocate the space for the value of a token. */char *make_token_value(const char *buffer, int started_at, int ended_at) { char *literal; size_t len; len = ended_at - started_at; if( (literal = (char *)malloc(sizeof(char) * (len + 1))) == NULL) FATAL_ERROR("Could not allocate memory for string literal."); /* Make the string. Copies the memory at the right start location by using pointer arithmetic. */ literal = (char *)memcpy(literal, buffer + started_at, len); literal[len] = '\0'; return literal;}/* Make a token for a string literal. This will work for single/double quotes and Python-like triple quoting. */SwToken *make_token_string_literal(const char q, const char *buffer, int *pos) { int i, started_at = 0, ended_at = 0; int is_triple = 0; /* start from the character *after* this quotation mark */ started_at = (*pos)+1; /* We could be dealing with a triple-quotes literal or an empty string. */ if(q == peek_next_token(buffer, pos)) { (*pos)++; /* yes, this is triple-quotes */ if(q == peek_next_token(buffer, pos)) { is_triple = 1; *pos++; /* no, this is an empty string */ } else { ended_at = (*pos)+1; goto SwTokenMakeLiteral; } } /* Go over the characters in the buffer and try to find the end of this string literal. */ for(i = started_at; (buffer[i] != '\0'); i++) { if(buffer[i] == q) { /* ignore this quote because the previous character negated it with a backslash */ if(buffer[i-1] == '\\') continue; /* If this is a triple, than this might be an everyday quote inside it which is allowed, otherwise it could be the ending one. Let's make sure. */ else { /* Closing of triple quotes OR closing of single quotes. */ if(!is_triple || (is_triple && (buffer[i+1] == q) && (buffer[i+2] == q))) { ended_at = i; break; } } } } /* change the pos to the end of the string */ *pos = is_triple ? i + 2 : i; /* We now have the start and end position of the string. Let's allocate space for it and then make the token. */ SwTokenMakeLiteral: return make_token(SW_TOKEN_STRING_LITERAL, make_token_value(buffer, \ started_at, ended_at));}/* Create a token for a text-only identifier. The identifier can have a lot in it but there are restrictions. This can find ints/floats, or normal identifiers. Note: we don't worry about numbers being negative. The parser can figure this out :) */SwToken *make_token_identifier(const char *buffer, int *pos) { int started_at = *pos, is_float = 0; char *val; TOKEN tk; /* number, could be a float, integer, etc. */ if(isdigit(buffer[*pos])) { for(; isdigit(buffer[*pos]) || buffer[*pos] == '.'; (*pos)++) { /* Check to see if we are dealing with a floating-point number, if so make sure it doesn't have more than one decimal place in it. */ if(buffer[*pos] == '.') { if(is_float) { (*pos)--; /* Rewind this so that we capture the period again later. */ break; } is_float = 1; } } tk = is_float ? SW_TOKEN_FLOAT : SW_TOKEN_INTEGER; val = make_token_value(buffer, started_at, *pos); /* identifier of some sort, could be a built-in construct */ } else if(char_allowed_in_identifier(buffer[*pos])) { /* go find the last valid character in this token */ for(; char_allowed_in_identifier(buffer[*pos]); (*pos)++); /* allocate the value */ val = make_token_value(buffer, started_at, *pos); /* fake switch for strings.. todo: remake this, it is fugly!! */ SWITCH_STR() CASE_STR(val, "for") { tk = SW_TOKEN_FOR; goto SwTokenMakeFreeValue; } CASE_STR(val, "do") { tk = SW_TOKEN_DO; goto SwTokenMakeFreeValue; } CASE_STR(val, "while") { tk = SW_TOKEN_WHILE; goto SwTokenMakeFreeValue; } /* flow control */ CASE_STR(val, "return") { tk = SW_TOKEN_RETURN; goto SwTokenMakeFreeValue; } CASE_STR(val, "continue") { tk = SW_TOKEN_CONT; goto SwTokenMakeFreeValue; } CASE_STR(val, "break") { tk = SW_TOKEN_BREAK; goto SwTokenMakeFreeValue; } /* types */ CASE_STR(val, "dict") { tk = SW_TOKEN_TYPE_DICT; goto SwTokenMakeFreeValue; } CASE_STR(val, "int") { tk = SW_TOKEN_TYPE_INT; goto SwTokenMakeFreeValue; } CASE_STR(val, "float") { tk = SW_TOKEN_TYPE_FLOAT; goto SwTokenMakeFreeValue; } CASE_STR(val, "list") { tk = SW_TOKEN_TYPE_LIST; goto SwTokenMakeFreeValue; } /* operators */ CASE_STR(val, "is") { tk = SW_TOKEN_IS; goto SwTokenMakeFreeValue; } CASE_STR(val, "in") { tk = SW_TOKEN_IN; goto SwTokenMakeFreeValue; } CASE_STR(val, "const") { tk = SW_TOKEN_CONST; goto SwTokenMakeFreeValue; } /* built-in constants */ CASE_STR(val, "true") { tk = SW_TOKEN_BOOL_TRUE; goto SwTokenMakeFreeValue; } CASE_STR(val, "false") { tk = SW_TOKEN_BOOL_FALSE; goto SwTokenMakeFreeValue; } /* other: could be functions/variables */ DEFAULT_STR() { tk = SW_TOKEN_IDENTIFIER; goto SwTokenMakeIdentifier; } /* if we don't need the value, free what we've stored */ SwTokenMakeFreeValue: val = (free(val), NULL); /* unknown character, todo: error or act as comment? */ } else PARSE_ERROR("Unknown character encountered."); /* make the token */ SwTokenMakeIdentifier: return make_token(tk, val);}/* Return a single token from the buffer. Note: the buffer and the current character position need to be initialized and maintained *outside* of this function. To work this function, use a while/for loop checking that this function doesn't return NULL. At the end of loop increment 'pos'!! */SwToken *get_token(const char *buffer, int *pos, const int len) { int i; char c, nc; TOKEN tk; /* Ignore all white space. */ for(; isspace(buffer[*pos]); (*pos)++); /* whoop, at the end of the string */ if(buffer[*pos] == '\0' || *pos > len) return NULL; /* Get the current and next characters */ c = buffer[*pos]; nc = peek_next_token(buffer, pos); /* We will do cases for every one-character token, then the default will be looked at as a possible identifier. The one-character tokens are used so that we can peek ahead to the next character(s) and see if we should collect more characters for the token on stay with the original single-character token. */ switch(c) { /* The following tokens have only a single character and no other tokens start with the same character, so we can handle these in the same way. */ case ':': tk = SW_TOKEN_COLON; goto SwTokenSimpleCreate; case ';': tk = SW_TOKEN_SEMICOLON; goto SwTokenSimpleCreate; case ',': tk = SW_TOKEN_COMMA; goto SwTokenSimpleCreate; case '_': tk = SW_TOKEN_JOIN; goto SwTokenSimpleCreate; case '/': tk = SW_TOKEN_DIVIDE; goto SwTokenSimpleCreate; case '+': tk = SW_TOKEN_ADD; goto SwTokenSimpleCreate; case '(': tk = SW_TOKEN_PAREN_OPEN; goto SwTokenSimpleCreate; case ')': tk = SW_TOKEN_PAREN_CLOSE; goto SwTokenSimpleCreate; case '[': tk = SW_TOKEN_BRACKET_OPEN; goto SwTokenSimpleCreate; case ']': tk = SW_TOKEN_BRACKET_CLOSE; goto SwTokenSimpleCreate; case '{': tk = SW_TOKEN_BRACE_OPEN; goto SwTokenSimpleCreate; case '}': tk = SW_TOKEN_BRACE_CLOSE; goto SwTokenSimpleCreate; case '~': tk = SW_TOKEN_BIT_COMPLEMENT; goto SwTokenSimpleCreate; case '%': tk = SW_TOKEN_SUBSTITUTE; goto SwTokenSimpleCreate; SwTokenSimpleCreate: /* goto's go here */ return make_token(tk, NULL); /* Make tokens given that the next character. The choice for these is between two possible tokens where there are only two tokens that start with the same character. */ case '!': return make_token_cond(nc != '=', SW_TOKEN_NOT, \ (*pos++, SW_TOKEN_NOT_EQUALS)); case '-': return make_token_cond(nc != '>', SW_TOKEN_SUBTRACT, \ (*pos++, SW_TOKEN_RELATE)); case '*': return make_token_cond(c != nc, SW_TOKEN_MULTIPLY, \ (*pos++, SW_TOKEN_EXPONENT)); case '|': return make_token_cond(c != nc, SW_TOKEN_BIT_OR, \ (*pos++, SW_TOKEN_LOGICAL_OR)); case '&': return make_token_cond(c != nc, SW_TOKEN_BIT_AND, \ (*pos++, SW_TOKEN_LOGICAL_AND)); case '^': return make_token_cond(c != nc, SW_TOKEN_BIT_XOR, \ (*pos++, SW_TOKEN_LOGICAL_XOR)); /* Make tokens given the next character, where there are three possible tokens that start with the same character. */ case '=': return nc == '>' ? make_token((*pos++, SW_TOKEN_PROCEDURE), NULL): make_token_cond(c != nc, SW_TOKEN_SET, \ (*pos++, SW_TOKEN_EQUALS)); case '<': return nc == '=' ? make_token((*pos++, SW_TOKEN_LT_EQUAL), NULL) : make_token_cond(c != nc, SW_TOKEN_LT, \ (*pos++, SW_TOKEN_BIT_LSHIFT)); case '>': return nc == '=' ? make_token((*pos++, SW_TOKEN_GT_EQUAL), NULL) : make_token_cond(c != nc, SW_TOKEN_GT, \ (*pos++, SW_TOKEN_BIT_RSHIFT)); /* Special case for the ellipsis signifying more arguments. */ case '.': if(nc != '.') return make_token(SW_TOKEN_PERIOD, NULL); else if(nc == '.' && buffer[*pos+2] == '.') { *pos += 2; return make_token(SW_TOKEN_MORE_ARGS, NULL); } else { return make_token(SW_TOKEN_RANGE, NULL); } /* Special cases, when we find a ' or " then we will try to find the string literal and we will consider that to be a token instead of tokenizing the actual ' or ". */ case '\'': case '"': return make_token_string_literal(c, buffer, pos); /* We will accept all other characters from here. We can assume that they will somehow be identifiers. In the parsing stage, if they are misplaced then the program will report an error. */ default: return make_token_identifier(buffer, pos); }}