#include <stdlib.h>#include <string.h>#include <ctype.h>#include "../types/tokens.h"/* Return a single token from the buffer. Note: the buffer and the current character position need to be initialized and maintained *outside* of this function. To work this function, use a while/for loop checking that this function doesn't return NULL. At the end of loop increment 'pos'!! */SwToken *get_token(char *buffer, unsigned int *pos) { unsigned int i; char ch, nch; TOKEN tk; /* Ignore all white space and control characters. */ for(i = *pos; (buffer[i] != '\0') && (iscntrl(buffer[i]) || \ isspace(buffer[i])); i++); *pos = i; /* Make sure we're not at the end of the string */ if(buffer[*pos] == '\0') return NULL; /* Get the current and next characters */ ch = buffer[*pos]; nch = peek_next_token(buffer, pos); /* We will do cases for every one-character token, then the default will be looked at as a possible identifier. The one-character tokens are used so that we can peek ahead to the next character(s) and see if we should collect more characters for the token on stay with the original single-character token. */ switch(ch) { /* The following tokens have only a single character and no other tokens start with the same character, so we can handle these in the same way. */ case ':': tk = SW_TOKEN_COLON; goto SwTokenSimpleCreate; case ';': tk = SW_TOKEN_SEMICOLON; goto SwTokenSimpleCreate; case '.': tk = SW_TOKEN_PERIOD; goto SwTokenSimpleCreate; case ',': tk = SW_TOKEN_COMMA; goto SwTokenSimpleCreate; case '_': tk = SW_TOKEN_JOIN; goto SwTokenSimpleCreate; case '/': tk = SW_TOKEN_DIVIDE; goto SwTokenSimpleCreate; case '+': tk = SW_TOKEN_ADD; goto SwTokenSimpleCreate; case '(': tk = SW_TOKEN_PAREN_OPEN; goto SwTokenSimpleCreate; case ')': tk = SW_TOKEN_PAREN_CLOSE; goto SwTokenSimpleCreate; case '[': tk = SW_TOKEN_BRACKET_OPEN; goto SwTokenSimpleCreate; case ']': tk = SW_TOKEN_BRACKET_CLOSE; goto SwTokenSimpleCreate; case '{': tk = SW_TOKEN_BRACE_OPEN; goto SwTokenSimpleCreate; case '}': tk = SW_TOKEN_BRACE_CLOSE; goto SwTokenSimpleCreate; case '~': tk = SW_TOKEN_BIT_COMPLEMENT; goto SwTokenSimpleCreate; case '%': tk = SW_TOKEN_SUBSTITUTE; goto SwTokenSimpleCreate; SwTokenSimpleCreate: /* goto's go here */ return make_token(tk, NULL); /* Make tokens given that the next character. The choice for these is between two possible tokens where there are only two tokens that start with the same character. */ case '!': return make_token_cond(nc != '=', SW_TOKEN_NOT, \ (*pos++, SW_TOKEN_NOT_EQUALS)); case '-': return make_token_cond(nc != '>', SW_TOKEN_SUBTRACT, \ (*pos++, SW_TOKEN_RELATE)); case '*': return make_token_cond(c != nc, SW_TOKEN_MULTIPLY, \ (*pos++, SW_TOKEN_EXPONENT)); case '|': return make_token_cond(c != nc, SW_TOKEN_BIT_OR, \ (*pos++, SW_TOKEN_LOGICAL_OR)); case '&': return make_token_cond(c != nc, SW_TOKEN_BIT_AND, \ (*pos++, SW_TOKEN_LOGICAL_AND)); case '^': return make_token_cond(c != nc, SW_TOKEN_BIT_XOR, \ (*pos++, SW_TOKEN_LOCIAL_XOR)); /* Make tokens given the next character, where there are three possible tokens that start with the same character. */ case '=': return nc == '>' ? make_token((*pos++, SW_TOKEN_PROCEDURE), NULL): make_token_cond(c != nc, SW_TOKEN_SET, \ (*pos++, SW_TOKEN_EQUALS)); case '<': return nc == '=' ? make_token((*pos++, SW_TOKEN_LT_EQUAL), NULL) : make_token_cond(c != nc, SW_TOKEN_LT, \ (*pos++, SW_TOKEN_BIT_LSHIFT)); case '>': return nc == '=' ? make_token((*pos++, SW_TOKEN_GT_EQUAL), NULL) : make_token_cond(c != nc, SW_TOKEN_GT, \ (*pos++, SW_TOKEN_BIT_RSHIFT)); /* Special case for the ellipsis signifying more arguments. */ case '.': if(nc != '.') return make_token(SW_TOKEN_PERIOD, NULL); else if(nc == '.' && buffer[*pos+2] == '.') { *pos += 2; return make_token(SW_TOKEN_MORE_ARGS, NULL); } else { return make_token(SW_TOKEN_RANGE, NULL); } /* Special cases, when we find a ' or " then we will try to find the string literal and we will consider that to be a token instead of tokenizing the actual ' or ". */ case '\'': case '"': return make_token_string_literal(ch, buffer, pos); /* We will accept all other characters from here. We can assume that they will somehow be identifiers. In the parsing stage, if they are misplaced then the program will report an error. */ default: return make_token_identifier(buffer, pos); }}/* Allocate and return a token. If no value is needed for the token, as is the case for most tokens, then NULL will be supplied. The 'val' is useful for numbers, string literals, and identifiers. */SwToken *make_token(TOKEN token, char *val) {}/* This is for when we find a character and are looking to see if the next character is the same. If the next character is the same, then we have found the 'xx' token, otherwise we stick with the original guess of having found the 'x' token. */SwToken *make_token_cond(int cond, TOKEN x, TOKEN xx) { return make_token((cond ? x : xx), NULL);}/* Make a token for a string literal. This will work for single/double quotes and Python-like triple quoting. */SwToken *make_token_string_literal(char q, char *buffer, unsigned int *pos) { unsigned int i, started_at = 0, ended_at = 0; int is_triple = 0; /* We could be dealing with a triple-quotes literal or an empty string. */ if(q == peek_next_token(buffer, pos)) { *pos++; /* yes, this is triple-quotes */ if(q == peek_next_token(buffer, pos)) { is_triple = 1; *pos++; /* no, this is an empty string */ } else { ended_at = *pos+1; goto SwTokenMakeLiteral; } } /* start from the character *after* this quotation mark */ started_at = *pos+1; /* Go over the characters in the buffer and try to find the end of this string literal. */ for(i = started_at; (buffer[i] != '\0'); i++) { if(buffer[i] == q) { /* ignore this quote because the previous character negated it with a backslash */ if(buffer[i-1] == '\\') continue; /* If this is a triple, than this might be an everyday quote inside it which is allowed, otherwise it could be the ending one. Let's make sure. */ else { /* The closing of the triple-quote. We don't need to worry about going out-of-bounds given the assumption that buffer is a properly formatted C string. */ if(is_triple && (buffer[i+1] == q) && (buffer[i+2] == q)) { ended_at = i; *pos += i + 2; break; /* Not a triple, we have found the end of the end of the string. */ } else if(!is_triple) { ended_at = i; *pos += i; break; } } } } /* We now have the start and end position of the string. Let's allocate space for it and then make the token. */ SwTokenMakeLiteral: return make_token(SW_TOKEN_STRING_LITERAL, make_token_value(buffer, \ started_at, ended_at));}char *make_token_value(char *buffer, int started_at, int ended_at) { char *literal; size_t len; len = ended_at - started_at; if( (literal = (char *)malloc(sizeof(char) * (len + 1))) == NULL) FATAL_ERROR("Could not allocate memory for string literal."); /* Make the string. Copies the memory at the right start location by using pointer arithmetic. */ literal = (char *)memcpy(literal, buffer + started_at, len); literal[len] = '\0'; return literal;}/* Create a token for a text-only identifier. The identifier can have a lot in it but there are restrictions. This can find ints/floats, or normal identifiers. Note: we don't worry about numbers being negative. The parser can figure this out :) */SwToken *make_token_identifier(char *buffer, unsigned int *pos) { unsigned int i = *pos, started_at = i; char *val; TOKEN tk; /* number, could be a float, integer, etc. */ if(buffer[i] >= '0' || buffer[i] <= '9') { /* identifier of some sort, could be a built-in construct */ } else if(char_allowed_in_identifier(buffer[i])) { for(i = *pos; char_allowed_in_identifier(buffer[i]); i++); val = make_token_value(buffer, started_at, i); switch(val) { case "for": case "do": case "while": case "map": case "reduce": case "yield": case "resume": case "transfer": case "return": case "continue": case "break": case "is": tk = SW_TOKEN_IS; goto SwTokenMakeFreeValue; case "in": tk = SW_TOKEN_IN; goto SwTokenMakeFreeValue; case "true": tk = SW_TOKEN_BOOL_TRUE; goto SwTokenMakeFreeValue; case "false": tk = SW_TOKEN_BOOL_FALSE; goto SwTokenMakeFreeValue; default: tk = SW_TOKEN_IDENTIFIER; goto SwTokenMakeIdentifier; } SwTokenMakeFreeValue: val = (free(val), NULL); SwTokenMakeIdentifier: /* goto comes here, skip the freeing of the val if this isn't something that we recognize. */ return make_token(SW_TOKEN_STRING_LITERAL, val); /* unknown character, todo: error or act as comment? */ } else PARSE_ERROR(sprintf("Unknown character [%c] found.", buffer[i]));}int char_allowed_in_identifier(char ch) { return ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || \ (ch >= '0' && ch <= '9') || ch == '_' || ch == '$');}/* Peek at the next character in the buffer */char peek_next_token(char *buffer, unsigned int *pos) { return buffer[*pos+1];}