k4st

#include <stdlib.h>#include <stdio.h>#include "../tokenizer.h"int main() {		char *stream = "identifier (arg1, arg2, ...) => do: while:;;";	SwToken *token;	SwTokenBuffer *buffer;			printf("\n\nTesting Swerve tokenizer...\n");	printf("---------------\n");	printf("buffer to parse: %s\n\n", stream);	printf("symbol\t\tvalue\n");	printf("------\t\t-----\n");		buffer = init_tokenizer(stream);			while((token = next_token(buffer)) != NULL) {						printf("%d\t\t%s\n", token->symbol, (token->value == NULL ? "" : token->value));	}		disable_tokenizer(buffer);		return 0;}/* output:Testing Swerve tokenizer...---------------buffer to parse: identifier (arg1, arg2, ...) => do: while:;;symbol          value------          -----9               identifier439               arg19               arg2234489               do9               while1818*/