Unnamed

In tokenizer.hpp:

#ifndef TOKENIZER_HPP
#define TOKENIZER_HPP

#include <string>
#include <vector>
#include <iostream>

namespace tokenizer {
  typedef std::string token;
  typedef std::vector<token> tokens;
  
  tokens tokenize(std::istream&);
  ...
}

#endif

In tokenizer.cpp:

#include <cctype>
#include <cassert>
#include <iostream>
#include <algorithm>

#include "util.hpp"
#include "tokenizer.hpp"


tokens tokenizer::tokenize(std::istream& is) {
  tokens toks;
  std::string line;
  while (is.getline(line)) {
    std::string::iterator start, stop, end;
    end = line.end();
    start = stop = line.begin();
    ...
  }
}