while (tok.next())
    {
      bool bad_token = false;
        tokenizer::token tk = tok.current_token();

        std::vector<tokenizer::token>::iterator it   = tokens.begin();
        std::vector<tokenizer::token>::iterator last = tokens.end();

        for (;it != last; ++it)
        {
            if (std::distance(it->last, it->first) == std::distance(tk.last, tk.first) && 
                std::string(it->first, it->last) == std::string(tk.first, tk.last))
            {
                tok.set_str(str.erase(tk.first, tk.last), str.end());
                bad_token = true;
                break;
            }
        }

        if (!bad_token)
            tokens.push_back(tok.current_token());
    }