diff --git a/include/boost/token_functions.hpp b/include/boost/token_functions.hpp
index e6b8bef5..4effb90e 100644
--- a/include/boost/token_functions.hpp
+++ b/include/boost/token_functions.hpp
@@ -40,6 +40,7 @@
 #include <boost/assert.hpp>
 #include <boost/type_traits/is_pointer.hpp>
 #include <boost/detail/workaround.hpp>
+#include <boost/move/utility_core.hpp>
 #include <boost/mpl/if.hpp>
 #include <boost/throw_exception.hpp>
 #if !defined(BOOST_NO_CWCTYPE)
@@ -88,7 +89,8 @@ namespace boost{
   // character (backslash \), can be assigned to other characters.
 
   struct escaped_list_error : public std::runtime_error{
-    escaped_list_error(const std::string& what_arg):std::runtime_error(what_arg) { }
+    explicit escaped_list_error(const char* what_arg) BOOST_NOEXCEPT_OR_NOTHROW
+      : std::runtime_error(what_arg) { }
   };
 
 
@@ -127,7 +129,7 @@ namespace boost{
     template <typename iterator, typename Token>
     void do_escape(iterator& next,iterator end,Token& tok) {
       if (++next == end)
-        BOOST_THROW_EXCEPTION(escaped_list_error(std::string("cannot end with escape")));
+        BOOST_THROW_EXCEPTION(escaped_list_error("cannot end with escape"));
       if (Traits::eq(*next,'n')) {
         tok+='\n';
         return;
@@ -145,7 +147,7 @@ namespace boost{
         return;
       }
       else
-        BOOST_THROW_EXCEPTION(escaped_list_error(std::string("unknown escape sequence")));
+        BOOST_THROW_EXCEPTION(escaped_list_error("unknown escape sequence"));
     }
 
     public:
@@ -155,7 +157,7 @@ namespace boost{
       : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { }
 
     escaped_list_separator(string_type e, string_type c, string_type q)
-      : escape_(e), c_(c), quote_(q), last_(false) { }
+      : escape_(boost::move(e)), c_(boost::move(c)), quote_(boost::move(q)), last_(false) { }
 
     void reset() {last_=false;}
 
@@ -216,11 +218,11 @@ namespace boost{
   template<typename traits, int N>
   struct traits_extension_details : public traits {
     typedef typename traits::char_type char_type;
-    static bool isspace(char_type c)
+    static bool isspace(char_type c) BOOST_NOEXCEPT_OR_NOTHROW
     {
        return std::iswspace(c) != 0;
     }
-    static bool ispunct(char_type c)
+    static bool ispunct(char_type c) BOOST_NOEXCEPT_OR_NOTHROW
     {
        return std::iswpunct(c) != 0;
     }
@@ -229,11 +231,11 @@ namespace boost{
   template<typename traits>
   struct traits_extension_details<traits, 1> : public traits {
     typedef typename traits::char_type char_type;
-    static bool isspace(char_type c)
+    static bool isspace(char_type c) BOOST_NOEXCEPT_OR_NOTHROW
     {
        return std::isspace(c) != 0;
     }
-    static bool ispunct(char_type c)
+    static bool ispunct(char_type c) BOOST_NOEXCEPT_OR_NOTHROW
     {
        return std::ispunct(c) != 0;
     }
@@ -246,7 +248,7 @@ namespace boost{
   template<typename traits>
   struct traits_extension : public traits {
     typedef typename traits::char_type char_type;
-    static bool isspace(char_type c)
+    static bool isspace(char_type c) BOOST_NOEXCEPT_OR_NOTHROW
     {
 #if !defined(BOOST_NO_CWCTYPE)
       return traits_extension_details<traits, sizeof(char_type)>::isspace(c);
@@ -255,7 +257,7 @@ namespace boost{
 #endif
     }
 
-    static bool ispunct(char_type c)
+    static bool ispunct(char_type c) BOOST_NOEXCEPT_OR_NOTHROW
     {
 #if !defined(BOOST_NO_CWCTYPE)
       return traits_extension_details<traits, sizeof(char_type)>::ispunct(c);
@@ -284,19 +286,19 @@ namespace boost{
     }
 
     template<class Token, class Value>
-    static void plus_equal(Token &, const Value &) { }
+    static void plus_equal(Token &, const Value &) BOOST_NOEXCEPT_OR_NOTHROW { }
 
     // If we are doing an assign, there is no need for the
     // the clear.
     //
     template<class Token>
-    static void clear(Token &) { }
+    static void clear(Token &) BOOST_NOEXCEPT_OR_NOTHROW { }
   };
 
   template <>
   struct assign_or_plus_equal<std::input_iterator_tag> {
     template<class Iterator, class Token>
-    static void assign(Iterator , Iterator , Token &) { }
+    static void assign(Iterator , Iterator , Token &) BOOST_NOEXCEPT_OR_NOTHROW { }
     template<class Token, class Value>
     static void plus_equal(Token &t, const Value &v) {
       t += v;
@@ -357,7 +359,7 @@ namespace boost{
         return_partial_last_(return_partial_last) { }
 
     offset_separator()
-      : offsets_(1,1), current_offset_(),
+      : offsets_(1,1), current_offset_(0),
         wrap_offsets_(true), return_partial_last_(true) { }
 
     void reset() {
@@ -435,16 +437,13 @@ namespace boost{
     char_separator(const Char* dropped_delims,
                    const Char* kept_delims = 0,
                    empty_token_policy empty_tokens = drop_empty_tokens)
-      : m_dropped_delims(dropped_delims),
+      : m_kept_delims(kept_delims, (kept_delims ? Traits::length(kept_delims) : 0)),
+        m_dropped_delims(dropped_delims),
         m_use_ispunct(false),
         m_use_isspace(false),
         m_empty_tokens(empty_tokens),
         m_output_done(false)
-    {
-      // Borland workaround
-      if (kept_delims)
-        m_kept_delims = kept_delims;
-    }
+    { }
 
                 // use ispunct() for kept delimiters and isspace for dropped.
     explicit
@@ -536,7 +535,7 @@ namespace boost{
     empty_token_policy m_empty_tokens;
     bool m_output_done;
 
-    bool is_kept(Char E) const
+    bool is_kept(Char E) const BOOST_NOEXCEPT_OR_NOTHROW
     {
       if (m_kept_delims.length())
         return m_kept_delims.find(E) != string_type::npos;
@@ -545,7 +544,7 @@ namespace boost{
       } else
         return false;
     }
-    bool is_dropped(Char E) const
+    bool is_dropped(Char E) const BOOST_NOEXCEPT_OR_NOTHROW
     {
       if (m_dropped_delims.length())
         return m_dropped_delims.find(E) != string_type::npos;
@@ -609,8 +608,8 @@ namespace boost{
     explicit char_delimiters_separator(bool return_delims = false,
                                        const Char* returnable = 0,
                                        const Char* nonreturnable = 0)
-      : returnable_(returnable ? returnable : string_type().c_str()),
-        nonreturnable_(nonreturnable ? nonreturnable:string_type().c_str()),
+      : returnable_(returnable, (returnable ? Traits::length(returnable) : 0)),
+        nonreturnable_(nonreturnable, (returnable ? Traits::length(nonreturnable) : 0)),
         return_delims_(return_delims), no_ispunct_(returnable!=0),
         no_isspace_(nonreturnable!=0) { }
 
@@ -633,7 +632,7 @@ namespace boost{
 
      // if we are to return delims and we are one a returnable one
      // move past it and stop
-     if (is_ret(*next) && return_delims_) {
+     if (is_ret(*next)) {
        tok+=*next;
        ++next;
      }
diff --git a/include/boost/token_iterator.hpp b/include/boost/token_iterator.hpp
index 42945d7e..98e7ccb2 100644
--- a/include/boost/token_iterator.hpp
+++ b/include/boost/token_iterator.hpp
@@ -68,7 +68,6 @@ namespace boost
       }
 
       void initialize(){
-          if(valid_) return;
           f_.reset();
           valid_ = (begin_ != end_)?
               f_(begin_,end_,tok_):false;
diff --git a/test/examples.cpp b/test/examples.cpp
index 1e0b69b8..84de7714 100644
--- a/test/examples.cpp
+++ b/test/examples.cpp
@@ -8,15 +8,45 @@
 
 // See http://www.boost.org for updates, documentation, and revision history.
 
+#include <cstring>
 #include <iostream>
 #include <iterator>
+#include <stdexcept>
 #include <string>
 #include <algorithm>
+
+#if defined(BOOST_ENABLE_ASSERT_HANDLER)
+# error "already defined BOOST_ENABLE_ASSERT_HANDLER somewhere"
+#else
+# define BOOST_ENABLE_ASSERT_HANDLER
+#endif
+
 #include <boost/tokenizer.hpp>
 #include <boost/array.hpp>
+#include <boost/range/algorithm/equal.hpp>
+#include <boost/range/begin.hpp>
+#include <boost/range/end.hpp>
 
 #include <boost/test/minimal.hpp>
 
+#define PP_REQUIRE_EXCEPTION(statement, exception_type, error_message) \
+  do {                                                                 \
+    bool caught_exception__ = false;                                   \
+    try {                                                              \
+      { statement }                                                    \
+    } catch (exception_type& e) {                                      \
+      BOOST_REQUIRE(std::strcmp(e.what(), error_message) == 0);        \
+      caught_exception__ = true;                                       \
+    }                                                                  \
+    BOOST_REQUIRE(caught_exception__);                                 \
+  } while (false)
+
+namespace boost {
+  void assertion_failed(char const* expr, char const *function, char const* file, long line) {
+    BOOST_THROW_EXCEPTION(std::runtime_error("assertion failed"));
+  }
+} // namespace boost
+
 int test_main( int /*argc*/, char* /*argv*/[] )
 {
   using namespace boost;
@@ -137,12 +167,313 @@ int test_main( int /*argc*/, char* /*argv*/[] )
     BOOST_REQUIRE(*other=="a");
   }
 
+  // Test token_iterator
+  {
+    // compare equality of two iterators: valid & valid (same target string)
+    {
+      const std::string test_string = "abc";
+      typedef char_separator<char> separator_type;
+      typedef token_iterator_generator<separator_type>::type token_iterator_type;
+      const std::string::const_iterator first = boost::begin(test_string);
+      const std::string::const_iterator last = boost::end(test_string);
+      const separator_type separator;
+      const token_iterator_type a = make_token_iterator<std::string>(first, last, separator);
+      const token_iterator_type b = make_token_iterator<std::string>(first, last, separator);
+      BOOST_REQUIRE(a == b);
+    }
+
+    // compare equality of two iterators: valid & valid (same but partial target string)
+    {
+      const std::string test_string = "abc";
+      typedef char_separator<char> separator_type;
+      typedef token_iterator_generator<separator_type>::type token_iterator_type;
+      const std::string::const_iterator first = boost::begin(test_string);
+      std::string::const_iterator last = boost::end(test_string);
+      const token_iterator_type a = make_token_iterator<std::string>(first, last, separator_type());
+      const token_iterator_type b = make_token_iterator<std::string>(first, --last, separator_type());
+      BOOST_REQUIRE(a != b);
+    }
+
+    // compare equality of two iterators: valid & valid (same but partial target string)
+    {
+      const std::string test_string = "abc,def";
+      typedef char_delimiters_separator<char> separator_type;
+      typedef token_iterator_generator<separator_type>::type token_iterator_type;
+      const std::string::const_iterator first = boost::begin(test_string);
+      std::string::const_iterator last = boost::end(test_string);
+      const token_iterator_type a = make_token_iterator<std::string>(first, last, separator_type());
+      const token_iterator_type b = make_token_iterator<std::string>(first, --last, separator_type());
+      BOOST_REQUIRE(a != b);
+    }
+
+    // compare equality of two iterators: invalid & valid
+    {
+      const std::string empty_string = "";
+      const std::string non_empty_string = "abc";
+      typedef char_delimiters_separator<char> separator_type;
+      typedef token_iterator_generator<separator_type>::type token_iterator_type;
+      const token_iterator_type a = make_token_iterator<std::string>(boost::begin(empty_string), boost::end(empty_string), separator_type());
+      const token_iterator_type b = make_token_iterator<std::string>(boost::begin(non_empty_string), boost::end(non_empty_string), separator_type());
+      BOOST_REQUIRE(a != b);
+    }
+  }
+
+  // Test escaped_list_separator
+  {
+    // input contains "\n"
+    {
+      const std::string test_string = "\\n";
+      const std::string answer[] = {"\n"};
+      tokenizer<escaped_list_separator<char> > tokenizer(test_string);
+      BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+    }
+
+    // input ends with escape
+    {
+      const std::string test_string = "\\";
+      tokenizer<escaped_list_separator<char> > t(test_string);
+
+      PP_REQUIRE_EXCEPTION(
+        { std::distance(boost::begin(t), boost::end(t)); },
+        boost::escaped_list_error,
+        "cannot end with escape"
+      );
+    }
+
+    // input contains unknown escape sequence
+    {
+      const std::string test_string = "\\q";
+      tokenizer<escaped_list_separator<char> > t(test_string);
+
+      PP_REQUIRE_EXCEPTION(
+        { std::distance(boost::begin(t), boost::end(t)); },
+        boost::escaped_list_error,
+        "unknown escape sequence"
+      );
+    }
+  }
+
+  // Test default constructed offset_separator
+  {
+    const std::string test_string = "1234567";
+    const std::string answer[] = {"1", "2", "3", "4", "5", "6", "7"};
+    typedef offset_separator separator_type;
+    tokenizer<separator_type> tokenizer(test_string);
+    BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+  }
+
+  // Test non-default constructed offset_separator
+  {
+    const std::string test_string = "1234567";
+
+    // empty offsets, wrap_offsets = false, return_partial_last = false
+    {
+      const int offset = 0;
+      typedef offset_separator separator_type;
+      offset_separator separator(&offset, &offset, false, false);
+      typedef token_iterator_generator<separator_type>::type token_iterator_type;
+
+      PP_REQUIRE_EXCEPTION(
+        { make_token_iterator<std::string>(boost::begin(test_string), boost::end(test_string), separator); },
+        std::runtime_error,
+        "assertion failed"
+      );
+    }
+
+    // wrap_offsets = false, return_partial_last = false
+    {
+      const std::string answer[] = {"1", "234"};
+
+      boost::array<int,3> offsets = {{1, 3, 5}};
+      typedef offset_separator separator_type;
+      offset_separator separator(offsets.begin(), offsets.end(), false, false);
+      tokenizer<separator_type> tokenizer(test_string, separator);
+      BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+    }
+
+    // wrap_offsets = false, return_partial_last = true
+    {
+      {
+        const std::string answer[] = {"1", "234", "567"};
+
+        boost::array<int,3> offsets = {{1, 3, 5}};
+        typedef offset_separator separator_type;
+        offset_separator separator(offsets.begin(), offsets.end(), false, true);
+        tokenizer<separator_type> tokenizer(test_string, separator);
+        BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+      }
+
+      {
+        const std::string answer[] = {"12345"};
+
+        boost::array<int,1> offsets = {{5}};
+        typedef offset_separator separator_type;
+        offset_separator separator(offsets.begin(), offsets.end(), false, true);
+        tokenizer<separator_type> tokenizer(test_string, separator);
+        BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+      }
+    }
+
+    // wrap_offsets = true, return_partial_last = false
+    {
+      const std::string answer[] = {"1", "234"};
+
+      boost::array<int,3> offsets = {{1, 3, 5}};
+      typedef offset_separator separator_type;
+      offset_separator separator(offsets.begin(), offsets.end(), true, false);
+      tokenizer<separator_type> tokenizer(test_string, separator);
+      BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+    }
+  }
+
+  // Test default constructed char_separator
+  {
+    const std::string test_string = ";Hello|world-";
+    const std::string answer[] = {";", "Hello", "|", "world", "-"};
+    tokenizer<char_separator<char> > tokenizer(test_string);
+    BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+  }
+
+  std::vector<int> a;
+  a.assign(10, 1);
+  // Test non-default contstructed char_separator
+  {
+    const std::string test_string = ";Hello||world-";
+
+    // dropped_delims = non-null, kept_delims = null, empty_tokens = drop_empty_tokens
+    {
+      const std::string answer[] = {"Hello||world"};
+      typedef char_separator<char> separator_type;
+      separator_type separator("-;", 0, boost::drop_empty_tokens);
+      tokenizer<separator_type> tokenizer(test_string, separator);
+      BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+    }
+
+    // dropped_delims = non-null, kept_delims = null, empty_tokens = keep_empty_tokens
+    {
+      const std::string answer[] = {"", "Hello", "", "world", ""};
+      typedef char_separator<char> separator_type;
+      separator_type separator("-;|", 0, boost::keep_empty_tokens);
+      tokenizer<separator_type> tokenizer(test_string, separator);
+      BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+    }
+
+    // dropped_delims = non-null, kept_delims = non-null, empty_tokens = drop_empty_tokens
+    {
+      const std::string answer[] = {"Hello", "|", "|", "world"};
+      typedef char_separator<char> separator_type;
+      separator_type separator("-;", "|", boost::drop_empty_tokens);
+      tokenizer<separator_type> tokenizer(test_string, separator);
+      BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+    }
+
+    // dropped_delims = non-null, kept_delims = non-null, empty_tokens = keep_empty_tokens
+    {
+      const std::string answer[] = {"", "Hello", "|", "", "|", "world", ""};
+      typedef char_separator<char> separator_type;
+      separator_type separator("-;", "|", boost::keep_empty_tokens);
+      tokenizer<separator_type> tokenizer(test_string, separator);
+      BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+    }
+
+    // dropped_delims = non-null (empty), kept_delims = null, empty_tokens = drop_empty_tokens
+    {
+      const std::string answer[] = {";Hello||world-"};
+      typedef char_separator<char> separator_type;
+      separator_type separator("", 0, boost::keep_empty_tokens);
+      tokenizer<separator_type> tokenizer(test_string, separator);
+      BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+    }
+  }
+
   // Test non-default constructed char_delimiters_separator
   {
-    const std::string test_string = "how,are you, doing";
-    std::string answer[] = {"how",",","are you",","," doing"};
-    tokenizer<> t(test_string,char_delimiters_separator<char>(true,",",""));
-    BOOST_REQUIRE(std::equal(t.begin(),t.end(),answer));
+    const std::string test_string = "how,are you, doing?";
+
+    // return_delims = true, returnable = non-null, nonreturnable = non-null
+    {
+      const std::string answer[] = {"how",",","are you",","," doing"};
+      tokenizer<> tokenizer(test_string,char_delimiters_separator<char>(true,",","?"));
+      BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+    }
+
+    // return_delims = true, returnable = non-null, nonreturnable = non-null (empty)
+    {
+      const std::string answer[] = {"how",",","are you",","," doing?"};
+      tokenizer<> tokenizer(test_string,char_delimiters_separator<char>(true,",",""));
+      BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+    }
+
+    // return_delims = true, returnable = non-null (empty), nonreturnable = non-null
+    {
+      const std::string answer[] = {"how,are you, doing"};
+      tokenizer<> tokenizer(test_string,char_delimiters_separator<char>(true,"","?"));
+      BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+    }
+
+    // return_delims = true, returnable = non-null (empty), nonreturnable = non-null (empty)
+    {
+      const std::string answer[] = {"how,are you, doing?"};
+      tokenizer<> tokenizer(test_string,char_delimiters_separator<char>(true,"",""));
+      BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+    }
+
+    // return_delims = false, returnable = non-null, nonreturnable = non-null
+    {
+      const std::string answer[] = {"how","are you"," doing"};
+      tokenizer<> tokenizer(test_string,char_delimiters_separator<char>(false,",","?"));
+      BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+    }
+
+    // return_delims = false, returnable = non-null, nonreturnable = non-null (empty)
+    {
+      const std::string answer[] = {"how","are you"," doing?"};
+      tokenizer<> tokenizer(test_string,char_delimiters_separator<char>(false,",",""));
+      BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+    }
+
+    // return_delims = false, returnable = non-null (empty), nonreturnable = non-null
+    {
+      const std::string answer[] = {"how,are you, doing"};
+      tokenizer<> tokenizer(test_string,char_delimiters_separator<char>(false,"","?"));
+      BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+    }
+
+    // return_delims = false, returnable = non-null (empty), nonreturnable = non-null (emppty)
+    {
+      const std::string answer[] = {"how,are you, doing?"};
+      tokenizer<> tokenizer(test_string,char_delimiters_separator<char>(false,"",""));
+      BOOST_REQUIRE(boost::range::equal(tokenizer, answer));
+    }
+  }
+
+  // Test iterator operations
+  {
+    const std::string test_string;
+
+    {
+      // increment invalid iterator
+      typedef tokenizer<> tokenizer_type;
+      tokenizer_type tokenizer(test_string);
+      tokenizer_type::iterator first = boost::begin(tokenizer);
+      PP_REQUIRE_EXCEPTION(
+        { std::advance(first, 1); },
+        std::runtime_error,
+        "assertion failed"
+      );
+    }
+
+    {
+      // dereference invalid iterator
+      typedef tokenizer<> tokenizer_type;
+      tokenizer_type tokenizer(test_string);
+      const tokenizer_type::iterator first = boost::begin(tokenizer);
+      PP_REQUIRE_EXCEPTION(
+        { *first; },
+        std::runtime_error,
+        "assertion failed"
+      );
+    }
   }
 
   return 0;