docs: string_token exposition

fix #845

doc/modules/ROOT/examples/unit/snippets.cpp

@@ -1575,6 +1575,57 @@ normalizing()
     }
 }
 
+void
+decode_with_token()
+{
+    {
+        // tag::snippet_string_token_1[]
+        url_view u("http://www.example.com/my%20file.txt");
+        pct_string_view sv = u.encoded_path();
+        assert(sv == "/my%20file.txt");
+        std::string s = u.path();
+        assert(s == "/my file.txt");
+        // end::snippet_string_token_1[]
+    }
+
+    {
+        // tag::snippet_string_token_2[]
+        url_view u("http://www.example.com/my%20file.txt");
+        std::string s = "existing string";
+        u.path(string_token::assign_to(s));
+        assert(s == "/my file.txt");
+        // end::snippet_string_token_2[]
+    }
+
+    {
+        // tag::snippet_string_token_3[]
+        url_view u("http://www.example.com/my%20file.txt");
+        std::string s = "existing string";
+        u.path(string_token::append_to(s));
+        assert(s == "existing string/my file.txt");
+        // end::snippet_string_token_3[]
+    }
+
+    {
+        // tag::snippet_string_token_4[]
+        url_view u("http://www.example.com/my%20file.txt");
+        std::string s = "existing string";
+        boost::core::string_view sv = u.path(string_token::preserve_size(s));
+        assert(sv == "/my file.txt");
+        // end::snippet_string_token_4[]
+    }
+
+    {
+#if defined(__cpp_static_assert) && __cpp_static_assert >= 201411L
+        // tag::snippet_string_token_5[]
+        static_assert(
+            string_token::is_token<string_token::return_string>::value);
+        // end::snippet_string_token_5[]
+#endif
+    }
+}
+
+
 void
 encoding()
 {
@@ -1792,6 +1843,7 @@ class snippets_test
         ignore_unused(&grammar_customization);
         ignore_unused(&modifying_path);
         normalizing();
+        decode_with_token();
         encoding();
         ignore_unused(&readme_snippets);
 

doc/modules/ROOT/pages/urls/stringtoken.adoc

@@ -11,15 +11,56 @@
 
 Functions which perform percent-decoding return values using cpp:std::string[] when called without special arguments.
 This is the best default for ergonomics, and a good enough default for performance considering that many decoded strings fit in the small buffer available to most standard implementations.
+
+[source,cpp]
+----
+include::example$unit/snippets.cpp[tag=snippet_string_token_1,indent=0]
+----
+
 Some use-cases may desire more control over how these algorithms acquire and store data in strings, for example:
 
 * Returning a string using a non-default allocator
 * Reusing the storage of an existing string
 * Appending to existing strings
 
 The library provides a special customization mechanism called
-__StringToken__ to control how algorithms which require an output buffer acquire their storage.
-The signature
+__StringToken__ to control how algorithms which require an output
+buffer acquire their storage.
+
+The __StringToken__ `string_token::assign_to` can be used to assign
+the decoded value to an existing string:
+
+[source,cpp]
+----
+include::example$unit/snippets.cpp[tag=snippet_string_token_2,indent=0]
+----
+
+The __StringToken__ `string_token::append_to` can be used to append
+the decoded value to an existing string:
+
+[source,cpp]
+----
+include::example$unit/snippets.cpp[tag=snippet_string_token_3,indent=0]
+----
+
+The __StringToken__ `string_token::preserve_size` can be used to
+return a `string_view` instead of a cpp:std::string[].
+The underlying storage for the `string_view` is
+provided to the token.
+
+[source,cpp]
+----
+include::example$unit/snippets.cpp[tag=snippet_string_token_4,indent=0]
+----
+
+When no customization is provided, the default behavior
+is to use the default `string_token::return_string` token which returns a cpp:std::string[].
 
+The trait `string_token::is_token` can be used to determine if a type
+is a __StringToken__:
 
+[source,cpp]
+----
+include::example$unit/snippets.cpp[tag=snippet_string_token_5,indent=0]
+----
 

doc/qbk/0.main.qbk

@@ -58,7 +58,7 @@
 [def __append_to__              [link url.ref.boost__urls__string_token__append_to `string_token::append_to`]]
 [def __assign_to__              [link url.ref.boost__urls__string_token__assign_to `string_token::assign_to`]]
 [def __preserve_size__          [link url.ref.boost__urls__string_token__preserve_size `string_token::preserve_size`]]
-[def __append_to__              [link url.ref.boost__urls__string_token__return_string `string_token::return_string`]]
+[def __return_string__          [link url.ref.boost__urls__string_token__return_string `string_token::return_string`]]
 
 
 [def __std_format__             [@https://en.cppreference.com/w/cpp/utility/format/format `std::format`]]

doc/qbk/3.6.stringtoken.qbk

@@ -14,6 +14,9 @@ __std_string__ when called without special arguments. This is
 the best default for ergonomics, and a good enough default for
 performance considering that many decoded strings fit in
 the small buffer available to most standard implementations.
+
+[snippet_string_token_1]
+
 Some use-cases may desire more control over how these
 algorithms acquire and store data in strings, for example:
 
@@ -23,7 +26,32 @@ algorithms acquire and store data in strings, for example:
 
 The library provides a special customization mechanism called
 __StringToken__ to control how algorithms which require an output
-buffer acquire their storage. The signature
+buffer acquire their storage.
+
+The __StringToken__ __assign_to__ can be used to assign
+the decoded value to an existing string:
+
+[snippet_string_token_2]
+
+The __StringToken__ __append_to__ can be used to append
+the decoded value to an existing string:
+
+[snippet_string_token_3]
+
+The __StringToken__ __preserve_size__ can be used to
+return a __string_view__ instead of a __std_string__.
+The underlying storage for the __string_view__ is
+provided to the token.
+
+[snippet_string_token_4]
+
+When no customization is provided, the default behavior
+is to use the default __return_string__ token which
+returns a __std_string__.
+
+The trait __is_token__ can be used to determine if a type
+is a __StringToken__:
 
+[snippet_string_token_5]
 
 [endsect]

test/unit/snippets.cpp

@@ -1576,6 +1576,56 @@ normalizing()
     }
 }
 
+void
+decode_with_token()
+{
+    {
+        //[snippet_string_token_1
+        url_view u("http://www.example.com/my%20file.txt");
+        pct_string_view sv = u.encoded_path();
+        assert(sv == "/my%20file.txt");
+        std::string s = u.path();
+        assert(s == "/my file.txt");
+        //]
+    }
+
+    {
+        //[snippet_string_token_2
+        url_view u("http://www.example.com/my%20file.txt");
+        std::string s = "existing string";
+        u.path(string_token::assign_to(s));
+        assert(s == "/my file.txt");
+        //]
+    }
+
+    {
+        //[snippet_string_token_3
+        url_view u("http://www.example.com/my%20file.txt");
+        std::string s = "existing string";
+        u.path(string_token::append_to(s));
+        assert(s == "existing string/my file.txt");
+        //]
+    }
+
+    {
+        //[snippet_string_token_4
+        url_view u("http://www.example.com/my%20file.txt");
+        std::string s = "existing string";
+        boost::core::string_view sv = u.path(string_token::preserve_size(s));
+        assert(sv == "/my file.txt");
+        //]
+    }
+
+    {
+#if defined(__cpp_static_assert) && __cpp_static_assert >= 201411L
+        //[snippet_string_token_5
+        static_assert(
+            string_token::is_token<string_token::return_string>::value);
+        //]
+#endif
+    }
+}
+
 void
 encoding()
 {
@@ -1793,6 +1843,7 @@ class snippets_test
         ignore_unused(&grammar_customization);
         ignore_unused(&modifying_path);
         normalizing();
+        decode_with_token();
         encoding();
         ignore_unused(&readme_snippets);