CppUtils::Strings::Tokenizer< CharTraits > Class Template Reference

#include <tokenizer.hpp>

Public Types
using	value_type = std::basic_string<char, CharTraits>
	Member types.
using	reference = std::basic_string<char, CharTraits>&
using	const_reference = const std::basic_string<char, CharTraits>&
using	iterator = typename std::basic_string<char, CharTraits>::iterator
using	const_iterator = typename std::basic_string<char, CharTraits>::const_iterator

Public Member Functions
constexpr	Tokenizer (const_iterator begin_, const_iterator end_, const_reference delimiters_=" \t\n", std::optional< value_type > keepDelimiters_=std::nullopt)
constexpr	Tokenizer (const_reference str_, const_reference delimiters_=" \t\n", std::optional< value_type > keepDelimiters_=std::nullopt)
template<typename Container = std::vector<value_type>, typename T = typename Container::value_type, typename = std::enable_if_t<std::conjunction_v<Meta::is_allocator_aware_container<Container>, std::negation<details::has_mapped_type<Container>>>>>
constexpr Container	split ()

Private Member Functions
constexpr bool	isDelimiter (const char ch_) const noexcept
constexpr value_type	nextToken ()

Private Attributes
const_iterator	str_current
const_iterator	str_end
value_type	delimiters
std::optional< value_type >	keepDelimiters

Detailed Description

template<typename CharTraits>
class CppUtils::Strings::Tokenizer< CharTraits >

A class template which separates a given string into "tokens", which fall between specified delimiters.

Template Parameters

CharTraits

- The character traits used when comparing strings

Note

We can make use of Class Template Argument Deduction (CTAD) in most cases with the provided deduction guides.

Member Typedef Documentation

value_type

template<typename CharTraits>

using CppUtils::Strings::Tokenizer< CharTraits >::value_type = std::basic_string<char, CharTraits>

reference

template<typename CharTraits>

using CppUtils::Strings::Tokenizer< CharTraits >::reference = std::basic_string<char, CharTraits>&

const_reference

template<typename CharTraits>

using CppUtils::Strings::Tokenizer< CharTraits >::const_reference = const std::basic_string<char, CharTraits>&

iterator

template<typename CharTraits>

using CppUtils::Strings::Tokenizer< CharTraits >::iterator = typename std::basic_string<char, CharTraits>::iterator

const_iterator

template<typename CharTraits>

using CppUtils::Strings::Tokenizer< CharTraits >::const_iterator = typename std::basic_string<char, CharTraits>::const_iterator

Constructor & Destructor Documentation

Tokenizer() [1/2]

template<typename CharTraits>

CppUtils::Strings::Tokenizer< CharTraits >::Tokenizer ( const_iterator begin_, const_iterator end_, const_reference delimiters_ = " \t\n", std::optional< value_type > keepDelimiters_ = std::nullopt )

inlineconstexpr

A constructor allowing for the user to pass a range to iterate through.

Parameters

begin_	- The start of our sequence to iterate over
end_	- The end of our sequence to iterate over
delimiters_	- The delimiters we are using as our splitting criterion
keepDelimiters_	- The delimiters to include as tokens (optional)

References delimiters, keepDelimiters, str_current, and str_end.

Tokenizer() [2/2]

template<typename CharTraits>

CppUtils::Strings::Tokenizer< CharTraits >::Tokenizer ( const_reference str_, const_reference delimiters_ = " \t\n", std::optional< value_type > keepDelimiters_ = std::nullopt )

inlineexplicitconstexpr

Delegating constructor taking a full string instead of just a range.

Parameters

str_	- The full string we will be splitting
delimiters_	- The delimiters we are using as our splitting criterion
keepDelimiters_	- The delimiters to include as tokens (optional)

References delimiters, keepDelimiters, str_current, and str_end.

Member Function Documentation

isDelimiter()

template<typename CharTraits>

bool CppUtils::Strings::Tokenizer< CharTraits >::isDelimiter ( const char ch_ ) const

inlineconstexprprivatenoexcept

A private helper function that is essentially just a wrapper for foundSubstr().

Parameters

ch	- The character we are evaluating from the list of delimiters

References delimiters, and CppUtils::Strings::foundSubstr().

Referenced by nextToken().

nextToken()

template<typename CharTraits>

value_type CppUtils::Strings::Tokenizer< CharTraits >::nextToken ( )

inlineconstexprprivate

A private helper function which iterates through the remaining string looking for the next token.

Returns

The next token we are looking for.

References CppUtils::Strings::foundSubstr(), isDelimiter(), keepDelimiters, str_current, and str_end.

Referenced by split().

split()

template<typename CharTraits>

template<typename Container = std::vector<value_type>, typename T = typename Container::value_type, typename = std::enable_if_t<std::conjunction_v<Meta::is_allocator_aware_container<Container>, std::negation<details::has_mapped_type<Container>>>>>

Container CppUtils::Strings::Tokenizer< CharTraits >::split ( )

inlineconstexpr

The main function we call to split our input string into separate tokens.

Template Parameters

Container	- The container type we wish to return
T	- The type of the value we will cast our tokens into

Returns

A container with the separated tokens cast to whatever type we specified

Note

A std::forward_list<> requires special treatment as it can only call insert_front

References CppUtils::Strings::lexical_cast(), nextToken(), str_current, and str_end.

Referenced by CppUtils::Files::AsColumns::operator()().

Member Data Documentation

str_current

template<typename CharTraits>

const_iterator CppUtils::Strings::Tokenizer< CharTraits >::str_current

private

Referenced by nextToken(), split(), Tokenizer(), and Tokenizer().

str_end

template<typename CharTraits>

const_iterator CppUtils::Strings::Tokenizer< CharTraits >::str_end

private

Referenced by nextToken(), split(), Tokenizer(), and Tokenizer().

delimiters

template<typename CharTraits>

value_type CppUtils::Strings::Tokenizer< CharTraits >::delimiters

private

Referenced by isDelimiter(), Tokenizer(), and Tokenizer().

keepDelimiters

template<typename CharTraits>

std::optional<value_type> CppUtils::Strings::Tokenizer< CharTraits >::keepDelimiters

private

Referenced by nextToken(), Tokenizer(), and Tokenizer().

---

The documentation for this class was generated from the following file:

• common-utilities/include/common-utils/strings/utils/tokenizer.hpp