tokenizer.hpp

Go to the documentation of this file.

// Copyright (c) 2020-2025 Cody R. Drisko. All rights reserved.
// Licensed under the MIT License. See the LICENSE file in the project root for more information.
//
// Name: tokenizer.hpp
// Author: crdrisko
// Date: 11/04/2020-07:04:33
// Description: A class dedicated to splitting strings (tokens) on selected delimiters
 
#ifndef DRYCHEM_COMMON_UTILITIES_INCLUDE_COMMON_UTILS_STRINGS_UTILS_TOKENIZER_HPP
#define DRYCHEM_COMMON_UTILITIES_INCLUDE_COMMON_UTILS_STRINGS_UTILS_TOKENIZER_HPP
 
#include <algorithm>
#include <forward_list>
#include <iterator>
#include <optional>
#include <string>
#include <type_traits>
#include <vector>
 
#include "common-utils/meta/traits/containerTraits.hpp"
#include "common-utils/strings/utils/lexicalCast.hpp"
#include "common-utils/strings/utils/stringUtils.hpp"
 
namespace CppUtils::Strings
{
    namespace details
    {
        template<typename, typename = std::void_t<>>
        struct has_mapped_type : std::false_type
        {
        };

        template<typename T>
        struct has_mapped_type<T, std::void_t<typename T::mapped_type>> : std::true_type
        {
        };
    }   // namespace details

    template<typename CharTraits>
    class Tokenizer
    {
    public:
        using value_type      = std::basic_string<char, CharTraits>;
        using reference       = std::basic_string<char, CharTraits>&;
        using const_reference = const std::basic_string<char, CharTraits>&;
        using iterator        = typename std::basic_string<char, CharTraits>::iterator;
        using const_iterator  = typename std::basic_string<char, CharTraits>::const_iterator;
 
    private:
        const_iterator str_current;
        const_iterator str_end;
        value_type delimiters;
        std::optional<value_type> keepDelimiters;

        constexpr bool isDelimiter(const char ch_) const noexcept { return foundSubstr(ch_, delimiters); }

        constexpr value_type nextToken()
        {
            value_type str {};
 
            if (str_current != str_end)
            {
                while (str_current != str_end && isDelimiter(*str_current))
                {
                    if (keepDelimiters && foundSubstr(*str_current, keepDelimiters.value()))
                        return value_type {*str_current++};
 
                    ++str_current;
                }
 
                while (str_current != str_end && !isDelimiter(*str_current))
                    str += value_type {*str_current++};
            }
 
            return str;
        }
 
    public:
        constexpr Tokenizer(const_iterator begin_, const_iterator end_, const_reference delimiters_ = " \t\n",
            std::optional<value_type> keepDelimiters_ = std::nullopt)
                : str_current {begin_}, str_end {end_}, delimiters {delimiters_}, keepDelimiters {keepDelimiters_} {}

        constexpr explicit Tokenizer(const_reference str_, const_reference delimiters_ = " \t\n",
            std::optional<value_type> keepDelimiters_ = std::nullopt)
                : str_current {str_.begin()}, str_end {str_.end()}, delimiters {delimiters_}, keepDelimiters {keepDelimiters_} {}

        template<typename Container = std::vector<value_type>, typename T = typename Container::value_type,
                 typename = std::enable_if_t<std::conjunction_v<Meta::is_allocator_aware_container<Container>,
                                                                std::negation<details::has_mapped_type<Container>>>>>
        constexpr Container split()
        {
            Container tokens;
 
            if (str_current != str_end)
            {
                if constexpr (std::is_same_v<Container, std::forward_list<T>>)
                {
                    std::front_insert_iterator<Container> tokensIter {tokens};
 
                    std::for_each(str_current, str_end, [&](const auto) {
                        auto str = this->nextToken();
 
                        if (!str.empty())
                            *tokensIter++ = lexical_cast<T>(str);
                    });
                }
                else
                {
                    std::insert_iterator<Container> tokensIter {tokens, tokens.begin()};
 
                    std::for_each(str_current, str_end, [&](const auto) {
                        value_type str = this->nextToken();
 
                        if (!str.empty())
                            *tokensIter++ = lexical_cast<T>(str);
                    });
                }
            }
 
            return tokens;
        }
    };

    template<typename CharTraits, typename... TArgs>
    Tokenizer(const std::basic_string<char, CharTraits>&, TArgs...) -> Tokenizer<CharTraits>;
}   // namespace CppUtils::Strings
 
#endif