tokenizer.inl

Go to the documentation of this file.

//==================================================================================================
/// \file
/// This header-file is part of module \alib_strings of the \aliblong.
///
/// \emoji :copyright: 2013-2025 A-Worx GmbH, Germany.
/// Published under \ref mainpage_license "Boost Software License".
//==================================================================================================
ALIB_EXPORT namespace alib {  namespace strings {
 
/// This sub-namespace  provides some utility classes which are related
/// to string classes found in namespace \ref alib::strings.
namespace util  {
 
//==================================================================================================
/// This class operates on strings which contains data separated by a delimiter character.
/// It identifies the substrings between the delimiters as \e tokens of type
/// \alib{strings;TSubstring;Substring}. After an instance of this class is constructed,
/// three methods are available:
/// - #HasNext: Indicates if there are further tokens available.
/// - #Next: Sets field #Actual (which is of type \b Substring) to reference the next token and
///   returns it.<br>
///   With each call to %Next, a different delimiter can be provided, which then serves as the
///   delimiter for this and subsequent tokens.<br>
///   The returned token by default will be trimmed according to the current trimable characters.
/// - #Rest:
///   Like #Next, however returns the complete remaining region without searching for
///   further delimiters (and tokens).<br>
///   After this method was invoked, #HasNext() will return \c false.
///
/// After a token was retrieved, it might be modified using the interface of class
/// \alib{strings;TSubstring;Substring} as the tokenizer does not rely on the bounds of
/// the current token when receiving the next. Furthermore, even field #Rest is allowed
/// to be changed using the interface of \b %Substring if it seems appropriate. The effect is the
/// same as if method #Set was invoked to apply a different source string.
///
/// Objects of this class can be reused by freshly initializing them using method #Set.
///
/// <b>Sample code</b>:<br>
/// The following code sample shows how to tokenize a string:
///
/// \snippet "DOX_TOKENIZER.cpp"     DOX_TOKENIZER
///
/// The output will be:
///
/// \verbinclude "DOX_TOKENIZER.txt"
///
/// @tparam TChar    The character type. Implementations for \c nchar and \c wchar are provided
///                  with type definitions \ref alib::TokenizerN and
///                  \ref alib::TokenizerW.
//==================================================================================================
template<typename TChar>
class TTokenizer
{
    // #############################################################################################
    // Public fields
    // #############################################################################################
    public:
        ///  A \alib{strings;TSubstring;Substring} that represents the part of
        ///  the underlying data that has not been tokenized, yet.
        ///  It is allowed to manipulate this public field, which has a similar effect as
        ///  using method #Set.<br>
        TSubstring<TChar>               Rest;
 
        ///  The actual token, which is returned with every invocation of #Next() or #Rest().
        ///  It is allowed to manipulate this field any time.<br>
        TSubstring<TChar>               Actual;
 
        /// The white spaces characters used to trim the tokens.
        /// Defaults to  \ref alib::DEFAULT_WHITESPACES
        TLocalString<TChar, 8>          TrimChars;
 
 
    // #############################################################################################
    // Internal fields
    // #############################################################################################
    protected:
        /// The most recently set delimiter used by default for the next token extraction.
        TChar           delim;
 
        /// If \c true, empty tokens are omitted.
        bool            skipEmpty;
 
 
    // #############################################################################################
    //  Constructors/Destructor
    // #############################################################################################
    public:
        //==========================================================================================
        /// Constructs an empty tokenizer. To initialize, method #Set needs to be invoked.
        //==========================================================================================
        TTokenizer()
        {}
 
        //==========================================================================================
        /// Constructs a tokenizer to work on a given string.
        ///
        /// @param  src             The string to be tokenized.
        /// @param  delimiter       The delimiter that separates the tokens. Can be changed with
        ///                         every next token.
        /// @param  skipEmptyTokens If \c true, empty tokens are omitted.
        ///                         Optional and defaults to \c false.
        //==========================================================================================
        TTokenizer( const TString<TChar>& src, TChar delimiter, bool skipEmptyTokens= false )
        : Rest  (src)
        , Actual(nullptr)
        , TrimChars( CStringConstantsTraits<TChar>::DefaultWhitespaces() )
        , delim(delimiter)
        , skipEmpty(skipEmptyTokens)
        {}
 
    // #############################################################################################
    //  Interface
    // #############################################################################################
    public:
        //==========================================================================================
        /// Resets a tokenizer to work on a given string.
        ///
        /// @param  src             The string to be tokenized
        /// @param  delimiter       The delimiter that separates the tokens. Can be changed with
        ///                         every next token.
        /// @param  skipEmptyTokens If \c true, empty tokens are omitted.
        ///                         Optional and defaults to \c false.
        //==========================================================================================
        void Set( const TString<TChar>& src, TChar delimiter, bool skipEmptyTokens= false )
        {
            Actual          = nullptr;
            Rest            = src;
            this->delim     = delimiter;
            this->skipEmpty = skipEmptyTokens;
        }
 
        //==========================================================================================
        /// Returns the next token, which is afterwards also available through field #Actual.
        /// If no further token  was available, the returned
        /// \alib{strings;TSubstring;Substring} will be \e nulled.
        /// (see \alib{strings;TString::IsNull;String::IsNull}).
        /// To prevent this, the availability of a next token should be
        /// checked using method #HasNext().
        ///
        /// For clarification, see the explanation and sample code in this classes documentation.
        ///
        ///  @param trimming  Determines if the token is trimmed in respect to the white space
        ///                   characters defined in field #TrimChars.
        ///                   Defaults to \b Whitespaces.Trim.
        ///  @param newDelim  The delimiter separates the tokens. Defaults to 0, which keeps the
        ///                   current delimiter intact.
        ///                   A new delimiter can be provided for every next token.
        /// @return The next token as \b %Substring. A nulled string is if no next token was
        ///         available.
        //==========================================================================================
        ALIB_DLL
        TSubstring<TChar>&  Next( lang::Whitespaces trimming= lang::Whitespaces::Trim,
                                  TChar             newDelim= '\0' );
 
        //==========================================================================================
        /// Returns the currently remaining string (without searching for further delimiter
        /// characters).
        /// After this call #HasNext will return \c false and #Next will return a \e nulled
        /// Substring.
        ///  @param trimming  Determines if the token is trimmed in respect to the white space
        ///                   characters defined in field #TrimChars.
        ///                   Defaults to \b Whitespaces.Trim.
        /// @return The rest of the original source string, which was not returned by #Next(), yet.
        //==========================================================================================
        TSubstring<TChar>&  GetRest( lang::Whitespaces trimming= lang::Whitespaces::Trim )
        {
            // set start, end and end of tokenizer
            Actual=  Rest;
            Rest  =  nullptr;
            if ( trimming == lang::Whitespaces::Trim )
                Actual.Trim( TrimChars );
            return Actual;
        }
 
        //==========================================================================================
        /// If this returns \c true, a call to #Next will be successful and will return a
        /// \b Substring which is not \e nulled.
        /// @return \c true if a next token is available.
        //==========================================================================================
        bool        HasNext()
        {
            return Rest.IsNotNull() && ( !skipEmpty || Rest.IsNotEmpty() );
        }
 
}; // class Tokenizer
 
 
extern template ALIB_DLL TSubstring<nchar>& TTokenizer<nchar>::Next( lang::Whitespaces, nchar );
extern template ALIB_DLL TSubstring<wchar>& TTokenizer<wchar>::Next( lang::Whitespaces, wchar );
 
}} // namespace alib[::strings::util]
 
/// Type alias in namespace \b alib.
using     Tokenizer=     strings::util::TTokenizer<character>;
 
/// Type alias in namespace \b alib.
using     TokenizerN=    strings::util::TTokenizer<nchar>;
 
/// Type alias in namespace \b alib.
using     TokenizerW=    strings::util::TTokenizer<wchar>;
 
 
 
} // namespace [alib]