EvoTokenizers

Evo string tokenizers. More...

Classes
class	StrTok
	String forward tokenizer. More...
class	StrTokBase
	Base tokenizer class – see StrTok and StrTokR. More...
class	StrTokLine
	String line tokenizer. More...
class	StrTokR
	String reverse tokenizer. More...
class	StrTokRS
	String reverse tokenizer (strict). More...
class	StrTokS
	String forward tokenizer (strict). More...
struct	StrTokVariant< T, NextCh, NextAny >
	String tokenizer adapter used internally to create variants of existing tokenizers – do not use directly. More...

Macros
#define	EVO_TOK_NEXT_OR_BREAK(TOK, DELIM)   if (!TOK.next(DELIM)) break
	Helper for tokenizing using a break-loop. More...
#define	EVO_TOK_OR_BREAK(EXPR)   if (!EXPR) break
	Helper for tokenizing using a break-loop. More...

Typedefs
typedef StrTokVariant< StrTok,&StrTok::nextq,&StrTok::nextanyq >	StrTokQ
	String forward tokenizer based on StrTok with quoted token support. More...
typedef StrTokVariant< StrTokR,&StrTokR::nextq,&StrTokR::nextanyq >	StrTokQR
	String reverse tokenizer based on StrTokR with quoted token support. More...
typedef StrTokVariant< StrTok,&StrTok::nextw >	StrTokWord
	String forward word tokenizer based on StrTok. More...
typedef StrTokVariant< StrTokR,&StrTokR::nextw >	StrTokWordR
	String reverse word tokenizer based on StrTokR. More...
typedef StrTokVariant< StrTokRS,&StrTokRS::nextw >	StrTokWordRS
	String reverse word tokenizer based on StrTokRS (strict). More...
typedef StrTokVariant< StrTokS,&StrTokS::nextw >	StrTokWordS
	String forward word tokenizer based on StrTokS (strict). More...

Detailed Description

Evo string tokenizers.

Macro Definition Documentation

EVO_TOK_NEXT_OR_BREAK

#define EVO_TOK_NEXT_OR_BREAK	(		TOK,
			DELIM
	)		if (!TOK.next(DELIM)) break

Helper for tokenizing using a break-loop.

• A break-loop is a loop that always breaks at the end so doesn't actually loop, but also allows an early break to skip remaining code in the loop
• This calls TOK.next() and if it fails this does a break to stop current loop
• This makes tokenizing and validating fields more readable by condensing repetitive if-statements
• See also: EVO_TOK_OR_BREAK()

Example

These examples use assert() to represent field validation.

#include <evo/strtok.h>
using namespace evo;

int main() {
    StrTok tok("1,2,3");
    for (;;) {
        // Field 1
        EVO_TOK_NEXT_OR_BREAK(tok, ',');
        assert(tok.value() == "1");

        // Field 2
        EVO_TOK_NEXT_OR_BREAK(tok, ',');
        assert(tok.value() == "2");

        // Field 3
        EVO_TOK_NEXT_OR_BREAK(tok, ',');
        assert(tok.value() == "3");

        // Field 4
        EVO_TOK_NEXT_OR_BREAK(tok, ',');
        assert(false);

        break;
    }

    return 0;
}

When used with StrTokLine, leave the DELIM argument empty.

#include <evo/strtok.h>
using namespace evo;

int main() {
    StrTokLine tok("1\n2");
    for (;;) {
        // Line 1
        EVO_TOK_NEXT_OR_BREAK(tok,);
        assert(tok.value() == "1");

        // Line 2
        EVO_TOK_NEXT_OR_BREAK(tok,);
        assert(tok.value() == "2");

        break;
    }

    return 0;
}

EVO_TOK_OR_BREAK

#define EVO_TOK_OR_BREAK

(

EXPR

)

if (!EXPR) break

Helper for tokenizing using a break-loop.

• A break-loop is a loop that always breaks at the end so doesn't actually loop, but also allows an early break to skip remaining code in the loop
• This is similar to EVO_TOK_NEXT_OR_BREAK(), but allows using a more generic expression

Example

These examples use assert() to represent field validation.

#include <evo/strtok.h>
using namespace evo;

int main() {
    StrTok tok("1,'2a,2b',3");
    for (;;) {
        // Field 1
        EVO_TOK_OR_BREAK(tok.nextq(','));
        assert(tok.value() == "1");

        // Field 2
        EVO_TOK_OR_BREAK(tok.nextq(','));
        assert(tok.value() == "2a,2b");

        // Field 3
        EVO_TOK_OR_BREAK(tok.nextq(','));
        assert(tok.value() == "3");

        // Field 4
        EVO_TOK_OR_BREAK(tok.nextq(','));
        assert(false);

        break;
    }

    return 0;
}

Typedef Documentation

StrTokQ

typedef StrTokVariant<StrTok,&StrTok::nextq,&StrTok::nextanyq> StrTokQ

String forward tokenizer based on StrTok with quoted token support.

• This is the same as StrTok, except next() is overridden to behave as StrTok::nextq()

Example:

#include <evo/strtok.h>
#include <evo/io.h>
using namespace evo;

int main() {
    Console& c = con();
    SubString str = "one, \"two\", three";

    // Tokens:
    //  one
    //  two
    //  three
    StrTokQ tok(str);
    while (tok.next(','))
        c.out << tok.value();

    return 0;
}

StrTokQR

typedef StrTokVariant<StrTokR,&StrTokR::nextq,&StrTokR::nextanyq> StrTokQR

String reverse tokenizer based on StrTokR with quoted token support.

• This is the same as StrTokR, except next() is overridden to behave as StrTokR::nextq()

Example:

#include <evo/strtok.h>
#include <evo/io.h>
using namespace evo;

int main() {
    Console& c = con();
    SubString str = "one, \"two\", three";

    // Tokens:
    //  three
    //  two
    //  one
    StrTokQR tok(str);
    while (tok.next(','))
        c.out << tok.value();

    return 0;
}

StrTokWord

typedef StrTokVariant<StrTok,&StrTok::nextw> StrTokWord

String forward word tokenizer based on StrTok.

• This is the same as StrTok, except next() is overridden to behave as StrTok::nextw()
• This uses StrTokVariant, which may prevent inlining optimization on next() – for absolute best performance use StrTok and StrTok::nextw() directly

Example:

#include <evo/strtok.h>
#include <evo/io.h>
using namespace evo;

int main() {
    Console& c = con();
    SubString str = "one, two, three";

    // Tokens:
    //  one
    //  two
    //  three
    StrTokWord tok(str);
    while (tok.next(','))
        c.out << tok.value();

    return 0;
}

StrTokWordR

typedef StrTokVariant<StrTokR,&StrTokR::nextw> StrTokWordR

String reverse word tokenizer based on StrTokR.

• This is the same as StrTokR, except next() is overridden to behave as StrTokR::nextw()
• This uses StrTokVariant, which may prevent inlining optimization on next() – for absolute best performance use StrTokR and StrTokR::nextw() directly

Example:

#include <evo/strtok.h>
#include <evo/io.h>
using namespace evo;

int main() {
    Console& c = con();
    SubString str = "one, two, three";

    // Tokens:
    //  three
    //  two
    //  one
    StrTokWordR tok(str);
    while (tok.next(','))
        c.out << tok.value();

    return 0;
}

StrTokWordRS

typedef StrTokVariant<StrTokRS,&StrTokRS::nextw> StrTokWordRS

String reverse word tokenizer based on StrTokRS (strict).

• This is the same as StrTokRS, except next() is overridden to behave as StrTokRS::nextw()
• This uses StrTokVariant, which may prevent inlining optimization on next() – for absolute best performance use StrTokRS and StrTokRS::nextw() directly

Example:

#include <evo/strtok.h>
#include <evo/io.h>
using namespace evo;

int main() {
    Console& c = con();
    SubString str = "one,two,three";

    // Tokens:
    //  three
    //  two
    //  one
    StrTokWordRS tok(str);
    while (tok.next(','))
        c.out << tok.value();

    return 0;
}

StrTokWordS

typedef StrTokVariant<StrTokS,&StrTokS::nextw> StrTokWordS

String forward word tokenizer based on StrTokS (strict).

• This is the same as StrTokS, except next() is overridden to behave as StrTokS::nextw()
• This uses StrTokVariant, which may prevent inlining optimization on next() – for absolute best performance use StrTokS and StrTokS::nextw() directly

Example:

#include <evo/strtok.h>
#include <evo/io.h>
using namespace evo;

int main() {
    Console& c = con();
    SubString str = "one,two,three";

    // Tokens:
    //  one
    //  two
    //  three
    StrTokWordS tok(str);
    while (tok.next(','))
        c.out << tok.value();

    return 0;
}