tokenizer.64.c File Reference

tokenizer interface implementation. More...

#include <tokenizer.h>

Classes
struct	tokenizer_iterator_ctx_t

Typedefs
typedef struct tokenizer_iterator_ctx_t	tokenizer_iterator_ctx_t

Functions
	MODULE ("turnstone.lib")
int8_t	tokenizer_destroy (iterator_t *iter)
int8_t	tokenizer_end_of_iterator (iterator_t *iter)
iterator_t *	tokenizer_next (iterator_t *iter)
const void *	tokenizer_get_item (iterator_t *iter)
const void *	tokenizer_get_extra_data (iterator_t *iter)
boolean_t	tokenizer_is_in_list (iterator_t *iter, uint8_t c, const token_delimiter_type_t *list, token_delimiter_type_t *type)
iterator_t *	tokenizer_new (buffer_t *buf, const token_delimiter_type_t *delimeters, const token_delimiter_type_t *whitespaces)

Variables
const uint8_t	token_delimeter_chars []

Detailed Description

tokenizer interface implementation.

This work is licensed under TURNSTONE OS Public License. Please read and understand latest version of Licence.

Variable Documentation

token_delimeter_chars

const uint8_t token_delimeter_chars[]

Initial value:

= {
    NULL, '\n', '\r', ' ', '\t', ',', ';', ':',
    '.', '\'', '"', '+', '-', '*', '/', '\\',
    '%', '|', '(', ')', '[', ']', '{', '}'
}