Struct StatefulTokenizer

pub struct StatefulTokenizer<D> { /* private fields */ }

Implementations

impl<D: DictionaryAccess + Clone> StatefulTokenizer<D>

pub fn dict_clone(&self) -> D

Get a clone of current dictionary

impl<D: DictionaryAccess> StatefulTokenizer<D>

pub fn new(dic: D, mode: Mode) -> Self

Create a new non-debug stateful tokenizer

pub fn create(dic: D, debug: bool, mode: Mode) -> Self

Create a new debug stateful tokenizer with the following options

pub fn set_debug(&mut self, debug: bool) -> bool

Set debug flag and returns the current one

pub fn set_mode(&mut self, mode: Mode) -> Mode

Set the analysis mode and returns the current one

pub fn mode(&self) -> Mode

Return current analysis mode

pub fn set_subset(&mut self, subset: InfoSubset) -> InfoSubset

Analyzer will read only following [WordInfo] field subset

pub fn reset(&mut self) -> &mut String

Prepare StatefulTokenizer for the next data. Data must be written in the returned reference.

pub fn dict(&self) -> &D

Borrow current dictionary

pub fn do_tokenize(&mut self) -> SudachiResult<()>

Perform the actual tokenization so the analysis result will be available for consumption

pub fn swap_result( &mut self, input: &mut InputBuffer, result: &mut Vec<ResultNode>, subset: &mut InfoSubset, )

Swap result data with the current analyzer

pub fn into_morpheme_list(self) -> SudachiResult<MorphemeList<D>>

Consume the Tokenizer and produce MorphemeList