Trait Tokenize

pub trait Tokenize {
    type Dictionary;

    // Required method
    fn tokenize(
        &self,
        input: &str,
        mode: Mode,
        enable_debug: bool,
    ) -> SudachiResult<MorphemeList<Self::Dictionary>>;
}

Able to tokenize Japanese text

Required Associated Types

type Dictionary

Required Methods

fn tokenize( &self, input: &str, mode: Mode, enable_debug: bool, ) -> SudachiResult<MorphemeList<Self::Dictionary>>

Break text into Morphemes

Implementors

impl<T> Tokenize for StatelessTokenizer<T>

where T: DictionaryAccess + Clone,

type Dictionary = T