Struct Lexicon 

pub struct Lexicon<'a> { /* private fields */ }

Dictionary lexicon

Contains trie, word_id, word_param, word_info

Implementations

impl<'a> Lexicon<'a>

pub fn from_binary(binary_lexicon: BinaryLexicon<'a>) -> Self

pub fn size(&self) -> u32

Returns the number of entries in the lexicon

pub fn entry_ids_in_order(&self) -> Vec<EntryId>

pub fn set_dic_id(&mut self, id: u8)

Assign lexicon id to the current Lexicon

pub fn lookup( &'a self, input: &'a [u8], offset: usize, ) -> impl Iterator<Item = LexiconEntry> + 'a

Returns an iterator of word_id and end of words that matches given input

pub fn lookup_batch<F: FnMut(usize, LexiconEntry)>( &self, input: &[u8], starts: &[usize], emit: F, )

Pipelined + prefetched batch form of Lexicon::lookup.

Calls emit(bucket, entry) for every match; bucket indexes starts. Within a bucket the order matches Lexicon::lookup, so grouping by bucket reproduces the scalar result.

pub fn get_word_info( &self, entry_id: EntryId, subset: InfoSubset, ) -> SudachiResult<WordInfoRefData>

Returns WordInfo for given word_id

WordInfo will contain only fields included in InfoSubset

pub fn get_word_param(&self, entry_id: EntryId) -> (i16, i16, i16)

Returns word_param for given word_id. Params are (left_id, right_id, cost).

pub fn get_word_param_checked( &self, entry_id: EntryId, ) -> Option<(i16, i16, i16)>

pub fn get_string(&self, strptr: StringPointer) -> SudachiResult<String>

pub fn update_cost<D: DictionaryAccess>( &mut self, dict: &D, ) -> SudachiResult<()>

update word_param cost based on current tokenizer