sudachi/dic/
dictionary.rs

1/*
2 *  Copyright (c) 2021-2026 Works Applications Co., Ltd.
3 *
4 *  Licensed under the Apache License, Version 2.0 (the "License");
5 *  you may not use this file except in compliance with the License.
6 *  You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 *   Unless required by applicable law or agreed to in writing, software
11 *  distributed under the License is distributed on an "AS IS" BASIS,
12 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 *  See the License for the specific language governing permissions and
14 *  limitations under the License.
15 */
16
17use std::fs::File;
18use std::path::Path;
19
20use memmap2::Mmap;
21
22use crate::analysis::morpheme::SingleMorpheme;
23use crate::config::Config;
24use crate::dic::binary_loader::BinaryDictionary;
25use crate::dic::character_category::CharacterCategory;
26use crate::dic::description::Description;
27use crate::dic::error::DictionaryCompatibilityError;
28use crate::dic::grammar::Grammar;
29use crate::dic::lexicon::Lexicon;
30use crate::dic::lexicon_set::LexiconSet;
31use crate::dic::storage::{Storage, SudachiDicData};
32use crate::dic::subset::InfoSubset;
33use crate::dic::{
34    lookup_all_entries, DescriptionAccess, DictionaryAccess, LexiconAccess, ReferenceIdAccess,
35};
36use crate::error::SudachiError;
37use crate::error::SudachiResult;
38use crate::plugin::input_text::InputTextPlugin;
39use crate::plugin::oov::OovProviderPlugin;
40use crate::plugin::path_rewrite::PathRewritePlugin;
41use crate::plugin::Plugins;
42
43// It is self-referential struct with 'static lifetime as a workaround
44// for the impossibility to specify the correct lifetime for
45// those fields. Accessor functions always provide the correct lifetime,
46// tied to the lifetime of the struct itself.
47// It is safe to move this structure around because the
48// pointers from memory mappings themselves are stable and
49// will not change if the structure will be moved around.
50// This structure is always read only after creation and is safe to share
51// between threads.
52pub struct JapaneseDictionary {
53    storage: SudachiDicData,
54    plugins: Plugins,
55    description: Description,
56    //'static is a a lie, lifetime is the same with StorageBackend
57    _grammar: Grammar<'static>,
58    //'static is a a lie, lifetime is the same with StorageBackend
59    _lexicon: LexiconSet<'static>,
60}
61
62fn map_file(path: &Path) -> SudachiResult<Storage> {
63    let file = File::open(path)?;
64    let mapping = unsafe { Mmap::map(&file) }?;
65    Ok(Storage::File(mapping))
66}
67
68fn load_system_dic(cfg: &Config) -> SudachiResult<Storage> {
69    let p = cfg.resolved_system_dict()?;
70    map_file(&p).map_err(|e| e.with_context(p.as_os_str().to_string_lossy()))
71}
72
73impl JapaneseDictionary {
74    /// Creates a dictionary from the specified configuration
75    /// Dictionaries will be read from disk
76    pub fn from_cfg(cfg: &Config) -> SudachiResult<JapaneseDictionary> {
77        let mut sb = SudachiDicData::new(load_system_dic(cfg)?);
78
79        for udic in cfg.resolved_user_dicts()? {
80            sb.add_user(
81                map_file(&udic).map_err(|e| e.with_context(udic.as_os_str().to_string_lossy()))?,
82            )
83        }
84
85        let chardef = CharacterCategory::from_bytes(
86            &cfg.resolve_resource(&cfg.character_definition_file)?
87                .read_bytes()?,
88        )?;
89
90        Self::from_cfg_storage_chardef(cfg, sb, chardef)
91    }
92
93    /// Creates a dictionary from the specified configuration and storage
94    pub fn from_cfg_storage(
95        cfg: &Config,
96        storage: SudachiDicData,
97    ) -> SudachiResult<JapaneseDictionary> {
98        let chardef = CharacterCategory::from_bytes(
99            &cfg.resolve_resource(&cfg.character_definition_file)?
100                .read_bytes()?,
101        )?;
102        Self::from_cfg_storage_chardef(cfg, storage, chardef)
103    }
104
105    #[deprecated(
106        since = "0.7.0",
107        note = "embedded resources are now resolved through Config; use from_cfg_storage instead"
108    )]
109    /// Creates a dictionary from the specified configuration and storage, with embedded character definition
110    pub fn from_cfg_storage_with_embedded_chardef(
111        cfg: &Config,
112        storage: SudachiDicData,
113    ) -> SudachiResult<JapaneseDictionary> {
114        let chardef = CharacterCategory::from_embedded();
115        Self::from_cfg_storage_chardef(cfg, storage, chardef)
116    }
117
118    pub fn from_cfg_storage_chardef(
119        cfg: &Config,
120        storage: SudachiDicData,
121        chardef: CharacterCategory,
122    ) -> SudachiResult<JapaneseDictionary> {
123        let system_binary =
124            BinaryDictionary::load_system(unsafe { storage.system_static_slice() })?;
125        let system_signature = system_binary.compatibility_key().to_owned();
126        let description = system_binary.description.clone();
127
128        let mut grammar = Grammar::from_system_binary(system_binary.grammar)?;
129        grammar.set_character_category(chardef);
130
131        let lexicon_set =
132            LexiconSet::from_system_binary(system_binary.lexicon, grammar.pos_list.len());
133
134        let plugins = { Plugins::load(cfg, &mut grammar)? };
135        if plugins.oov.is_empty() {
136            return Err(SudachiError::NoOOVPluginProvided);
137        }
138        for p in plugins.connect_cost.plugins() {
139            p.edit(&mut grammar);
140        }
141
142        let mut dic = JapaneseDictionary {
143            storage,
144            plugins,
145            description,
146            _grammar: grammar,
147            _lexicon: lexicon_set,
148        };
149
150        // this Vec is needed to prevent double borrowing of dic
151        let user_dicts: Vec<_> = dic.storage.user_static_slice();
152        for (user_index, udic) in user_dicts.into_iter().enumerate() {
153            let user_dict = BinaryDictionary::load_user(udic)?;
154            if user_dict.compatibility_key() != system_signature {
155                return Err(DictionaryCompatibilityError::UserDictionary {
156                    user_index,
157                    system_signature: system_signature.clone(),
158                    user_reference: user_dict.compatibility_key().to_owned(),
159                }
160                .into());
161            }
162            dic = dic.merge_user_dictionary(user_dict)?;
163        }
164
165        Ok(dic)
166    }
167
168    /// Returns grammar with the correct lifetime
169    pub fn grammar(&self) -> &Grammar<'_> {
170        &self._grammar
171    }
172
173    /// Returns lexicon with the correct lifetime
174    pub fn lexicon(&self) -> &LexiconSet<'_> {
175        &self._lexicon
176    }
177
178    pub fn description(&self) -> &Description {
179        &self.description
180    }
181
182    /// Iterates over dictionary entries as standalone morphemes.
183    ///
184    /// This corresponds to public lexicon CSV rows. It includes entries that
185    /// are referred to from other entries, such as split or constituent units,
186    /// even when they are not indexed for normal lookup. Internal entries
187    /// automatically generated for literal normalized forms are not exposed.
188    /// The iteration order is not part of the public contract.
189    pub fn entries(&self) -> impl Iterator<Item = SudachiResult<SingleMorpheme<&Self>>> + '_ {
190        self.entries_subset(InfoSubset::all())
191    }
192
193    /// Iterates over dictionary entries, loading only the requested word-info fields.
194    pub fn entries_subset(
195        &self,
196        subset: InfoSubset,
197    ) -> impl Iterator<Item = SudachiResult<SingleMorpheme<&Self>>> + '_ {
198        self.lexicon()
199            .word_ids()
200            .map(move |word_id| SingleMorpheme::from_word_id(self, word_id?, subset))
201    }
202
203    /// Looks up all dictionary entries whose normalized surface matches `surface`.
204    ///
205    /// This normalizes the query using dictionary input-text plugins and scans
206    /// every public lexicon entry. It can find entries that are not indexed for
207    /// normal lookup. Use `lookup` for normal indexed lookup.
208    pub fn lookup_all_entries(&self, surface: &str) -> SudachiResult<Vec<SingleMorpheme<&Self>>> {
209        self.lookup_all_entries_subset(surface, InfoSubset::all())
210    }
211
212    /// Looks up all matching dictionary entries, loading only requested fields.
213    pub fn lookup_all_entries_subset(
214        &self,
215        surface: &str,
216        subset: InfoSubset,
217    ) -> SudachiResult<Vec<SingleMorpheme<&Self>>> {
218        lookup_all_entries(self, surface, subset)
219    }
220
221    /// Creates an out-of-vocabulary standalone morpheme from the pos id and the surface.
222    ///
223    /// Uses the surface for reading, normalized, and dictionary forms.
224    pub fn oov_morpheme(&self, pos_id: u16, surface: &str) -> SudachiResult<SingleMorpheme<&Self>> {
225        self.oov_morpheme_with_forms(pos_id, surface, surface, surface, surface)
226    }
227
228    /// Creates an out-of-vocabulary standalone morpheme from the pos id and string forms.
229    pub fn oov_morpheme_with_forms(
230        &self,
231        pos_id: u16,
232        surface: &str,
233        reading: &str,
234        normalized_form: &str,
235        dictionary_form: &str,
236    ) -> SudachiResult<SingleMorpheme<&Self>> {
237        SingleMorpheme::oov(
238            self,
239            pos_id,
240            surface.to_owned(),
241            reading.to_owned(),
242            normalized_form.to_owned(),
243            dictionary_form.to_owned(),
244        )
245    }
246
247    fn merge_user_dictionary(
248        mut self,
249        user_dict: BinaryDictionary<'static>,
250    ) -> SudachiResult<Self> {
251        // we need to update lexicon first, since it needs the current number of pos
252        let mut user_lexicon = Lexicon::from_binary(user_dict.lexicon);
253        user_lexicon.update_cost(&self)?;
254        self._lexicon
255            .append(user_lexicon, self._grammar.pos_list.len())?;
256
257        self._grammar.merge_binary(user_dict.grammar);
258
259        Ok(self)
260    }
261}
262
263impl LexiconAccess for JapaneseDictionary {
264    fn lexicon(&self) -> &LexiconSet<'_> {
265        self.lexicon()
266    }
267}
268
269impl DictionaryAccess for JapaneseDictionary {
270    fn grammar(&self) -> &Grammar<'_> {
271        self.grammar()
272    }
273
274    fn input_text_plugins(&self) -> &[Box<dyn InputTextPlugin + Sync + Send>] {
275        self.plugins.input_text.plugins()
276    }
277
278    fn oov_provider_plugins(&self) -> &[Box<dyn OovProviderPlugin + Sync + Send>] {
279        self.plugins.oov.plugins()
280    }
281
282    fn path_rewrite_plugins(&self) -> &[Box<dyn PathRewritePlugin + Sync + Send>] {
283        self.plugins.path_rewrite.plugins()
284    }
285}
286
287impl DescriptionAccess for JapaneseDictionary {
288    fn description(&self) -> &Description {
289        &self.description
290    }
291}
292
293impl ReferenceIdAccess for JapaneseDictionary {
294    fn reference_ids(&self) -> std::collections::HashMap<u32, String> {
295        BinaryDictionary::load_system(unsafe { self.storage.system_static_slice() })
296            .and_then(|dict| dict.reference_id_table())
297            .unwrap_or_default()
298    }
299}
sudachi/dic/dictionary.rs

sudachi/dic/
dictionary.rs
