sudachi/plugin/oov/
mecab_oov.rs

1/*
2 * Copyright (c) 2021-2026 Works Applications Co., Ltd.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17use crate::analysis::created::CreatedWords;
18use crate::util::user_pos::{UserPosMode, UserPosSupport};
19use serde::Deserialize;
20use serde_json::Value;
21use std::collections::HashMap;
22use std::io::BufRead;
23use std::path::PathBuf;
24
25use crate::analysis::Node;
26use crate::config::{Config, DEFAULT_CHAR_DEF_FILE, DEFAULT_UNK_DEF_FILE};
27use crate::dic::category_type::CategoryType;
28use crate::dic::character_category::Error as CharacterCategoryError;
29use crate::dic::grammar::Grammar;
30use crate::dic::word_id::WordId;
31use crate::hash::RoMu;
32use crate::input_text::InputBuffer;
33use crate::input_text::InputTextIndex;
34use crate::plugin::oov::OovProviderPlugin;
35use crate::plugin::PluginError;
36use crate::prelude::*;
37
38#[cfg(test)]
39mod test;
40
41/// provides MeCab oov nodes
42#[derive(Default)]
43pub struct MeCabOovPlugin {
44    categories: HashMap<CategoryType, CategoryInfo, RoMu>,
45    oov_list: HashMap<CategoryType, Vec<Oov>, RoMu>,
46}
47
48/// Struct corresponds with raw config json file.
49#[allow(non_snake_case)]
50#[derive(Deserialize)]
51struct PluginSettings {
52    charDef: Option<PathBuf>,
53    unkDef: Option<PathBuf>,
54    #[serde(default)]
55    userPOS: UserPosMode,
56}
57
58impl MeCabOovPlugin {
59    /// Loads character category definition
60    ///
61    /// See resources/char.def for the syntax
62    fn read_character_property<T: BufRead>(
63        reader: T,
64    ) -> SudachiResult<HashMap<CategoryType, CategoryInfo, RoMu>> {
65        let mut categories = HashMap::with_hasher(RoMu::new());
66        for (i, line) in reader.lines().enumerate() {
67            let line = line?;
68            let line = line.trim();
69            if line.is_empty()
70                || line.starts_with('#')
71                || line.chars().take(2).collect::<Vec<_>>() == vec!['0', 'x']
72            {
73                continue;
74            }
75
76            let cols: Vec<_> = line.split_whitespace().collect();
77            if cols.len() < 4 {
78                return Err(SudachiError::InvalidCharacterCategory(
79                    CharacterCategoryError::InvalidFormat(i),
80                ));
81            }
82            let category_type: CategoryType = match cols[0].parse() {
83                Ok(t) => t,
84                Err(_) => {
85                    return Err(SudachiError::InvalidCharacterCategory(
86                        CharacterCategoryError::InvalidCategoryType(i, cols[0].to_string()),
87                    ))
88                }
89            };
90            if categories.contains_key(&category_type) {
91                return Err(SudachiError::InvalidCharacterCategory(
92                    CharacterCategoryError::MultipleTypeDefinition(i, cols[0].to_string()),
93                ));
94            }
95
96            categories.insert(
97                category_type,
98                CategoryInfo {
99                    category_type,
100                    is_invoke: cols[1] == "1",
101                    is_group: cols[2] == "1",
102                    length: cols[3].parse()?,
103                },
104            );
105        }
106
107        Ok(categories)
108    }
109
110    /// Load OOV definition
111    ///
112    /// Each line contains: CategoryType, left_id, right_id, cost, and pos
113    fn read_oov<T: BufRead>(
114        reader: T,
115        categories: &HashMap<CategoryType, CategoryInfo, RoMu>,
116        mut grammar: &mut Grammar,
117        user_pos: UserPosMode,
118    ) -> SudachiResult<HashMap<CategoryType, Vec<Oov>, RoMu>> {
119        let mut oov_list: HashMap<CategoryType, Vec<Oov>, RoMu> = HashMap::with_hasher(RoMu::new());
120        for (i, line) in reader.lines().enumerate() {
121            let line = line?;
122            let line = line.trim();
123            if line.is_empty() || line.starts_with('#') {
124                continue;
125            }
126
127            let cols: Vec<_> = line.split(',').collect();
128            if cols.len() < 10 {
129                return Err(SudachiError::PluginError(
130                    PluginError::InvalidDataFormatWithLine {
131                        line: i,
132                        message: format!("Invalid number of columns ({})", line),
133                    },
134                ));
135            }
136            let category_type: CategoryType = cols[0].parse()?;
137            if !categories.contains_key(&category_type) {
138                return Err(SudachiError::PluginError(
139                    PluginError::InvalidDataFormatWithLine {
140                        line: i,
141                        message: format!("{} is undefined in char definition", cols[0]),
142                    },
143                ));
144            }
145
146            let oov = Oov {
147                left_id: cols[1].parse()?,
148                right_id: cols[2].parse()?,
149                cost: cols[3].parse()?,
150                pos_id: grammar.handle_user_pos(&cols[4..10], user_pos)?,
151            };
152
153            if oov.left_id as usize > grammar.conn_matrix().num_left() {
154                return Err(SudachiError::PluginError(
155                    PluginError::InvalidDataFormatWithLine {
156                        line: i,
157                        message: format!(
158                            "max grammar left_id is {}, was {}",
159                            grammar.conn_matrix().num_left(),
160                            oov.left_id
161                        ),
162                    },
163                ));
164            }
165
166            if oov.right_id as usize > grammar.conn_matrix().num_right() {
167                return Err(SudachiError::PluginError(
168                    PluginError::InvalidDataFormatWithLine {
169                        line: i,
170                        message: format!(
171                            "max grammar right_id is {}, was {}",
172                            grammar.conn_matrix().num_right(),
173                            oov.right_id
174                        ),
175                    },
176                ));
177            }
178
179            match oov_list.get_mut(&category_type) {
180                None => {
181                    oov_list.insert(category_type, vec![oov]);
182                }
183                Some(l) => {
184                    l.push(oov);
185                }
186            };
187        }
188
189        Ok(oov_list)
190    }
191
192    /// Creates a new oov node
193    fn get_oov_node(&self, oov: &Oov, start: usize, end: usize) -> Node {
194        Node::new(
195            start as u16,
196            end as u16,
197            oov.left_id as u16,
198            oov.right_id as u16,
199            oov.cost,
200            WordId::oov(oov.pos_id as u32),
201        )
202    }
203
204    fn provide_oov_gen<T: InputTextIndex>(
205        &self,
206        input: &T,
207        offset: usize,
208        other_words: CreatedWords,
209        nodes: &mut Vec<Node>,
210    ) -> SudachiResult<usize> {
211        let char_len = input.cat_continuous_len(offset);
212        if char_len == 0 {
213            return Ok(0);
214        }
215        let mut num_created = 0;
216
217        for ctype in input.cat_at_char(offset).iter() {
218            let cinfo = match self.categories.get(&ctype) {
219                Some(ci) => ci,
220                None => continue,
221            };
222
223            if !cinfo.is_invoke && other_words.not_empty() {
224                continue;
225            }
226
227            let mut llength = char_len;
228            let oovs = match self.oov_list.get(&cinfo.category_type) {
229                Some(v) => v,
230                None => continue,
231            };
232
233            if cinfo.is_group {
234                for oov in oovs {
235                    nodes.push(self.get_oov_node(oov, offset, offset + char_len));
236                    num_created += 1;
237                }
238                llength -= 1;
239            }
240            for i in 1..=cinfo.length {
241                let sublength = input.char_distance(offset, i as usize);
242                if sublength > llength {
243                    break;
244                }
245                for oov in oovs {
246                    nodes.push(self.get_oov_node(oov, offset, offset + sublength));
247                    num_created += 1;
248                }
249            }
250        }
251        Ok(num_created)
252    }
253}
254
255impl OovProviderPlugin for MeCabOovPlugin {
256    fn set_up(
257        &mut self,
258        settings: &Value,
259        config: &Config,
260        grammar: &mut Grammar,
261    ) -> SudachiResult<()> {
262        let settings: PluginSettings =
263            serde_json::from_value(settings.clone()).map_err(PluginError::from)?;
264
265        let char_def = config.resolve_resource(
266            settings
267                .charDef
268                .unwrap_or_else(|| PathBuf::from(DEFAULT_CHAR_DEF_FILE)),
269        )?;
270        let categories = MeCabOovPlugin::read_character_property(char_def.reader()?)?;
271
272        let unk_def = config.resolve_resource(
273            settings
274                .unkDef
275                .unwrap_or_else(|| PathBuf::from(DEFAULT_UNK_DEF_FILE)),
276        )?;
277        let oov_list =
278            MeCabOovPlugin::read_oov(unk_def.reader()?, &categories, grammar, settings.userPOS)?;
279
280        self.categories = categories;
281        self.oov_list = oov_list;
282
283        Ok(())
284    }
285
286    fn provide_oov(
287        &self,
288        input_text: &InputBuffer,
289        offset: usize,
290        other_words: CreatedWords,
291        result: &mut Vec<Node>,
292    ) -> SudachiResult<usize> {
293        self.provide_oov_gen(input_text, offset, other_words, result)
294    }
295}
296
297/// The character category definition
298#[derive(Debug)]
299struct CategoryInfo {
300    category_type: CategoryType,
301    is_invoke: bool,
302    is_group: bool,
303    length: u32,
304}
305
306/// The OOV definition
307#[derive(Debug, Default, Clone)]
308struct Oov {
309    left_id: i16,
310    right_id: i16,
311    cost: i16,
312    pos_id: u16,
313}
sudachi/plugin/oov/mecab_oov.rs

sudachi/plugin/oov/
mecab_oov.rs
