sudachi/plugin/oov/
regex_oov.rs

1/*
2 *  Copyright (c) 2022-2026 Works Applications Co., Ltd.
3 *
4 *  Licensed under the Apache License, Version 2.0 (the "License");
5 *  you may not use this file except in compliance with the License.
6 *  You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 *   Unless required by applicable law or agreed to in writing, software
11 *  distributed under the License is distributed on an "AS IS" BASIS,
12 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 *  See the License for the specific language governing permissions and
14 *  limitations under the License.
15 */
16
17use crate::analysis::created::{CreatedWords, HasWord};
18use crate::analysis::node::LatticeNode;
19use crate::analysis::Node;
20use crate::config::Config;
21use crate::dic::grammar::Grammar;
22use crate::dic::word_id::WordId;
23use crate::error::{SudachiError, SudachiResult};
24use crate::input_text::{InputBuffer, InputTextIndex};
25use crate::plugin::oov::OovProviderPlugin;
26use crate::plugin::PluginError;
27use crate::util::check_params::CheckParams;
28use crate::util::user_pos::{UserPosMode, UserPosSupport};
29use regex::{Regex, RegexBuilder};
30use serde::Deserialize;
31use serde_json::Value;
32
33#[cfg(test)]
34mod test;
35
36#[derive(Default)]
37pub(crate) struct RegexOovProvider {
38    regex: Option<Regex>,
39    left_id: u16,
40    right_id: u16,
41    cost: i16,
42    pos: u16,
43    max_length: usize,
44    debug: bool,
45    boundaries: BoundaryMode,
46}
47
48#[derive(Deserialize, Eq, PartialEq, Debug, Copy, Clone, Default)]
49#[serde(rename_all = "lowercase")]
50pub enum BoundaryMode {
51    #[default]
52    Strict,
53    Relaxed,
54}
55
56fn default_max_length() -> usize {
57    32
58}
59
60#[derive(Deserialize)]
61#[allow(non_snake_case)]
62struct RegexProviderConfig {
63    #[serde(alias = "oovPOS")]
64    pos: Vec<String>,
65    leftId: i64,
66    rightId: i64,
67    cost: i64,
68    regex: String,
69    #[serde(default = "default_max_length")]
70    maxLength: usize,
71    #[serde(default)]
72    debug: bool,
73    #[serde(default)]
74    userPOS: UserPosMode,
75    #[serde(default)]
76    boundaries: BoundaryMode,
77}
78
79impl OovProviderPlugin for RegexOovProvider {
80    fn set_up(
81        &mut self,
82        settings: &Value,
83        _config: &Config,
84        mut grammar: &mut Grammar,
85    ) -> SudachiResult<()> {
86        let mut parsed: RegexProviderConfig =
87            serde_json::from_value(settings.clone()).map_err(PluginError::from)?;
88
89        if !parsed.regex.starts_with('^') {
90            parsed.regex.insert(0, '^');
91        }
92
93        self.left_id = grammar.check_left_id(parsed.leftId)?;
94        self.right_id = grammar.check_right_id(parsed.rightId)?;
95        self.cost = grammar.check_cost(parsed.cost)?;
96        self.max_length = parsed.maxLength;
97        self.debug = parsed.debug;
98        self.pos = grammar.handle_user_pos(&parsed.pos, parsed.userPOS)?;
99        self.boundaries = parsed.boundaries;
100
101        match RegexBuilder::new(&parsed.regex).build() {
102            Ok(re) => self.regex = Some(re),
103            Err(e) => {
104                return Err(SudachiError::PluginError(PluginError::InvalidDataFormat(
105                    format!("regex {:?} is invalid: {:?}", &parsed.regex, e),
106                )))
107            }
108        };
109
110        Ok(())
111    }
112
113    fn provide_oov(
114        &self,
115        input_text: &InputBuffer,
116        offset: usize,
117        other_words: CreatedWords,
118        result: &mut Vec<Node>,
119    ) -> SudachiResult<usize> {
120        if self.boundaries == BoundaryMode::Strict && offset > 0 {
121            // check that we have discontinuity in character categories
122            let this_cat = input_text.cat_continuous_len(offset);
123            let prev_cat = input_text.cat_continuous_len(offset - 1);
124            if this_cat + 1 == prev_cat {
125                // no discontinuity
126                return Ok(0);
127            }
128        }
129
130        let regex = self
131            .regex
132            .as_ref()
133            .ok_or_else(|| SudachiError::InvalidDictionaryGrammar)?;
134
135        let end = input_text
136            .current_chars()
137            .len()
138            .min(offset + self.max_length);
139        let text_data = input_text.curr_slice_c(offset..end);
140        match regex.find(text_data) {
141            None => Ok(0),
142            Some(m) => {
143                if m.start() != 0 {
144                    return if self.debug {
145                        Err(SudachiError::InvalidDataFormat(m.start(), format!("in input {:?} regex {:?} matched non-starting text in non-starting position: {}", text_data, regex, m.as_str())))
146                    } else {
147                        Ok(0)
148                    };
149                }
150
151                let byte_offset = input_text.to_curr_byte_idx(offset);
152                let match_start = offset;
153                let match_end = input_text.ch_idx(byte_offset + m.end());
154
155                let match_length = match_end - match_start;
156
157                match other_words.has_word(match_length as i64) {
158                    HasWord::Yes => return Ok(0),
159                    HasWord::No => {} // do nothing
160                    HasWord::Maybe => {
161                        // need to check actual lengths for long words
162                        for node in result.iter() {
163                            if node.end() == match_end {
164                                return Ok(0);
165                            }
166                        }
167                    }
168                }
169
170                let node = Node::new(
171                    match_start as _,
172                    match_end as _,
173                    self.left_id,
174                    self.right_id,
175                    self.cost,
176                    WordId::oov(self.pos as _),
177                );
178                result.push(node);
179                Ok(1)
180            }
181        }
182    }
183}
sudachi/plugin/oov/regex_oov.rs

sudachi/plugin/oov/
regex_oov.rs
