sudachi/plugin/input_text/
default_input_text.rs

1/*
2 * Copyright (c) 2021-2026 Works Applications Co., Ltd.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17use std::collections::{HashMap, HashSet};
18use std::io::BufRead;
19use std::path::PathBuf;
20
21use aho_corasick::{
22    AhoCorasick, AhoCorasickBuilder, AhoCorasickKind, Anchored, MatchKind, StartKind,
23};
24use serde::Deserialize;
25use serde_json::Value;
26use unicode_normalization::{is_nfkc_quick, IsNormalized, UnicodeNormalization};
27
28use crate::config::{Config, DEFAULT_REWRITE_DEF_FILE};
29use crate::dic::grammar::Grammar;
30use crate::hash::RoMu;
31use crate::input_text::{InputBuffer, InputEditor};
32use crate::plugin::input_text::InputTextPlugin;
33use crate::plugin::PluginError;
34use crate::prelude::*;
35
36#[cfg(test)]
37mod tests;
38
39/// Provides basic normalization of the input text
40#[derive(Default)]
41pub struct DefaultInputTextPlugin {
42    /// Set of characters to skip normalization
43    ignore_normalize_set: HashSet<char, RoMu>,
44    /// Mapping from a character to the maximum char_length of possible replacement
45    key_lengths: HashMap<char, usize>,
46    /// Replacement mapping
47    replace_char_map: HashMap<String, String>,
48    /// Checks whether the string contains symbols to normalize
49    checker: Option<AhoCorasick>,
50    replacements: Vec<String>,
51}
52
53/// Struct corresponds with raw config json file.
54#[allow(non_snake_case)]
55#[derive(Deserialize)]
56struct PluginSettings {
57    rewriteDef: Option<PathBuf>,
58}
59
60impl DefaultInputTextPlugin {
61    /// Loads rewrite definition
62    ///
63    /// Definition syntax:
64    ///     Ignored normalize:
65    ///         Each line contains a character
66    ///     Replace char list:
67    ///         Each line contains two strings separated by white spaces
68    ///         Plugin replaces the first by the second
69    ///         Same target string cannot be defined multiple times
70    ///     Empty or line starts with "#" will be ignored
71    fn read_rewrite_lists<T: BufRead>(&mut self, reader: T) -> SudachiResult<()> {
72        let mut ignore_normalize_set = HashSet::with_hasher(RoMu::new());
73        let mut key_lengths = HashMap::new();
74        let mut replace_char_map = HashMap::new();
75        for (i, line) in reader.lines().enumerate() {
76            let line = line?;
77            let line = line.trim();
78            if line.is_empty() || line.starts_with('#') {
79                continue;
80            }
81            let cols: Vec<_> = line.split_whitespace().collect();
82
83            // ignored normalize list
84            if cols.len() == 1 {
85                if cols[0].chars().count() != 1 {
86                    return Err(SudachiError::PluginError(
87                        PluginError::InvalidDataFormatWithLine {
88                            line: i,
89                            message: format!("{} is not character", cols[0]),
90                        },
91                    ));
92                }
93                ignore_normalize_set.insert(cols[0].chars().next().unwrap());
94                continue;
95            }
96            // replace char list
97            if cols.len() == 2 {
98                if replace_char_map.contains_key(cols[0]) {
99                    return Err(SudachiError::PluginError(
100                        PluginError::InvalidDataFormatWithLine {
101                            line: i,
102                            message: format!("{} is already defined", cols[0]),
103                        },
104                    ));
105                }
106                let first_char = cols[0].chars().next().unwrap();
107                let n_char = cols[0].chars().count();
108                if key_lengths.get(&first_char).copied().unwrap_or(0) < n_char {
109                    key_lengths.insert(first_char, n_char);
110                }
111                replace_char_map.insert(cols[0].to_string(), cols[1].to_string());
112                continue;
113            }
114            return Err(SudachiError::PluginError(
115                PluginError::InvalidDataFormatWithLine {
116                    line: i,
117                    message: String::new(),
118                },
119            ));
120        }
121
122        self.ignore_normalize_set = ignore_normalize_set;
123        self.key_lengths = key_lengths;
124        self.replace_char_map = replace_char_map;
125
126        let mut values: Vec<String> = Vec::new();
127        let mut keys: Vec<String> = Vec::new();
128
129        for (k, v) in self.replace_char_map.iter() {
130            keys.push(k.clone());
131            values.push(v.clone());
132        }
133
134        self.checker = Some(
135            AhoCorasickBuilder::new()
136                .kind(Some(AhoCorasickKind::DFA))
137                .match_kind(MatchKind::LeftmostLongest)
138                .start_kind(StartKind::Both)
139                .build(keys.clone())
140                .map_err(|e| {
141                    PluginError::InvalidDataFormat(format!("failed to parse rewrite.def: {e:?}"))
142                })?,
143        );
144
145        self.replacements = values;
146
147        Ok(())
148    }
149
150    #[inline]
151    fn should_ignore(&self, ch: char) -> bool {
152        self.ignore_normalize_set.contains(&ch)
153    }
154
155    /// Fast case: lowercasing is not needed and the string is already in NFKC
156    /// Use AhoCorasick automaton to find all replacements and replace them
157    ///
158    /// Ignores are not used here, forced replacements have higher priority
159    /// Fast version does not need to walk every character!
160    fn replace_fast<'a>(
161        &'a self,
162        buffer: &InputBuffer,
163        mut replacer: InputEditor<'a>,
164    ) -> SudachiResult<InputEditor<'a>> {
165        let cur = buffer.current();
166        let checker = self.checker.as_ref().unwrap();
167
168        let ac_input = aho_corasick::Input::new(cur).anchored(Anchored::No);
169
170        for m in checker.find_iter(ac_input) {
171            let replacement = self.replacements[m.pattern()].as_str();
172            replacer.replace_ref(m.start()..m.end(), replacement);
173        }
174
175        Ok(replacer)
176    }
177
178    /// Slow case: need to handle lowercasing or NFKC normalization
179    /// Slow version needs to walk every character
180    fn replace_slow<'a>(
181        &'a self,
182        buffer: &InputBuffer,
183        mut replacer: InputEditor<'a>,
184    ) -> SudachiResult<InputEditor<'a>> {
185        let cur = buffer.current();
186        let checker = self.checker.as_ref().unwrap();
187        let mut min_offset = 0;
188
189        let mut ac_input = aho_corasick::Input::new(cur)
190            .anchored(Anchored::Yes)
191            .earliest(true);
192
193        for (offset, ch) in cur.char_indices() {
194            if offset < min_offset {
195                continue;
196            }
197            ac_input.set_start(offset);
198            // 1. replacement as defined by char.def
199            if let Some(m) = checker.find(ac_input.clone()) {
200                let range = m.range();
201                let replacement = self.replacements[m.pattern()].as_str();
202                min_offset = range.end;
203                replacer.replace_ref(range, replacement);
204                continue;
205            }
206
207            // 2. handle normalization
208            let need_lowercase = ch.is_uppercase();
209            let need_nkfc =
210                !self.should_ignore(ch) && is_nfkc_quick(std::iter::once(ch)) != IsNormalized::Yes;
211
212            // iterator types are incompatible, so calls can't be moved outside branches
213            match (need_lowercase, need_nkfc) {
214                //no need to do anything
215                (false, false) => continue,
216                // only lowercasing
217                (true, false) => {
218                    let chars = ch.to_lowercase();
219                    self.handle_normalization_slow(chars, &mut replacer, offset, ch.len_utf8(), ch)
220                }
221                // only normalization
222                (false, true) => {
223                    let chars = std::iter::once(ch).nfkc();
224                    self.handle_normalization_slow(chars, &mut replacer, offset, ch.len_utf8(), ch)
225                }
226                // both
227                (true, true) => {
228                    let chars = ch.to_lowercase().nfkc();
229                    self.handle_normalization_slow(chars, &mut replacer, offset, ch.len_utf8(), ch)
230                }
231            }
232        }
233        Ok(replacer)
234    }
235
236    fn handle_normalization_slow<'a, I: Iterator<Item = char>>(
237        &'a self,
238        mut data: I,
239        replacer: &mut InputEditor<'a>,
240        start: usize,
241        len: usize,
242        ch: char,
243    ) {
244        if let Some(ch2) = data.next() {
245            if ch2 != ch {
246                replacer.replace_char_iter(start..start + len, ch2, data)
247            }
248        }
249    }
250}
251
252impl InputTextPlugin for DefaultInputTextPlugin {
253    fn set_up(
254        &mut self,
255        settings: &Value,
256        config: &Config,
257        _grammar: &Grammar,
258    ) -> SudachiResult<()> {
259        let settings: PluginSettings =
260            serde_json::from_value(settings.clone()).map_err(PluginError::from)?;
261
262        let rewrite_def = config.resolve_resource(
263            settings
264                .rewriteDef
265                .unwrap_or_else(|| DEFAULT_REWRITE_DEF_FILE.into()),
266        )?;
267        self.read_rewrite_lists(rewrite_def.reader()?)?;
268
269        Ok(())
270    }
271
272    fn uses_chars(&self) -> bool {
273        true
274    }
275
276    fn rewrite_impl<'a>(
277        &'a self,
278        buffer: &InputBuffer,
279        edit: InputEditor<'a>,
280    ) -> SudachiResult<InputEditor<'a>> {
281        let chars = buffer.current_chars();
282        let need_nkfc = is_nfkc_quick(chars.iter().cloned()) != IsNormalized::Yes;
283
284        let need_lowercase = chars.iter().any(|c| c.is_uppercase());
285
286        if need_nkfc || need_lowercase {
287            self.replace_slow(buffer, edit)
288        } else {
289            self.replace_fast(buffer, edit)
290        }
291    }
292}
sudachi/plugin/input_text/default_input_text.rs

sudachi/plugin/input_text/
default_input_text.rs
