sudachi/plugin/path_rewrite/
join_numeric.rs

1/*
2 * Copyright (c) 2021-2026 Works Applications Co., Ltd.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17use serde::Deserialize;
18use serde_json::Value;
19
20use self::numeric_parser::NumericParser;
21use crate::analysis::lattice::Lattice;
22use crate::analysis::node::{concat_nodes, LatticeNode, ResultNode};
23use crate::config::Config;
24use crate::dic::category_type::CategoryType;
25use crate::dic::grammar::Grammar;
26use crate::dic::word_info::WordInfoResolver;
27use crate::input_text::InputBuffer;
28use crate::input_text::InputTextIndex;
29use crate::plugin::path_rewrite::PathRewritePlugin;
30use crate::plugin::PluginError;
31use crate::prelude::*;
32
33mod numeric_parser;
34#[cfg(test)]
35mod test;
36
37/// Concatenates numeric nodes as one
38#[derive(Default)]
39pub struct JoinNumericPlugin {
40    /// The pos_id to concatenate
41    numeric_pos_id: u16,
42    /// Whether if to normalize the normalized_form
43    enable_normalize: bool,
44}
45
46/// Struct corresponds with raw config json file.
47#[allow(non_snake_case)]
48#[derive(Deserialize)]
49struct PluginSettings {
50    enableNormalize: Option<bool>,
51}
52
53impl JoinNumericPlugin {
54    fn concat(
55        &self,
56        mut path: Vec<ResultNode>,
57        begin: usize,
58        end: usize,
59        parser: &mut NumericParser,
60        resolver: &dyn WordInfoResolver,
61    ) -> SudachiResult<Vec<ResultNode>> {
62        let word_info = path[begin].word_info();
63
64        if word_info.pos_id() != self.numeric_pos_id {
65            return Ok(path);
66        }
67
68        if self.enable_normalize {
69            let normalized_form = parser.get_normalized();
70            if end - begin > 1 || normalized_form != word_info.normalized_form(resolver) {
71                path = concat_nodes(path, begin, end, Some(normalized_form), resolver)?;
72            }
73            return Ok(path);
74        }
75
76        if end - begin > 1 {
77            path = concat_nodes(path, begin, end, None, resolver)?;
78        }
79        Ok(path)
80    }
81
82    fn rewrite_gen<T: InputTextIndex>(
83        &self,
84        text: &T,
85        mut path: Vec<ResultNode>,
86        resolver: &dyn WordInfoResolver,
87    ) -> SudachiResult<Vec<ResultNode>> {
88        let mut begin_idx = -1;
89        let mut comma_as_digit = true;
90        let mut period_as_digit = true;
91        let mut parser = NumericParser::new();
92        let mut i = -1;
93        while i < path.len() as i32 - 1 {
94            i += 1;
95            let node = &path[i as usize];
96            let ctypes = text.cat_of_range(node.char_range());
97            let s = node.word_info().normalized_form(resolver);
98            if ctypes.intersects(CategoryType::NUMERIC | CategoryType::KANJINUMERIC)
99                || (comma_as_digit && s == ",")
100                || (period_as_digit && s == ".")
101            {
102                if begin_idx < 0 {
103                    parser.clear();
104                    begin_idx = i;
105                }
106                for c in s.chars() {
107                    if !parser.append(&c) {
108                        if begin_idx >= 0 {
109                            if parser.error_state == numeric_parser::Error::Comma {
110                                comma_as_digit = false;
111                                i = begin_idx - 1;
112                            } else if parser.error_state == numeric_parser::Error::Point {
113                                period_as_digit = false;
114                                i = begin_idx - 1;
115                            }
116                            begin_idx = -1;
117                        }
118                        break;
119                    }
120                }
121                continue;
122            }
123
124            let c = if s.len() == 1 {
125                // must be 1 byte utf-8: ASCII
126                s.as_bytes()[0] as char
127            } else {
128                char::MAX
129            };
130
131            // can't use s below this line
132
133            if begin_idx >= 0 {
134                if parser.done() {
135                    path =
136                        self.concat(path, begin_idx as usize, i as usize, &mut parser, resolver)?;
137                    i = begin_idx + 1;
138                } else {
139                    let ss = path[i as usize - 1].word_info().normalized_form(resolver);
140                    if (parser.error_state == numeric_parser::Error::Comma && ss == ",")
141                        || (parser.error_state == numeric_parser::Error::Point && ss == ".")
142                    {
143                        path = self.concat(
144                            path,
145                            begin_idx as usize,
146                            i as usize - 1,
147                            &mut parser,
148                            resolver,
149                        )?;
150                        i = begin_idx + 2;
151                    }
152                }
153            }
154            begin_idx = -1;
155            if !comma_as_digit && c != ',' {
156                comma_as_digit = true;
157            }
158            if !period_as_digit && c != '.' {
159                period_as_digit = true;
160            }
161        }
162
163        // process last part
164        if begin_idx >= 0 {
165            let len = path.len();
166            if parser.done() {
167                path = self.concat(path, begin_idx as usize, len, &mut parser, resolver)?;
168            } else {
169                let ss = path[len - 1].word_info().normalized_form(resolver);
170                if (parser.error_state == numeric_parser::Error::Comma && ss == ",")
171                    || (parser.error_state == numeric_parser::Error::Point && ss == ".")
172                {
173                    path = self.concat(path, begin_idx as usize, len - 1, &mut parser, resolver)?;
174                }
175            }
176        }
177
178        Ok(path)
179    }
180}
181
182impl PathRewritePlugin for JoinNumericPlugin {
183    fn set_up(
184        &mut self,
185        settings: &Value,
186        _config: &Config,
187        grammar: &Grammar,
188    ) -> SudachiResult<()> {
189        let settings: PluginSettings =
190            serde_json::from_value(settings.clone()).map_err(PluginError::from)?;
191
192        // this pos is fixed
193        let numeric_pos_string = vec!["名詞", "数詞", "*", "*", "*", "*"];
194        let numeric_pos_id = grammar.get_part_of_speech_id(&numeric_pos_string).ok_or(
195            SudachiError::InvalidPartOfSpeech(format!("{:?}", numeric_pos_string)),
196        )?;
197        let enable_normalize = settings.enableNormalize;
198
199        self.numeric_pos_id = numeric_pos_id;
200        self.enable_normalize = enable_normalize.unwrap_or(true);
201
202        Ok(())
203    }
204
205    fn rewrite(
206        &self,
207        text: &InputBuffer,
208        path: Vec<ResultNode>,
209        _lattice: &Lattice,
210        resolver: &dyn WordInfoResolver,
211    ) -> SudachiResult<Vec<ResultNode>> {
212        self.rewrite_gen(text, path, resolver)
213    }
214}
sudachi/plugin/path_rewrite/join_numeric.rs

sudachi/plugin/path_rewrite/
join_numeric.rs
