sudachi/plugin/path_rewrite/
join_katakana_oov.rs

1/*
2 * Copyright (c) 2021-2026 Works Applications Co., Ltd.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17use serde::Deserialize;
18use serde_json::Value;
19
20use crate::analysis::lattice::Lattice;
21use crate::analysis::node::{concat_oov_nodes, LatticeNode, ResultNode};
22use crate::config::Config;
23use crate::dic::category_type::CategoryType;
24use crate::dic::grammar::Grammar;
25use crate::dic::word_info::WordInfoResolver;
26use crate::input_text::InputBuffer;
27use crate::input_text::InputTextIndex;
28use crate::plugin::path_rewrite::PathRewritePlugin;
29use crate::plugin::PluginError;
30use crate::prelude::*;
31
32#[cfg(test)]
33mod tests;
34
35/// Concatenates katakana oov nodes into one
36#[derive(Default)]
37pub struct JoinKatakanaOovPlugin {
38    /// The pos_id used for concatenated node
39    oov_pos_id: u16,
40    /// The minimum node char_length to concatenate even if it is not oov
41    min_length: usize,
42}
43
44/// Struct corresponds with raw config json file.
45#[allow(non_snake_case)]
46#[derive(Deserialize)]
47struct PluginSettings {
48    oovPOS: Vec<String>,
49    minLength: usize,
50}
51
52impl JoinKatakanaOovPlugin {
53    fn is_katakana_node<T: InputTextIndex>(&self, text: &T, node: &ResultNode) -> bool {
54        text.cat_of_range(node.begin()..node.end())
55            .contains(CategoryType::KATAKANA)
56    }
57
58    // fn is_one_char(&self, text: &Utf8InputText, node: &Node) -> bool {
59    //     let b = node.begin;
60    //     b + text.get_code_points_offset_length(b, 1) == node.end
61    // }
62
63    fn can_oov_bow_node<T: InputTextIndex>(&self, text: &T, node: &ResultNode) -> bool {
64        !text
65            .cat_at_char(node.begin())
66            .contains(CategoryType::NOOOVBOW)
67    }
68
69    fn is_shorter(&self, node: &ResultNode) -> bool {
70        node.num_codepts() < self.min_length
71    }
72
73    fn rewrite_gen(
74        &self,
75        text: &InputBuffer,
76        mut path: Vec<ResultNode>,
77        lattice: &Lattice,
78        resolver: &dyn WordInfoResolver,
79    ) -> SudachiResult<Vec<ResultNode>> {
80        let mut i = 0;
81        loop {
82            if i >= path.len() {
83                break;
84            }
85
86            let node = &path[i];
87            if !(node.is_oov() || self.is_shorter(node)) || !self.is_katakana_node(text, node) {
88                i += 1;
89                continue;
90            }
91            let mut begin = i as i32 - 1;
92            loop {
93                if begin < 0 {
94                    break;
95                }
96                if !self.is_katakana_node(text, &path[begin as usize]) {
97                    begin += 1;
98                    break;
99                }
100                begin -= 1;
101            }
102            let mut begin = if begin < 0 { 0 } else { begin as usize };
103            let mut end = i + 1;
104            loop {
105                if end >= path.len() {
106                    break;
107                }
108                if !self.is_katakana_node(text, &path[end]) {
109                    break;
110                }
111                end += 1;
112            }
113            while begin != end && !self.can_oov_bow_node(text, &path[begin]) {
114                begin += 1;
115            }
116
117            if (end - begin) > 1 {
118                path =
119                    concat_oov_nodes(path, begin, end, self.oov_pos_id, text, lattice, resolver)?;
120                // skip next node, as we already know it is not a joinable katakana
121                i = begin + 1;
122            }
123            i += 1;
124        }
125
126        Ok(path)
127    }
128}
129
130impl PathRewritePlugin for JoinKatakanaOovPlugin {
131    fn set_up(
132        &mut self,
133        settings: &Value,
134        _config: &Config,
135        grammar: &Grammar,
136    ) -> SudachiResult<()> {
137        let settings: PluginSettings =
138            serde_json::from_value(settings.clone()).map_err(PluginError::from)?;
139
140        let oov_pos_string: Vec<&str> = settings.oovPOS.iter().map(|s| s.as_str()).collect();
141        let oov_pos_id = grammar.get_part_of_speech_id(&oov_pos_string).ok_or(
142            SudachiError::InvalidPartOfSpeech(format!("{:?}", oov_pos_string)),
143        )?;
144        let min_length = settings.minLength;
145
146        self.oov_pos_id = oov_pos_id;
147        self.min_length = min_length;
148
149        Ok(())
150    }
151
152    fn rewrite(
153        &self,
154        text: &InputBuffer,
155        path: Vec<ResultNode>,
156        lattice: &Lattice,
157        resolver: &dyn WordInfoResolver,
158    ) -> SudachiResult<Vec<ResultNode>> {
159        self.rewrite_gen(text, path, lattice, resolver)
160    }
161}
sudachi/plugin/path_rewrite/join_katakana_oov.rs

sudachi/plugin/path_rewrite/
join_katakana_oov.rs
