sudachi/analysis/
mlist.rs

1/*
2 *  Copyright (c) 2021-2026 Works Applications Co., Ltd.
3 *
4 *  Licensed under the Apache License, Version 2.0 (the "License");
5 *  you may not use this file except in compliance with the License.
6 *  You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 *   Unless required by applicable law or agreed to in writing, software
11 *  distributed under the License is distributed on an "AS IS" BASIS,
12 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 *  See the License for the specific language governing permissions and
14 *  limitations under the License.
15 */
16
17use std::cell::{Ref, RefCell};
18use std::iter::FusedIterator;
19use std::ops::{Deref, DerefMut, Index};
20use std::rc::Rc;
21
22use crate::analysis::morpheme::Morpheme;
23use crate::analysis::node::{PathCost, ResultNode};
24use crate::analysis::stateful_tokenizer::StatefulTokenizer;
25use crate::analysis::{Mode, Node};
26use crate::dic::subset::InfoSubset;
27use crate::dic::{normalize_input_text, DictionaryAccess};
28use crate::error::{SudachiError, SudachiResult};
29use crate::input_text::InputBuffer;
30
31struct InputPart {
32    input: InputBuffer,
33    subset: InfoSubset,
34}
35
36impl Default for InputPart {
37    fn default() -> Self {
38        let mut input = InputBuffer::new();
39        input.start_build().unwrap();
40        Self {
41            input,
42            subset: Default::default(),
43        }
44    }
45}
46
47#[derive(Default)]
48struct Nodes {
49    data: Vec<ResultNode>,
50}
51
52impl Nodes {
53    fn mut_data(&mut self) -> &mut Vec<ResultNode> {
54        &mut self.data
55    }
56}
57
58pub struct MorphemeList<D> {
59    dict: D,
60    input: Rc<RefCell<InputPart>>,
61    nodes: Nodes,
62}
63
64impl<D: DictionaryAccess> MorphemeList<D> {
65    /// Returns an empty morpheme list
66    pub fn empty(dict: D) -> Self {
67        let input = Default::default();
68        Self {
69            dict,
70            input: Rc::new(RefCell::new(input)),
71            nodes: Default::default(),
72        }
73    }
74
75    /// Creates MorphemeList from components
76    pub fn from_components(
77        dict: D,
78        input: InputBuffer,
79        path: Vec<ResultNode>,
80        subset: InfoSubset,
81    ) -> Self {
82        let input = InputPart { input, subset };
83        Self {
84            dict,
85            input: Rc::new(RefCell::new(input)),
86            nodes: Nodes { data: path },
87        }
88    }
89
90    pub fn collect_results<U: DictionaryAccess>(
91        &mut self,
92        analyzer: &mut StatefulTokenizer<U>,
93    ) -> SudachiResult<()> {
94        match self.input.try_borrow_mut() {
95            Ok(mut i) => {
96                let mref = i.deref_mut();
97                analyzer.swap_result(&mut mref.input, self.nodes.mut_data(), &mut mref.subset);
98                Ok(())
99            }
100            Err(_) => Err(SudachiError::MorphemeListBorrowed),
101        }
102    }
103
104    /// Splits morphemes and writes them into the resulting list
105    /// The resulting list is _not_ cleared before that
106    /// Returns true if split produced more than two elements
107    pub fn split_into(&self, mode: Mode, index: usize, out: &mut Self) -> SudachiResult<bool> {
108        let node = self.node(index);
109        let num_splits = node.num_splits(mode);
110
111        if num_splits == 0 {
112            Ok(false)
113        } else {
114            out.assign_input(self);
115            let data = out.nodes.mut_data();
116            let input = self.input();
117            let subset = self.subset();
118            data.reserve(num_splits);
119            for n in node.split(mode, self.dict().lexicon(), subset, input.deref()) {
120                data.push(n);
121            }
122            Ok(true)
123        }
124    }
125
126    /// Clears morphemes from analysis result
127    pub fn clear(&mut self) {
128        self.nodes.mut_data().clear();
129    }
130
131    pub fn len(&self) -> usize {
132        self.nodes.data.len()
133    }
134
135    pub fn is_empty(&self) -> bool {
136        self.nodes.data.is_empty()
137    }
138
139    pub fn get(&self, idx: usize) -> Morpheme<'_, D> {
140        Morpheme::for_list(self, idx)
141    }
142
143    pub fn surface(&self) -> Ref<'_, str> {
144        let inp = self.input();
145        Ref::map(inp, |i| i.original())
146    }
147
148    pub fn iter(&self) -> MorphemeIter<'_, D> {
149        MorphemeIter {
150            index: 0,
151            list: self,
152        }
153    }
154
155    /// Gets the whole cost of the path
156    pub fn get_internal_cost(&self) -> i32 {
157        let len = self.len();
158        if len == 0 {
159            return 0;
160        }
161
162        let first_node = self.node(0);
163        let last_node = self.node(len - 1);
164        last_node.total_cost() - first_node.total_cost()
165    }
166
167    pub(crate) fn node(&self, idx: usize) -> &ResultNode {
168        self.nodes.data.index(idx)
169    }
170
171    pub fn dict(&self) -> &D {
172        &self.dict
173    }
174
175    pub(crate) fn input(&self) -> Ref<'_, InputBuffer> {
176        Ref::map(self.input.deref().borrow(), |x| &x.input)
177    }
178
179    /// Makes this point to the input of another MorphemeList
180    pub(crate) fn assign_input(&mut self, other: &Self) {
181        if self.input.as_ptr() != other.input.as_ptr() {
182            self.input = other.input.clone();
183        }
184    }
185
186    pub fn subset(&self) -> InfoSubset {
187        self.input.deref().borrow().subset
188    }
189
190    pub fn copy_slice(&self, start: usize, end: usize, out: &mut Self) {
191        let out_data = out.nodes.mut_data();
192        out_data.extend_from_slice(&self.nodes.data[start..end]);
193    }
194
195    /// Looks up the given query and reset the morpheme list to the result.
196    ///
197    /// The query is normalized with dictionary input-text plugins before lookup.
198    /// Returns the number of found entries.
199    pub fn lookup(&mut self, query: &str, subset: InfoSubset) -> SudachiResult<usize> {
200        let (normalized, end_chars) = {
201            let dict = &self.dict;
202            let input = &mut self.input.borrow_mut().input;
203            normalize_input_text(dict, query, input)?;
204            let normalized = input.current().to_owned();
205            input.reset().push_str(&normalized);
206            input.start_build()?;
207            input.build(dict.grammar())?;
208            let end_chars = input.ch_idx(normalized.len());
209            (normalized, end_chars)
210        };
211
212        let mut result = 0;
213        let lex = self.dict.lexicon();
214        for entry in lex.lookup(normalized.as_bytes(), 0) {
215            if entry.end != normalized.len() {
216                continue;
217            }
218            let info = lex.get_word_info_subset(entry.word_id, subset)?;
219            let node = Node::new(0, end_chars as _, 0, 0, 0, entry.word_id);
220            self.nodes
221                .data
222                .push(ResultNode::new(node, 0, 0, normalized.len() as _, info));
223            result += 1;
224        }
225        Ok(result)
226    }
227}
228
229impl<T: DictionaryAccess + Clone> MorphemeList<T> {
230    pub fn empty_clone(&self) -> Self {
231        Self {
232            dict: self.dict.clone(),
233            input: self.input.clone(),
234            nodes: Default::default(),
235        }
236    }
237
238    /// Returns a new morpheme list splitting the morpheme with a given mode.
239    /// Returns an empty list if there was no splits
240    #[deprecated(note = "use split_into", since = "0.6.1")]
241    pub fn split(&self, mode: Mode, index: usize) -> SudachiResult<MorphemeList<T>> {
242        let mut list = self.empty_clone();
243        if !self.split_into(mode, index, &mut list)? {
244            list.nodes.mut_data().push(self.node(index).clone())
245        }
246        Ok(list)
247    }
248}
249
250/// Iterates over morpheme list
251pub struct MorphemeIter<'a, T> {
252    list: &'a MorphemeList<T>,
253    index: usize,
254}
255
256impl<'a, T: DictionaryAccess> Iterator for MorphemeIter<'a, T> {
257    type Item = Morpheme<'a, T>;
258
259    fn next(&mut self) -> Option<Self::Item> {
260        if self.index >= self.list.len() {
261            return None;
262        }
263
264        let morpheme = Morpheme::for_list(self.list, self.index);
265
266        self.index += 1;
267        Some(morpheme)
268    }
269
270    fn size_hint(&self) -> (usize, Option<usize>) {
271        let rem = self.list.len() - self.index;
272        (rem, Some(rem))
273    }
274}
275
276impl<'a, T: DictionaryAccess> FusedIterator for MorphemeIter<'a, T> {}
277
278impl<'a, T: DictionaryAccess> ExactSizeIterator for MorphemeIter<'a, T> {
279    fn len(&self) -> usize {
280        self.size_hint().0
281    }
282}
sudachi/analysis/mlist.rs

sudachi/analysis/
mlist.rs
