sudachi/dic/word_info/
parse.rs

1/*
2 *  Copyright (c) 2021-2026 Works Applications Co., Ltd.
3 *
4 *  Licensed under the Apache License, Version 2.0 (the "License");
5 *  you may not use this file except in compliance with the License.
6 *  You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 *   Unless required by applicable law or agreed to in writing, software
11 *  distributed under the License is distributed on an "AS IS" BASIS,
12 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 *  See the License for the specific language governing permissions and
14 *  limitations under the License.
15 */
16
17use crate::dic::subset::InfoSubset;
18use crate::dic::word_info::layout;
19use crate::dic::word_info::{
20    parse_i32_array, parse_u32_array, parse_user_data, WordInfoFixedData, WordInfoRawData,
21};
22use crate::error::SudachiResult;
23
24#[derive(Clone, Copy, Debug, Eq, PartialEq)]
25struct SplitReadPlan {
26    c_unit_split: bool,
27    b_unit_split: bool,
28    a_unit_split: bool,
29    word_structure: bool,
30}
31
32impl SplitReadPlan {
33    fn from_subset(subset: InfoSubset) -> Self {
34        Self {
35            c_unit_split: subset.intersects(
36                InfoSubset::SPLIT_C
37                    | InfoSubset::SPLIT_B
38                    | InfoSubset::SPLIT_A
39                    | InfoSubset::WORD_STRUCTURE,
40            ),
41            b_unit_split: subset
42                .intersects(InfoSubset::SPLIT_B | InfoSubset::SPLIT_A | InfoSubset::WORD_STRUCTURE),
43            a_unit_split: subset.intersects(InfoSubset::SPLIT_A | InfoSubset::WORD_STRUCTURE),
44            word_structure: subset.contains(InfoSubset::WORD_STRUCTURE),
45        }
46    }
47}
48
49pub struct WordInfoParser {
50    info: WordInfoRawData,
51    flds: InfoSubset,
52}
53
54impl Default for WordInfoParser {
55    #[inline]
56    fn default() -> Self {
57        Self::subset(InfoSubset::all())
58    }
59}
60
61impl WordInfoParser {
62    #[inline]
63    pub fn subset(flds: InfoSubset) -> WordInfoParser {
64        // normalize() adds the fixed fields required to interpret requested views.
65        // For example, NORMALIZED_FORM/DICTIONARY_FORM require HEADWORD, and split
66        // requests require INDEX_FORM_LENGTH.
67        Self {
68            info: Default::default(),
69            flds: flds.normalize(),
70        }
71    }
72
73    #[inline]
74    pub fn embedded_c_unit_split_length(&self) -> usize {
75        self.info.c_unit_split_length as usize
76    }
77
78    #[inline]
79    pub fn embedded_b_unit_split_length(&self) -> usize {
80        layout::embedded_len(self.info.b_unit_split_length)
81    }
82
83    #[inline]
84    pub fn embedded_a_unit_split_length(&self) -> usize {
85        layout::embedded_len(self.info.a_unit_split_length)
86    }
87
88    #[inline]
89    pub fn embedded_word_structure_length(&self) -> usize {
90        layout::embedded_len(self.info.word_structure_length)
91    }
92
93    #[inline]
94    pub fn embedded_synonym_group_ids_length(&self) -> usize {
95        self.info.synonym_group_ids_length as usize
96    }
97
98    #[inline]
99    pub fn parse(mut self, data: &[u8]) -> SudachiResult<WordInfoRawData> {
100        let (data, _) = nom::bytes::complete::take(layout::PARAMS_SIZE)(data)?;
101        let (data, fixed) = WordInfoFixedData::parse(data)?;
102        let split_plan = SplitReadPlan::from_subset(self.flds);
103        self.copy_fixed_fields(&fixed);
104
105        let (data, c_unit_split) = parse_u32_array(
106            data,
107            self.embedded_c_unit_split_length(),
108            split_plan.c_unit_split,
109        )?;
110        if split_plan.c_unit_split {
111            self.info.c_unit_split = c_unit_split;
112        }
113
114        let (data, b_unit_split) = parse_u32_array(
115            data,
116            self.embedded_b_unit_split_length(),
117            split_plan.b_unit_split,
118        )?;
119        if fixed.b_unit_split_length < 0 {
120            if split_plan.b_unit_split {
121                self.info.b_unit_split = self.info.c_unit_split.clone();
122            }
123        } else if split_plan.b_unit_split {
124            self.info.b_unit_split = b_unit_split;
125        }
126
127        let (data, a_unit_split) = parse_u32_array(
128            data,
129            self.embedded_a_unit_split_length(),
130            split_plan.a_unit_split,
131        )?;
132        if fixed.a_unit_split_length < 0 {
133            if split_plan.a_unit_split {
134                self.info.a_unit_split = self.info.b_unit_split.clone();
135            }
136        } else if split_plan.a_unit_split {
137            self.info.a_unit_split = a_unit_split;
138        }
139
140        let (data, word_structure) = parse_u32_array(
141            data,
142            self.embedded_word_structure_length(),
143            split_plan.word_structure,
144        )?;
145        if fixed.word_structure_length < 0 {
146            if split_plan.word_structure {
147                self.info.word_structure = self.info.a_unit_split.clone();
148            }
149        } else if split_plan.word_structure {
150            self.info.word_structure = word_structure;
151        }
152
153        let (data, synonym_group_ids) = parse_i32_array(
154            data,
155            self.embedded_synonym_group_ids_length(),
156            self.keep_synonym_group_ids(),
157        )?;
158        if self.keep_synonym_group_ids() {
159            self.info.synonym_group_ids = synonym_group_ids;
160        }
161
162        if fixed.has_user_data() {
163            let (_, user_data) = parse_user_data(data, self.keep_user_data())?;
164            if self.keep_user_data() {
165                self.info.user_data = user_data;
166            }
167        }
168        Ok(self.info)
169    }
170
171    fn copy_fixed_fields(&mut self, fixed: &WordInfoFixedData) {
172        if self.flds.contains(InfoSubset::POS_ID) {
173            self.info.pos_id = fixed.pos_id;
174        }
175        if self.flds.contains(InfoSubset::HEADWORD) {
176            self.info.headword_strptr = fixed.headword_strptr;
177        }
178        if self.flds.contains(InfoSubset::READING_FORM) {
179            self.info.reading_form_strptr = fixed.reading_form_strptr;
180        }
181        if self.flds.contains(InfoSubset::NORMALIZED_FORM) {
182            self.info.normalized_form = fixed.normalized_form;
183        }
184        if self.flds.contains(InfoSubset::DICTIONARY_FORM) {
185            self.info.dictionary_form = fixed.dictionary_form;
186        }
187        if self.flds.contains(InfoSubset::INDEX_FORM_LENGTH) {
188            self.info.index_form_length = fixed.index_form_length;
189        }
190        self.info.c_unit_split_length = fixed.c_unit_split_length;
191        self.info.b_unit_split_length = fixed.b_unit_split_length;
192        self.info.a_unit_split_length = fixed.a_unit_split_length;
193        self.info.word_structure_length = fixed.word_structure_length;
194        self.info.synonym_group_ids_length = fixed.synonym_group_ids_length;
195        self.info.user_data_flag = fixed.user_data_flag;
196    }
197
198    #[inline]
199    fn keep_synonym_group_ids(&self) -> bool {
200        self.flds.contains(InfoSubset::SYNONYM_GROUP_IDS)
201    }
202
203    #[inline]
204    fn keep_user_data(&self) -> bool {
205        self.flds.contains(InfoSubset::USER_DATA)
206    }
207}
208
209#[cfg(test)]
210mod tests {
211    use super::*;
212    use crate::dic::lexicon::strings::StringPointer;
213
214    fn push_u32s(buf: &mut Vec<u8>, data: &[u32]) {
215        for value in data {
216            buf.extend_from_slice(&value.to_le_bytes());
217        }
218    }
219
220    fn push_i32s(buf: &mut Vec<u8>, data: &[i32]) {
221        for value in data {
222            buf.extend_from_slice(&value.to_le_bytes());
223        }
224    }
225
226    fn push_utf16(buf: &mut Vec<u8>, data: &str) {
227        let utf16: Vec<u16> = data.encode_utf16().collect();
228        buf.extend_from_slice(&(utf16.len() as i16).to_le_bytes());
229        for unit in utf16 {
230            buf.extend_from_slice(&unit.to_le_bytes());
231        }
232    }
233
234    #[test]
235    fn parses_embedded_variable_length_fields() {
236        let fixed = WordInfoFixedData {
237            pos_id: 5,
238            headword_strptr: StringPointer::unchecked(2, 4),
239            reading_form_strptr: StringPointer::unchecked(3, 8),
240            normalized_form: 11,
241            dictionary_form: 12,
242            index_form_length: 6,
243            c_unit_split_length: 2,
244            b_unit_split_length: 1,
245            a_unit_split_length: 3,
246            word_structure_length: 2,
247            synonym_group_ids_length: 2,
248            user_data_flag: 1,
249        };
250
251        let mut bytes = vec![0u8; layout::PARAMS_SIZE];
252        fixed.write_to(&mut bytes).unwrap();
253        push_u32s(&mut bytes, &[100, 101]);
254        push_u32s(&mut bytes, &[200]);
255        push_u32s(&mut bytes, &[300, 301, 302]);
256        push_u32s(&mut bytes, &[400, 401]);
257        push_i32s(&mut bytes, &[7, 8]);
258        push_utf16(&mut bytes, "meta");
259
260        let parsed = WordInfoParser::default().parse(&bytes).unwrap();
261        assert_eq!(parsed.pos_id, fixed.pos_id);
262        assert_eq!(parsed.c_unit_split, vec![100, 101]);
263        assert_eq!(parsed.b_unit_split, vec![200]);
264        assert_eq!(parsed.a_unit_split, vec![300, 301, 302]);
265        assert_eq!(parsed.word_structure, vec![400, 401]);
266        assert_eq!(parsed.synonym_group_ids, vec![7, 8]);
267        assert_eq!(parsed.user_data, "meta");
268    }
269
270    #[test]
271    fn expands_shared_split_arrays() {
272        let fixed = WordInfoFixedData {
273            pos_id: 9,
274            headword_strptr: StringPointer::unchecked(1, 2),
275            reading_form_strptr: StringPointer::unchecked(1, 4),
276            normalized_form: 21,
277            dictionary_form: 22,
278            index_form_length: 3,
279            c_unit_split_length: 2,
280            b_unit_split_length: -1,
281            a_unit_split_length: -1,
282            word_structure_length: -1,
283            synonym_group_ids_length: 1,
284            user_data_flag: 0,
285        };
286
287        let mut bytes = vec![0u8; layout::PARAMS_SIZE];
288        fixed.write_to(&mut bytes).unwrap();
289        push_u32s(&mut bytes, &[10, 11]);
290        push_i32s(&mut bytes, &[99]);
291
292        let parsed = WordInfoParser::default().parse(&bytes).unwrap();
293        assert_eq!(parsed.c_unit_split, vec![10, 11]);
294        assert_eq!(parsed.b_unit_split, vec![10, 11]);
295        assert_eq!(parsed.a_unit_split, vec![10, 11]);
296        assert_eq!(parsed.word_structure, vec![10, 11]);
297        assert_eq!(parsed.synonym_group_ids, vec![99]);
298        assert!(parsed.user_data.is_empty());
299    }
300
301    #[test]
302    fn subset_clears_unrequested_fields() {
303        let fixed = WordInfoFixedData {
304            pos_id: 9,
305            headword_strptr: StringPointer::unchecked(1, 2),
306            reading_form_strptr: StringPointer::unchecked(1, 4),
307            normalized_form: 21,
308            dictionary_form: 22,
309            index_form_length: 3,
310            c_unit_split_length: 2,
311            b_unit_split_length: 1,
312            a_unit_split_length: 1,
313            word_structure_length: 1,
314            synonym_group_ids_length: 1,
315            user_data_flag: 1,
316        };
317
318        let mut bytes = vec![0u8; layout::PARAMS_SIZE];
319        fixed.write_to(&mut bytes).unwrap();
320        push_u32s(&mut bytes, &[10, 11]);
321        push_u32s(&mut bytes, &[20]);
322        push_u32s(&mut bytes, &[30]);
323        push_u32s(&mut bytes, &[40]);
324        push_i32s(&mut bytes, &[99]);
325        push_utf16(&mut bytes, "meta");
326
327        let parsed = WordInfoParser::subset(InfoSubset::READING_FORM)
328            .parse(&bytes)
329            .unwrap();
330        assert_eq!(parsed.reading_form_strptr, fixed.reading_form_strptr);
331        assert_eq!(parsed.pos_id, 0);
332        assert_eq!(parsed.headword_strptr, Default::default());
333        assert_eq!(parsed.normalized_form, 0);
334        assert_eq!(parsed.dictionary_form, 0);
335        assert_eq!(parsed.index_form_length, 0);
336        assert!(parsed.c_unit_split.is_empty());
337        assert!(parsed.b_unit_split.is_empty());
338        assert!(parsed.a_unit_split.is_empty());
339        assert!(parsed.word_structure.is_empty());
340        assert!(parsed.synonym_group_ids.is_empty());
341        assert!(parsed.user_data.is_empty());
342    }
343
344    #[test]
345    fn subset_normalize_keeps_required_dependencies() {
346        let fixed = WordInfoFixedData {
347            pos_id: 7,
348            headword_strptr: StringPointer::unchecked(2, 6),
349            reading_form_strptr: StringPointer::unchecked(1, 2),
350            normalized_form: 33,
351            dictionary_form: 44,
352            index_form_length: 5,
353            c_unit_split_length: 1,
354            b_unit_split_length: 0,
355            a_unit_split_length: 0,
356            word_structure_length: 0,
357            synonym_group_ids_length: 0,
358            user_data_flag: 0,
359        };
360
361        let mut bytes = vec![0u8; layout::PARAMS_SIZE];
362        fixed.write_to(&mut bytes).unwrap();
363        push_u32s(&mut bytes, &[10]);
364
365        let parsed = WordInfoParser::subset(InfoSubset::NORMALIZED_FORM | InfoSubset::SPLIT_C)
366            .parse(&bytes)
367            .unwrap();
368        assert_eq!(parsed.normalized_form, fixed.normalized_form);
369        assert_eq!(parsed.headword_strptr, fixed.headword_strptr);
370        assert_eq!(parsed.index_form_length, fixed.index_form_length);
371        assert_eq!(parsed.c_unit_split, vec![10]);
372    }
373}
sudachi/dic/word_info/parse.rs

sudachi/dic/word_info/
parse.rs
