sudachi/dic/word_info/
binary.rs

1/*
2 * Copyright (c) 2026 Works Applications Co., Ltd.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17use std::io::Write;
18
19use nom::number::complete::{le_i16, le_i32, le_i8, le_u32};
20
21use crate::dic::lexicon::strings::StringPointer;
22use crate::dic::read::error::SudachiNomResult;
23use crate::dic::read::utf16_string::{skip_utf16_string, utf16_string};
24use crate::dic::word_info::layout;
25use crate::dic::word_info::{WordInfoFixedData, WordInfoVariableData};
26use crate::error::SudachiResult;
27
28impl WordInfoFixedData {
29    pub fn parse(input: &[u8]) -> SudachiNomResult<&[u8], Self> {
30        let (input, pos_id) = le_i16(input)?;
31        let (input, headword_strptr) =
32            le_u32(input).map(|(rest, pointer)| (rest, StringPointer::decode(pointer)))?;
33        let (input, reading_form_strptr) =
34            le_u32(input).map(|(rest, pointer)| (rest, StringPointer::decode(pointer)))?;
35        let (input, normalized_form) = le_u32(input)?;
36        let (input, dictionary_form) = le_u32(input)?;
37        let (input, index_form_length) = le_i16(input)?;
38        let (input, c_unit_split_length) = le_i8(input)?;
39        let (input, b_unit_split_length) = le_i8(input)?;
40        let (input, a_unit_split_length) = le_i8(input)?;
41        let (input, word_structure_length) = le_i8(input)?;
42        let (input, synonym_group_ids_length) = le_i8(input)?;
43        let (input, user_data_flag) = le_i8(input)?;
44        Ok((
45            input,
46            Self {
47                pos_id,
48                headword_strptr,
49                reading_form_strptr,
50                normalized_form,
51                dictionary_form,
52                index_form_length,
53                c_unit_split_length,
54                b_unit_split_length,
55                a_unit_split_length,
56                word_structure_length,
57                synonym_group_ids_length,
58                user_data_flag,
59            },
60        ))
61    }
62
63    pub fn from_entry_bytes(data: &[u8]) -> Option<Self> {
64        let fixed = data.get(..layout::FIXED_PART_SIZE)?;
65        Some(Self {
66            pos_id: i16::from_le_bytes([
67                fixed[layout::PARAMS_SIZE],
68                fixed[layout::PARAMS_SIZE + 1],
69            ]),
70            headword_strptr: StringPointer::decode(u32::from_le_bytes([
71                fixed[8], fixed[9], fixed[10], fixed[11],
72            ])),
73            reading_form_strptr: StringPointer::decode(u32::from_le_bytes([
74                fixed[12], fixed[13], fixed[14], fixed[15],
75            ])),
76            normalized_form: u32::from_le_bytes([fixed[16], fixed[17], fixed[18], fixed[19]]),
77            dictionary_form: u32::from_le_bytes([fixed[20], fixed[21], fixed[22], fixed[23]]),
78            index_form_length: i16::from_le_bytes([fixed[24], fixed[25]]),
79            c_unit_split_length: fixed[layout::OFFSET_C_UNIT_SPLIT_LENGTH] as i8,
80            b_unit_split_length: fixed[layout::OFFSET_B_UNIT_SPLIT_LENGTH] as i8,
81            a_unit_split_length: fixed[layout::OFFSET_A_UNIT_SPLIT_LENGTH] as i8,
82            word_structure_length: fixed[layout::OFFSET_WORD_STRUCTURE_LENGTH] as i8,
83            synonym_group_ids_length: fixed[layout::OFFSET_SYNONYM_GROUP_IDS_LENGTH] as i8,
84            user_data_flag: fixed[layout::OFFSET_USER_DATA_FLAG] as i8,
85        })
86    }
87
88    pub fn has_user_data(&self) -> bool {
89        self.user_data_flag == 1
90    }
91
92    pub fn write_to<W: Write>(&self, w: &mut W) -> std::io::Result<usize> {
93        w.write_all(&self.pos_id.to_le_bytes())?;
94        w.write_all(&self.headword_strptr.encode().to_le_bytes())?;
95        w.write_all(&self.reading_form_strptr.encode().to_le_bytes())?;
96        w.write_all(&self.normalized_form.to_le_bytes())?;
97        w.write_all(&self.dictionary_form.to_le_bytes())?;
98        w.write_all(&self.index_form_length.to_le_bytes())?;
99        w.write_all(&self.c_unit_split_length.to_le_bytes())?;
100        w.write_all(&self.b_unit_split_length.to_le_bytes())?;
101        w.write_all(&self.a_unit_split_length.to_le_bytes())?;
102        w.write_all(&self.word_structure_length.to_le_bytes())?;
103        w.write_all(&self.synonym_group_ids_length.to_le_bytes())?;
104        w.write_all(&self.user_data_flag.to_le_bytes())?;
105        Ok(layout::WORD_INFO_FIXED_SIZE)
106    }
107}
108
109pub(crate) fn parse_u32_array(
110    input: &[u8],
111    length: usize,
112    keep: bool,
113) -> SudachiResult<(&[u8], Vec<u32>)> {
114    if keep {
115        let (rest, values) = nom::multi::count(le_u32, length)(input)?;
116        Ok((rest, values))
117    } else {
118        let bytes = length * 4;
119        let (rest, _) = nom::bytes::complete::take(bytes)(input)?;
120        Ok((rest, Vec::new()))
121    }
122}
123
124pub(crate) fn parse_i32_array(
125    input: &[u8],
126    length: usize,
127    keep: bool,
128) -> SudachiResult<(&[u8], Vec<i32>)> {
129    if keep {
130        let (rest, values) = nom::multi::count(le_i32, length)(input)?;
131        Ok((rest, values))
132    } else {
133        let bytes = length * 4;
134        let (rest, _) = nom::bytes::complete::take(bytes)(input)?;
135        Ok((rest, Vec::new()))
136    }
137}
138
139pub(crate) fn parse_user_data(input: &[u8], keep: bool) -> SudachiResult<(&[u8], String)> {
140    if keep {
141        utf16_string(input).map_err(Into::into)
142    } else {
143        skip_utf16_string(input).map_err(Into::into)
144    }
145}
146
147pub(crate) fn write_u32_slice<W: Write>(w: &mut W, data: &[u32]) -> std::io::Result<usize> {
148    let mut size = 0;
149    for value in data {
150        w.write_all(&value.to_le_bytes())?;
151        size += 4;
152    }
153    Ok(size)
154}
155
156pub(crate) fn write_i32_slice<W: Write>(w: &mut W, data: &[i32]) -> std::io::Result<usize> {
157    let mut size = 0;
158    for value in data {
159        w.write_all(&value.to_le_bytes())?;
160        size += 4;
161    }
162    Ok(size)
163}
164
165pub(crate) fn write_utf16_string<W: Write>(w: &mut W, data: &str) -> std::io::Result<usize> {
166    let utf16: Vec<u16> = data.encode_utf16().collect();
167    let mut size = 0;
168    w.write_all(&(utf16.len() as i16).to_le_bytes())?;
169    size += 2;
170    for unit in utf16 {
171        w.write_all(&unit.to_le_bytes())?;
172        size += 2;
173    }
174    Ok(size)
175}
176
177impl<'a> WordInfoVariableData<'a> {
178    pub fn write_to<W: Write>(
179        &self,
180        w: &mut W,
181        fixed: &WordInfoFixedData,
182    ) -> std::io::Result<usize> {
183        let mut size = 0;
184        size += write_u32_slice(w, self.c_unit_split)?;
185        if fixed.b_unit_split_length > 0 {
186            size += write_u32_slice(w, self.b_unit_split)?;
187        }
188        if fixed.a_unit_split_length > 0 {
189            size += write_u32_slice(w, self.a_unit_split)?;
190        }
191        if fixed.word_structure_length > 0 {
192            size += write_u32_slice(w, self.word_structure)?;
193        }
194        size += write_i32_slice(w, self.synonym_group_ids)?;
195        if fixed.has_user_data() {
196            size += write_utf16_string(w, self.user_data)?;
197        }
198        Ok(size)
199    }
200}
201
202#[cfg(test)]
203mod tests {
204    use super::*;
205    use crate::dic::word_info::WordInfoParser;
206
207    fn sample_fixed() -> WordInfoFixedData {
208        WordInfoFixedData {
209            pos_id: 123,
210            headword_strptr: StringPointer::unchecked(3, 8),
211            reading_form_strptr: StringPointer::unchecked(5, 16),
212            normalized_form: 42,
213            dictionary_form: 84,
214            index_form_length: 9,
215            c_unit_split_length: 2,
216            b_unit_split_length: -1,
217            a_unit_split_length: 4,
218            word_structure_length: -1,
219            synonym_group_ids_length: 3,
220            user_data_flag: 1,
221        }
222    }
223
224    #[test]
225    fn fixed_data_write_and_parse_round_trip() {
226        let fixed = sample_fixed();
227        let mut bytes = Vec::new();
228        let written = fixed.write_to(&mut bytes).unwrap();
229        assert_eq!(written, layout::WORD_INFO_FIXED_SIZE);
230
231        let (rest, parsed) = WordInfoFixedData::parse(&bytes).unwrap();
232        assert!(rest.is_empty());
233        assert_eq!(parsed.pos_id, fixed.pos_id);
234        assert_eq!(parsed.headword_strptr, fixed.headword_strptr);
235        assert_eq!(parsed.reading_form_strptr, fixed.reading_form_strptr);
236        assert_eq!(parsed.normalized_form, fixed.normalized_form);
237        assert_eq!(parsed.dictionary_form, fixed.dictionary_form);
238        assert_eq!(parsed.index_form_length, fixed.index_form_length);
239        assert_eq!(parsed.c_unit_split_length, fixed.c_unit_split_length);
240        assert_eq!(parsed.b_unit_split_length, fixed.b_unit_split_length);
241        assert_eq!(parsed.a_unit_split_length, fixed.a_unit_split_length);
242        assert_eq!(parsed.word_structure_length, fixed.word_structure_length);
243        assert_eq!(
244            parsed.synonym_group_ids_length,
245            fixed.synonym_group_ids_length
246        );
247        assert_eq!(parsed.user_data_flag, fixed.user_data_flag);
248    }
249
250    #[test]
251    fn fixed_data_reads_from_entry_bytes_after_params() {
252        let fixed = sample_fixed();
253        let mut entry = vec![0u8; layout::PARAMS_SIZE];
254        fixed.write_to(&mut entry).unwrap();
255        assert_eq!(entry.len(), layout::FIXED_PART_SIZE);
256
257        let scanned = WordInfoFixedData::from_entry_bytes(&entry).unwrap();
258        assert_eq!(scanned.pos_id, fixed.pos_id);
259        assert_eq!(scanned.headword_strptr, fixed.headword_strptr);
260        assert_eq!(scanned.reading_form_strptr, fixed.reading_form_strptr);
261        assert_eq!(scanned.normalized_form, fixed.normalized_form);
262        assert_eq!(scanned.dictionary_form, fixed.dictionary_form);
263        assert_eq!(scanned.index_form_length, fixed.index_form_length);
264        assert_eq!(scanned.c_unit_split_length, fixed.c_unit_split_length);
265        assert_eq!(scanned.b_unit_split_length, fixed.b_unit_split_length);
266        assert_eq!(scanned.a_unit_split_length, fixed.a_unit_split_length);
267        assert_eq!(scanned.word_structure_length, fixed.word_structure_length);
268        assert_eq!(
269            scanned.synonym_group_ids_length,
270            fixed.synonym_group_ids_length
271        );
272        assert_eq!(scanned.user_data_flag, fixed.user_data_flag);
273    }
274
275    #[test]
276    fn fixed_and_variable_round_trip_into_raw_data() {
277        let fixed = WordInfoFixedData {
278            pos_id: 15,
279            headword_strptr: StringPointer::unchecked(4, 12),
280            reading_form_strptr: StringPointer::unchecked(5, 18),
281            normalized_form: 123,
282            dictionary_form: 456,
283            index_form_length: 9,
284            c_unit_split_length: 2,
285            b_unit_split_length: 1,
286            a_unit_split_length: 3,
287            word_structure_length: 2,
288            synonym_group_ids_length: 2,
289            user_data_flag: 1,
290        };
291        let variable = WordInfoVariableData {
292            c_unit_split: &[10, 11],
293            b_unit_split: &[20],
294            a_unit_split: &[30, 31, 32],
295            word_structure: &[40, 41],
296            synonym_group_ids: &[7, 8],
297            user_data: "meta",
298        };
299
300        let mut bytes = vec![0u8; layout::PARAMS_SIZE];
301        fixed.write_to(&mut bytes).unwrap();
302        variable.write_to(&mut bytes, &fixed).unwrap();
303
304        let parsed = WordInfoParser::default().parse(&bytes).unwrap();
305        assert_eq!(parsed.pos_id, fixed.pos_id);
306        assert_eq!(parsed.headword_strptr, fixed.headword_strptr);
307        assert_eq!(parsed.reading_form_strptr, fixed.reading_form_strptr);
308        assert_eq!(parsed.normalized_form, fixed.normalized_form);
309        assert_eq!(parsed.dictionary_form, fixed.dictionary_form);
310        assert_eq!(parsed.index_form_length, fixed.index_form_length);
311        assert_eq!(parsed.c_unit_split_length, fixed.c_unit_split_length);
312        assert_eq!(parsed.b_unit_split_length, fixed.b_unit_split_length);
313        assert_eq!(parsed.a_unit_split_length, fixed.a_unit_split_length);
314        assert_eq!(parsed.word_structure_length, fixed.word_structure_length);
315        assert_eq!(
316            parsed.synonym_group_ids_length,
317            fixed.synonym_group_ids_length
318        );
319        assert_eq!(parsed.user_data_flag, fixed.user_data_flag);
320        assert_eq!(parsed.c_unit_split, variable.c_unit_split);
321        assert_eq!(parsed.b_unit_split, variable.b_unit_split);
322        assert_eq!(parsed.a_unit_split, variable.a_unit_split);
323        assert_eq!(parsed.word_structure, variable.word_structure);
324        assert_eq!(parsed.synonym_group_ids, variable.synonym_group_ids);
325        assert_eq!(parsed.user_data, variable.user_data);
326    }
327}
sudachi/dic/word_info/binary.rs

sudachi/dic/word_info/
binary.rs
