sudachi/dic/word_info/
infos.rs

1/*
2 * Copyright (c) 2021-2026 Works Applications Co., Ltd.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17use std::iter::FusedIterator;
18
19use crate::dic::subset::InfoSubset;
20use crate::dic::word_id::EntryId;
21use crate::dic::word_info::layout;
22use crate::dic::word_info::{WordInfoFixedData, WordInfoParser, WordInfoRefData};
23use crate::prelude::*;
24use thiserror::Error;
25
26/// Errors returned while scanning binary WordInfo entry boundaries.
27#[derive(Error, Debug, Clone, Eq, PartialEq)]
28#[non_exhaustive]
29pub enum WordInfoError {
30    /// WordInfo entry offsets must be aligned because entry ids are derived from them.
31    #[error("word info entry id at byte offset {0} is not aligned")]
32    EntryIdNotAligned(usize),
33
34    /// The scanner could not determine a complete, valid entry size at the offset.
35    #[error("failed to load word info entry size at byte offset {0}")]
36    FailedToLoadEntrySize(usize),
37
38    /// The entry offset cannot be represented as a valid Sudachi entry id.
39    #[error("word info entry id at byte offset {0} is too large")]
40    EntryIdTooLarge(usize),
41}
42
43pub struct WordInfos<'a> {
44    bytes: &'a [u8],
45}
46
47impl<'a> WordInfos<'a> {
48    pub const ENTRIES_INITIAL_OFFSET: usize = layout::ENTRY_INITIAL_OFFSET;
49    pub const WORD_ID_ALIGNMENT_BITS: usize = layout::WORD_ID_ALIGNMENT_BITS;
50    pub const WORD_INFO_OFFSET_ALIGNMENT: usize = layout::WORD_INFO_OFFSET_ALIGNMENT;
51
52    pub fn from_bytes(bytes: &'a [u8]) -> WordInfos<'a> {
53        WordInfos { bytes }
54    }
55
56    pub fn entry_id_to_offset(entry_id: EntryId) -> usize {
57        (entry_id.as_raw() as usize) << Self::WORD_ID_ALIGNMENT_BITS
58    }
59
60    pub fn entry_ids_in_order(&self, num_total_entries: u32) -> Option<Vec<EntryId>> {
61        self.entry_ids(num_total_entries)
62            .collect::<SudachiResult<Vec<_>>>()
63            .ok()
64    }
65
66    pub(crate) fn entry_ids(&self, num_total_entries: u32) -> WordInfoEntryIdIter<'_, '_> {
67        WordInfoEntryIdIter {
68            infos: self,
69            cursor: Self::entry_id_cursor(num_total_entries),
70        }
71    }
72
73    pub(crate) fn entry_id_cursor(num_total_entries: u32) -> WordInfoEntryIdCursor {
74        WordInfoEntryIdCursor {
75            remaining: num_total_entries,
76            offset: Self::ENTRIES_INITIAL_OFFSET,
77        }
78    }
79
80    /// Validate that all WordInfo entries can be scanned from their binary boundaries.
81    ///
82    /// Returns [`WordInfoError`] through [`SudachiError`] when an entry id cannot be
83    /// derived or an entry size cannot be read.
84    pub fn validate_entry_boundaries(&self, num_total_entries: u32) -> SudachiResult<()> {
85        let mut cursor = Self::entry_id_cursor(num_total_entries);
86        while self.next_entry_id(&mut cursor)?.is_some() {}
87        Ok(())
88    }
89
90    pub(crate) fn next_entry_id(
91        &self,
92        cursor: &mut WordInfoEntryIdCursor,
93    ) -> SudachiResult<Option<EntryId>> {
94        if cursor.remaining == 0 {
95            return Ok(None);
96        }
97
98        let entry_id = Self::entry_id_from_offset(cursor.offset)?;
99        let size = self
100            .entry_size_at(cursor.offset)
101            .ok_or(WordInfoError::FailedToLoadEntrySize(cursor.offset))?;
102
103        cursor.offset = match cursor.offset.checked_add(size) {
104            Some(offset) => offset,
105            None => return Err(WordInfoError::EntryIdTooLarge(cursor.offset).into()),
106        };
107        cursor.remaining -= 1;
108        Ok(Some(entry_id))
109    }
110
111    fn entry_id_from_offset(offset: usize) -> SudachiResult<EntryId> {
112        if offset % Self::WORD_INFO_OFFSET_ALIGNMENT != 0 {
113            return Err(WordInfoError::EntryIdNotAligned(offset).into());
114        }
115
116        let raw = offset >> Self::WORD_ID_ALIGNMENT_BITS;
117        if raw > EntryId::MAX as usize {
118            return Err(WordInfoError::EntryIdTooLarge(offset).into());
119        }
120
121        Ok(EntryId::new(raw as u32))
122    }
123
124    fn entry_size_at(&self, offset: usize) -> Option<usize> {
125        let entry_bytes = self.bytes.get(offset..)?;
126        let fixed = WordInfoFixedData::from_entry_bytes(entry_bytes)?;
127
128        if !layout::is_valid_user_data_flag(fixed.user_data_flag) {
129            return None;
130        }
131
132        let mut user_data_units = None;
133        if fixed.has_user_data() {
134            let user_data_offset = offset.checked_add(layout::unaligned_size_from_lengths(
135                fixed.c_unit_split_length,
136                fixed.b_unit_split_length,
137                fixed.a_unit_split_length,
138                fixed.word_structure_length,
139                fixed.synonym_group_ids_length,
140                None,
141            )?)?;
142            let user_data_len_end = user_data_offset.checked_add(2)?;
143            let user_len_bytes = self.bytes.get(user_data_offset..user_data_len_end)?;
144            let user_len = i16::from_le_bytes([user_len_bytes[0], user_len_bytes[1]]);
145            user_data_units = Some(user_len);
146        }
147
148        let aligned = layout::size_from_lengths(
149            fixed.c_unit_split_length,
150            fixed.b_unit_split_length,
151            fixed.a_unit_split_length,
152            fixed.word_structure_length,
153            fixed.synonym_group_ids_length,
154            user_data_units,
155        )?;
156        let end = offset.checked_add(aligned)?;
157        self.bytes.get(offset..end)?;
158        Some(aligned)
159    }
160
161    pub fn get_word_info(
162        &self,
163        entry_id: EntryId,
164        subset: InfoSubset,
165    ) -> SudachiResult<WordInfoRefData> {
166        let offset = Self::entry_id_to_offset(entry_id);
167        let parser = WordInfoParser::subset(subset);
168        let bytes = self.bytes.get(offset..).ok_or_else(|| {
169            SudachiError::InvalidDataFormat(
170                0,
171                format!("invalid word info entry id: {}", entry_id.as_raw()),
172            )
173        })?;
174        let word_info = parser.parse(bytes)?;
175        Ok(WordInfoRefData::from_raw(word_info))
176    }
177}
178
179pub(crate) struct WordInfoEntryIdCursor {
180    remaining: u32,
181    offset: usize,
182}
183
184pub(crate) struct WordInfoEntryIdIter<'a, 'b> {
185    infos: &'a WordInfos<'b>,
186    cursor: WordInfoEntryIdCursor,
187}
188
189impl Iterator for WordInfoEntryIdIter<'_, '_> {
190    type Item = SudachiResult<EntryId>;
191
192    fn next(&mut self) -> Option<Self::Item> {
193        match self.infos.next_entry_id(&mut self.cursor) {
194            Ok(Some(entry_id)) => Some(Ok(entry_id)),
195            Ok(None) => None,
196            Err(error) => {
197                self.cursor.remaining = 0;
198                Some(Err(error))
199            }
200        }
201    }
202
203    fn size_hint(&self) -> (usize, Option<usize>) {
204        let remaining = self.cursor.remaining as usize;
205        (0, Some(remaining))
206    }
207}
208
209impl FusedIterator for WordInfoEntryIdIter<'_, '_> {}
210
211#[cfg(test)]
212mod tests {
213    use super::*;
214    use crate::dic::lexicon::strings::StringPointer;
215    use crate::dic::word_id::DictId;
216    use crate::dic::word_info::WordInfoVariableData;
217
218    fn assert_word_info_error(error: SudachiError, expected: WordInfoError) {
219        match error {
220            SudachiError::WordInfo(actual) => assert_eq!(actual, expected),
221            other => panic!("expected word info error {expected:?}, got {other:?}"),
222        }
223    }
224
225    fn sample_fixed() -> WordInfoFixedData {
226        WordInfoFixedData {
227            pos_id: 3,
228            headword_strptr: StringPointer::unchecked(2, 4),
229            reading_form_strptr: StringPointer::unchecked(3, 8),
230            normalized_form: 10,
231            dictionary_form: 11,
232            index_form_length: 6,
233            c_unit_split_length: 2,
234            b_unit_split_length: -1,
235            a_unit_split_length: 1,
236            word_structure_length: -1,
237            synonym_group_ids_length: 2,
238            user_data_flag: 1,
239        }
240    }
241
242    fn make_entry(fixed: &WordInfoFixedData) -> Vec<u8> {
243        let variable = WordInfoVariableData {
244            c_unit_split: &[100, 101],
245            b_unit_split: &[100, 101],
246            a_unit_split: &[200],
247            word_structure: &[200],
248            synonym_group_ids: &[7, 8],
249            user_data: "meta",
250        };
251        let mut bytes = vec![0u8; layout::ENTRY_INITIAL_OFFSET + layout::PARAMS_SIZE];
252        fixed.write_to(&mut bytes).unwrap();
253        variable.write_to(&mut bytes, fixed).unwrap();
254        let aligned = layout::aligned_size(bytes.len());
255        bytes.resize(aligned, 0);
256        bytes
257    }
258
259    #[test]
260    fn rejects_invalid_user_data_flag() {
261        let mut fixed = sample_fixed();
262        fixed.user_data_flag = 2;
263        let bytes = make_entry(&fixed);
264        let infos = WordInfos::from_bytes(&bytes);
265        assert!(infos.entry_size_at(layout::ENTRY_INITIAL_OFFSET).is_none());
266    }
267
268    #[test]
269    fn rejects_truncated_user_data_length() {
270        let fixed = sample_fixed();
271        let mut bytes = make_entry(&fixed);
272        let user_len_offset = layout::ENTRY_INITIAL_OFFSET
273            + layout::unaligned_size_from_lengths(
274                fixed.c_unit_split_length,
275                fixed.b_unit_split_length,
276                fixed.a_unit_split_length,
277                fixed.word_structure_length,
278                fixed.synonym_group_ids_length,
279                None,
280            )
281            .unwrap();
282        bytes.truncate(user_len_offset + 1);
283        let infos = WordInfos::from_bytes(&bytes);
284        assert!(infos.entry_size_at(layout::ENTRY_INITIAL_OFFSET).is_none());
285    }
286
287    #[test]
288    fn rejects_split_payload_shorter_than_length() {
289        let fixed = WordInfoFixedData {
290            user_data_flag: 0,
291            synonym_group_ids_length: 0,
292            word_structure_length: 0,
293            a_unit_split_length: 0,
294            b_unit_split_length: 0,
295            c_unit_split_length: 2,
296            ..sample_fixed()
297        };
298        let mut bytes = vec![0u8; layout::ENTRY_INITIAL_OFFSET + layout::PARAMS_SIZE];
299        fixed.write_to(&mut bytes).unwrap();
300        bytes.extend_from_slice(&10u32.to_le_bytes());
301        let infos = WordInfos::from_bytes(&bytes);
302        assert!(infos.entry_size_at(layout::ENTRY_INITIAL_OFFSET).is_none());
303    }
304
305    #[test]
306    fn parser_and_scanner_agree_on_entry_boundaries() {
307        let first = make_entry(&sample_fixed());
308        let second_fixed = WordInfoFixedData {
309            pos_id: 9,
310            headword_strptr: StringPointer::unchecked(1, 2),
311            reading_form_strptr: StringPointer::unchecked(1, 4),
312            normalized_form: 21,
313            dictionary_form: 22,
314            index_form_length: 3,
315            c_unit_split_length: 1,
316            b_unit_split_length: 0,
317            a_unit_split_length: 0,
318            word_structure_length: 0,
319            synonym_group_ids_length: 0,
320            user_data_flag: 0,
321        };
322        let mut second = vec![0u8; layout::PARAMS_SIZE];
323        second_fixed.write_to(&mut second).unwrap();
324        second.extend_from_slice(&55u32.to_le_bytes());
325        second.resize(layout::aligned_size(second.len()), 0);
326
327        let mut bytes = first.clone();
328        bytes.extend_from_slice(&second);
329
330        let infos = WordInfos::from_bytes(&bytes);
331        let ids = infos.entry_ids_in_order(2).unwrap();
332        assert_eq!(ids[0], EntryId::new(4));
333        let second_offset = WordInfos::entry_id_to_offset(ids[1]);
334        assert_eq!(second_offset, first.len());
335
336        let first_info = infos.get_word_info(ids[0], InfoSubset::all()).unwrap();
337        let second_info = infos.get_word_info(ids[1], InfoSubset::all()).unwrap();
338        assert_eq!(
339            first_info
340                .resolve(DictId::SYSTEM, 0, &[0], InfoSubset::all())
341                .index_form_length(),
342            6
343        );
344        assert_eq!(
345            second_info
346                .resolve(DictId::SYSTEM, 0, &[0], InfoSubset::all())
347                .c_unit_split()
348                .len(),
349            1
350        );
351    }
352
353    #[test]
354    fn validate_entry_boundaries_rejects_malformed_entries() {
355        let mut bytes = make_entry(&sample_fixed());
356        bytes.truncate(bytes.len() - 1);
357        let infos = WordInfos::from_bytes(&bytes);
358
359        let err = infos.validate_entry_boundaries(1).unwrap_err();
360        assert_word_info_error(
361            err,
362            WordInfoError::FailedToLoadEntrySize(layout::ENTRY_INITIAL_OFFSET),
363        );
364    }
365
366    #[test]
367    fn validate_entry_boundaries_rejects_short_entry_block() {
368        let bytes = vec![0; layout::ENTRY_INITIAL_OFFSET - 1];
369        let infos = WordInfos::from_bytes(&bytes);
370
371        let err = infos.validate_entry_boundaries(1).unwrap_err();
372        assert_word_info_error(
373            err,
374            WordInfoError::FailedToLoadEntrySize(layout::ENTRY_INITIAL_OFFSET),
375        );
376    }
377
378    #[test]
379    fn next_entry_id_rejects_misaligned_entry_offset() {
380        let bytes = make_entry(&sample_fixed());
381        let infos = WordInfos::from_bytes(&bytes);
382        let mut cursor = WordInfoEntryIdCursor {
383            remaining: 1,
384            offset: layout::ENTRY_INITIAL_OFFSET + 1,
385        };
386
387        let err = infos.next_entry_id(&mut cursor).unwrap_err();
388        assert_word_info_error(
389            err,
390            WordInfoError::EntryIdNotAligned(layout::ENTRY_INITIAL_OFFSET + 1),
391        );
392    }
393
394    #[test]
395    fn next_entry_id_rejects_too_large_entry_id() {
396        let bytes = make_entry(&sample_fixed());
397        let infos = WordInfos::from_bytes(&bytes);
398        let too_large_offset = ((EntryId::MAX as usize) + 1) << WordInfos::WORD_ID_ALIGNMENT_BITS;
399        let mut cursor = WordInfoEntryIdCursor {
400            remaining: 1,
401            offset: too_large_offset,
402        };
403
404        let err = infos.next_entry_id(&mut cursor).unwrap_err();
405        assert_word_info_error(err, WordInfoError::EntryIdTooLarge(too_large_offset));
406    }
407
408    #[test]
409    fn entry_id_iterator_reports_malformed_entries() {
410        let mut bytes = make_entry(&sample_fixed());
411        bytes.truncate(bytes.len() - 1);
412        let infos = WordInfos::from_bytes(&bytes);
413        let mut entries = infos.entry_ids(1);
414
415        assert!(entries.next().unwrap().is_err());
416        assert!(entries.next().is_none());
417    }
418}
sudachi/dic/word_info/infos.rs

sudachi/dic/word_info/
infos.rs
