sudachi/dic/lexicon/
word_id_table.rs

1/*
2 * Copyright (c) 2021-2026 Works Applications Co., Ltd.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17use std::iter::FusedIterator;
18
19use crate::dic::read::varint::varint32;
20use crate::dic::word_id::EntryId;
21
22pub struct WordIdTable<'a> {
23    bytes: &'a [u8],
24}
25
26impl<'a> WordIdTable<'a> {
27    pub fn from_bytes(bytes: &'a [u8]) -> WordIdTable<'a> {
28        WordIdTable { bytes }
29    }
30
31    pub fn new(bytes: &'a [u8], size: u32, offset: usize) -> WordIdTable<'a> {
32        Self::from_bytes(&bytes[offset..offset + size as usize])
33    }
34
35    #[inline]
36    pub fn entries(&self, index: usize) -> DeltaCompressedEntryIdIter<'a> {
37        debug_assert!(index < self.bytes.len());
38        DeltaCompressedEntryIdIter::new(&self.bytes[index..])
39    }
40
41    pub fn all_entries(&self) -> EntryIdIter<'a> {
42        EntryIdIter {
43            inner: DeltaCompressedEntryIdIter::new(self.bytes),
44        }
45    }
46}
47
48/// Iterator over word ids in a delta-compressed varint32 format.
49pub struct DeltaCompressedEntryIdIter<'a> {
50    pub(crate) rest: &'a [u8],
51    remining: u32,
52    sum: u32,
53}
54
55impl<'a> DeltaCompressedEntryIdIter<'a> {
56    pub fn new(bytes: &'a [u8]) -> Self {
57        let (rest, remining) = varint32(bytes).expect("Failed to parse length in WordIdTable");
58
59        DeltaCompressedEntryIdIter {
60            rest,
61            remining,
62            sum: 0,
63        }
64    }
65}
66
67impl Iterator for DeltaCompressedEntryIdIter<'_> {
68    type Item = EntryId;
69
70    #[inline]
71    fn next(&mut self) -> Option<Self::Item> {
72        if self.remining == 0 {
73            return None;
74        }
75
76        let (rest, delta) = varint32(self.rest).expect("Failed to parse next word id delta");
77
78        self.rest = rest;
79        self.remining -= 1;
80        self.sum += delta;
81        Some(EntryId::new(self.sum))
82    }
83}
84
85impl FusedIterator for DeltaCompressedEntryIdIter<'_> {}
86
87/// Iterator over all word ids in the table.
88pub struct EntryIdIter<'a> {
89    inner: DeltaCompressedEntryIdIter<'a>,
90}
91
92impl Iterator for EntryIdIter<'_> {
93    type Item = EntryId;
94
95    #[inline]
96    fn next(&mut self) -> Option<Self::Item> {
97        self.inner.next().or_else(|| {
98            // If we reached the end of the inner iterator, move to the next list (if exists)
99            if self.inner.rest.is_empty() {
100                None
101            } else {
102                self.inner = DeltaCompressedEntryIdIter::new(self.inner.rest);
103                self.next()
104            }
105        })
106    }
107}
108
109impl FusedIterator for EntryIdIter<'_> {}
sudachi/dic/lexicon/word_id_table.rs

sudachi/dic/lexicon/
word_id_table.rs
