sudachi/dic/
word_id.rs

1/*
2 *  Copyright (c) 2021-2026 Works Applications Co., Ltd.
3 *
4 *  Licensed under the Apache License, Version 2.0 (the "License");
5 *  you may not use this file except in compliance with the License.
6 *  You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 *   Unless required by applicable law or agreed to in writing, software
11 *  distributed under the License is distributed on an "AS IS" BASIS,
12 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 *  See the License for the specific language governing permissions and
14 *  limitations under the License.
15 */
16
17use crate::dic::lexicon_set::LexiconSetError;
18use crate::error::{SudachiError, SudachiResult};
19use std::fmt::{Debug, Display, Formatter};
20
21/// Bit mask for the entry id part of the WordId
22const WORD_MASK: u32 = 0x0fff_ffff;
23
24/// Dictionary ID
25///
26/// Id of the binary dictionary in a combined dictionary.
27///
28/// 0: system dictionary
29/// 1-14: user dictionary
30/// 15: OOV and other special nodes
31#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)]
32#[repr(transparent)]
33pub struct DictId {
34    raw: u8,
35}
36
37impl DictId {
38    /// Create DictId from the compressed representation
39    const fn from_raw(raw: u8) -> DictId {
40        DictId { raw }
41    }
42
43    /// Create a new DictId from parts
44    pub fn new(dict: u8) -> DictId {
45        debug_assert!(dict <= 15);
46        Self::from_raw(dict)
47    }
48
49    /// Create a new DictId with correctness checking
50    pub fn checked(dict: u8) -> SudachiResult<DictId> {
51        if dict > 15 {
52            return Err(SudachiError::LexiconSetError(
53                LexiconSetError::TooLargeDictionaryId(dict as usize),
54            ));
55        }
56        Ok(DictId::new(dict))
57    }
58
59    /// Get the raw value of the DictId
60    pub const fn as_raw(&self) -> u8 {
61        self.raw
62    }
63
64    /// Check if the word comes from the system dictionary
65    pub fn is_system(&self) -> bool {
66        self.raw == 0
67    }
68
69    /// Check if the word comes from the user dictionary
70    pub fn is_user(&self) -> bool {
71        !matches!(self.raw, 0 | 0xf)
72    }
73
74    /// Check if the word is OOV
75    /// An OOV node can come of OOV handlers or be a special system node like BOS or EOS
76    pub fn is_oov(&self) -> bool {
77        self.raw == 0xf
78    }
79
80    pub const SYSTEM: Self = DictId::from_raw(0);
81    pub const MAX_USER: Self = DictId::from_raw(14);
82    pub const SPECIAL: Self = DictId::from_raw(15);
83}
84
85/// Entry id
86///
87/// Id of the entry in a single binary dictionary.
88/// Top 4 bits are always 0.
89#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)]
90#[repr(transparent)]
91pub struct EntryId {
92    raw: u32,
93}
94
95impl EntryId {
96    /// Create WordId from the compressed representation
97    const fn from_raw(raw: u32) -> Self {
98        EntryId { raw }
99    }
100
101    /// Create a new EntryId from parts
102    pub fn new(entry: u32) -> Self {
103        debug_assert_eq!(entry & (!WORD_MASK), 0);
104        Self::from_raw(entry)
105    }
106
107    /// Create a new EntryId with correctness checking
108    pub fn checked(entry: u32) -> SudachiResult<Self> {
109        if entry & !WORD_MASK != 0 {
110            return Err(SudachiError::LexiconSetError(
111                LexiconSetError::TooLargeWordId(entry, WORD_MASK as usize),
112            ));
113        }
114        Ok(Self::new(entry))
115    }
116
117    /// Get the raw value of the EntryId
118    pub const fn as_raw(&self) -> u32 {
119        self.raw
120    }
121
122    pub const MAX: u32 = 0x0fff_ffff;
123}
124
125/// Dictionary Word ID
126///
127/// Id of the word in a combined dictionary.
128/// Encode dictionary ID and entry ID as 4 bits and 28 bits respectively.
129#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
130#[repr(transparent)]
131pub struct WordId {
132    raw: u32,
133}
134
135impl Default for WordId {
136    fn default() -> Self {
137        Self::INVALID
138    }
139}
140
141impl Debug for WordId {
142    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
143        Display::fmt(self, f)
144    }
145}
146
147impl Display for WordId {
148    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
149        let fmtdic = if self.is_oov() {
150            -1
151        } else {
152            self.dict().as_raw() as i32
153        };
154        write!(f, "({}, {})", fmtdic, self.entry().as_raw())
155    }
156}
157
158impl WordId {
159    /// Create WordId from the compressed representation
160    pub(crate) const fn from_raw(raw: u32) -> WordId {
161        WordId { raw }
162    }
163
164    /// Create WordId from Dict and Entry parts.
165    pub fn from_parts(dict: DictId, entry: EntryId) -> WordId {
166        Self::new(dict.as_raw(), entry.as_raw())
167    }
168
169    /// Create WordId from parts
170    pub fn new(dict: u8, entry: u32) -> WordId {
171        debug_assert_eq!(entry & (!WORD_MASK), 0);
172        debug_assert_eq!(dict & (!0xf), 0);
173        let dic_part = ((dict & 0xf) as u32) << 28;
174        let entry_part = entry & WORD_MASK;
175        let raw = dic_part | entry_part;
176        Self::from_raw(raw)
177    }
178
179    /// Creates the WordId with correctness checking
180    pub fn checked(dic: u8, entry: u32) -> SudachiResult<WordId> {
181        if dic > 15 {
182            return Err(SudachiError::LexiconSetError(
183                LexiconSetError::TooLargeDictionaryId(dic as usize),
184            ));
185        }
186
187        if entry & !WORD_MASK != 0 {
188            return Err(SudachiError::LexiconSetError(
189                LexiconSetError::TooLargeWordId(entry, WORD_MASK as usize),
190            ));
191        }
192
193        Ok(Self::new(dic, entry))
194    }
195
196    /// Creates an OOV node for pos_id
197    pub fn oov(pos_id: u32) -> WordId {
198        Self::new(0xf, pos_id)
199    }
200
201    /// Extract Dictionary ID
202    pub fn dict(&self) -> DictId {
203        DictId::new((self.raw >> 28) as u8)
204    }
205
206    /// Extract Word ID
207    pub fn entry(&self) -> EntryId {
208        EntryId::from_raw(self.raw & WORD_MASK)
209    }
210
211    /// Convert to raw representation
212    pub fn as_raw(&self) -> u32 {
213        self.raw
214    }
215
216    /// Check if the word comes from the system dictionary
217    pub fn is_system(&self) -> bool {
218        self.dict().is_system()
219    }
220
221    /// Check if the word comes from the user dictionary
222    pub fn is_user(&self) -> bool {
223        self.dict().is_user()
224    }
225
226    /// Check if the word is OOV
227    /// An OOV node can come of OOV handlers or be a special system node like BOS or EOS
228    pub fn is_oov(&self) -> bool {
229        self.dict().is_oov()
230    }
231
232    /// Checks if the WordId corresponds to a special node
233    pub fn is_special(&self) -> bool {
234        // only beginning-of-sentence and end-of-sentence are special.
235        self == &Self::BOS || self == &Self::EOS
236    }
237
238    pub const INVALID: WordId = WordId::from_raw(0xffff_ffff);
239    pub const OOV_NOPOS: WordId = WordId::from_raw(0xf000_ffff);
240    pub const BOS: WordId = WordId::from_raw(0xffff_fff0);
241    pub const EOS: WordId = WordId::from_raw(0xffff_fff1);
242}
243
244/// Word reference
245///
246/// Reference which points to a entry in the system or user dictionary.
247/// Similar to the WordId but the dict id part is a flag which indicates if it is a system or user word.
248///
249/// Top 4 bit is 0000 - points to the word in the system dictionary.
250/// Top 4 bit is 0001 - points to the word in the user dictionary which this wordref is used in.
251#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
252#[repr(transparent)]
253pub struct WordRef {
254    raw: u32,
255}
256
257impl Default for WordRef {
258    fn default() -> Self {
259        Self::INVALID
260    }
261}
262
263impl Debug for WordRef {
264    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
265        Display::fmt(self, f)
266    }
267}
268
269impl Display for WordRef {
270    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
271        write!(
272            f,
273            "({}, {})",
274            if self.is_system() { "sys" } else { "usr" },
275            self.entry().as_raw()
276        )
277    }
278}
279
280impl WordRef {
281    /// Create WordRef from the compressed representation
282    pub const fn from_raw(raw: u32) -> WordRef {
283        WordRef { raw }
284    }
285
286    /// Create a new WordRef
287    pub fn new(is_system: bool, entry: u32) -> WordRef {
288        debug_assert_eq!(entry & (!WORD_MASK), 0);
289        let dic_part = if is_system { 0 } else { 1 << 28 };
290        let word_part = entry & WORD_MASK;
291        WordRef {
292            raw: dic_part | word_part,
293        }
294    }
295
296    /// Create a new WordRef with correctness checking
297    pub fn checked(is_system: bool, entry: u32) -> SudachiResult<WordRef> {
298        if entry & !WORD_MASK != 0 {
299            return Err(SudachiError::LexiconSetError(
300                LexiconSetError::TooLargeWordId(entry, WORD_MASK as usize),
301            ));
302        }
303        Ok(Self::new(is_system, entry))
304    }
305
306    /// Check if the WordRef points to a system word
307    pub fn is_system(&self) -> bool {
308        self.raw >> 28 == 0
309    }
310
311    /// Check if the WordRef points to a user word
312    pub fn is_user(&self) -> bool {
313        self.raw >> 28 == 1
314    }
315
316    /// Extract Entry ID
317    pub fn entry(&self) -> EntryId {
318        EntryId::from_raw(self.raw & WORD_MASK)
319    }
320
321    /// Convert to raw representation
322    pub fn as_raw(&self) -> u32 {
323        self.raw
324    }
325
326    /// Resolve the WordRef with its DictId in the dictionary
327    pub fn resolve(&self, dict: DictId) -> WordId {
328        if self.is_system() {
329            // dict part of system wordref is 0 and it is already resolved.
330            WordId::from_raw(self.as_raw())
331        } else {
332            // set actual dict id for user wordref.
333            WordId::from_parts(dict, self.entry())
334        }
335    }
336
337    /// resolve raw
338    pub fn resolve_raw(raw: u32, dict: DictId) -> u32 {
339        WordRef::from_raw(raw).resolve(dict).as_raw()
340    }
341
342    pub const INVALID: WordRef = WordRef::from_raw(0xffff_ffff);
343}
344
345#[cfg(test)]
346mod test {
347    use super::*;
348
349    fn assert_create(dic: u8, word: u32) {
350        let id = WordId::new(dic, word);
351        assert_eq!(dic, id.dict().as_raw());
352        assert_eq!(word, id.entry().as_raw());
353    }
354
355    #[test]
356    fn create() {
357        assert_create(0, 0);
358        assert_create(0, 1);
359        assert_create(0, 0x0fffffff);
360        assert_create(14, 0x0fffffff);
361        assert_create(1, 0);
362        assert_create(1, 0x0fffffff);
363        assert_create(15, 3121);
364        assert_create(15, 0);
365        assert_create(15, 0x0fffffff);
366    }
367
368    #[test]
369    fn display() {
370        let id1 = WordId::new(0, 521321);
371        assert_eq!("(0, 521321)", format!("{}", id1));
372    }
373
374    #[test]
375    fn debug() {
376        let id1 = WordId::new(0, 521321);
377        assert_eq!("(0, 521321)", format!("{:?}", id1));
378    }
379
380    #[test]
381    fn is_system() {
382        assert!(WordId::new(0, 0).is_system());
383        assert!(!WordId::new(1, 0).is_system());
384        assert!(!WordId::new(14, 0).is_system());
385        assert!(!WordId::new(15, 0).is_system());
386    }
387
388    #[test]
389    fn is_user() {
390        assert!(!WordId::new(0, 0).is_user());
391        assert!(WordId::new(1, 0).is_user());
392        assert!(WordId::new(14, 0).is_user());
393        assert!(!WordId::new(15, 0).is_user());
394    }
395
396    #[test]
397    fn is_oov() {
398        assert!(!WordId::new(0, 0).is_oov());
399        assert!(!WordId::new(1, 0).is_oov());
400        assert!(!WordId::new(14, 0).is_oov());
401        assert!(WordId::new(15, 0).is_oov());
402    }
403
404    #[test]
405    fn is_special() {
406        assert!(WordId::EOS.is_special());
407        assert!(WordId::BOS.is_special());
408        assert!(!WordId::INVALID.is_special());
409        assert!(!WordId::new(0, 0).is_special());
410    }
411}
sudachi/dic/word_id.rs

sudachi/dic/
word_id.rs
