sudachi/dic/lexicon/
strings.rs

1/*
2 * Copyright (c) 2026 Works Applications Co., Ltd.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17use crate::dic::lexicon_set::LexiconSetError;
18use crate::dic::read::utf16_string::utf16_string_of_length;
19use crate::error::{SudachiError, SudachiResult};
20
21pub struct CompactedStrings<'a> {
22    bytes: &'a [u8],
23}
24
25impl<'a> CompactedStrings<'a> {
26    pub fn from_bytes(bytes: &'a [u8]) -> CompactedStrings<'a> {
27        CompactedStrings { bytes }
28    }
29
30    pub fn get_string(&self, pointer: StringPointer) -> SudachiResult<String> {
31        let (_, parsed) = utf16_string_of_length(
32            &self.bytes[(pointer.offset as usize * 2)..],
33            pointer.length as usize,
34        )?;
35        Ok(parsed)
36    }
37}
38
39#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Default)]
40pub struct StringPointer {
41    /// length of the string (in utf16 codepoint)
42    pub length: u32,
43    /// offset in the CompactedStrings (in utf16 codepoint)
44    pub offset: u32,
45}
46
47impl StringPointer {
48    /// bit count of the base part
49    pub const BASE_LENGTH_BITS: u32 = 5;
50    /// offset to the base part
51    pub const BASE_LENGTH_OFFSET: u32 = 32 - Self::BASE_LENGTH_BITS;
52    /// max bit count of the additional length part (in value)
53    /// note that its top 1 bit is not stored in byte representation.
54    pub const MAX_VARIABLE_LENGTH_BITS: u32 = 12;
55    /// max string length that can be stored using base part only
56    pub const MAX_SIMPLE_LENGTH: u32 =
57        2u32.pow(Self::BASE_LENGTH_BITS) - 1 - Self::MAX_VARIABLE_LENGTH_BITS;
58    /// max string length that can be stored
59    pub const MAX_LENGTH: u32 =
60        2u32.pow(Self::MAX_VARIABLE_LENGTH_BITS) - 1 + Self::MAX_SIMPLE_LENGTH;
61
62    /// Check if the given length and offset satisfy the constraints for a valid StringPointer.
63    fn is_valid(length: u32, offset: u32) -> bool {
64        if length > Self::MAX_LENGTH {
65            return false;
66        }
67        let alignment = Self::required_alignment(length);
68        if alignment == 0 {
69            return true;
70        }
71        let alignment_mask = (1 << (alignment - 1)) - 1;
72        (offset & alignment_mask) == 0
73    }
74
75    /// Calculate the required alignment bits for the offset from the length.
76    fn required_alignment(length: u32) -> u32 {
77        if length <= Self::MAX_SIMPLE_LENGTH {
78            0
79        } else {
80            let remaining = length - Self::MAX_SIMPLE_LENGTH;
81            32 - remaining.leading_zeros()
82        }
83    }
84
85    pub fn unchecked(length: u32, offset: u32) -> Self {
86        StringPointer { length, offset }
87    }
88
89    pub fn checked(length: u32, offset: u32) -> SudachiResult<Self> {
90        if !Self::is_valid(length, offset) {
91            return Err(SudachiError::LexiconSetError(
92                LexiconSetError::InvalidStringPointer(
93                    length as usize,
94                    offset as usize,
95                    Self::required_alignment(length) as usize,
96                ),
97            ));
98        }
99        Ok(Self::unchecked(length, offset))
100    }
101
102    pub fn encode(&self) -> u32 {
103        let additional_length_bits = Self::required_alignment(self.length);
104        let base_length = std::cmp::min(self.length, Self::MAX_SIMPLE_LENGTH);
105        let base_part = (base_length + additional_length_bits) << Self::BASE_LENGTH_OFFSET;
106
107        let additional_length = self.length - base_length;
108        let implicit_bit = (1 << Self::MAX_VARIABLE_LENGTH_BITS) >> (13 - additional_length_bits);
109        let non_fixed_length = additional_length ^ implicit_bit;
110        let variable_part =
111            non_fixed_length << (16 + Self::MAX_VARIABLE_LENGTH_BITS - additional_length_bits);
112
113        let offset_part = self.offset >> additional_length_bits.saturating_sub(1);
114
115        debug_assert!(base_part & variable_part == 0);
116        debug_assert!(base_part & offset_part == 0);
117        debug_assert!(variable_part & offset_part == 0);
118        base_part | variable_part | offset_part
119    }
120
121    pub fn decode(encoded: u32) -> Self {
122        // first 5 bits are length and marker values for additional length bits
123        let base_value = encoded >> Self::BASE_LENGTH_OFFSET;
124        let additional_length_bits = base_value.saturating_sub(Self::MAX_SIMPLE_LENGTH);
125        // additional length bits are stored in the following
126        let non_fixed_length = (encoded & 0x07ff_0000)
127            >> (16 + Self::MAX_VARIABLE_LENGTH_BITS - additional_length_bits);
128        // compute the non-stored first bit which is implicitly one
129        let implicit_bit = (1 << Self::MAX_VARIABLE_LENGTH_BITS) >> (13 - additional_length_bits);
130        let length = (base_value - additional_length_bits) + (non_fixed_length | implicit_bit);
131        // offset are aligned based on the additional length bits
132        let alignment = additional_length_bits.saturating_sub(1);
133        let offset = (encoded & (0x07ff_ffff >> alignment)) << alignment;
134
135        StringPointer { length, offset }
136    }
137}
sudachi/dic/lexicon/strings.rs

sudachi/dic/lexicon/
strings.rs
