sudachi/dic/
character_category.rs

1/*
2 * Copyright (c) 2021-2026 Works Applications Co., Ltd.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17use std::collections::BTreeSet;
18use std::fs;
19use std::io::{BufRead, BufReader};
20use std::iter::FusedIterator;
21use std::ops::Range;
22use std::path::Path;
23
24use thiserror::Error;
25
26use crate::config::DEFAULT_CHAR_DEF_BYTES;
27use crate::dic::category_type::CategoryType;
28use crate::prelude::*;
29
30/// Sudachi error
31#[derive(Error, Debug, Eq, PartialEq)]
32#[non_exhaustive]
33pub enum Error {
34    #[error("Invalid format at line {0}")]
35    InvalidFormat(usize),
36
37    #[error("Invalid type {1} at line {0}")]
38    InvalidCategoryType(usize, String),
39
40    #[error("Multiple definition for type {1} at line {0}")]
41    MultipleTypeDefinition(usize, String),
42
43    #[error("Invalid character {0:X} at line {1}")]
44    InvalidChar(u32, usize),
45}
46
47#[derive(Debug, Clone, PartialEq, Eq)]
48struct CatRange {
49    begin: u32,
50    end: u32,
51    categories: CategoryType,
52}
53
54/// CharacterCategory holds mapping from character to character category type
55#[derive(Debug, Clone)]
56pub struct CharacterCategory {
57    /// Split the whole domain of codepoints into ranges,
58    /// limited by boundaries.
59    ///
60    /// Ranges are half-open: `[boundaries[i], boundaries[i + 1])`
61    /// meaning that the right bound is not included.
62    /// 0 and u32::MAX are not stored, they are included implicitly
63    /// as if they would have indices of `-1` and `boundaries.len()`.
64    boundaries: Vec<u32>,
65
66    /// Stores the category for each range.
67    /// `categories[i]` is for the range `[boundaries[i - 1], boundaries[i])`.
68    /// Plays well with [`std::slice::binary_search`], see [`get_category_types()`].
69    /// This should be always true: `boundaries.len() + 1 == categories.len()`.
70    categories: Vec<CategoryType>,
71}
72
73impl Default for CharacterCategory {
74    fn default() -> Self {
75        CharacterCategory {
76            boundaries: Vec::new(),
77            categories: vec![CategoryType::DEFAULT],
78        }
79    }
80}
81
82impl CharacterCategory {
83    /// Creates a character category from file
84    pub fn from_file(path: &Path) -> SudachiResult<CharacterCategory> {
85        let reader = BufReader::new(fs::File::open(path)?);
86        Self::from_reader(reader)
87    }
88
89    pub fn from_bytes(bytes: &[u8]) -> SudachiResult<CharacterCategory> {
90        let reader = BufReader::new(bytes);
91        Self::from_reader(reader)
92    }
93
94    pub fn from_reader<T: BufRead>(data: T) -> SudachiResult<CharacterCategory> {
95        let ranges = Self::read_character_definition(data)?;
96        Ok(Self::compile(&ranges))
97    }
98
99    pub fn from_embedded() -> CharacterCategory {
100        Self::from_bytes(DEFAULT_CHAR_DEF_BYTES).unwrap()
101    }
102
103    /// Reads character type definition as a list of Ranges
104    ///
105    /// Definition file syntax:
106    ///     Each line contains [TARGET_CHARACTER_CODE_POINT] [TYPES], where
107    ///     TARGET_CHARACTER_CODE_POINT:
108    ///         a code_point in hexadecimal format or two separated by ".."
109    ///     TYPES:
110    ///         one or more Category_types separated by white space
111    ///     Loads only lines start with "0x" are loaded and ignore others
112    ///
113    /// Definition example:
114    ///     "0x0030..0x0039 NUMERIC"
115    ///     "0x3008         KANJI KANJINUMERIC"
116    fn read_character_definition<T: BufRead>(reader: T) -> SudachiResult<Vec<CatRange>> {
117        let mut ranges: Vec<CatRange> = Vec::new();
118        for (i, line) in reader.lines().enumerate() {
119            let line = line?;
120            let line = line.trim();
121            if line.is_empty() || line.starts_with('#') || !line.starts_with("0x") {
122                continue;
123            }
124
125            let cols: Vec<_> = line.split_whitespace().collect();
126            if cols.len() < 2 {
127                return Err(SudachiError::InvalidCharacterCategory(
128                    Error::InvalidFormat(i),
129                ));
130            }
131
132            let r: Vec<_> = cols[0].split("..").collect();
133            let begin = u32::from_str_radix(String::from(r[0]).trim_start_matches("0x"), 16)?;
134            let end = if r.len() > 1 {
135                u32::from_str_radix(String::from(r[1]).trim_start_matches("0x"), 16)? + 1
136            } else {
137                begin + 1
138            };
139            if begin >= end {
140                return Err(SudachiError::InvalidCharacterCategory(
141                    Error::InvalidFormat(i),
142                ));
143            }
144            if char::from_u32(begin).is_none() {
145                return Err(SudachiError::InvalidCharacterCategory(Error::InvalidChar(
146                    begin, i,
147                )));
148            }
149
150            if char::from_u32(end).is_none() {
151                return Err(SudachiError::InvalidCharacterCategory(Error::InvalidChar(
152                    end, i,
153                )));
154            }
155
156            let mut categories = CategoryType::empty();
157            for elem in cols[1..].iter().take_while(|elem| !elem.starts_with('#')) {
158                categories.insert(match elem.parse() {
159                    Ok(t) => t,
160                    Err(_) => {
161                        return Err(SudachiError::InvalidCharacterCategory(
162                            Error::InvalidCategoryType(i, elem.to_string()),
163                        ))
164                    }
165                });
166            }
167
168            ranges.push(CatRange {
169                begin,
170                end,
171                categories,
172            });
173        }
174
175        Ok(ranges)
176    }
177
178    /// Creates a character category from given range_list
179    ///
180    /// Transforms given range_list to non overlapped range list
181    /// to apply binary search in get_category_types
182    fn compile(ranges: &Vec<CatRange>) -> CharacterCategory {
183        if ranges.is_empty() {
184            return CharacterCategory::default();
185        }
186
187        let boundaries = Self::collect_boundaries(ranges);
188        let mut categories = vec![CategoryType::empty(); boundaries.len()];
189
190        for range in ranges {
191            let start_idx = match boundaries.binary_search(&range.begin) {
192                Ok(i) => i + 1,
193                Err(_) => panic!("there can not be not found boundaries"),
194            };
195            // apply category to all splits which are included in the current range
196            for i in start_idx..boundaries.len() {
197                if boundaries[i] > range.end {
198                    break;
199                }
200                categories[i] |= range.categories;
201            }
202        }
203
204        // first category is always default (it is impossible to get it assigned above)
205        debug_assert_eq!(categories[0], CategoryType::empty());
206        categories[0] = CategoryType::DEFAULT;
207        // merge successive ranges of the same category
208        let mut final_boundaries = Vec::with_capacity(boundaries.len());
209        let mut final_categories = Vec::with_capacity(categories.len());
210
211        let mut last_category = categories[0];
212        let mut last_boundary = boundaries[0];
213        for i in 1..categories.len() {
214            if categories[i] == last_category {
215                last_boundary = boundaries[i];
216                continue;
217            }
218            final_boundaries.push(last_boundary);
219            final_categories.push(last_category);
220            last_category = categories[i];
221            last_boundary = boundaries[i];
222        }
223
224        final_categories.push(last_category);
225        final_boundaries.push(last_boundary);
226
227        // replace empty categories with default
228        for cat in final_categories.iter_mut() {
229            if cat.is_empty() {
230                *cat = CategoryType::DEFAULT;
231            }
232        }
233
234        // and add the category after the last boundary
235        final_categories.push(CategoryType::DEFAULT);
236
237        final_boundaries.shrink_to_fit();
238        final_categories.shrink_to_fit();
239
240        CharacterCategory {
241            boundaries: final_boundaries,
242            categories: final_categories,
243        }
244    }
245
246    /// Find sorted list of all boundaries
247    fn collect_boundaries(data: &Vec<CatRange>) -> Vec<u32> {
248        let mut boundaries = BTreeSet::new();
249        for i in data {
250            boundaries.insert(i.begin);
251            boundaries.insert(i.end);
252        }
253        boundaries.into_iter().collect()
254    }
255
256    /// Returns a set of category types which given char has
257    pub fn get_category_types(&self, c: char) -> CategoryType {
258        if self.boundaries.is_empty() {
259            return CategoryType::DEFAULT;
260        }
261        let cint = c as u32;
262        match self.boundaries.binary_search(&cint) {
263            //Ok means the index in boundaries, so the next category
264            Ok(idx) => self.categories[idx + 1],
265            //Err means the insertion index, so the current category
266            Err(idx) => self.categories[idx],
267        }
268    }
269
270    pub fn iter(&self) -> CharCategoryIter<'_> {
271        CharCategoryIter {
272            categories: self,
273            current: 0,
274        }
275    }
276}
277
278pub struct CharCategoryIter<'a> {
279    categories: &'a CharacterCategory,
280    current: usize,
281}
282
283impl Iterator for CharCategoryIter<'_> {
284    type Item = (Range<char>, CategoryType);
285
286    fn next(&mut self) -> Option<Self::Item> {
287        if self.current == self.categories.boundaries.len() + 1 {
288            return None;
289        }
290
291        // char casts are safe, we are checking for correctness during the data load
292        let range = if self.current == self.categories.boundaries.len() {
293            let left = char::from_u32(*self.categories.boundaries.last().unwrap()).unwrap();
294            (left..char::MAX, *self.categories.categories.last().unwrap())
295        } else if self.current == 0 {
296            let right = char::from_u32(*self.categories.boundaries.first().unwrap()).unwrap();
297            let r = (0 as char)..right;
298            (r, self.categories.categories[0])
299        } else {
300            let left = char::from_u32(self.categories.boundaries[self.current - 1]).unwrap();
301            let right = char::from_u32(self.categories.boundaries[self.current]).unwrap();
302            let cat = self.categories.categories[self.current];
303            (left..right, cat)
304        };
305
306        self.current += 1;
307        Some(range)
308    }
309}
310
311impl FusedIterator for CharCategoryIter<'_> {}
312
313#[cfg(test)]
314mod tests {
315    use super::*;
316    use claim::assert_matches;
317    use std::path::PathBuf;
318
319    const TEST_RESOURCE_DIR: &str = "./tests/resources/";
320    const TEST_CHAR_DEF_FILE: &str = "char.def";
321    type CT = CategoryType;
322
323    #[test]
324    fn get_category_types() {
325        let path = PathBuf::from(TEST_RESOURCE_DIR).join(TEST_CHAR_DEF_FILE);
326        let cat = CharacterCategory::from_file(&path).expect("failed to load char.def for test");
327        let cats = cat.get_category_types('熙');
328        assert_eq!(1, cats.count());
329        assert!(cats.contains(CT::KANJI));
330    }
331
332    fn read_categories(data: &str) -> CharacterCategory {
333        let ranges = CharacterCategory::read_character_definition(data.as_bytes())
334            .expect("error when parsing character categories");
335        CharacterCategory::compile(&ranges)
336    }
337
338    #[test]
339    fn read_cdef_1() {
340        let cat = read_categories(
341            "
342            0x0030..0x0039 NUMERIC
343            0x0032         KANJI",
344        );
345        assert_eq!(cat.get_category_types('\u{0030}'), CT::NUMERIC);
346        assert_eq!(cat.get_category_types('\u{0031}'), CT::NUMERIC);
347        assert_eq!(cat.get_category_types('\u{0032}'), CT::NUMERIC | CT::KANJI);
348        assert_eq!(cat.get_category_types('\u{0033}'), CT::NUMERIC);
349        assert_eq!(cat.get_category_types('\u{0039}'), CT::NUMERIC);
350    }
351
352    #[test]
353    fn read_cdef_2() {
354        let cat = read_categories(
355            "
356            0x0030..0x0039 NUMERIC
357            0x0070..0x0079 ALPHA
358            0x3007         KANJI
359            0x0030         KANJI",
360        );
361        assert_eq!(cat.get_category_types('\u{0030}'), CT::NUMERIC | CT::KANJI);
362        assert_eq!(cat.get_category_types('\u{0039}'), CT::NUMERIC);
363        assert_eq!(cat.get_category_types('\u{3007}'), CT::KANJI);
364        assert_eq!(cat.get_category_types('\u{0069}'), CT::DEFAULT);
365        assert_eq!(cat.get_category_types('\u{0070}'), CT::ALPHA);
366        assert_eq!(cat.get_category_types('\u{0080}'), CT::DEFAULT);
367    }
368
369    #[test]
370    fn read_cdef_3() {
371        let cat = read_categories(
372            "
373            0x0030..0x0039 KATAKANA
374            0x3007         KANJI KANJINUMERIC
375            0x3008         KANJI KANJINUMERIC
376            0x3009         KANJI KANJINUMERIC
377            0x0039..0x0040 ALPHA
378            0x0030..0x0039 NUMERIC
379            0x0030         KANJI",
380        );
381        assert_eq!(cat.get_category_types('\u{0029}'), CT::DEFAULT);
382        assert_eq!(
383            cat.get_category_types('\u{0030}'),
384            CT::NUMERIC | CT::KATAKANA | CT::KANJI
385        );
386        assert_eq!(
387            cat.get_category_types('\u{0039}'),
388            CT::NUMERIC | CT::ALPHA | CT::KATAKANA
389        );
390        assert_eq!(cat.get_category_types('\u{0040}'), CT::ALPHA);
391        assert_eq!(cat.get_category_types('\u{0041}'), CT::DEFAULT);
392        assert_eq!(
393            cat.get_category_types('\u{3007}'),
394            CT::KANJI | CT::KANJINUMERIC
395        );
396        assert_eq!(cat.get_category_types('\u{4007}'), CT::DEFAULT);
397    }
398
399    #[test]
400    fn read_cdef_4() {
401        let cat = read_categories(
402            "
403            0x4E00..0x9FFF KANJI
404            0x4E8C         KANJI KANJINUMERIC",
405        );
406        assert_eq!(cat.get_category_types('男'), CT::KANJI);
407        assert_eq!(cat.get_category_types('\u{4E8B}'), CT::KANJI);
408        assert_eq!(
409            cat.get_category_types('\u{4E8C}'),
410            CT::KANJI | CT::KANJINUMERIC
411        );
412        assert_eq!(cat.get_category_types('\u{4E8D}'), CT::KANJI);
413    }
414
415    #[test]
416    fn read_cdef_holes_1() {
417        let cat = read_categories(
418            "
419            0x0030 USER1
420            0x0032 USER2",
421        );
422        assert_eq!(cat.get_category_types('\u{0029}'), CT::DEFAULT);
423        assert_eq!(cat.get_category_types('\u{0030}'), CT::USER1);
424        assert_eq!(cat.get_category_types('\u{0031}'), CT::DEFAULT);
425        assert_eq!(cat.get_category_types('\u{0032}'), CT::USER2);
426        assert_eq!(cat.get_category_types('\u{0033}'), CT::DEFAULT);
427    }
428
429    #[test]
430    fn read_cdef_merge_1() {
431        let cat = read_categories(
432            "
433            0x0030 USER1
434            0x0031 USER1",
435        );
436        assert_eq!(cat.boundaries.len(), 2);
437        assert_eq!(cat.categories.len(), 3);
438        assert_eq!(cat.get_category_types('\u{0029}'), CT::DEFAULT);
439        assert_eq!(cat.get_category_types('\u{0030}'), CT::USER1);
440        assert_eq!(cat.get_category_types('\u{0031}'), CT::USER1);
441        assert_eq!(cat.get_category_types('\u{0032}'), CT::DEFAULT);
442    }
443
444    #[test]
445    fn read_cdef_merge_2() {
446        let cat = read_categories(
447            "
448            0x0030          USER1
449            0x0031..0x0032  USER1",
450        );
451        assert_eq!(cat.boundaries.len(), 2);
452        assert_eq!(cat.categories.len(), 3);
453        assert_eq!(cat.get_category_types('\u{0029}'), CT::DEFAULT);
454        assert_eq!(cat.get_category_types('\u{0030}'), CT::USER1);
455        assert_eq!(cat.get_category_types('\u{0031}'), CT::USER1);
456        assert_eq!(cat.get_category_types('\u{0032}'), CT::USER1);
457        assert_eq!(cat.get_category_types('\u{0033}'), CT::DEFAULT);
458    }
459
460    #[test]
461    fn read_cdef_merge_3() {
462        let cat = read_categories(
463            "
464            0x0030..0x0031  USER1
465            0x0032..0x0033  USER1",
466        );
467        assert_eq!(cat.boundaries.len(), 2);
468        assert_eq!(cat.categories.len(), 3);
469        assert_eq!(cat.get_category_types('\u{0029}'), CT::DEFAULT);
470        assert_eq!(cat.get_category_types('\u{0030}'), CT::USER1);
471        assert_eq!(cat.get_category_types('\u{0031}'), CT::USER1);
472        assert_eq!(cat.get_category_types('\u{0032}'), CT::USER1);
473        assert_eq!(cat.get_category_types('\u{0033}'), CT::USER1);
474        assert_eq!(cat.get_category_types('\u{0034}'), CT::DEFAULT);
475    }
476
477    #[test]
478    fn read_character_definition_with_invalid_format() {
479        let data = "0x0030..0x0039";
480        let result = CharacterCategory::read_character_definition(data.as_bytes());
481        assert_matches!(
482            result,
483            Err(SudachiError::InvalidCharacterCategory(
484                Error::InvalidFormat(0)
485            ))
486        );
487    }
488
489    #[test]
490    fn read_character_definition_with_invalid_range() {
491        let data = "0x0030..0x0029 NUMERIC";
492        let result = CharacterCategory::read_character_definition(data.as_bytes());
493        assert_matches!(
494            result,
495            Err(SudachiError::InvalidCharacterCategory(
496                Error::InvalidFormat(0)
497            ))
498        );
499    }
500
501    #[test]
502    fn read_character_definition_with_invalid_type() {
503        let data = "0x0030..0x0039 FOO";
504        let result = CharacterCategory::read_character_definition(data.as_bytes());
505        assert_matches!(result, Err(SudachiError::InvalidCharacterCategory(Error::InvalidCategoryType(0, s))) if s == "FOO");
506    }
507
508    #[test]
509    fn check_test_cdef() {
510        let data: &[u8] = include_bytes!("../../tests/resources/char.def");
511        let c = CharacterCategory::from_reader(data).expect("failed to read chars");
512        assert_eq!(c.get_category_types('â'), CT::ALPHA);
513        assert_eq!(c.get_category_types('ｂ'), CT::ALPHA);
514        assert_eq!(c.get_category_types('C'), CT::ALPHA);
515        assert_eq!(c.get_category_types('漢'), CT::KANJI);
516        assert_eq!(c.get_category_types('々'), CT::KANJI | CT::SYMBOL);
517        assert_eq!(c.get_category_types('𡈽'), CT::DEFAULT);
518        assert_eq!(c.get_category_types('ア'), CT::KATAKANA);
519        assert_eq!(c.get_category_types('ｺ'), CT::KATAKANA);
520        assert_eq!(c.get_category_types('ﾞ'), CT::KATAKANA);
521    }
522
523    #[test]
524    fn iter_cdef_holes_1() {
525        let cat = read_categories(
526            "
527            0x0030 USER1
528            0x0032 USER2",
529        );
530        let mut iter = cat.iter();
531        assert_matches!(
532            iter.next(),
533            Some((
534                Range {
535                    start: '\x00',
536                    end: '\x30'
537                },
538                CT::DEFAULT
539            ))
540        );
541        assert_matches!(
542            iter.next(),
543            Some((
544                Range {
545                    start: '\x30',
546                    end: '\x31'
547                },
548                CT::USER1
549            ))
550        );
551        assert_matches!(
552            iter.next(),
553            Some((
554                Range {
555                    start: '\x31',
556                    end: '\x32'
557                },
558                CT::DEFAULT
559            ))
560        );
561        assert_matches!(
562            iter.next(),
563            Some((
564                Range {
565                    start: '\x32',
566                    end: '\x33'
567                },
568                CT::USER2
569            ))
570        );
571        assert_matches!(
572            iter.next(),
573            Some((
574                Range {
575                    start: '\x33',
576                    end: char::MAX
577                },
578                CT::DEFAULT
579            ))
580        );
581        assert_eq!(iter.next(), None);
582    }
583}
sudachi/dic/character_category.rs

sudachi/dic/
character_category.rs
