sudachi/analysis/
stateless_tokenizer.rs

1/*
2 *  Copyright (c) 2021-2026 Works Applications Co., Ltd.
3 *
4 *  Licensed under the Apache License, Version 2.0 (the "License");
5 *  you may not use this file except in compliance with the License.
6 *  You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 *   Unless required by applicable law or agreed to in writing, software
11 *  distributed under the License is distributed on an "AS IS" BASIS,
12 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 *  See the License for the specific language governing permissions and
14 *  limitations under the License.
15 */
16
17use std::ops::Deref;
18
19use crate::analysis::mlist::MorphemeList;
20use crate::analysis::node::ResultNode;
21use crate::analysis::stateful_tokenizer::StatefulTokenizer;
22use crate::analysis::{Mode, Tokenize};
23use crate::dic::subset::InfoSubset;
24use crate::dic::DictionaryAccess;
25use crate::error::SudachiResult;
26use crate::input_text::InputBuffer;
27
28/// Implementation of a Tokenizer which does not have tokenization state.
29///
30/// This is a wrapper which is generic over dictionary pointers.
31/// Usable where dictionary is a struct itself, &, &mut, Rc<.>, Arc<.>.
32pub struct StatelessTokenizer<T> {
33    dict: T,
34}
35
36impl<T: DictionaryAccess> StatelessTokenizer<T> {
37    pub fn new(dict: T) -> StatelessTokenizer<T> {
38        StatelessTokenizer { dict }
39    }
40}
41
42impl<T> StatelessTokenizer<T>
43where
44    T: Deref,
45    <T as Deref>::Target: DictionaryAccess,
46{
47    pub fn as_dict(&self) -> &<T as Deref>::Target {
48        Deref::deref(&self.dict)
49    }
50}
51
52impl<T> Tokenize for StatelessTokenizer<T>
53where
54    T: DictionaryAccess + Clone,
55{
56    type Dictionary = T;
57
58    fn tokenize<'a>(
59        &'a self,
60        input: &'a str,
61        mode: Mode,
62        enable_debug: bool,
63    ) -> SudachiResult<MorphemeList<Self::Dictionary>> {
64        let mut tok = StatefulTokenizer::create(self.dict.clone(), enable_debug, mode);
65        tok.reset().push_str(input);
66        tok.do_tokenize()?;
67        tok.into_morpheme_list()
68    }
69}
70
71pub(super) fn split_path<T: DictionaryAccess + ?Sized>(
72    dict: &T,
73    path: Vec<ResultNode>,
74    mode: Mode,
75    subset: InfoSubset,
76    input: &InputBuffer,
77) -> SudachiResult<Vec<ResultNode>> {
78    if mode == Mode::C {
79        return Ok(path);
80    }
81
82    let mut new_path = Vec::with_capacity(path.len() * 3 / 2);
83    for node in path {
84        let split_len = node.num_splits(mode);
85        if split_len <= 1 {
86            new_path.push(node);
87        } else {
88            new_path.extend(node.split(mode, dict.lexicon(), subset, input));
89        }
90    }
91
92    Ok(new_path)
93}
94
95pub(super) fn dump_path(path: &[ResultNode]) {
96    for (i, node) in path.iter().enumerate() {
97        println!("{}: {}", i, node);
98    }
99}
sudachi/analysis/stateless_tokenizer.rs

sudachi/analysis/
stateless_tokenizer.rs
