{
    "mode": "perldoc",
    "parameter": "Plucene::Analysis::Tokenizer",
    "section": "",
    "url": "https://www.chedong.com/phpMan.php/perldoc/Plucene%3A%3AAnalysis%3A%3ATokenizer/json",
    "generated": "2026-06-14T00:14:34Z",
    "synopsis": "my $tokenizer = Plucene::Analysis::Tokenizer::Subclass\n->new({ reader => $reader });",
    "sections": {
        "NAME": {
            "content": "Plucene::Analysis::Tokenizer - base class for tokenizers\n",
            "subsections": []
        },
        "SYNOPSIS": {
            "content": "my $tokenizer = Plucene::Analysis::Tokenizer::Subclass\n->new({ reader => $reader });\n",
            "subsections": []
        },
        "DESCRIPTION": {
            "content": "This is an abstract base class for tokenizers.\n\nA Tokenizer is a TokenStream whose input is a Reader.\n",
            "subsections": []
        },
        "METHODS": {
            "content": "new\nmy $tokenizer = Plucene::Analysis::Tokenizer::Subclass\n->new({ reader => $reader });\n\nThis will create a new tokenizer.\n\nnext\nThis must be defined in a subclass\n\nclose\nClose the input reader.\n",
            "subsections": []
        }
    },
    "summary": "Plucene::Analysis::Tokenizer - base class for tokenizers",
    "flags": [],
    "examples": [],
    "see_also": []
}