{
    "mode": "perldoc",
    "parameter": "Plucene::Analysis::CharTokenizer",
    "section": "",
    "url": "https://www.chedong.com/phpMan.php/perldoc/Plucene%3A%3AAnalysis%3A%3ACharTokenizer/json",
    "generated": "2026-06-13T10:15:46Z",
    "synopsis": "# isa Plucene::Analysis::Tokenizer\nmy $next = $chartokenizer->next;",
    "sections": {
        "NAME": {
            "content": "Plucene::Analysis::CharTokenizer - base class for character tokenisers\n",
            "subsections": []
        },
        "SYNOPSIS": {
            "content": "# isa Plucene::Analysis::Tokenizer\n\nmy $next = $chartokenizer->next;\n",
            "subsections": []
        },
        "DESCRIPTION": {
            "content": "This is an abstract base class for simple, character-oriented tokenizers.\n",
            "subsections": []
        },
        "METHODS": {
            "content": "tokenre\nThis should be defined in subclasses.\n\nnormalize\nThis will normalise the character before it is added to the token.\n\nnext\nmy $next = $chartokenizer->next;\n\nThis will return the next token in the string, or undef at the end of the string.\n",
            "subsections": []
        }
    },
    "summary": "Plucene::Analysis::CharTokenizer - base class for character tokenisers",
    "flags": [],
    "examples": [],
    "see_also": []
}