CharTokenizer.h

Go to the documentation of this file.

// Copyright (c) 2009-2014 Alan Wright. All rights reserved.
// Distributable under the terms of either the Apache License (Version 2.0)
// or the GNU Lesser General Public License.
 
#ifndef CHARTOKENIZER_H
#define CHARTOKENIZER_H
 
#include "Tokenizer.h"
 
namespace Lucene {

class LPPAPI CharTokenizer : public Tokenizer {
public:
    CharTokenizer(const ReaderPtr& input);
    CharTokenizer(const AttributeSourcePtr& source, const ReaderPtr& input);
    CharTokenizer(const AttributeFactoryPtr& factory, const ReaderPtr& input);
    virtual ~CharTokenizer();
 
    LUCENE_CLASS(CharTokenizer);
 
protected:
    int32_t offset;
    int32_t bufferIndex;
    int32_t dataLen;
 
    static const int32_t MAX_WORD_LEN;
    static const int32_t IO_BUFFER_SIZE;
 
    CharArray ioBuffer;
    TermAttributePtr termAtt;
    OffsetAttributePtr offsetAtt;
 
public:
    virtual bool incrementToken();
    virtual void end();
    virtual void reset(const ReaderPtr& input);
 
protected:
    virtual bool isTokenChar(wchar_t c) = 0;

    virtual wchar_t normalize(wchar_t c);
};
 
}
 
#endif