You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
29 lines
599 B
29 lines
599 B
#ifndef LPC_TOKENIZER_H1118429480_INCLUDE_GUARD_
|
|
#define LPC_TOKENIZER_H1118429480_INCLUDE_GUARD_
|
|
|
|
#include <vector>
|
|
#include <string>
|
|
#include <memory>
|
|
#include <assert.h>
|
|
|
|
namespace indexlib { namespace detail {
|
|
|
|
class tokenizer {
|
|
public:
|
|
virtual ~tokenizer() { }
|
|
std::vector<std::string> string_to_words( const char* str ) {
|
|
assert( str );
|
|
return do_string_to_words( str );
|
|
}
|
|
|
|
private:
|
|
virtual std::vector<std::string> do_string_to_words( const char* ) = 0;
|
|
};
|
|
|
|
std::unique_ptr<tokenizer> get_tokenizer( std::string );
|
|
}}
|
|
|
|
|
|
|
|
#endif /* LPC_TOKENIZER_H1118429480_INCLUDE_GUARD_ */
|