Commit ed71cd97 authored by Igor Danielewicz's avatar Igor Danielewicz

Fixed bug with empty files

parent ba40439e
......@@ -14,6 +14,6 @@ public:
}
bool has_more()
{
return !input.eof();
return input.peek() != EOF;
}
};
\ No newline at end of file
......@@ -13,6 +13,8 @@ public:
{
// std::cout << "GET" << std::endl;
auto& tokens = getSentence();
if(tokens.empty())
return std::string();
std::string tokenstr = token2string(*tokens.front());
tokens.erase(tokens.begin());
return tokenstr;
......
......@@ -8,7 +8,8 @@ public:
Tokenizer(std::stringstream& stream) : layer_tokenizer(stream, Toki::Config::from_file("nkjp.ini")), tokenizer(layer_tokenizer)
{
sentence = tokenizer.get_next_sentence();
sentence->tokens()[0]->set_preceeding_whitespace(PwrNlp::Whitespace::Enum::None);
if(!sentence->tokens().empty())
sentence->tokens()[0]->set_preceeding_whitespace(PwrNlp::Whitespace::Enum::None);
}
std::vector<Toki::Token*>& getSentence()
{
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment