From 31efb090f6deb0f5b94efe9cf211bc905268cb0f Mon Sep 17 00:00:00 2001
From: Adam Radziszewski <adam.radziszewski@pwr.wroc.pl>
Date: Mon, 30 Jul 2012 15:49:15 +0200
Subject: [PATCH] fix get_next_token having sentences delete their tokens
before giving tokens out
---
libcorpus2/io/reader.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/libcorpus2/io/reader.cpp b/libcorpus2/io/reader.cpp
index 211a85a..d156887 100644
--- a/libcorpus2/io/reader.cpp
+++ b/libcorpus2/io/reader.cpp
@@ -290,6 +290,7 @@ Token* BufferedSentenceReader::get_next_token()
if (s) {
std::copy(s->tokens().begin(), s->tokens().end(),
std::back_inserter(token_buf_));
+ s->release_tokens();
} else {
more = false;
}
--
GitLab