From a7763da66cbdb4bd329f106d0b4bf3536ad363e5 Mon Sep 17 00:00:00 2001
From: ilor <kailoran@gmail.com>
Date: Wed, 22 Jun 2011 14:43:28 +0200
Subject: [PATCH] proper, if ugly fix for no-break spaces
---
libtoki/tokenizer/whitespacetokenizer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libtoki/tokenizer/whitespacetokenizer.cpp b/libtoki/tokenizer/whitespacetokenizer.cpp
index 97e56f3..9f1a032 100644
--- a/libtoki/tokenizer/whitespacetokenizer.cpp
+++ b/libtoki/tokenizer/whitespacetokenizer.cpp
@@ -178,7 +178,7 @@ namespace Toki {
// << input().peek_begins_sentence() << " "
// << next_token_begins_sentence << "\n";
u = input().peek_next_char();
- if (u_isUWhiteSpace(u)) {
+ if (u_isUWhiteSpace(u) || u == 0xfeff || u == 0x200b) {
if (input().peek_begins_sentence()) {
next_token_begins_sentence = true;
}
--
GitLab