Commit fb529811 authored by ilor's avatar ilor

Handle U+200B (zero width space) in Toki

parent ad4f1d62
......@@ -119,7 +119,9 @@ namespace Toki {
int ws = 0; int nl = 0;
while (input().has_more_chars()) {
UChar u = input().peek_next_char();
if (u == 0xfeff) { //BOM mark (aka ZERO WIDTH NO-BREAK SPACE)
if (u == 0xfeff || u == 0x200b) {
//U+FEFF BOM mark (aka ZERO WIDTH NO-BREAK SPACE)
//U+200B ZERO WIDTH SPACE
//do not increment ws. BOM's are skipped entirely
input().get_next_char();
} else if (!u_isUWhiteSpace(u)) {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment