diff --git a/cclutils/extras/annotations.py b/cclutils/extras/annotations.py
index 8ae3504073bf405b0c7c93310015f029edbc5c50..21304efa9ea231928c26b0cd5640fe10eba0c6d4 100644
--- a/cclutils/extras/annotations.py
+++ b/cclutils/extras/annotations.py
@@ -459,6 +459,8 @@ class DocumentAnnotations(object):
                     doc=self._doc,
                 )
                 self._ann_dict[ann_descr] = related_ann
+            else:
+                return
             self._tok_dict[tok_pos].append(related_ann)
             self._tok_pos_to_tok[tok_pos] = token