diff --git a/gensim/corpora/wikicorpus.py b/gensim/corpora/wikicorpus.py
index 9696f843d9..09802b0e24 100644
--- a/gensim/corpora/wikicorpus.py
+++ b/gensim/corpora/wikicorpus.py
@@ -43,7 +43,7 @@
RE_P0 = re.compile(r'', re.DOTALL | re.UNICODE)
"""Comments."""
-RE_P1 = re.compile(r'[ ].*?)(]|/>)', re.DOTALL | re.UNICODE)
+RE_P1 = re.compile(r'[]*?)?/>|][]*?)?>.*?]', re.DOTALL | re.UNICODE)
"""Footnotes."""
RE_P2 = re.compile(r'(\n\[\[[a-z][a-z][\w-]*:[^:\]]+\]\])+$', re.UNICODE)
"""Links to languages."""
diff --git a/gensim/test/test_corpora.py b/gensim/test/test_corpora.py
index 3e47531a2c..874bc45fba 100644
--- a/gensim/test/test_corpora.py
+++ b/gensim/test/test_corpora.py
@@ -640,6 +640,17 @@ def test_default_preprocessing(self):
first_text = next(corpus.get_texts())
self.assertEqual(expected, first_text)
+ def test_filter_wiki_ref_with_nested_self_closing_tag(self):
+ # Regression test for #3520: a [...] footnote that contains a nested
+ # self-closing tag (e.g. or
) must be removed entirely, not
+ # truncated at the nested tag's "/>", which used to leak the rest of the footnote.
+ text = "total[notehttps://example.com/x]end"
+ self.assertEqual(wikicorpus.filter_wiki(text), "totalend")
+
+ # Self-closing and plain paired refs are still removed.
+ self.assertEqual(wikicorpus.filter_wiki('ab'), "ab")
+ self.assertEqual(wikicorpus.filter_wiki("a[note]b"), "ab")
+
def test_len(self):
# When there is no min_token limit all 9 articles must be registered.
corpus = self.corpus_class(self.fname, article_min_tokens=0)