diff --git a/gensim/corpora/wikicorpus.py b/gensim/corpora/wikicorpus.py index 9696f843d9..09802b0e24 100644 --- a/gensim/corpora/wikicorpus.py +++ b/gensim/corpora/wikicorpus.py @@ -43,7 +43,7 @@ RE_P0 = re.compile(r'', re.DOTALL | re.UNICODE) """Comments.""" -RE_P1 = re.compile(r' ].*?)(|/>)', re.DOTALL | re.UNICODE) +RE_P1 = re.compile(r']*?)?/>|]*?)?>.*?', re.DOTALL | re.UNICODE) """Footnotes.""" RE_P2 = re.compile(r'(\n\[\[[a-z][a-z][\w-]*:[^:\]]+\]\])+$', re.UNICODE) """Links to languages.""" diff --git a/gensim/test/test_corpora.py b/gensim/test/test_corpora.py index 3e47531a2c..874bc45fba 100644 --- a/gensim/test/test_corpora.py +++ b/gensim/test/test_corpora.py @@ -640,6 +640,17 @@ def test_default_preprocessing(self): first_text = next(corpus.get_texts()) self.assertEqual(expected, first_text) + def test_filter_wiki_ref_with_nested_self_closing_tag(self): + # Regression test for #3520: a ... footnote that contains a nested + # self-closing tag (e.g. or
) must be removed entirely, not + # truncated at the nested tag's "/>", which used to leak the rest of the footnote. + text = "totalnotehttps://example.com/xend" + self.assertEqual(wikicorpus.filter_wiki(text), "totalend") + + # Self-closing and plain paired refs are still removed. + self.assertEqual(wikicorpus.filter_wiki('ab'), "ab") + self.assertEqual(wikicorpus.filter_wiki("anoteb"), "ab") + def test_len(self): # When there is no min_token limit all 9 articles must be registered. corpus = self.corpus_class(self.fname, article_min_tokens=0)