From 8b8fc0888c222fb2eb19f699a485f84e5b9269dd Mon Sep 17 00:00:00 2001 From: Nirmeet-kamble Date: Mon, 13 Oct 2025 22:35:24 +0530 Subject: [PATCH 1/2] Add Finnish Wikipedia wordlist generator script --- .gitignore | 1 + app/reviews/create_fiwiki_wordlist.py | 141 ++++++++++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 app/reviews/create_fiwiki_wordlist.py diff --git a/.gitignore b/.gitignore index 60875cea..d4669512 100644 --- a/.gitignore +++ b/.gitignore @@ -211,3 +211,4 @@ pywikibot-Zache-test.lwp user-config.py app/user-config.py *.lwp +app/reviews/fiwiki_wordlist.txt.gz diff --git a/app/reviews/create_fiwiki_wordlist.py b/app/reviews/create_fiwiki_wordlist.py new file mode 100644 index 00000000..e4248013 --- /dev/null +++ b/app/reviews/create_fiwiki_wordlist.py @@ -0,0 +1,141 @@ +""" +Finnish Wikipedia Wordlist Generator +Processes Finnish Wikipedia dump to create a compressed wordlist for sanity checking. + +Usage: + python fiwiki_wordlist_generator.py fiwiki-20251001-pages-meta-current.xml.bz2 +""" + +import bz2 +import gzip +import re +import sys +from xml.etree import ElementTree as ET + + +def extract_text_from_dump(dump_path): + """ + Parse Wikipedia XML dump and extract all text content. + + Args: + dump_path: Path to the bz2-compressed Wikipedia dump + + Yields: + Text content from each page + """ + print(f"Opening dump file: {dump_path}") + + # Open bz2-compressed file correctly + with bz2.open(dump_path, 'rt', encoding='utf-8') as f: + # Parse XML iteratively to handle large files + context = ET.iterparse(f, events=('end',)) + + page_count = 0 + for event, elem in context: + # Look for text elements (Wikipedia page content) + # Tag will be like {http://www.mediawiki.org/xml/export-0.10/}text + if elem.tag.endswith('text'): + if elem.text: + yield elem.text + page_count += 1 + + if page_count % 10000 == 0: + print(f"Processed {page_count} pages...", file=sys.stderr) + + # Clear element to free memory + elem.clear() + + # Also clear parent elements periodically + if elem.tag.endswith('page'): + elem.clear() + + print(f"Total pages processed: {page_count}", file=sys.stderr) + + +def process_text_to_words(text_generator): + """ + Process text content into individual words. + + Args: + text_generator: Generator yielding text strings + + Yields: + Individual words (3+ characters, must contain at least one letter) + """ + # Compile regex patterns + non_alnum_pattern = re.compile(r'[^a-zA-ZäöåÄÖÅ0-9]+') + has_letter_pattern = re.compile(r'[a-zA-ZäöåÄÖÅ]') + + for text in text_generator: + # Replace non-alphanumeric with spaces + text = non_alnum_pattern.sub(' ', text) + + # Split on spaces and filter + for word in text.split(): + # Must be 3+ chars AND contain at least one letter + if len(word) >= 3 and has_letter_pattern.search(word): + yield word.lower() # Normalize to lowercase + + +def create_wordlist(dump_path, output_path): + """ + Create compressed wordlist from Wikipedia dump. + + Args: + dump_path: Path to Wikipedia dump file + output_path: Path for output gzip file + """ + print("Step 1-3: Extracting and parsing wikitext...") + text_gen = extract_text_from_dump(dump_path) + + print("Step 4-6: Processing text into words (3+ chars, must contain letters)...") + words = process_text_to_words(text_gen) + + print("Step 7: Collecting unique words...") + unique_words = set() + word_count = 0 + + for word in words: + unique_words.add(word) + word_count += 1 + + if word_count % 100000 == 0: + print(f"Processed {word_count} words, {len(unique_words)} unique...", file=sys.stderr) + + print(f"Total words: {word_count}", file=sys.stderr) + print(f"Unique words: {len(unique_words)}", file=sys.stderr) + + print("Sorting words...") + sorted_words = sorted(unique_words) + + print(f"Step 8: Writing compressed output to {output_path}...") + with gzip.open(output_path, 'wt', encoding='utf-8') as f: + for word in sorted_words: + f.write(word + '\n') + + print(f"Done! Wordlist saved to {output_path}") + print(f"Final count: {len(sorted_words)} unique words") + + +def main(): + if len(sys.argv) != 2: + print("Usage: python fiwiki_wordlist_generator.py ") + print("\nExample:") + print(" python fiwiki_wordlist_generator.py fiwiki-20251001-pages-meta-current.xml.bz2") + sys.exit(1) + + dump_path = sys.argv[1] + output_path = "fiwiki_wordlist.txt.gz" + + try: + create_wordlist(dump_path, output_path) + except FileNotFoundError: + print(f"Error: File not found: {dump_path}", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file From 2baba9546f7ac3a70890bf5ca525dffacda67a6b Mon Sep 17 00:00:00 2001 From: Nirmeet-kamble Date: Sun, 19 Oct 2025 11:30:03 +0530 Subject: [PATCH 2/2] Update app/reviews/create_fiwiki_wordlist.py Co-authored-by: Ezra Yendau <125133850+Ezi-code@users.noreply.github.com> --- app/reviews/create_fiwiki_wordlist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/reviews/create_fiwiki_wordlist.py b/app/reviews/create_fiwiki_wordlist.py index e4248013..e3db9dc0 100644 --- a/app/reviews/create_fiwiki_wordlist.py +++ b/app/reviews/create_fiwiki_wordlist.py @@ -110,7 +110,7 @@ def create_wordlist(dump_path, output_path): print(f"Step 8: Writing compressed output to {output_path}...") with gzip.open(output_path, 'wt', encoding='utf-8') as f: - for word in sorted_words: + f.writelines(f"{word}\n" for word in sorted_words) f.write(word + '\n') print(f"Done! Wordlist saved to {output_path}")