John
→ (removed) +#
John Doe
\s*(?:
) + for chunk in self._extract_text_chunks(markup): + names |= self._discover_names(chunk) + return names def anonymize(self, text: str) -> str: """ Anonymize names, emails, usernames, greetings, and sign-offs from the text. """ - doc = self.nlp(text) - names_to_replace = { - ent.text for ent in doc.ents - if ent.label_ == "PERSON" and ent.text not in self.EXCLUDED_WORDS - } + names_to_replace = self._discover_names(text) # Remove email addresses and usernames text = self.EMAIL_PATTERN.sub("", text) text = self.USERNAME_PATTERN.sub("", text) - # Remove greetings and sign-offs + text = self._strip_greetings_signoffs(text) + return self._replace_names(text, names_to_replace) + + def anonymize_markup(self, markup: str) -> str: + """ + Anonymize names, emails, usernames, greetings, and sign-offs from the markup. + including html, rss, and other markup formats. (especially twiki and discourse markup) + """ + names_to_replace = self._discover_names_markup(markup) + # Remove email addresses and usernames + markup = self.EMAIL_PATTERN.sub("", markup) + markup = self.USERNAME_PATTERN.sub("", markup) + markup = _DC_CREATOR_RE.sub(r'\1\2', markup) + markup = _DEFAULT_GENERIC_MARKUP_AUTHOR_ELEMENT_RE.sub("", markup) + markup = _DEFAULT_GENERIC_MARKUP_USER_LINK_RE.sub("", markup) + markup = _DEFAULT_MARKUP_SIGNOFF_TAG_RE.sub("", markup) + markup = _DEFAULT_MARKUP_TRAILING_SIGNOFF_TAG_RE.sub("", markup) + markup = _DEFAULT_MARKUP_TWIKI_USER_LINK_RE.sub("", markup) + markup = self._strip_greetings_signoffs(markup) + return self._replace_names(markup, names_to_replace) + + def _strip_greetings_signoffs(self, text: str) -> str: lines = text.splitlines() - filtered_lines: List[str] = [] + filtered = [] for line in lines: - stripped_line = line.strip() - if any(p.match(stripped_line) for p in self.GREETING_PATTERNS): + stripped = line.strip() + if any(p.match(stripped) for p in self.GREETING_PATTERNS): continue - if any(p.match(stripped_line) for p in self.SIGNOFF_PATTERNS): + if any(p.match(stripped) for p in self.SIGNOFF_PATTERNS): continue - filtered_lines.append(line) - text = "\n".join(filtered_lines) - - # Remove names (case-insensitive) - for name in sorted(names_to_replace, key=len, reverse=True): - pattern = re.compile(r'\b' + re.escape(name) + r'\b', re.IGNORECASE) - text = pattern.sub("", text) - - # Remove extra whitespace - text = "\n".join(line for line in text.splitlines() if line.strip()) - - return text + filtered.append(line) + return "\n".join(filtered) + + def _replace_names(self, text: str, names: set) -> str: + for name in sorted(names, key=len, reverse=True): + text = re.compile(r'\b' + re.escape(name) + r'\b', re.IGNORECASE).sub("", text) + return "\n".join(line for line in text.splitlines() if line.strip()) + + def _extract_text(self, markup: str) -> str: + """Strip markup to plain text for NER. Format-agnostic.""" + attrs = " ".join(_ATTR_TEXT_RE.findall(markup)) + clean = _CDATA_RE.sub(" ", markup) + clean = _TAG_RE.sub(" ", clean) + clean = unescape(clean) + return re.sub(r"\s+", " ", f"{clean} {attrs}").strip() + + def _extract_text_chunks(self, markup: str) -> list: + chunks = [] + # Text content from tags + for match in _CONTENT_TAG_RE.finditer(markup): + inner = _CDATA_RE.sub(" ", match.group(1)) + clean = _TAG_RE.sub(" ", inner) + clean = unescape(clean).strip() + if clean: + chunks.append(clean) + # Text from attributes + attr_text = " ".join(_ATTR_TEXT_RE.findall(markup)) + if attr_text.strip(): + chunks.append(attr_text.strip()) + return chunks diff --git a/src/data_manager/collectors/utils/markitdown_convertor.py b/src/data_manager/collectors/utils/markitdown_convertor.py new file mode 100644 index 000000000..5cbadd603 --- /dev/null +++ b/src/data_manager/collectors/utils/markitdown_convertor.py @@ -0,0 +1,38 @@ +import io +from markitdown import MarkItDown +from src.utils.logging import get_logger +# from src.interfaces.llm.llm_client import LLMClient + +logger = get_logger(__name__) + +def to_valid_file_extension(file_extension: str) -> str: + """ + Convert the file extension to a valid MarkItDown file extension. + """ + return "." + file_extension.lstrip(".") + +class MarkitdownConvertor: + + def __init__(self): + self.markitdown = MarkItDown( + enable_plugins=True, + # llm_client=llm_client, + # llm_model=llm_model, + ) + + def convert(self, content: str, file_extension: str = ".html") -> str: + """ + Convert the content to markdown using MarkItDown. + Args: + content: The content to convert. + file_extension: The file extension of the content. + Returns: + The converted content. + """ + logger.debug(f"Converting content to markdown: {content}") + result = self.markitdown.convert_stream( + io.BytesIO(content.encode("utf-8")), + file_extension=to_valid_file_extension(file_extension), + ) + logger.debug(f"Markitdown result: {result.text_content if hasattr(result, 'text_content') else str(result)}") + return result.text_content if hasattr(result, 'text_content') else str(result)