Source code for geeViz.eeCatalogHelpers.text
"""Text normalization helpers."""
from __future__ import annotations
import re
__all__ = ["clean_text", "normalize_for_matching"]
_DEFAULT_NULL_TOKENS = {"NA", "N/A", "NONE", "NULL", ""}
[docs]
def clean_text(value: str | None) -> str:
"""Normalize BOM/whitespace and return an empty string for nulls."""
if value is None:
return ""
text = value.replace("\ufeff", "")
return re.sub(r"\s+", " ", text).strip()
[docs]
def normalize_for_matching(value: str | None, null_tokens: set[str] | None = None) -> str:
"""Normalize text and collapse common null-like tokens to empty strings."""
tokens = _DEFAULT_NULL_TOKENS if null_tokens is None else null_tokens
normalized_tokens = {token.upper() for token in tokens}
text = clean_text(value)
if text.upper() in normalized_tokens:
return ""
return text