数据规范化

pip install semantica
# 或安装所有可选依赖：
pip install semantica[all]

# 安装 semantica 包
!pip install semantica

# 使用 TextNormalizer 清洗并规范化文本
from semantica.normalize import TextNormalizer

text_normalizer = TextNormalizer()

# 含多余空格与标点的示例文本
sample_text = "Hello   World!!!  This is a test."

# 转为小写并清洗文本
normalized = text_normalizer.normalize_text(sample_text, case="lower")
cleaned = text_normalizer.clean_text(sample_text, remove_special_chars=False)

sample_text, normalized, cleaned

# 将同一实体的不同写法规范化为统一形式
from semantica.normalize import EntityNormalizer

entity_normalizer = EntityNormalizer()

# 同一组织的多种名称变体
entity_variants = ["Apple Inc.", "Apple Inc", "Apple", "Apple Incorporated"]

normalized_entities = []
for entity in entity_variants:
    # 按组织类型逐条规范化
    normalized = entity_normalizer.normalize_entity(entity, entity_type="Organization")
    normalized_entities.append(normalized)
    print(f"{entity} -> {normalized}")

# 将多种日期写法规范化为标准格式
from semantica.normalize import DateNormalizer

date_normalizer = DateNormalizer()

# 不同风格的日期字符串
date_formats = ["2023-12-25", "12/25/2023", "December 25, 2023", "25 Dec 2023"]

for date_str in date_formats:
    normalized = date_normalizer.normalize_date(date_str)
    print(f"{date_str} -> {normalized}")

# 规范化数字与数量表示（千分位、单位、百分比、科学计数法等）
import importlib
import semantica.normalize.number_normalizer
# 重新加载模块以确保使用最新实现
importlib.reload(semantica.normalize.number_normalizer)

from semantica.normalize import NumberNormalizer

number_normalizer = NumberNormalizer()

# 各种数字书写形式
numbers = ["1,000", "1.5M", "$100", "50%", "3.14e2"]

for num_str in numbers:
    normalized = number_normalizer.normalize_number(num_str)
    print(f"{num_str} -> {normalized}")

# 使用 DataCleaner 清洗记录并去除重复项
from semantica.normalize import DataCleaner

data_cleaner = DataCleaner()

# 含重复记录（前两条实为同一公司）的示例数据
data = [
    {"name": "Apple Inc.", "value": 100},
    {"name": "Apple Inc", "value": 100},
    {"name": "Microsoft", "value": 200}
]

cleaned_data = data_cleaner.clean_data(data, remove_duplicates=True)

print(f"Original records: {len(data)}")
print(f"Cleaned records: {len(cleaned_data)}")

# 检测文本语言与字节编码，保证数据质量
from semantica.normalize import LanguageDetector, EncodingHandler

language_detector = LanguageDetector()
encoding_handler = EncodingHandler()

# 多语言文本样本
text_samples = [
    "Hello, this is English text.",
    "Bonjour, ceci est du texte français.",
    "Hola, este es texto en español."
]

# 逐条检测语言
for text in text_samples:
    detected_lang = language_detector.detect(text)
    print(f"Text: {text[:30]}... -> Language: {detected_lang}")

# 检测字节串的编码
sample_bytes = "Hello World".encode('utf-8')
detected_encoding = encoding_handler.detect(sample_bytes)
print(f"\nDetected encoding: {detected_encoding}")