pip install semantica
# 或安装所有可选依赖：
pip install semantica[all]

# 安装 Semantica
!pip install -q semantica

from semantica.split import TextSplitter

# 示例长文本
text = """
Apple Inc. is a technology company founded by Steve Jobs, Steve Wozniak, and Ronald Wayne 
in Cupertino, California on April 1, 1976. The company's current CEO is Tim Cook, who took 
over from Steve Jobs in August 2011. Apple is headquartered at One Apple Park Way in Cupertino.

Apple develops and sells consumer electronics, computer software, and online services. The company's 
hardware products include the iPhone smartphone, the iPad tablet computer, the Mac personal computer, 
the iPod portable media player, the Apple Watch smartwatch, the Apple TV digital media player, and the 
HomePod smart speaker.

Apple's software includes the macOS and iOS operating systems, the iTunes media player, the Safari web 
browser, and the iLife and iWork creativity and productivity suites. Its online services include the 
iTunes Store, the iOS App Store and Mac App Store, Apple Music, and iCloud.
"""

# 基础递归切分
splitter = TextSplitter(
    method="recursive",
    chunk_size=200,
    chunk_overlap=50
)

chunks = splitter.split(text)

print(f"Split into {len(chunks)} chunks using recursive method\n")
print("=" * 80)

for i, chunk in enumerate(chunks, 1):
    print(f"\nChunk {i}:")
    print(f"  Length: {len(chunk.text)} characters")
    print(f"  Start: {chunk.start_index}, End: {chunk.end_index}")
    print(f"  Text: {chunk.text[:100]}...")

print("\n" + "=" * 80)

# 比较不同方法
methods = ["recursive", "sentence", "paragraph"]

print("Comparing Standard Splitting Methods:\n")
print("=" * 80)

for method in methods:
    splitter = TextSplitter(
        method=method,
        chunk_size=200,
        chunk_overlap=50
    )

    chunks = splitter.split(text)

    print(f"\nMethod: {method.upper()}")
    print("-" * 40)
    print(f"  Chunks created: {len(chunks)}")
    print(f"  Avg chunk size: {sum(len(c.text) for c in chunks) / len(chunks):.0f} chars")
    print(f"  First chunk: {chunks[0].text[:80]}...")

print("\n" + "=" * 80)

from semantica.split import split_by_tokens

# 基于 token 的切分
chunks = split_by_tokens(
    text,
    chunk_size=100,  # 100 个 token
    chunk_overlap=20,
    tokenizer="tiktoken",
    model="gpt-4"
)

print("Token-Based Splitting Results:\n")
print("=" * 80)

for i, chunk in enumerate(chunks, 1):
    token_count = chunk.metadata.get('token_count', 'N/A')
    print(f"\nChunk {i}:")
    print(f"  Tokens: {token_count}")
    print(f"  Characters: {len(chunk.text)}")
    print(f"  Ratio: {len(chunk.text)/token_count if token_count != 'N/A' else 'N/A':.2f} chars/token")

print("\n" + "=" * 80)

from semantica.split import SemanticChunker

# 语义分块
semantic_chunker = SemanticChunker(
    chunk_size=200,
    chunk_overlap=50,
    embedding_model="all-MiniLM-L6-v2",
    similarity_threshold=0.7
)

chunks = semantic_chunker.chunk(text)

print("Semantic Chunking Results:\n")
print("=" * 80)

for i, chunk in enumerate(chunks, 1):
    coherence = chunk.metadata.get('coherence_score', 'N/A')
    print(f"\nChunk {i}:")
    print(f"  Length: {len(chunk.text)} chars")
    print(f"  Coherence: {coherence}")
    print(f"  Text: {chunk.text[:100]}...")

print("\n" + "=" * 80)

from semantica.split import EntityAwareChunker

# 实体感知分块
entity_chunker = EntityAwareChunker(
    chunk_size=200,
    chunk_overlap=50,
    ner_method="ml",  # "ml"（spaCy）、"pattern" 或 "llm"
    preserve_entities=True
)

chunks = entity_chunker.chunk(text)

print("Entity-Aware Chunking Results:\n")
print("=" * 80)

for i, chunk in enumerate(chunks, 1):
    entities = chunk.metadata.get('entities', [])
    print(f"\nChunk {i}:")
    print(f"  Length: {len(chunk.text)} chars")
    print(f"  Entities: {len(entities)}")

    if entities:
        # 同时处理 Entity 对象和字典
        entity_texts = [e.get('text', e.get('entity', '')) if isinstance(e, dict) else str(e) for e in entities[:3]]
        print(f"  Sample entities: {entity_texts}")

print("\n" + "=" * 80)

from semantica.split import RelationAwareChunker

# 关系感知分块
relation_chunker = RelationAwareChunker(
    chunk_size=200,
    chunk_overlap=50,
    preserve_triplets=True
)

chunks = relation_chunker.chunk(text)

print("Relation-Aware Chunking Results:\n")
print("=" * 80)

for i, chunk in enumerate(chunks, 1):
    triplets = chunk.metadata.get('triplets', [])
    relationships = chunk.metadata.get('relationships', [])

    print(f"\nChunk {i}:")
    print(f"  Length: {len(chunk.text)} chars")
    print(f"  Triplets: {len(triplets)}")
    print(f"  Relationships: {len(relationships)}")

print("\n" + "=" * 80)

print(f"\nChunk {i}:")
print(f"  Section: {section}")
print(f"  Level: {level}")
print(f"  Text: {chunk.text[:80]}...")

from semantica.split import HierarchicalChunker

# 层级分块
hierarchical_chunker = HierarchicalChunker(
    chunk_sizes=[400, 200, 100],  # 3 个层级
    chunk_overlaps=[80, 40, 20],
    create_parent_chunks=True
)

chunks = hierarchical_chunker.chunk(text)

print("Hierarchical Chunking Results:\n")
print("=" * 80)

for i, chunk in enumerate(chunks, 1):
    level = chunk.metadata.get('level', 'N/A')
    parent_id = chunk.metadata.get('parent_id', None)
    child_ids = chunk.metadata.get('child_ids', [])

    print(f"\nChunk {i}:")
    print(f"  Level: {level}")
    print(f"  Length: {len(chunk.text)} chars")
    print(f"  Parent: {parent_id if parent_id else 'None (root)'}")
    print(f"  Children: {len(child_ids)}")

print("\n" + "=" * 80)

from semantica.split import SlidingWindowChunker

# 滑动窗口分块
sliding_chunker = SlidingWindowChunker(
    chunk_size=150,
    overlap=50
)

chunks = sliding_chunker.chunk(text)

print("Sliding Window Chunking Results:\n")
print("=" * 80)

for i, chunk in enumerate(chunks, 1):
    # 手动计算重叠
    overlap = 0
    if i > 1:
        prev_chunk = chunks[i-2]
        overlap = max(0, prev_chunk.end_index - chunk.start_index)

    print(f"\nWindow {i}:")
    print(f"  Position: {chunk.start_index}-{chunk.end_index}")
    print(f"  Length: {len(chunk.text)} chars")
    print(f"  Overlap with previous: {overlap} chars")

print("\n" + "=" * 80)

from semantica.split import TableChunker

# 带表格的文本
text_with_table = """
Apple's product lineup includes:

| Product | Category | Release Year |
|---------|----------|-------------|
| iPhone | Smartphone | 2007 |
| iPad | Tablet | 2010 |
| Mac | Computer | 1984 |
| Apple Watch | Wearable | 2015 |
| AirPods | Audio | 2016 |

These products have revolutionized their respective categories.
"""

# 表格分块
table_chunker = TableChunker(
    preserve_headers=True,
    max_rows_per_chunk=3,
    include_context=True,
    table_format="markdown"
)

chunks = table_chunker.chunk(text_with_table)

print("Table Chunking Results:\n")
print("=" * 80)

for i, chunk in enumerate(chunks, 1):
    is_table = chunk.metadata.get('is_table', False)

    print(f"\nChunk {i}:")
    print(f"  Type: {'Table' if is_table else 'Text'}")

    if is_table:
        rows = chunk.metadata.get('row_count', 'N/A')
        cols = chunk.metadata.get('column_count', 'N/A')
        print(f"  Rows: {rows}, Columns: {cols}")

    print(f"  Content: {chunk.text[:100]}...")

print("\n" + "=" * 80)

import sys
import os
import importlib

# 1. 确保本地包在路径中
project_root = os.path.abspath(os.path.join(os.getcwd(), "../.."))
if project_root not in sys.path:
    sys.path.insert(0, project_root)

# 2. 强制卸载模块以确保干净地重新加载
modules_to_unload = [
    'semantica.split.semantic_chunker', 
    'semantica.split.splitter', 
    'semantica.split.provenance_tracker',
    'semantica.split'
]
for module in modules_to_unload:
    if module in sys.modules:
        del sys.modules[module]

# 3. 导入全新的模块
import semantica.split.semantic_chunker
import semantica.split.splitter
import semantica.split.provenance_tracker
from semantica.split import ProvenanceTracker, TextSplitter

# 4. 验证 Chunk 类具有 id 字段
from semantica.split.semantic_chunker import Chunk
print(f"Chunk class fields: {Chunk.__annotations__}")
if 'id' not in Chunk.__annotations__:
    print("WARNING: Chunk class still missing 'id' field. Kernel restart required.")

# 创建分块
splitter = TextSplitter(method="recursive", chunk_size=200, chunk_overlap=50)
chunks = splitter.split(text)

# 跟踪溯源
tracker = ProvenanceTracker()

for chunk in chunks:
    tracker.track_chunk(
        chunk=chunk,
        source_document="apple_doc_001",
        source_path="data/apple.txt",
        timestamp="2024-01-01T00:00:00Z",
        method="recursive"
    )

print("Provenance Tracking Results:\n")
print("=" * 80)

# 获取第一个分块的血缘
if chunks:
    # 使用分块的 ID 获取溯源信息
    chunk_id = getattr(chunks[0], 'id', None)
    print(f"Chunk ID: {chunk_id}")

    if chunk_id:
        prov_info = tracker.get_provenance(chunk_id)

        if prov_info:
            print(f"\nLineage for Chunk 1:")
            print(f"  Source Document: {prov_info.source_document}")
            print(f"  File Path: {prov_info.source_path}")
            print(f"  Method: {prov_info.metadata.get('method')}")
            print(f"  Timestamp: {prov_info.timestamp}")
    else:
        print("Error: Chunk ID not found. The Chunk class definition might still be cached.")
        print("Please click 'Kernel' -> 'Restart Kernel' in the menu and run all cells again.")

print("\n" + "=" * 80)

import time

# 要比较的方法
methods_to_compare = [
    ("recursive", {}),
    ("sentence", {}),
    ("paragraph", {}),
    ("token", {"tokenizer": "tiktoken"}),
]

print("Method Comparison:\n")
print("=" * 80)
print(f"{'Method':<15} {'Chunks':<10} {'Avg Size':<12} {'Time (ms)':<12}")
print("-" * 80)

for method, kwargs in methods_to_compare:
    try:
        start_time = time.time()

        splitter = TextSplitter(
            method=method,
            chunk_size=200,
            chunk_overlap=50,
            **kwargs
        )

        chunks = splitter.split(text)

        elapsed = (time.time() - start_time) * 1000
        avg_size = sum(len(c.text) for c in chunks) / len(chunks) if chunks else 0

        print(f"{method:<15} {len(chunks):<10} {avg_size:<12.0f} {elapsed:<12.2f}")

    except Exception as e:
        print(f"{method:<15} Error: {str(e)[:40]}")

print("=" * 80)