Coverage for src/qdrant_loader/config/chunking.py: 100%
69 statements
« prev ^ index » next coverage.py v7.15.0, created at 2026-07-20 10:15 +0000
1"""Configuration for text chunking."""
3from typing import Literal
5from pydantic import BaseModel, Field, ValidationInfo, field_validator
8class DefaultStrategyConfig(BaseModel):
9 """Configuration for default text chunking strategy."""
11 min_chunk_size: int = Field(
12 default=100, description="Minimum chunk size in characters", gt=0
13 )
14 enable_entity_extraction: bool = Field(
15 default=True, description="Enable entity extraction from text"
16 )
19class HtmlStrategyConfig(BaseModel):
20 """Configuration for HTML chunking strategy."""
22 simple_parsing_threshold: int = Field(
23 default=100000,
24 description="Size threshold for simple vs complex HTML parsing",
25 gt=0,
26 )
27 max_html_size_for_parsing: int = Field(
28 default=500000,
29 description="Maximum HTML size for complex parsing (bytes)",
30 gt=0,
31 )
32 max_sections_to_process: int = Field(
33 default=200, description="Maximum number of sections to process", gt=0
34 )
35 max_chunk_size_for_nlp: int = Field(
36 default=20000,
37 description="Maximum chunk size for NLP processing (characters)",
38 gt=0,
39 )
40 preserve_semantic_structure: bool = Field(
41 default=True, description="Preserve HTML semantic structure in chunks"
42 )
45class CodeStrategyConfig(BaseModel):
46 """Configuration for code chunking strategy."""
48 max_file_size_for_ast: int = Field(
49 default=75000,
50 description="Maximum file size for AST parsing (characters)",
51 gt=0,
52 )
53 max_elements_to_process: int = Field(
54 default=800, description="Maximum number of code elements to process", gt=0
55 )
56 max_recursion_depth: int = Field(
57 default=8, description="Maximum AST recursion depth", gt=0
58 )
59 max_element_size: int = Field(
60 default=20000,
61 description="Maximum size for individual code elements (characters)",
62 gt=0,
63 )
64 enable_ast_parsing: bool = Field(
65 default=True, description="Enable AST parsing for code analysis"
66 )
67 enable_dependency_analysis: bool = Field(
68 default=True, description="Enable dependency analysis for code"
69 )
72class JsonStrategyConfig(BaseModel):
73 """Configuration for JSON chunking strategy."""
75 max_json_size_for_parsing: int = Field(
76 default=1000000, description="Maximum JSON size for parsing (bytes)", gt=0
77 )
78 max_objects_to_process: int = Field(
79 default=200, description="Maximum number of JSON objects to process", gt=0
80 )
81 max_chunk_size_for_nlp: int = Field(
82 default=20000,
83 description="Maximum chunk size for NLP processing (characters)",
84 gt=0,
85 )
86 max_recursion_depth: int = Field(
87 default=5, description="Maximum recursion depth for nested structures", gt=0
88 )
89 max_array_items_per_chunk: int = Field(
90 default=50, description="Maximum array items to include per chunk", gt=0
91 )
92 max_object_keys_to_process: int = Field(
93 default=100, description="Maximum object keys to process", gt=0
94 )
95 enable_schema_inference: bool = Field(
96 default=True, description="Enable JSON schema inference"
97 )
100class MarkdownStrategyConfig(BaseModel):
101 """Configuration for Markdown chunking strategy."""
103 min_content_length_for_nlp: int = Field(
104 default=100,
105 description="Minimum content length for NLP processing (characters)",
106 gt=0,
107 )
108 min_word_count_for_nlp: int = Field(
109 default=20, description="Minimum word count for NLP processing", gt=0
110 )
111 min_line_count_for_nlp: int = Field(
112 default=3, description="Minimum line count for NLP processing", gt=0
113 )
114 min_section_size: int = Field(
115 default=500, description="Minimum characters for a standalone section", gt=0
116 )
117 max_chunks_per_section: int = Field(
118 default=1000,
119 description="Maximum chunks per section (prevents runaway chunking)",
120 gt=0,
121 )
122 max_overlap_percentage: float = Field(
123 default=0.25,
124 description="Maximum overlap between chunks as percentage (0.25 = 25%)",
125 ge=0.0,
126 le=1.0,
127 )
128 max_workers: int = Field(
129 default=4, description="Maximum worker threads for parallel processing", gt=0
130 )
131 estimation_buffer: float = Field(
132 default=0.2,
133 description="Buffer factor for chunk count estimation (0.2 = 20%)",
134 ge=0.0,
135 le=1.0,
136 )
137 words_per_minute_reading: int = Field(
138 default=200, description="Words per minute for reading time estimation", gt=0
139 )
140 header_analysis_threshold_h1: int = Field(
141 default=3,
142 description="H1 header count threshold for split level decisions",
143 gt=0,
144 )
145 header_analysis_threshold_h3: int = Field(
146 default=8,
147 description="H3 header count threshold for split level decisions",
148 gt=0,
149 )
150 enable_hierarchical_metadata: bool = Field(
151 default=True, description="Enable extraction of hierarchical section metadata"
152 )
155class DoclingStrategyConfig(BaseModel):
156 """Configuration for the docling structure-aware chunking strategy.
158 Used for files converted by the docling engine. Unlike the char-based strategies,
159 docling sizes chunks by a *token* budget aligned to the embedding model's tokenizer
160 (the tokenizer identity still comes from ``embedding.tokenizer``).
161 """
163 max_tokens: int | None = Field(
164 default=None,
165 gt=0,
166 description="Per-chunk token budget for docling chunking. "
167 "Unset inherits embedding.max_tokens_per_chunk.",
168 )
169 include_context_in_embed: bool = Field(
170 default=False,
171 description="Prepend the heading breadcrumb to each chunk's content (and thus "
172 "its embedding). Adds context for retrieval; may push a chunk past max_tokens.",
173 )
174 table_serialization: Literal["triplets", "markdown"] = Field(
175 default="triplets",
176 description="How tables are rendered into chunk text: 'triplets' "
177 "('row, column = value' lines — docling's embedding-friendly default) or "
178 "'markdown' (pipe-delimited rows — more readable for LLMs and display).",
179 )
182class StrategySpecificConfig(BaseModel):
183 """Strategy-specific configuration settings."""
185 default: DefaultStrategyConfig = Field(
186 default_factory=DefaultStrategyConfig,
187 description="Configuration for default text chunking strategy",
188 )
189 html: HtmlStrategyConfig = Field(
190 default_factory=HtmlStrategyConfig,
191 description="Configuration for HTML chunking strategy",
192 )
193 code: CodeStrategyConfig = Field(
194 default_factory=CodeStrategyConfig,
195 description="Configuration for code chunking strategy",
196 )
197 json_strategy: JsonStrategyConfig = Field(
198 default_factory=JsonStrategyConfig,
199 description="Configuration for JSON chunking strategy",
200 alias="json",
201 )
202 markdown: MarkdownStrategyConfig = Field(
203 default_factory=MarkdownStrategyConfig,
204 description="Configuration for Markdown chunking strategy",
205 )
206 docling: DoclingStrategyConfig = Field(
207 default_factory=DoclingStrategyConfig,
208 description="Configuration for docling structure-aware chunking strategy",
209 )
212class ChunkingConfig(BaseModel):
213 """Configuration for text chunking."""
215 chunk_size: int = Field(
216 default=1500,
217 description="Size of text chunks in characters",
218 gt=0,
219 title="Chunk Size",
220 )
221 chunk_overlap: int = Field(
222 default=200,
223 description="Overlap between chunks in characters",
224 ge=0,
225 title="Chunk Overlap",
226 )
227 max_chunks_per_document: int = Field(
228 default=500,
229 description="Maximum number of chunks per document (safety limit)",
230 gt=0,
231 title="Max Chunks Per Document",
232 )
233 enable_semantic_analysis: bool = Field(
234 default=True,
235 description="Master switch for semantic analysis (spaCy + LDA) across all chunking strategies. "
236 "Disable for faster ingestion when NLP enrichment is not needed.",
237 )
238 enable_enhanced_semantic_analysis: bool = Field(
239 default=False,
240 description="Enable advanced NLP fields: pos_tags, dependencies, document_similarity. "
241 "Requires enable_semantic_analysis=true. "
242 "Increases payload size and ingestion time.",
243 )
245 # Strategy-specific configurations
246 strategies: StrategySpecificConfig = Field(
247 default_factory=StrategySpecificConfig,
248 description="Strategy-specific configuration settings",
249 )
251 @field_validator("chunk_overlap")
252 def validate_chunk_overlap(cls, v: int, info: ValidationInfo) -> int:
253 """Validate that chunk overlap is less than chunk size."""
254 chunk_size = info.data.get("chunk_size", 1500)
255 if v >= chunk_size:
256 raise ValueError("Chunk overlap must be less than chunk size")
257 return v
259 @field_validator("enable_enhanced_semantic_analysis")
260 def validate_enhanced_semantic_analysis_dependency(
261 cls, v: bool, info: ValidationInfo
262 ) -> bool:
263 """Validate enhanced semantic analysis requires base semantic analysis."""
264 enable_semantic_analysis = info.data.get("enable_semantic_analysis", True)
265 if v and enable_semantic_analysis is not True:
266 raise ValueError(
267 "enable_enhanced_semantic_analysis requires enable_semantic_analysis=True"
268 )
269 return v