Coverage for src/qdrant_loader/core/conversion/service.py: 94%
82 statements
« prev ^ index » next coverage.py v7.15.0, created at 2026-07-20 10:15 +0000
1"""The conversion composition root: pick an engine from config, convert a file,
2hand the connector one uniform result.
4This is where the ``markitdown`` | ``docling`` choice is resolved (config-driven, not
5a feature flag). Connectors depend on :class:`ConversionService` and
6:class:`ConvertedFile`; they never branch on the engine themselves. The two engines
7produce fundamentally different artifacts — markitdown a markdown string, docling a
8structured ``DoclingDocument`` — and :class:`ConvertedFile` carries both shapes so a
9single downstream contract works for either:
11* ``content`` — markdown for *both* engines (docling exports its structure to markdown
12 for display, state and the content hash), so existing consumers keep working.
13* ``converted_document`` — the structured artifact, present only on the docling path.
14 This is what the docling chunking strategy consumes for structure-aware chunking;
15 it rides the in-process ``Document`` by reference to the chunker (never serialized).
17Per-document engine failures are NOT raised here — docling reports them as a status on
18its outcome, and this service projects that into a fallback ``ConvertedFile`` mirroring
19today's ``conversion_method="*_fallback"`` behavior, so a connector never has to special
20-case the engine. Policy/precondition errors (unsupported format, too large) remain the
21caller's gate, exactly as today.
23A docling ``PARTIAL_SUCCESS`` (e.g. a document_timeout truncating the document mid-parse)
24yields usable-but-incomplete content: it is still indexed as ``conversion_method="docling"``
25(content is real), but it is logged as a distinct warning so a truncated document never
26passes silently as if it were complete. The fallback-document template and
27``original_file_type`` classification are shared with the markitdown path (the canonical
28:class:`FileConverter`/:class:`FileDetector`) so the two engines stamp identical metadata.
29"""
31from __future__ import annotations
33import os
34from dataclasses import dataclass
35from functools import cached_property
36from pathlib import Path
37from typing import TYPE_CHECKING, Protocol
39from qdrant_loader.utils.logging import LoggingConfig
41from .engine import EngineKind, build_engine
42from .formats import FormatPolicy
43from .outcome import ConversionStatus
45if TYPE_CHECKING:
46 from qdrant_loader.core.file_conversion.conversion_config import (
47 FileConversionConfig,
48 )
49 from qdrant_loader.core.file_conversion.file_detector import FileDetector
51 from .engine import ConversionEngine
52 from .outcome import ConvertedDocument
54logger = LoggingConfig.get_logger(__name__)
57class MarkitdownConverter(Protocol):
58 """The slice of ``FileConverter`` the markitdown path depends on.
60 Depending on this Protocol rather than the concrete ``FileConverter`` keeps the
61 legacy engine injectable (and therefore testable without the markitdown library).
62 """
64 def convert_file(self, file_path: str) -> str: ...
66 def create_fallback_document(self, file_path: str, error: Exception) -> str: ...
69@dataclass(frozen=True, slots=True)
70class ConvertedFile:
71 """The uniform conversion result a connector turns into a ``Document``.
73 ``conversion_method`` and ``conversion_failed`` map straight onto the existing
74 metadata contract (``"markitdown"`` / ``"markitdown_fallback"`` / ``"docling"`` /
75 ``"docling_fallback"``); ``converted_document`` is the structured artifact for the
76 docling chunking path (``None`` for markitdown and for any fallback).
77 """
79 content: str
80 conversion_method: str
81 conversion_failed: bool
82 original_file_type: str
83 converted_document: ConvertedDocument | None = None
86class ConversionService:
87 """Selects the configured engine and converts files to a :class:`ConvertedFile`."""
89 def __init__(
90 self,
91 config: FileConversionConfig,
92 *,
93 markitdown_converter: MarkitdownConverter | None = None,
94 ) -> None:
95 self._config = config
96 self._injected_markitdown = markitdown_converter
98 @cached_property
99 def _docling_engine(self) -> ConversionEngine:
100 """Build the docling engine once (it caches pipelines internally)."""
101 return build_engine(EngineKind.DOCLING, self._config.docling.to_config())
103 @cached_property
104 def _markitdown_converter(self) -> MarkitdownConverter:
105 """The injected converter, or a lazily-built ``FileConverter``.
107 Imported lazily so the docling path never drags in markitdown.
108 """
109 if self._injected_markitdown is not None:
110 return self._injected_markitdown
111 from qdrant_loader.core.file_conversion.file_converter import FileConverter
113 return FileConverter(self._config)
115 @cached_property
116 def _file_detector(self) -> FileDetector:
117 """One shared ``FileDetector`` for ``original_file_type`` classification.
119 Held lazily (built once, reused) so the docling path stamps the same type
120 the rest of the system does, rather than a raw suffix.
121 """
122 from qdrant_loader.core.file_conversion.file_detector import FileDetector
124 return FileDetector()
126 def convert(self, file_path: str) -> ConvertedFile:
127 """Convert ``file_path`` using the configured engine.
129 The caller is responsible for the supported-type / size gate (via
130 :meth:`is_supported`), so this dispatches purely on the configured engine.
131 """
132 match self._config.engine:
133 case EngineKind.DOCLING:
134 return self._convert_with_docling(file_path)
135 case EngineKind.MARKITDOWN:
136 return self._convert_with_markitdown(file_path)
137 raise ValueError(f"unknown conversion engine: {self._config.engine!r}")
139 def is_supported(self, file_path: str) -> bool:
140 """Whether the configured engine can convert ``file_path`` — one gate.
142 Connectors call this instead of reaching for ``FileDetector`` directly, so
143 eligibility tracks the *active* engine rather than a single static MIME table:
145 * ``markitdown`` — preserves today's behaviour exactly: delegate to
146 ``FileDetector.is_supported_for_conversion``.
147 * ``docling`` — supported only when the FileDetector says convertible (keeping
148 the natively-handled exclusions like ``.md``/``.html``) AND the file falls
149 within docling's own :class:`~.formats.FormatPolicy` (enabled format + size
150 cap), so the two sources of truth can no longer disagree.
151 """
152 detector = self._file_detector
153 if self._config.engine is EngineKind.MARKITDOWN:
154 return detector.is_supported_for_conversion(file_path)
156 if not detector.is_supported_for_conversion(file_path):
157 return False
158 policy = self._format_policy
159 mime_type, _ = detector.detect_file_type(file_path)
160 if mime_type not in policy.supported_mime_types():
161 return False
162 try:
163 size_bytes = os.path.getsize(file_path)
164 except OSError:
165 return False
166 return policy.is_within_size_limit(size_bytes)
168 @cached_property
169 def _format_policy(self) -> FormatPolicy:
170 """docling's conversion-eligibility policy, built once from the engine config."""
171 return FormatPolicy(self._config.docling.to_config())
173 # ── docling path (full Option B) ────────────────────────────────────────────
174 def _convert_with_docling(self, file_path: str) -> ConvertedFile:
175 outcome = self._docling_engine.convert(file_path)
176 original_file_type = self._file_type(file_path)
178 if outcome.succeeded and outcome.document is not None:
179 if outcome.status is ConversionStatus.PARTIAL:
180 # PARTIAL_SUCCESS (e.g. document_timeout mid-document): content is
181 # real and usable, so we still index it as a docling document — but
182 # it is truncated/incomplete, so it must NOT pass silently.
183 logger.warning(
184 "Docling conversion was partial; indexing truncated content",
185 file_path=file_path,
186 error=outcome.error,
187 )
188 return ConvertedFile(
189 content=outcome.document.to_markdown(),
190 conversion_method="docling",
191 conversion_failed=False,
192 original_file_type=original_file_type,
193 converted_document=outcome.document,
194 )
196 # FAILED: fall back to the canonical document, mirroring markitdown_fallback.
197 logger.warning(
198 "Docling conversion did not succeed; emitting fallback document",
199 file_path=file_path,
200 status=str(outcome.status),
201 error=outcome.error,
202 )
203 # Reuse FileConverter's canonical fallback so the docling and markitdown
204 # paths emit identical fallback documents (no divergent local stub). This
205 # lazily builds FileConverter, but only on a docling FAILURE (rare).
206 error = Exception(outcome.error or "conversion failed")
207 return ConvertedFile(
208 content=self._markitdown_converter.create_fallback_document(
209 file_path, error
210 ),
211 conversion_method="docling_fallback",
212 conversion_failed=True,
213 original_file_type=original_file_type,
214 converted_document=None,
215 )
217 # ── markitdown path (legacy behavior, now behind the service) ────────────────
218 def _convert_with_markitdown(self, file_path: str) -> ConvertedFile:
219 """Delegate to the markitdown converter, mirroring the connectors' contract.
221 Success -> ``conversion_method="markitdown"``; any ``FileConversionError``
222 (including timeouts) -> a fallback document with ``"markitdown_fallback"`` and
223 ``conversion_failed=True`` — exactly the three-way branch the connectors use
224 today. There is no structured artifact on this path.
225 """
226 from qdrant_loader.core.file_conversion.exceptions import FileConversionError
228 converter = self._markitdown_converter
229 original_file_type = self._file_type(file_path)
230 try:
231 return ConvertedFile(
232 content=converter.convert_file(file_path),
233 conversion_method="markitdown",
234 conversion_failed=False,
235 original_file_type=original_file_type,
236 converted_document=None,
237 )
238 except FileConversionError as error:
239 return ConvertedFile(
240 content=converter.create_fallback_document(file_path, error),
241 conversion_method="markitdown_fallback",
242 conversion_failed=True,
243 original_file_type=original_file_type,
244 converted_document=None,
245 )
247 def _file_type(self, file_path: str) -> str:
248 """The ``original_file_type`` contract, via the shared FileDetector.
250 Reuses ``FileDetector.get_file_type_info`` so both engines stamp the same
251 normalized type the rest of the system uses, with a suffix-based fallback
252 for files the detector can't classify (e.g. unknown extensions).
253 """
254 normalized = self._file_detector.get_file_type_info(file_path).get(
255 "normalized_type"
256 )
257 return normalized or Path(file_path).suffix.lstrip(".").lower() or "unknown"