Coverage for src/qdrant_loader/config/global_config.py: 100%

1"""Global configuration settings. 

2 

3This module defines the global configuration settings that apply across the application, 

4including chunking, embedding, and logging configurations. 

5""" 

6 

7from typing import Any 

8 

9from pydantic import Field 

10 

11from qdrant_loader.config.base import BaseConfig 

12from qdrant_loader.config.chunking import ChunkingConfig 

13from qdrant_loader.config.concurrency import ConcurrencyConfig 

14from qdrant_loader.config.embedding import EmbeddingConfig 

15from qdrant_loader.config.graph import GraphConfig 

16from qdrant_loader.config.qdrant import QdrantConfig 

17from qdrant_loader.config.sources import SourcesConfig 

18from qdrant_loader.config.state import StateManagementConfig 

19from qdrant_loader.config.workers import WorkersConfig 

20from qdrant_loader.core.file_conversion import FileConversionConfig 

21 

22 

23class SemanticAnalysisConfig(BaseConfig): 

24 """Configuration for semantic analysis.""" 

25 

26 num_topics: int = Field( 

27 default=3, description="Number of topics to extract using LDA" 

28 ) 

29 

30 lda_passes: int = Field(default=10, description="Number of passes for LDA training") 

31 

32 spacy_model: str = Field( 

33 default="en_core_web_md", 

34 description="spaCy model to use for text processing. Options: en_core_web_sm (15MB, no vectors), en_core_web_md (50MB, 20k vectors), en_core_web_lg (750MB, 514k vectors)", 

35 ) 

36 

37 

38class GlobalConfig(BaseConfig): 

39 """Global configuration settings.""" 

40 

41 chunking: ChunkingConfig = Field(default_factory=ChunkingConfig) 

42 embedding: EmbeddingConfig = Field(default_factory=EmbeddingConfig) 

43 llm: dict[str, Any] | None = Field( 

44 default=None, description="Unified LLM configuration (provider-agnostic)" 

45 ) 

46 semantic_analysis: SemanticAnalysisConfig = Field( 

47 default_factory=SemanticAnalysisConfig, 

48 description="Semantic analysis configuration", 

49 ) 

50 state_management: StateManagementConfig = Field( 

51 default_factory=StateManagementConfig, 

52 description="State management configuration", 

53 ) 

54 sources: SourcesConfig = Field(default_factory=SourcesConfig) 

55 file_conversion: FileConversionConfig = Field( 

56 default_factory=FileConversionConfig, 

57 description="File conversion configuration", 

58 ) 

59 qdrant: QdrantConfig = Field( 

60 default_factory=QdrantConfig, description="Qdrant configuration" 

61 ) 

62 workers: WorkersConfig = Field( 

63 default_factory=WorkersConfig, 

64 description="Worker scheduling and runtime configuration", 

65 ) 

66 concurrency: ConcurrencyConfig = Field( 

67 default_factory=ConcurrencyConfig, 

68 description="Ingestion pipeline concurrency configuration (chunk/embed/upsert)", 

69 ) 

70 graph: GraphConfig = Field( 

71 default_factory=GraphConfig, description="Graph configuration" 

72 ) 

73 

74 def __init__(self, **data): 

75 """Initialize global configuration.""" 

76 # If skip_validation is True and no state_management is provided, use in-memory database 

77 skip_validation = data.pop("skip_validation", False) 

78 if skip_validation and "state_management" not in data: 

79 data["state_management"] = { 

80 "database_path": "./state.db", 

81 "table_prefix": "qdrant_loader_", 

82 "connection_pool": {"size": 5, "timeout": 30}, 

83 } 

84 super().__init__(**data) 

85 

86 def to_dict(self) -> dict[str, Any]: 

87 """Convert the configuration to a dictionary.""" 

88 return { 

89 "chunking": { 

90 "chunk_size": self.chunking.chunk_size, 

91 "chunk_overlap": self.chunking.chunk_overlap, 

92 "enable_semantic_analysis": self.chunking.enable_semantic_analysis, 

93 "enable_enhanced_semantic_analysis": self.chunking.enable_enhanced_semantic_analysis, 

94 }, 

95 "embedding": self.embedding.model_dump(), 

96 "llm": self.llm, 

97 "semantic_analysis": { 

98 "num_topics": self.semantic_analysis.num_topics, 

99 "lda_passes": self.semantic_analysis.lda_passes, 

100 "spacy_model": self.semantic_analysis.spacy_model, 

101 }, 

102 "sources": self.sources.to_dict(), 

103 "state_management": self.state_management.to_dict(), 

104 "file_conversion": { 

105 "max_file_size": self.file_conversion.max_file_size, 

106 "conversion_timeout": self.file_conversion.conversion_timeout, 

107 # EngineKind is a StrEnum; emit the plain string so the merged 

108 # dict re-parses cleanly through the parser merge path. 

109 "engine": self.file_conversion.engine.value, 

110 "markitdown": { 

111 "enable_llm_descriptions": self.file_conversion.markitdown.enable_llm_descriptions, 

112 "llm_model": self.file_conversion.markitdown.llm_model, 

113 "llm_endpoint": self.file_conversion.markitdown.llm_endpoint, 

114 "llm_api_key": self.file_conversion.markitdown.llm_api_key, 

115 }, 

116 "docling": self.file_conversion.docling.model_dump(mode="json"), 

117 }, 

118 "qdrant": self.qdrant.to_dict(), 

119 "workers": self.workers.to_dict(), 

120 "concurrency": self.concurrency.to_dict(), 

121 "graph": self.graph.to_dict(), 

122 }