Coverage for src/qdrant_loader/cli/cli.py: 87%

1"""CLI module for QDrant Loader."""

3from pathlib import Path

5import click

6from click.decorators import group, option

7from click.exceptions import ClickException

8from click.types import Choice

9from click.types import Path as ClickPath

10from click.utils import echo

12# async_command is needed at module level for @async_command decorator on commands.

13from qdrant_loader.cli.asyncio import async_command # noqa: F401

14from qdrant_loader.cli.commands.jobs_cmd import jobs_cmd

15from qdrant_loader.cli.commands.serve_cmd import serve_cmd as serve_command

16from qdrant_loader.utils.sensitive import sanitize_exception_message

18# Heavy modules are lazy-imported to keep CLI startup fast.

21def _get_version_str():

22 from qdrant_loader.cli.version import get_version_str

24 return get_version_str()

27def _check_updates_helper(version):

28 from qdrant_loader.cli.update_check import check_for_updates

30 check_for_updates(version)

33def _setup_workspace_impl(workspace_path):

34 from qdrant_loader.cli.config_loader import setup_workspace

36 return setup_workspace(workspace_path)

39def _create_db_dir_helper(abs_path):

40 from qdrant_loader.cli.path_utils import create_database_directory

42 return create_database_directory(abs_path)

45async def _commands_run_init(settings, force):

46 from qdrant_loader.cli.commands import run_init

48 return await run_init(settings, force)

51# Use minimal imports at startup to improve CLI responsiveness.

52logger = None # Logger will be initialized when first accessed.

55def _get_version() -> str:

56 try:

57 return _get_version_str()

58 except Exception:

59 # Maintain CLI resilience: if version lookup fails for any reason,

60 # surface as 'unknown' rather than crashing the CLI.

61 return "unknown"

64# Back-compat helpers for tests: implement wrappers that operate on this module's global logger

67def _get_logger():

68 global logger

69 if logger is None:

70 from qdrant_loader.utils.logging import LoggingConfig

72 logger = LoggingConfig.get_logger(__name__)

73 return logger

76def _setup_logging(log_level: str, workspace_config=None) -> None:

77 try:

78 from qdrant_loader.utils.logging import LoggingConfig

80 log_format = "console"

81 log_file = (

82 str(workspace_config.logs_path / "cli.log")

83 if workspace_config

84 else "qdrant-loader.log"

85 )

86 LoggingConfig.setup(level=log_level, format=log_format, file=log_file)

87 # update module-global logger

88 global logger

89 logger = LoggingConfig.get_logger(__name__)

90 except Exception as e: # pragma: no cover - exercised via tests with mock

91 from click.exceptions import ClickException

93 raise ClickException(f"Failed to setup logging: {str(e)!s}") from e

96def _check_for_updates() -> None:

97 try:

98 _check_updates_helper(_get_version())

99 except Exception:

100 pass

101

102

103def _setup_workspace(workspace_path):

104 try:

105 return _setup_workspace_impl(workspace_path)

106 except ValueError as e:

107 raise ClickException(str(e)) from e

108 except Exception as e: # pragma: no cover - handled by CLI tests

109 raise ClickException(f"Failed to setup workspace: {str(e)!s}") from e

110

111

112@group(name="qdrant-loader")

113@option(

114 "--log-level",

115 type=Choice(

116 ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False

117 ),

118 default="INFO",

119 help="Set the logging level.",

120)

121@click.version_option(

122 version=_get_version(),

123 message="qDrant Loader v.%(version)s",

124)

125def cli(log_level: str = "INFO") -> None:

126 """QDrant Loader CLI."""

127 # Check for available updates in background without blocking CLI startup.

128 _check_for_updates()

129

130

131cli.add_command(serve_command)

132cli.add_command(jobs_cmd)

133

134

135def _create_database_directory(path: Path) -> bool:

136 """Create database directory with user confirmation.

137

138 Args:

139 path: Path to the database directory

140

141 Returns:

142 bool: True if directory was created, False if user declined

143 """

144 try:

145 abs_path = path.resolve()

146 _get_logger().info("The database directory does not exist", path=str(abs_path))

147 created = _create_db_dir_helper(abs_path)

148 if created:

149 _get_logger().info(f"Created directory: {abs_path}")

150 return created

151 except ClickException:

152 # Propagate ClickException from helper directly

153 raise

154 except Exception as e:

155 # Wrap any other unexpected errors

156 raise ClickException(f"Failed to create directory: {str(e)!s}") from e

157

158

159def _load_config(

160 config_path: Path | None = None,

161 env_path: Path | None = None,

162 skip_validation: bool = False,

163) -> None:

164 """Load configuration from file.

165

166 Args:

167 config_path: Optional path to config file

168 env_path: Optional path to .env file

169 skip_validation: If True, skip directory validation and creation

170 """

171 try:

172 # Lazy import to avoid slow startup

173 from qdrant_loader.config import initialize_config

174

175 # Step 1: If config path is provided, use it

176 if config_path is not None:

177 if not config_path.exists():

178 _get_logger().error("config_not_found", path=str(config_path))

179 raise ClickException(f"Config file not found: {str(config_path)!s}")

180 initialize_config(config_path, env_path, skip_validation=skip_validation)

181 return

182

183 # Step 2: If no config path, look for config.yaml in current folder

184 default_config = Path("config.yaml")

185 if default_config.exists():

186 initialize_config(default_config, env_path, skip_validation=skip_validation)

187 return

188

189 # Step 4: If no file is found, raise an error

190 raise ClickException(

191 f"No config file found. Please specify a config file or create config.yaml in the current directory: {str(default_config)!s}"

192 )

193

194 except Exception as e:

195 # Handle DatabaseDirectoryError and other exceptions

196 from qdrant_loader.config.state import DatabaseDirectoryError

197

198 if isinstance(e, DatabaseDirectoryError):

199 if skip_validation:

200 # For config display, we don't need to create the directory

201 return

202

203 # Get the path from the error - it's already a Path object

204 error_path = e.path

205 # Resolve to absolute path for consistency

206 abs_path = error_path.resolve()

207

208 if not _create_database_directory(abs_path):

209 raise ClickException(

210 "Database directory creation declined. Exiting."

211 ) from e

212

213 # No need to retry _load_config since the directory is now created

214 # Just initialize the config with the expanded path

215 if config_path is not None:

216 initialize_config(

217 config_path, env_path, skip_validation=skip_validation

218 )

219 else:

220 initialize_config(

221 Path("config.yaml"), env_path, skip_validation=skip_validation

222 )

223 elif isinstance(e, ClickException):

224 raise e from None

225 else:

226 safe_error = sanitize_exception_message(e) or type(e).__name__

227 _get_logger().error("config_load_failed", error=safe_error)

228 raise ClickException(f"Failed to load configuration: {safe_error}") from e

229

230

231def _check_settings():

232 """Check if settings are available."""

233 # Lazy import to avoid slow startup

234 from qdrant_loader.config import get_settings

235

236 settings = get_settings()

237 if settings is None:

238 _get_logger().error("settings_not_available")

239 raise ClickException("Settings not available")

240 return settings

241

242

243def _load_config_with_workspace(

244 workspace_config,

245 config_path: Path | None = None,

246 env_path: Path | None = None,

247 skip_validation: bool = False,

248):

249 """Compatibility wrapper used by tests and project commands.

250

251 Delegates to qdrant_loader.cli.config_loader.load_config_with_workspace.

252 """

253 from qdrant_loader.cli.config_loader import (

254 load_config_with_workspace as _load_with_ws,

255 )

256

257 _load_with_ws(

258 workspace_config,

259 config_path=config_path,

260 env_path=env_path,

261 skip_validation=skip_validation,

262 )

263

264

265async def _run_init(settings, force: bool) -> None:

266 """Run initialization process via command helper, keeping existing logging."""

267 try:

268 await _commands_run_init(settings, force)

269 if force:

270 _get_logger().info(

271 "Collection recreated successfully",

272 collection=settings.qdrant_collection_name,

273 )

274 else:

275 _get_logger().info(

276 "Collection initialized successfully",

277 collection=settings.qdrant_collection_name,

278 )

279 except Exception as e:

280 safe_error = sanitize_exception_message(e) or type(e).__name__

281 _get_logger().error("init_failed", error=safe_error)

282 raise ClickException(f"Failed to initialize collection: {safe_error}") from e

283

284

285@cli.command()

286@option(

287 "--workspace",

288 type=ClickPath(path_type=Path),

289 help="Workspace directory containing config.yaml and .env files. All output will be stored here.",

290)

291@option(

292 "--config", type=ClickPath(exists=True, path_type=Path), help="Path to config file."

293)

294@option("--env", type=ClickPath(exists=True, path_type=Path), help="Path to .env file.")

295@option("--force", is_flag=True, help="Force reinitialization of collection.")

296@option(

297 "--log-level",

298 type=Choice(

299 ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False

300 ),

301 default="INFO",

302 help="Set the logging level.",

303)

304@async_command

305async def init(

306 workspace: Path | None,

307 config: Path | None,

308 env: Path | None,

309 force: bool,

310 log_level: str,

311):

312 """Initialize QDrant collection."""

313 from qdrant_loader.cli.commands.init_cmd import run_init_command

314

315 await run_init_command(workspace, config, env, force, log_level)

316

317

318async def _cancel_all_tasks():

319 from qdrant_loader.cli.async_utils import cancel_all_tasks

320

321 await cancel_all_tasks()

322

323

324@cli.command()

325@option(

326 "--workspace",

327 type=ClickPath(path_type=Path),

328 help="Workspace directory containing config.yaml and .env files. All output will be stored here.",

329)

330@option(

331 "--config", type=ClickPath(exists=True, path_type=Path), help="Path to config file."

332)

333@option("--env", type=ClickPath(exists=True, path_type=Path), help="Path to .env file.")

334@option(

335 "--project",

336 type=str,

337 help="Project ID to process. If specified, --source-type and --source will filter within this project.",

338)

339@option(

340 "--source-type",

341 type=str,

342 help="Source type to process (e.g., confluence, jira, git). If --project is specified, filters within that project; otherwise applies to all projects.",

343)

344@option(

345 "--source",

346 type=str,

347 help="Source name to process. If --project is specified, filters within that project; otherwise applies to all projects.",

348)

349@option(

350 "--log-level",

351 type=Choice(

352 ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False

353 ),

354 default="INFO",

355 help="Set the logging level.",

356)

357@option(

358 "--profile/--no-profile",

359 default=False,

360 help="Run the ingestion under cProfile and save output to 'profile.out' (for performance analysis).",

361)

362@option(

363 "--force",

364 is_flag=True,

365 help="Force processing of all documents, bypassing change detection and ignoring checkpoints. Warning: May significantly increase processing time and costs.",

366)

367@option(

368 "--resume/--no-resume",

369 default=True,

370 help="Resume from the last checkpoint if available (default: enabled). Use --no-resume to start fresh from the beginning.",

371)

372@async_command

373async def ingest(

374 workspace: Path | None,

375 config: Path | None,

376 env: Path | None,

377 project: str | None,

378 source_type: str | None,

379 source: str | None,

380 log_level: str,

381 profile: bool,

382 force: bool,

383 resume: bool,

384):

385 """Ingest documents from configured sources.

386

387 Examples:

388 # Ingest all projects (with checkpoint resume if available)

389 qdrant-loader ingest

390

391 # Ingest with checkpoint resume disabled

392 qdrant-loader ingest --no-resume

393

394 # Force full re-ingestion (ignores checkpoints)

395 qdrant-loader ingest --force

396

397 # Ingest specific project

398 qdrant-loader ingest --project my-project

399

400 # Ingest specific source type from all projects

401 qdrant-loader ingest --source-type git

402

403 # Ingest specific source type from specific project

404 qdrant-loader ingest --project my-project --source-type git

405

406 # Ingest specific source from specific project

407 qdrant-loader ingest --project my-project --source-type git --source my-repo

408

409 # Force processing of all documents (bypass change detection)

410 qdrant-loader ingest --force

411 """

412 from qdrant_loader.cli.commands.ingest_cmd import run_ingest_command

413

414 await run_ingest_command(

415 workspace,

416 config,

417 env,

418 project,

419 source_type,

420 source,

421 log_level,

422 profile,

423 force,

424 resume,

425 )

426

427

428async def _start_webhook_server(

429 workspace: Path | None,

430 config: Path | None,

431 env: Path | None,

432 host: str,

433 port: int,

434 log_level: str,

435) -> None:

436 """Start webhook server for receiving connector events and triggering ingestion.

437

438 Internal function called by the future `serve` command. Not exposed as a user CLI command in v1.1.

439 """

440 from qdrant_loader.cli.commands.webhook_cmd import run_webhook_command

441

442 await run_webhook_command(workspace, config, env, host, port, log_level)

443

444

445@cli.command()

446@option(

447 "--output-dir",

448 type=ClickPath(path_type=Path),

449 default=None,

450 help="Workspace directory to write config.yaml and .env files to. "

451 "If omitted, you will be prompted to choose a workspace folder.",

452)

453@option(

454 "--mode",

455 type=Choice(["default", "normal", "advanced"], case_sensitive=False),

456 default=None,

457 help="Setup mode: default (quick start), normal (interactive wizard), advanced (full control). "

458 "If omitted, you will be prompted to choose.",

459)

460def setup(output_dir: Path | None, mode: str | None) -> None:

461 """Setup wizard to generate config.yaml and .env files.

462

463 When run without flags, presents a TUI to choose a workspace folder and

464 setup mode (Default / Normal / Advanced).

465 """

466 from qdrant_loader.cli.commands.setup_cmd import run_setup

467

468 run_setup(output_dir, mode=mode)

469

470

471@cli.command()

472@option(

473 "--workspace",

474 type=ClickPath(path_type=Path),

475 help="Workspace directory containing config.yaml and .env files. All output will be stored here.",

476)

477@option(

478 "--log-level",

479 type=Choice(

480 ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False

481 ),

482 default="INFO",

483 help="Set the logging level.",

484)

485@option(

486 "--config", type=ClickPath(exists=True, path_type=Path), help="Path to config file."

487)

488@option("--env", type=ClickPath(exists=True, path_type=Path), help="Path to .env file.")

489def config(

490 workspace: Path | None, log_level: str, config: Path | None, env: Path | None

491):

492 """Display current configuration."""

493 try:

494 echo("Current Configuration:")

495 from qdrant_loader.cli.commands.config import (

496 run_show_config as _run_show_config,

497 )

498

499 output = _run_show_config(workspace, config, env, log_level)

500 echo(output)

501 except ClickException:

502 raise

503 except Exception as e:

504 from qdrant_loader.utils.logging import LoggingConfig

505

506 safe_error = sanitize_exception_message(e) or type(e).__name__

507 LoggingConfig.get_logger(__name__).error("config_failed", error=safe_error)

508 raise ClickException(f"Failed to display configuration: {safe_error}") from e

509

510

511# Add project management commands with lazy import

512def _add_project_commands():

513 """Lazily add project commands to avoid slow startup."""

514 from qdrant_loader.cli.project_commands import project_cli

515

516 cli.add_command(project_cli)

517

518

519# Only add project commands when CLI is actually used

520if __name__ == "__main__":

521 _add_project_commands()

522 cli()

523else:

524 # Register project commands immediately so they are available when CLI parses args

525 _add_project_commands()