Deployment Documentation

┌─────────────────────────────────────────────────────────────┐
│ QDrant Loader Deployment │
├─────────────────────────────────────────────────────────────┤
│ CLI Tool │ MCP Server (Optional) │
│ ┌───────────────┐ │ ┌─────────────────────────────────┐ │
│ │ qdrant-loader │ │ │ mcp-qdrant-loader │ │
│ │ │ │ │ (AI Assistant Integration) │ │
│ │ - init │ │ │ │ │
│ │ - ingest │ │ │ - semantic_search │ │
│ │ - config │ │ │ - hierarchy_search │ │
│ │ - project │ │ │ - attachment_search │ │
│ └───────────────┘ │ └─────────────────────────────────┘ │
├─────────────────────────────────────────────────────────────┤
│ External Dependencies │
│ ┌───────────────┐ │ ┌─────────────────────────────────┐ │
│ │ QDrant │ │ │ OpenAI API │ │
│ │ Vector DB │ │ │ (Embeddings) │ │
│ └───────────────┘ │ └─────────────────────────────────┘ │
└─────────────────────────────────────────────────────────────┘

# Create deployment directory
mkdir qdrant-loader-deployment
cd qdrant-loader-deployment
# Create virtual environment
python -m venv venv
source venv/bin/activate # On Windows: venv\Scripts\activate
# Install QDrant Loader (and optional MCP server)
pip install qdrant-loader
pip install qdrant-loader-mcp-server  # optional
# Create workspace structure
mkdir -p {data,logs}
# Create configuration files
cat > config.yaml << EOF
global:
  qdrant:
    url: "http://localhost:6333"
    collection_name: "documents"
  openai:
    api_key: "${OPENAI_API_KEY}"
  state_management:
    state_db_path: "./data/state.db"

projects:
  docs:
    display_name: "Documentation"
    sources:
      git:
        main-docs:
          base_url: "https://github.com/company/docs"
          branch: "main"
          token: "${REPO_TOKEN}"
EOF
# Create environment file
cat > .env << EOF
QDRANT_URL=http://localhost:6333
QDRANT_COLLECTION_NAME=documents
OPENAI_API_KEY=your-openai-key
REPO_TOKEN=your-github-token
EOF
# Initialize and start
qdrant-loader init --workspace .
qdrant-loader ingest --workspace .

# Create production user
sudo useradd -m -s /bin/bash qdrant-loader
sudo su - qdrant-loader
# Setup application directory
mkdir -p /opt/qdrant-loader/{data,logs}
cd /opt/qdrant-loader
# Install Python and dependencies
python -m venv venv
source venv/bin/activate
pip install qdrant-loader qdrant-loader-mcp-server
# Setup configuration (see Configuration section below)
# Edit config.yaml and .env with your settings
# Initialize workspace
qdrant-loader init --workspace /opt/qdrant-loader

# Ubuntu/Debian
sudo apt update
sudo apt install -y python3.12 python3.12-venv python3.12-dev git curl
# CentOS/RHEL
sudo yum install -y python3.12 python3.12-venv python3.12-devel git curl
# macOS (with Homebrew)
brew install python@3.12 git curl

# Core dependencies are automatically installed
pip install qdrant-loader qdrant-loader-mcp-server
# Optional development dependencies
pip install qdrant-loader[dev] qdrant-loader-mcp-server[dev]

# Using Docker (recommended)
docker run -p 6333:6333 -p 6334:6334 \ -v $(pwd)/qdrant_storage:/qdrant/storage:z \ qdrant/qdrant
# Using binary installation
wget https://github.com/qdrant/qdrant/releases/latest/download/qdrant-x86_64-unknown-linux-gnu.tar.gz
tar xzf qdrant-x86_64-unknown-linux-gnu.tar.gz
./qdrant

# QDrant Cloud configuration
export QDRANT_URL="https://your-cluster.qdrant.io"
export QDRANT_API_KEY="your-api-key"
export QDRANT_COLLECTION_NAME="documents"

# Production environment variables
cat > /opt/qdrant-loader/.env << EOF
# QDrant Configuration
QDRANT_URL=http://localhost:6333
QDRANT_COLLECTION_NAME=documents
QDRANT_API_KEY=your-api-key
# OpenAI Configuration
OPENAI_API_KEY=your-openai-api-key
# Data Source Credentials
REPO_TOKEN=your-github-token
CONFLUENCE_TOKEN=your-confluence-token
CONFLUENCE_EMAIL=your-email@domain.com
JIRA_TOKEN=your-jira-token
JIRA_EMAIL=your-email@domain.com
# Application Settings
STATE_DB_PATH=./data/state.db
EOF

# /opt/qdrant-loader/config.yaml
global:
  qdrant:
    url: "${QDRANT_URL}"
    api_key: "${QDRANT_API_KEY}"
    collection_name: "${QDRANT_COLLECTION_NAME}"
  openai:
    api_key: "${OPENAI_API_KEY}"
  state_management:
    state_db_path: "${STATE_DB_PATH}"
  chunking:
    chunk_size: 1200
    chunk_overlap: 300
  file_conversion:
    max_file_size: "100MB"
    conversion_timeout: 300

projects:
  production:
    project_id: "production"
    display_name: "Production Documentation"
    description: "Production documentation and knowledge base"
    sources:
      git:
        docs-repo:
          source_type: "git"
          source: "docs-repo"
          base_url: "https://github.com/company/docs"
          branch: "main"
          token: "${REPO_TOKEN}"
          include_paths:
            - "**/*.md"
            - "**/*.rst"
      confluence:
        company-wiki:
          source_type: "confluence"
          source: "company-wiki"
          base_url: "https://company.atlassian.net/wiki"
          deployment_type: "cloud"
          space_key: "DOCS"
          token: "${CONFLUENCE_TOKEN}"
          email: "${CONFLUENCE_EMAIL}"

# /etc/systemd/system/qdrant-loader.service
[Unit]
Description=QDrant Loader Service
After=network.target
Wants=network.target
[Service]
Type=simple
User=qdrant-loader
Group=qdrant-loader
WorkingDirectory=/opt/qdrant-loader
Environment=PATH=/opt/qdrant-loader/venv/bin
ExecStart=/opt/qdrant-loader/venv/bin/qdrant-loader ingest --workspace /opt/qdrant-loader
Restart=always
RestartSec=10
StandardOutput=journal
StandardError=journal
[Install]
WantedBy=multi-user.target

# /etc/systemd/system/mcp-qdrant-loader.service
[Unit]
Description=QDrant Loader MCP Server
After=network.target
Wants=network.target
[Service]
Type=simple
User=qdrant-loader
Group=qdrant-loader
WorkingDirectory=/opt/qdrant-loader
Environment=PATH=/opt/qdrant-loader/venv/bin
ExecStart=/opt/qdrant-loader/venv/bin/mcp-qdrant-loader --workspace /opt/qdrant-loader/config
Restart=always
RestartSec=10
StandardOutput=journal
StandardError=journal
[Install]
WantedBy=multi-user.target

# Enable and start services
sudo systemctl enable qdrant-loader
sudo systemctl enable mcp-qdrant-loader
sudo systemctl start qdrant-loader
sudo systemctl start mcp-qdrant-loader
# Check status
sudo systemctl status qdrant-loader
sudo systemctl status mcp-qdrant-loader
# View logs
sudo journalctl -u qdrant-loader -f
sudo journalctl -u mcp-qdrant-loader -f
# Restart services
sudo systemctl restart qdrant-loader
sudo systemctl restart mcp-qdrant-loader

# logging.yaml
version: 1
formatters: default: format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s' json: format: '{"timestamp": "%(asctime)s", "logger": "%(name)s", "level": "%(levelname)s", "message": "%(message)s"}'
handlers: console: class: logging.StreamHandler level: INFO formatter: json stream: ext://sys.stdout file: class: logging.handlers.RotatingFileHandler level: DEBUG formatter: default filename: /opt/qdrant-loader/logs/app.log maxBytes: 10485760 # 10MB backupCount: 5
loggers: qdrant_loader: level: DEBUG handlers: [console, file] propagate: false
root: level: INFO handlers: [console]

# /etc/logrotate.d/qdrant-loader
/opt/qdrant-loader/logs/*.log { daily missingok rotate 30 compress delaycompress notifempty create 644 qdrant-loader qdrant-loader postrotate systemctl reload qdrant-loader endscript }

#!/bin/bash
# /opt/qdrant-loader/bin/health-check.sh
set -e
WORKSPACE="/opt/qdrant-loader/config"
LOG_FILE="/opt/qdrant-loader/logs/health-check.log"
# Function to log with timestamp
log() { echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
}
# Check QDrant Loader configuration
if qdrant-loader config --workspace "$WORKSPACE" >/dev/null 2>&1; then
  log "QDrant Loader: HEALTHY - Configuration valid"
  exit 0
else
  log "QDrant Loader: UNHEALTHY - Configuration invalid"
  exit 1
fi

# Add to crontab
*/5 * * * * /opt/qdrant-loader/bin/health-check.sh

# Monitor system resources
htop
iostat -x 1
free -h
df -h

# Check project status
qdrant-loader project --workspace /opt/qdrant-loader/config list
# Check project status with JSON output
qdrant-loader project --workspace /opt/qdrant-loader/config status --format json
# Monitor system services
systemctl status qdrant-loader
systemctl status mcp-qdrant-loader

# Available metrics (from prometheus_metrics.py)
INGESTED_DOCUMENTS = Counter("qdrant_ingested_documents_total", "Total number of documents ingested")
CHUNKING_DURATION = Histogram("qdrant_chunking_duration_seconds", "Time spent chunking documents")
EMBEDDING_DURATION = Histogram("qdrant_embedding_duration_seconds", "Time spent embedding chunks")
UPSERT_DURATION = Histogram("qdrant_upsert_duration_seconds", "Time spent upserting to Qdrant")
CHUNK_QUEUE_SIZE = Gauge("qdrant_chunk_queue_size", "Current size of the chunk queue")
EMBED_QUEUE_SIZE = Gauge("qdrant_embed_queue_size", "Current size of the embedding queue")
CPU_USAGE = Gauge("qdrant_cpu_usage_percent", "CPU usage percent")
MEMORY_USAGE = Gauge("qdrant_memory_usage_percent", "Memory usage percent")

# Set proper file permissions
sudo chown -R qdrant-loader:qdrant-loader /opt/qdrant-loader
sudo chmod 750 /opt/qdrant-loader
sudo chmod 640 /opt/qdrant-loader/config/.env
sudo chmod 644 /opt/qdrant-loader/config/config.yaml
sudo chmod 755 /opt/qdrant-loader/bin/health-check.sh

# Ubuntu/Debian (ufw)
sudo ufw allow ssh
sudo ufw allow 6333/tcp # QDrant HTTP
sudo ufw allow 6334/tcp # QDrant gRPC
sudo ufw enable
# CentOS/RHEL (firewalld)
sudo firewall-cmd --permanent --add-service=ssh
sudo firewall-cmd --permanent --add-port=6333/tcp
sudo firewall-cmd --permanent --add-port=6334/tcp
sudo firewall-cmd --reload

# Generate SSL certificates for QDrant
openssl req -x509 -newkey rsa:4096 -keyout qdrant-key.pem -out qdrant-cert.pem -days 365 -nodes
# Configure QDrant with SSL
# Add to QDrant configuration

# Run multiple ingestion processes for different projects
qdrant-loader ingest --workspace /opt/qdrant-loader --project project1 &
qdrant-loader ingest --workspace /opt/qdrant-loader --project project2 &
qdrant-loader ingest --workspace /opt/qdrant-loader --project project3 &
wait

# Use nginx for load balancing MCP servers
# /etc/nginx/sites-available/qdrant-loader
upstream mcp_servers { server 127.0.0.1:8001; server 127.0.0.1:8002; server 127.0.0.1:8003;
}
server { listen 80; server_name qdrant-loader.example.com; location / { proxy_pass http://mcp_servers; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; }
}

# Optimize for high-memory systems
# Configure larger chunk sizes in config.yaml:
# global:
# chunking:
# chunk_size: 2000
# chunk_overlap: 400
# Run ingestion with specific projectqdrant-loader ingest --workspace /opt/qdrant-loader/config --project high-priority

# config.yaml - Performance tuning
global:
  chunking:
    chunk_size: 1200
    chunk_overlap: 300
  file_conversion:
    max_file_size: "100MB"
    conversion_timeout: 300