Fix: Core APIs

.env.example

@@ -1,51 +1,182 @@
-# Copy this file to .env and update with your values
+# Rendiff FFmpeg API - Production Environment Configuration
+# Copy this file to .env and configure for your environment
 
-# Database Configuration
-POSTGRES_PASSWORD=your_secure_password_here
-POSTGRES_USER=ffmpeg_user
-POSTGRES_DB=ffmpeg_api
-DATABASE_URL=postgresql://ffmpeg_user:your_secure_password_here@postgres:5432/ffmpeg_api
+# =============================================================================
+# CORE APPLICATION SETTINGS
+# =============================================================================
 
-# Redis/Queue Configuration
-REDIS_URL=redis://redis:6379/0
+# Application Environment
+DEBUG=false
+TESTING=false
+VERSION=1.0.0
 
-# API Configuration
+# API Server Configuration
 API_HOST=0.0.0.0
 API_PORT=8000
 API_WORKERS=4
-LOG_LEVEL=info
-DEBUG=false
+API_RELOAD=false
+API_LOG_LEVEL=info
+
+# =============================================================================
+# DATABASE CONFIGURATION
+# =============================================================================
+
+# Production PostgreSQL (Recommended)
+DATABASE_URL=postgresql://ffmpeg_user:your_secure_password@postgres:5432/ffmpeg_api
+
+# Alternative: SQLite (Development Only)
+# DATABASE_URL=sqlite+aiosqlite:///data/rendiff.db
+
+# Database Pool Settings
+DATABASE_POOL_SIZE=20
+DATABASE_MAX_OVERFLOW=40
+
+# =============================================================================
+# QUEUE & CACHE CONFIGURATION
+# =============================================================================
+
+# Redis Configuration
+REDIS_URL=redis://redis:6379/0
+REDIS_MAX_CONNECTIONS=100
 
 # Worker Configuration
 WORKER_CONCURRENCY=4
-CPU_WORKERS=2
-GPU_WORKERS=0
+WORKER_PREFETCH_MULTIPLIER=1
+WORKER_MAX_TASKS_PER_CHILD=100
+WORKER_TASK_TIME_LIMIT=21600
 
-# Storage Configuration
-STORAGE_PATH=./storage
+# =============================================================================
+# STORAGE CONFIGURATION
+# =============================================================================
+
+# Storage Paths
 STORAGE_CONFIG=/app/config/storage.yml
+STORAGE_PATH=./storage
+TEMP_PATH=/tmp/rendiff
+
+# Data Persistence Paths (for Docker volumes)
+POSTGRES_DATA_PATH=./data/postgres
+REDIS_DATA_PATH=./data/redis
+PROMETHEUS_DATA_PATH=./data/prometheus
+GRAFANA_DATA_PATH=./data/grafana
+
+# =============================================================================
+# FFMPEG & PROCESSING CONFIGURATION
+# =============================================================================
 
-# Security Configuration
-ADMIN_API_KEYS=your_admin_key_1,your_admin_key_2
-RENDIFF_API_KEYS=your_client_key_1,your_client_key_2,your_client_key_3
+# FFmpeg Settings
+FFMPEG_THREADS=0
+FFMPEG_PRESET=medium
+FFMPEG_CRF=23
+FFMPEG_HARDWARE_ACCELERATION=auto
+
+# =============================================================================
+# SECURITY & AUTHENTICATION
+# =============================================================================
+
+# API Security
+API_KEY_HEADER=X-API-Key
 ENABLE_API_KEYS=true
-CORS_ORIGINS=https://localhost,http://localhost
+ENABLE_IP_WHITELIST=false
+IP_WHITELIST=10.0.0.0/8,192.168.0.0/16
+
+# Admin API Keys (comma-separated)
+ADMIN_API_KEYS=your_admin_key_here
+
+# Rate Limiting
+ENABLE_RATE_LIMITING=true
+RATE_LIMIT_CALLS=2000
+RATE_LIMIT_PERIOD=3600
 
-# SSL/HTTPS Configuration (for production)
-DOMAIN_NAME=localhost
-CERTBOT_EMAIL=admin@example.com
-CERT_RESOLVER=letsencrypt
-LETSENCRYPT_STAGING=false
+# CORS Origins (comma-separated)
+API_CORS_ORIGINS=http://localhost,https://localhost,https://yourdomain.com
+API_TRUSTED_HOSTS=localhost,yourdomain.com
 
-# Monitoring Configuration
-GRAFANA_PASSWORD=your_grafana_password_here
-PROMETHEUS_AUTH=admin:your_prometheus_password_here
-TRAEFIK_AUTH=admin:your_traefik_password_here
+# Database Passwords
+POSTGRES_PASSWORD=your_secure_postgres_password
 
-# Resource Limits
+# =============================================================================
+# MONITORING & OBSERVABILITY
+# =============================================================================
+
+# Metrics & Monitoring
+ENABLE_METRICS=true
+METRICS_PORT=9000
+ENABLE_TRACING=false
+TRACING_ENDPOINT=
+
+# Grafana Configuration
+GRAFANA_PASSWORD=your_secure_grafana_password
+
+# =============================================================================
+# RESOURCE LIMITS
+# =============================================================================
+
+# Upload & Processing Limits
 MAX_UPLOAD_SIZE=10737418240
-MAX_CONCURRENT_JOBS_PER_KEY=10
 MAX_JOB_DURATION=21600
+MAX_CONCURRENT_JOBS_PER_KEY=10
+JOB_RETENTION_DAYS=7
+
+# =============================================================================
+# WEBHOOKS & NOTIFICATIONS
+# =============================================================================
+
+# Webhook Configuration
+WEBHOOK_TIMEOUT=30
+WEBHOOK_MAX_RETRIES=3
+WEBHOOK_RETRY_DELAY=60
+
+# =============================================================================
+# OPTIONAL SERVICES
+# =============================================================================
+
+# Virus Scanning (Optional)
+ENABLE_VIRUS_SCAN=false
+CLAMAV_HOST=
+CLAMAV_PORT=3310
+
+# =============================================================================
+# DEPLOYMENT SPECIFIC
+# =============================================================================
+
+# Docker Compose Profiles
+# Uncomment the profile you want to use:
+# COMPOSE_PROFILES=standard        # Standard CPU-only deployment
+# COMPOSE_PROFILES=gpu            # GPU-accelerated deployment
+# COMPOSE_PROFILES=monitoring     # Include Prometheus/Grafana
+# COMPOSE_PROFILES=gpu,monitoring # GPU + Monitoring
+
+# Network Configuration
+# COMPOSE_PROJECT_NAME=ffmpeg-api
+
+# =============================================================================
+# CLOUD STORAGE (Optional)
+# =============================================================================
+
+# AWS S3
+# AWS_ACCESS_KEY_ID=your_access_key
+# AWS_SECRET_ACCESS_KEY=your_secret_key
+# AWS_DEFAULT_REGION=us-west-2
+# S3_BUCKET_NAME=your-video-bucket
+
+# Azure Blob Storage
+# AZURE_STORAGE_ACCOUNT=your_account
+# AZURE_STORAGE_KEY=your_key
+# AZURE_CONTAINER_NAME=videos
+
+# Google Cloud Storage
+# GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json
+# GCS_BUCKET_NAME=your-video-bucket
+
+# =============================================================================
+# SSL/TLS CONFIGURATION
+# =============================================================================
+
+# SSL Certificate Paths (for production)
+# SSL_CERT_PATH=/etc/ssl/certs/your-cert.crt
+# SSL_KEY_PATH=/etc/ssl/private/your-key.key
 
-# External URLs
-EXTERNAL_URL=https://localhost
+# Let's Encrypt (for automatic SSL)
+# LETSENCRYPT_EMAIL=admin@yourdomain.com
+# LETSENCRYPT_HOST=yourdomain.com

.github/workflows/yaml-lint.yml

@@ -0,0 +1,133 @@
+name: YAML Lint
+
+on:
+  push:
+    branches: [ main, develop ]
+    paths:
+      - '**/*.yml'
+      - '**/*.yaml'
+      - '.github/workflows/*.yml'
+  pull_request:
+    branches: [ main, develop ]
+    paths:
+      - '**/*.yml'
+      - '**/*.yaml'
+      - '.github/workflows/*.yml'
+
+jobs:
+  yaml-lint:
+    name: YAML Lint
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+
+    - name: Install yamllint
+      run: |
+        python -m pip install --upgrade pip
+        pip install yamllint
+
+    - name: Create yamllint config
+      run: |
+        cat > .yamllint.yml << EOF
+        extends: default
+
+        rules:
+          # Allow longer lines for docker compose
+          line-length:
+            max: 120
+            level: warning
+
+          # Allow multiple spaces after operators
+          colons:
+            max-spaces-before: 0
+            max-spaces-after: -1
+
+          # Allow indentation of 2 or 4 spaces
+          indentation:
+            spaces: consistent
+            indent-sequences: true
+            check-multi-line-strings: false
+
+          # Don't require document start markers
+          document-start: disable
+
+          # Allow empty values
+          empty-values:
+            forbid-in-block-mappings: false
+            forbid-in-flow-mappings: false
+
+          # Allow trailing spaces in comments
+          trailing-spaces:
+            level: warning
+
+          # Allow truthy values like 'yes', 'on', etc.
+          truthy:
+            allowed-values: ['true', 'false', 'yes', 'no', 'on', 'off']
+            check-keys: false
+        EOF
+
+    - name: Lint YAML files
+      run: |
+        echo "Linting YAML files..."
+        find . -type f \( -name "*.yml" -o -name "*.yaml" \) -not -path "./.git/*" | while read file; do
+          echo "Checking: $file"
+          yamllint "$file"
+        done
+
+    - name: Validate Docker Compose files
+      run: |
+        echo "Validating Docker Compose syntax..."
+
+        # Check main compose file
+        if [ -f "compose.yml" ]; then
+          echo "Validating compose.yml..."
+          docker compose -f compose.yml config > /dev/null
+        fi
+
+        # Check production compose file
+        if [ -f "compose.prod.yml" ]; then
+          echo "Validating compose.prod.yml..."
+          docker compose -f compose.prod.yml config > /dev/null
+        fi
+
+        # Check stable compose file
+        if [ -f "compose.stable.yml" ]; then
+          echo "Validating compose.stable.yml..."
+          docker compose -f compose.stable.yml config > /dev/null
+        fi
+
+        # Check override file
+        if [ -f "compose.override.yml" ]; then
+          echo "Validating compose.override.yml..."
+          docker compose -f compose.yml -f compose.override.yml config > /dev/null
+        fi
+
+        echo "All Docker Compose files are valid!"
+
+    - name: Check for common issues
+      run: |
+        echo "Checking for common Docker Compose issues..."
+
+        # Check for deprecated version field
+        if grep -r "version:" . --include="*.yml" --include="*.yaml" --exclude-dir=.git; then
+          echo "::warning::Found 'version:' field in compose files. This is deprecated in modern Docker Compose."
+        fi
+
+        # Check for hardcoded localhost
+        if grep -r "localhost" . --include="compose*.yml" --exclude-dir=.git; then
+          echo "::warning::Found hardcoded 'localhost' in compose files. Consider using service names."
+        fi
+
+        # Check for missing health checks on databases
+        if grep -A 10 "image.*postgres" . --include="compose*.yml" --exclude-dir=.git | grep -v "healthcheck:" > /dev/null; then
+          echo "::warning::PostgreSQL services should have health checks defined."
+        fi
+
+        echo "Common issues check completed!"

.structure.md

@@ -0,0 +1,29 @@
+# Optimized Repository Structure
+
+## Core Structure
+```
+ffmpeg-api/
+├── 📁 src/                    # Source code (renamed from api/)
+│   ├── 📁 api/               # API layer
+│   ├── 📁 core/              # Core business logic
+│   ├── 📁 models/            # Data models
+│   ├── 📁 services/          # Business services
+│   └── 📁 utils/             # Utilities
+├── 📁 workers/               # Worker processes
+├── 📁 tests/                 # Test suite
+├── 📁 deployment/            # Deployment configs
+│   ├── 📁 docker/           # Docker configurations
+│   ├── 📁 k8s/              # Kubernetes manifests
+│   └── 📁 compose/          # Docker Compose files
+├── 📁 config/                # Configuration files
+├── 📁 docs/                  # Documentation
+├── 📁 scripts/               # Utility scripts
+└── 📁 monitoring/            # Monitoring and observability
+
+## Changes Made:
+1. Consolidated API code under src/
+2. Moved deployment files to deployment/
+3. Cleaned up root directory
+4. Better separation of concerns
+5. Removed redundant files
+```