diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8936174 --- /dev/null +++ b/.gitignore @@ -0,0 +1,193 @@ +# ============================================================================== +# PYTHON + NEXT.JS UNIFIED .GITIGNORE +# ============================================================================== + +# ---------------------------------------------------------------------------- +# Python: Byte-compiled & Cache +# ---------------------------------------------------------------------------- +__pycache__/ +*.py[cod] +*$py.class +*.so +.cython_debug/ +cython_debug/ + +# ---------------------------------------------------------------------------- +# Python: Packaging & Distribution +# ---------------------------------------------------------------------------- +build/ +dist/ +develop-eggs/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# ---------------------------------------------------------------------------- +# Python: Virtual Environments & Dependency Managers +# ---------------------------------------------------------------------------- +# Virtual environments +.venv +venv/ +env/ +ENV/ +env.bak/ +venv.bak/ +.pixi/ +__pypackages__/ + +# pyenv +.python-version + +# Dependency lock files (usually committed, uncomment if you prefer to ignore) +# Pipfile.lock +# poetry.lock +# uv.lock +# pdm.lock +# pixi.lock + +# Tool-specific +.tox/ +.nox/ +.pdm-python +.pdm-build/ +.poetry.toml +.pdm.toml + +# ---------------------------------------------------------------------------- +# Python: Testing & Coverage +# ---------------------------------------------------------------------------- +htmlcov/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ +.ruff_cache/ + +# ---------------------------------------------------------------------------- +# Python: Development & IDE +# ---------------------------------------------------------------------------- +# Jupyter / IPython +.ipynb_checkpoints +profile_default/ +ipython_config.py + +# Type checkers & linters +.mypy_cache/ +.dmypy.json +dmypy.json +.pyre/ +.pytype/ + +# Project / IDE settings +.spyderproject +.spyproject +.ropeproject + +# PyCharm / JetBrains (uncomment to ignore entire folder) +# .idea/ + +# VS Code (uncomment to ignore entire folder) +# .vscode/ + +# ---------------------------------------------------------------------------- +# Python: Frameworks & Tools +# ---------------------------------------------------------------------------- +# Django +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask +instance/ +.webassets-cache + +# Scrapy +.scrapy + +# Celery +celerybeat-schedule +celerybeat-schedule.* +celerybeat.pid + +# Sphinx / MkDocs / Marimo +docs/_build/ +/site +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# Streamlit secrets +.streamlit/secrets.toml + +# ---------------------------------------------------------------------------- +# Next.js / Node.js +# ---------------------------------------------------------------------------- +# Dependencies +/node_modules +/.pnp +.pnp.js +.pnp.loader.mjs + +# Build outputs +.next/ +out/ +build/ + +# TypeScript +*.tsbuildinfo +next-env.d.ts + +# Testing (Jest, etc.) +/coverage + +# Vercel +.vercel + +# ---------------------------------------------------------------------------- +# General / OS / Security +# ---------------------------------------------------------------------------- +# Environment variables +.env +.env*.local +.envrc + +# OS generated files +.DS_Store +Thumbs.db +*.pem + +# Logs & debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +*.log + +# PyPI config +.pypirc + +# ============================================================================== +# End of file +# ============================================================================== diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..e559576 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,10 @@ +{ + "recommendations": [ + "ms-python.python", + "ms-python.vscode-pylance", + "charliermarsh.ruff", + "tamasfe.even-better-toml", + "aaron-bond.better-comments", + "bierner.markdown-mermaid", + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..0c4d369 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,13 @@ +{ + // Workspace settings: Apply to a specific project or workspace. Overrides User Settings, but only for that workspace. + // Python settings + "python.envFile": "${workspaceFolder}/.env", + "python.terminal.activateEnvironment": true, + "python.defaultInterpreterPath": "${workspaceFolder}/backend/.venv/bin/python", + // Test settings + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false, + "python.testing.cwd": "${workspaceFolder}/", + "python.testing.pytestPath": "${workspaceFolder}/.venv/bin/pytest", + "python.testing.autoTestDiscoverOnSaveEnabled": true, +} diff --git a/.vscode/user-settings.json b/.vscode/user-settings.json new file mode 100644 index 0000000..008c85b --- /dev/null +++ b/.vscode/user-settings.json @@ -0,0 +1,168 @@ +{ + // User Settings: Personal preferences that apply globally across all VS Code workspaces for that user. + // General settings + "security.workspace.trust.untrustedFiles": "newWindow", + "window.zoomLevel": 2, + "files.exclude": { + "**/.git": true + }, + "extensions.autoUpdate": "onlyEnabledExtensions", + "chat.disableAIFeatures": true, + // ChatGPT Codex + "chatgpt.openOnStartup": true, + // Git settings + "git.autofetch": true, + "git.confirmSync": false, + "git.enableSmartCommit": true, + "git.showActionButton": { + "commit": false, + "publish": false, + "sync": false + }, + // Explorer settings + "explorer.excludeGitIgnore": true, + "explorer.autoReveal": true, + "explorer.confirmDelete": false, + "explorer.confirmDragAndDrop": false, + "explorer.sortOrder": "filesFirst", + // Workbench settings + "workbench.colorTheme": "Default Dark+", + "workbench.editor.enablePreview": false, + "workbench.editor.tabSizing": "shrink", + "workbench.settings.editor": "json", + // Editor settings + "ruff.importStrategy": "useBundled", + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.formatOnPaste": true, + "editor.formatOnSave": true, + "editor.formatOnSaveMode": "file", + "editor.codeActionsOnSave": { + "source.organizeImports": "always", + "source.fixAll": "always" + }, + "files.autoSave": "onFocusChange", + "[json]": { + "editor.defaultFormatter": "vscode.json-language-features" + }, + "[jsonc]": { + "editor.defaultFormatter": "vscode.json-language-features" + }, + // Debug settings + "debug.toolBarLocation": "docked", + // Terminal settings + "terminal.integrated.tabs.enabled": true, + "terminal.integrated.tabs.hideCondition": "never", + "terminal.integrated.tabs.location": "right", + // Markdown settings + "markdown.preview.scrollEditorWithPreview": true, + "markdown.preview.scrollPreviewWithEditor": true, + // Color customization settings + "workbench.colorCustomizations": { + // Status bar + "statusBar.background": "#00D396", + "statusBar.foreground": "#0c1b29", + "statusBar.noFolderBackground": "#2A5677", + "statusBar.debuggingBackground": "#511f1f", + "statusBarItem.remoteBackground": "#00D396", + "statusBarItem.remoteForeground": "#0c1b29", + // Activity Bar (right bar) + "activityBar.background": "#0c1b29", + "activityBar.foreground": "#A1F7DB", + "activityBarBadge.background": "#00D396", + "activityBarBadge.foreground": "#0c1b29", + // Side bar (left panel) + "sideBar.background": "#0c1b29", + "sideBar.foreground": "#EDEDF0", + "sideBarTitle.foreground": "#A1F7DB", + "sideBarSectionHeader.background": "#2A5677", + // Editor + "editor.background": "#0c1b29", + "editor.foreground": "#EDEDF0", + "editor.lineHighlightBackground": "#005bd330", + "editor.selectionBackground": "#2A567780", + "editorCursor.foreground": "#A1F7DB", + // Tab colors + "tab.activeBackground": "#0c1b29", + "tab.activeBorderTop": "#00D396", + "tab.activeForeground": "#A1F7DB", + "tab.unfocusedActiveBorder": "#ffffff", + "tab.inactiveBackground": "#0c1b29", + "tab.inactiveForeground": "#ffffff", + // Editor group header + "editorGroupHeader.tabsBackground": "#0c1b29", + "editorGroupHeader.tabsBorder": "#00D396", + "editorGroupHeader.noTabsBackground": "#2A5677", + // Scrollbar + "scrollbarSlider.background": "#A1F7DB90", + "scrollbarSlider.hoverBackground": "#00D39690", + // Terminal + "terminal.background": "#0c1b29", + "terminal.tab.activeBorder": "#00D396", + "terminal.tab.background": "#2A5677", + "terminal.tab.activeForeground": "#00D396", + "terminal.tab.inactiveForeground": "#A1F7DB", + // Panel + "panelTitle.activeBorder": "#00D396", + "panel.background": "#0c1b29", + // Notifications + "notification.background": "#2A5677", + "notification.foreground": "#EDEDF0", + "notification.infoBackground": "#00D396", + "notification.warningBackground": "#A1F7DB", + "notification.errorBackground": "#511f1f", + // Window + "window.activeBorder": "#0c1b29", + "window.inactiveBorder": "#00D396", + "titleBar.activeBackground": "#0c1b29", + "titleBar.activeForeground": "#A1F7DB", + "titleBar.inactiveBackground": "#2A5677", + "titleBar.inactiveForeground": "#A1F7DB", + // Button styles + "button.background": "#00D396", + "button.foreground": "#0c1b29", + "button.hoverBackground": "#00B386", + // Input styles + "input.background": "#0c1b29", + "input.foreground": "#ffffff", + "input.placeholderForeground": "#A1F7DB80", + "inputValidation.errorBackground": "#511f1f", + "inputValidation.errorForeground": "#EDEDF0", + "inputValidation.errorBorder": "#FF5555", + // Quick Open / Command Palette input box + "quickInput.background": "#0c1b29", + "quickInput.foreground": "#ffffff", + "quickInputTitle.background": "#0F2436", + "pickerGroup.foreground": "#ffffff", + "pickerGroup.border": "#00D396", + "pickerGroup.background": "#00D396", + // Icons and decorations for quick + "keybindingLabel.foreground": "#1E3A57", + "keybindingLabel.background": "#00D396", + "keybindingLabel.border": "#00D396", + "keybindingLabel.bottomBorder": "#00D396", + // Quick Open/Command Palette selected item + "list.activeSelectionBackground": "#2a567775", + "list.activeSelectionForeground": "#ffffff", + "list.activeSelectionIconForeground": "#A1F7DB", + "list.hoverBackground": "#1E3A57", + "list.inactiveSelectionBackground": "#2A5677", + "list.inactiveSelectionForeground": "#A1F7DB", + // Editor widget (Quick Open, Search, Replace) + "editorWidget.background": "#0c1b29", + "editorWidget.border": "#00D396", + "editorWidget.foreground": "#EDEDF0", + "editor.findMatchBackground": "#00D39630", + "editor.findMatchHighlightBackground": "#2A567780", + "editor.findRangeHighlightBackground": "#2A567780", + "editor.findMatchBorder": "#00D396", + "editor.findMatchHighlightBorder": "#00D396" + }, + "workbench.startupEditor": "none", + "python.analysis.typeCheckingMode": "strict", + "markdown-pdf.displayHeaderFooter": false, + "markdown-pdf.highlightStyle": "github.css", + "markdown-mermaid.darkModeTheme": "forest", + "markdown-mermaid.lightModeTheme": "forest", + "markdown-pdf.mermaidServer": "https://unpkg.com/mermaid@11.12.1/dist/mermaid.js", + "markdown-pdf.executablePath": "/opt/google/chrome/google-chrome" +} \ No newline at end of file diff --git a/README.md b/README.md index a251b6b..4db80da 100644 --- a/README.md +++ b/README.md @@ -71,114 +71,3 @@ | **⭐ Shine** *(Recommended)* | 50 | **kr 5 999** | The sweet spot for building natural fluency and confidence. | | **Radiance** | 200 | **kr 17 999** | Designed for dedicated learners seeking transformation. | -## 4. Project Structure - -```bash -avaaz.ai/ -├── .dockerignore # Specifies files and directories to exclude from Docker builds, such as .git, node_modules, and build artifacts, to optimize image sizes. -├── .gitignore # Lists files and patterns to ignore in Git, including .env, __pycache__, node_modules, and logs, preventing sensitive or temporary files from being committed. -├── .gitattributes # Controls Git’s handling of files across platforms (e.g. normalizing line endings with * text=auto), and can force certain files to be treated as binary or configure diff/merge drivers. -│ -├── .env.example # Template for environment variables, showing required keys like DATABASE_URL, GEMINI_API_KEY, LIVEKIT_API_KEY without actual values. -├── docker-compose.dev.yml # Docker Compose file for development environment: defines services for local frontend, backend, postgres, livekit with volume mounts for hot-reloading. -├── docker-compose.prod.yml # Docker Compose file for production: defines services for caddy, gitea (if integrated), frontend, backend, postgres, livekit with optimized settings and no volumes for code. -├── README.md # Project overview: includes setup instructions, architecture diagram (embed the provided Mermaid), contribution guidelines, and deployment steps. -│ -├── .gitea/ # Directory for Gitea-specific configurations, as the repo is hosted on Gitea. -│ └── workflows/ # Contains YAML files for Gitea Actions workflows, enabling CI/CD pipelines. -│ ├── ci.yml # Workflow for continuous integration: runs tests, linting (Ruff), type checks, and builds on pull requests or pushes. -│ └── cd.yml # Workflow for continuous deployment: triggers builds and deploys Docker images to the VPS on merges to main. -│ -├── .vscode/ # Editor configuration for VS Code to standardize the development environment for all contributors. -│ ├── extensions.json # Recommends VS Code extensions (e.g. Python, ESLint, Docker, GitLens) so developers get linting, formatting, and container tooling out of the box. -│ └── settings.json # Workspace-level VS Code settings: formatter on save, path aliases, Python/TypeScript language server settings, lint integration (Ruff, ESLint), and file exclusions. -│ -├── backend/ # Root for the FastAPI backend, following Python best practices for scalable applications (inspired by FastAPI's "Bigger Applications" guide). -│ ├── Dockerfile # Builds the backend container: installs UV, copies pyproject.toml, syncs dependencies, copies source code, sets entrypoint to Gunicorn/Uvicorn. -│ ├── pyproject.toml # Project metadata and dependencies: uses UV for dependency management, specifies FastAPI, SQLAlchemy, Pydantic, LiveKit-Agent, etc.; includes [tool.uv], [tool.ruff] sections for config. -│ ├── uv.lock # Lockfile generated by UV, ensuring reproducible dependencies across environments. -│ ├── ruff.toml # Configuration for Ruff linter and formatter (can be in pyproject.toml): sets rules for Python code style, ignoring certain errors if needed. -│ ├── alembic.ini # Configuration for Alembic migrations: points to SQLAlchemy URL and script location. -│ ├── alembic/ # Directory for database migrations using Alembic, integrated with SQLAlchemy. -│ │ ├── env.py # Alembic environment script: sets up the migration context with SQLAlchemy models and pgvector support. -│ │ ├── script.py.mako # Template for generating migration scripts. -│ │ └── versions/ # Auto-generated migration files: each represents a database schema change, e.g., create_tables.py. -│ ├── src/ # Source code package: keeps business logic isolated, importable as 'from src import ...'. -│ │ ├── __init__.py # Makes src a package. -│ │ ├── main.py # FastAPI app entrypoint: initializes app, includes routers, sets up middleware, connects to Gemini Live via prompts. -│ │ ├── config.py # Application settings: uses Pydantic-settings to load from .env, e.g., DB_URL, API keys for Gemini, LiveKit, Stripe (for pricing plans). -│ │ ├── api/ # API-related modules: organizes endpoints and dependencies. -│ │ │ ├── __init__.py # Package init. -│ │ │ ├── dependencies.py # Global dependencies: e.g., current_user via FastAPI Users, database session. -│ │ │ └── v1/ # Versioned API: allows future versioning without breaking changes. -│ │ │ └── routers/ # API routers: modular endpoints. -│ │ │ ├── auth.py # Handles authentication: uses FastAPI Users for JWT/OAuth, user registration/login. -│ │ │ ├── users.py # User management: progress tracking, plan subscriptions. -│ │ │ ├── lessons.py # Lesson endpoints: structured oral language lessons, progress tracking. -│ │ │ ├── chat.py # Integration with LiveKit and Gemini: handles conversational AI tutor sessions. -│ │ │ └── documents.py # Document upload and processing: endpoints for file uploads, using Docling for parsing and semantic search prep. -│ │ ├── core/ # Core utilities: shared across the app. -│ │ │ ├── __init__.py # Package init. -│ │ │ └── security.py # Security functions: hashing, JWT handling via FastAPI Users. -│ │ ├── db/ # Database layer: SQLAlchemy setup with pgvector for vector embeddings (e.g., for AI tutor memory). -│ │ │ ├── __init__.py # Package init. -│ │ │ ├── base.py # Base model class for SQLAlchemy declarative base. -│ │ │ ├── session.py # Database session management: async session maker. -│ │ │ └── models/ # SQLAlchemy models. -│ │ │ ├── __init__.py # Exports all models. -│ │ │ ├── user.py # User model: includes fields for progress, plan, proficiency. -│ │ │ ├── lesson.py # Lesson and session models: tracks user interactions, B2 exam prep. -│ │ │ └── document.py # Document chunk model: for semantic search, with text, metadata, embedding (pgvector). -│ │ ├── schemas/ # Pydantic schemas: for API validation and serialization. -│ │ │ ├── __init__.py # Exports schemas. -│ │ │ ├── user.py # User schemas: create, read, update. -│ │ │ ├── lesson.py # Lesson schemas: input/output for AI interactions. -│ │ │ └── document.py # Document schemas: for upload responses and search queries. -│ │ └── services/ # Business logic services: decoupled from API. -│ │ ├── __init__.py # Package init. -│ │ ├── llm.py # Gemini Live integration: prompt engineering for conversational tutor. -│ │ ├── payment.py # Handles pricing plans: integrates with Stripe for subscriptions (Spark, Glow, etc.). -│ │ └── document.py # Docling processing: parses files, chunks, embeds (via Gemini), stores for semantic search. -│ └── tests/ # Unit and integration tests: uses pytest, Hypothesis for property-based testing, httpx for API testing. -│ ├── __init__.py # Package init. -│ ├── conftest.py # Pytest fixtures: e.g., test database, mock Gemini. -│ └── test_users.py # Example test file: tests user endpoints. -│ -├── frontend/ # Root for Next.js frontend and PWA, following Next.js app router best practices (2025 standards: improved SSR, layouts). -│ ├── Dockerfile # Builds the frontend container: installs dependencies, builds Next.js, serves with Node. -│ ├── .eslintrc.json # ESLint configuration: extends next/core-web-vitals, adds rules for code quality. -│ ├── next.config.js # Next.js config: enables PWA, images optimization, API routes if needed. -│ ├── package.json # Node dependencies: includes next, react, @livekit/client for WebRTC, axios or fetch for API calls. -│ ├── package-lock.json # Lockfile for reproducible npm installs. -│ ├── tsconfig.json # TypeScript config: targets ES2022, includes paths for components. -│ ├── app/ # App router directory: pages, layouts, loading states. -│ │ ├── globals.css # Global styles: Tailwind or CSS modules. -│ │ ├── layout.tsx # Root layout: includes providers, navigation. -│ │ ├── page.tsx # Home page: landing for avaaz.ai. -│ │ └── components/ # Reusable UI components. -│ │ ├── ChatInterface.tsx # Component for conversational tutor using LiveKit WebRTC. -│ │ └── ProgressTracker.tsx # Tracks user progress toward B2 exam. -│ ├── lib/ # Utility functions: API clients, hooks. -│ │ └── api.ts # API client: typed fetches to backend endpoints. -│ └── public/ # Static assets. -│ ├── favicon.ico # Site icon. -│ └── manifest.json # PWA manifest: for mobile app-like experience. -│ -├── infra/ # Infrastructure configurations: Dockerfiles and configs for supporting services, keeping them separate for scalability. -│ ├── caddy/ # Caddy reverse proxy setup. -│ │ ├── Dockerfile # Extends official Caddy image, copies Caddyfile. -│ │ └── Caddyfile # Caddy config: routes www.avaaz.ai to frontend, api.avaaz.ai to backend, WSS to LiveKit; auto HTTPS. -│ ├── gitea/ # Gitea git server (added for customization if needed; otherwise use official image directly in Compose). -│ │ ├── Dockerfile # Optional: Extends official Gitea image, copies custom config for Actions integration. -│ │ └── app.ini # Gitea config: sets up server, database, Actions runner. -│ ├── livekit/ # LiveKit server for real-time audio/video in tutor sessions. -│ │ ├── Dockerfile # Extends official LiveKit image, copies config. -│ │ └── livekit.yaml # LiveKit config: API keys, room settings, agent integration for AI tutor. -│ └── postgres/ # PostgreSQL with pgvector. -│ ├── Dockerfile # Extends postgres image, installs pgvector extension. -│ └── init/ # Initialization scripts. -│ └── 00-pgvector.sql # SQL to create pgvector extension on db init. -│ -└── docs/ # Documentation: architecture, APIs, etc. - └── architecture.md # Detailed system explanation, including the provided Mermaid diagram. -``` diff --git a/app/.dockerignore b/app/.dockerignore new file mode 100644 index 0000000..16bd95a --- /dev/null +++ b/app/.dockerignore @@ -0,0 +1,126 @@ +# ============================================================================== +# .dockerignore – Python + Next.js (Docker Compose) +# ============================================================================== + +# ---------------------------------------------------------------------------- +# Git & Version Control +# ---------------------------------------------------------------------------- +.git +.gitignore +.gitattributes +.github +.gitpod.yml + +# ---------------------------------------------------------------------------- +# Python-specific (already in .gitignore, but repeat for safety) +# ---------------------------------------------------------------------------- +__pycache__/ +*.py[cod] +*$py.class +*.so +*.egg-info/ +.installed.cfg +*.egg +*.manifest +*.spec + +# Virtual environments & caches +.venv +venv/ +env/ +ENV/ +.pixi/ +__pypackages__/ +.tox/ +.nox/ +.pdm-python +.pdm-build/ + +# Testing & coverage +htmlcov/ +.coverage +.coverage.* +.pytest_cache/ +.coverage/ +.ruff_cache/ +.mypy_cache/ +.pyre/ +.pytype/ + +# Jupyter / notebooks +.ipynb_checkpoints + +# IDEs & editors +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# ---------------------------------------------------------------------------- +# Next.js / Node.js +# ---------------------------------------------------------------------------- +node_modules/ +.next/ +out/ +build/ +dist/ +.npm +.pnp.* +.yarn/ +.yarn-cache/ +.yarn-unplugged/ + +# TypeScript build info +*.tsbuildinfo +next-env.d.ts + +# Logs & debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* + +# ---------------------------------------------------------------------------- +# Environment & Secrets (never send to Docker daemon) +# ---------------------------------------------------------------------------- +.env +.env.local +.env*.local +.env.production +.env.development +.envrc +*.pem +*.key +*.crt +*.secrets +.streamlit/secrets.toml + +# ---------------------------------------------------------------------------- +# Docker & Compose (avoid recursive inclusion) +# ---------------------------------------------------------------------------- +Dockerfile* +docker-compose*.yml +docker-compose*.yaml +.dockerignore + +# ---------------------------------------------------------------------------- +# Misc / OS +# ---------------------------------------------------------------------------- +.DS_Store +Thumbs.db +desktop.ini + +# Local documentation builds +/site +docs/_build/ + +# Temporary files +tmp/ +temp/ +*.tmp +*.log + +# ============================================================================== +# End of file +# ============================================================================== diff --git a/app/.env.example b/app/.env.example new file mode 100644 index 0000000..fa6ad91 --- /dev/null +++ b/app/.env.example @@ -0,0 +1,11 @@ +APP_ENV=development +COMPOSE_PROFILES=development +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=app +DATABASE_URL=postgresql://postgres:postgres@postgres:5432/app +LIVEKIT_URL=http://livekit:7880 +LIVEKIT_API_KEY=devkey +LIVEKIT_API_SECRET=devsecret +OPENAI_API_KEY=change_me +GOOGLE_API_KEY=change_me diff --git a/app/backend/Dockerfile b/app/backend/Dockerfile new file mode 100644 index 0000000..754aa51 --- /dev/null +++ b/app/backend/Dockerfile @@ -0,0 +1,63 @@ +# Backend image for FastAPI + LiveKit Agent runtime +# Supports dev (uvicorn reload) and production (gunicorn) via APP_ENV. + +# Builder: install deps with uv into an isolated venv +FROM python:3.12-slim AS builder + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + UV_PROJECT_ENV=/opt/venv + +# System packages needed for common Python builds + uv installer +RUN apt-get update && \ + apt-get install -y --no-install-recommends build-essential curl && \ + rm -rf /var/lib/apt/lists/* + +# Install uv (fast Python package manager) as the dependency tool +RUN curl -LsSf https://astral.sh/uv/install.sh | sh +ENV PATH="/root/.local/bin:${PATH}" + +# Create virtualenv and prepare workspace +RUN python -m venv ${UV_PROJECT_ENV} +ENV PATH="${UV_PROJECT_ENV}/bin:${PATH}" +WORKDIR /app + +# Copy the full source; dependency install is guarded to avoid failures while the app is scaffolded +COPY . /app +RUN if [ -f uv.lock ] && [ -f pyproject.toml ]; then \ + uv sync --frozen --no-dev; \ + elif ls requirements*.txt >/dev/null 2>&1; then \ + for req in requirements*.txt; do \ + uv pip install --no-cache -r "$req"; \ + done; \ + elif [ -f pyproject.toml ]; then \ + uv pip install --no-cache .; \ + else \ + echo "No dependency manifest found; skipping install"; \ + fi && \ + if [ -f pyproject.toml ]; then \ + uv pip install --no-cache -e .; \ + fi + +# Runtime: slim image with non-root user and prebuilt venv +FROM python:3.12-slim AS runtime + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH="/opt/venv/bin:${PATH}" + +# Create unprivileged user +RUN addgroup --system app && adduser --system --ingroup app app + +WORKDIR /app + +# Copy runtime artifacts from builder +COPY --from=builder /opt/venv /opt/venv +COPY --from=builder /app /app +RUN chown -R app:app /app /opt/venv + +USER app +EXPOSE 8000 + +# Default command switches between uvicorn (dev) and gunicorn (prod) based on APP_ENV +CMD ["sh", "-c", "if [ \"$APP_ENV\" = \"production\" ]; then exec gunicorn app.main:app -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 --workers 4 --timeout 120; else exec uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload; fi"] diff --git a/app/docker-compose.yml b/app/docker-compose.yml new file mode 100644 index 0000000..be490fd --- /dev/null +++ b/app/docker-compose.yml @@ -0,0 +1,202 @@ +version: "3.9" + +# A single compose file that supports development and production. +# Switch modes by setting APP_ENV and COMPOSE_PROFILES to either +# "development" (default) or "production" before running docker compose up. + +x-backend-common: &backend-common + build: + context: ./backend + dockerfile: Dockerfile + env_file: + - .env + environment: + APP_ENV: ${APP_ENV:-development} + DATABASE_URL: ${DATABASE_URL} + LIVEKIT_URL: ${LIVEKIT_URL} + LIVEKIT_API_KEY: ${LIVEKIT_API_KEY} + LIVEKIT_API_SECRET: ${LIVEKIT_API_SECRET} + restart: unless-stopped + +x-frontend-common: &frontend-common + build: + context: ./frontend + dockerfile: Dockerfile + env_file: + - .env + environment: + APP_ENV: ${APP_ENV:-development} + # Server-side calls from Next.js hit the backend by container name + BACKEND_URL: http://backend:8000 + restart: unless-stopped + +x-postgres-common: &postgres-common + image: pgvector/pgvector:pg16 + environment: + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + POSTGRES_DB: ${POSTGRES_DB:-app} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-app}"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + +x-livekit-common: &livekit-common + image: livekit/livekit-server:latest + env_file: + - .env + environment: + # Keys are passed in via env; LiveKit will refuse to start without them. + LIVEKIT_KEYS: "${LIVEKIT_API_KEY:-devkey}:${LIVEKIT_API_SECRET:-devsecret}" + LIVEKIT_PORT: 7880 + LIVEKIT_RTC_PORT_RANGE_START: 50000 + LIVEKIT_RTC_PORT_RANGE_END: 60000 + restart: unless-stopped + +services: + backend-dev: + <<: *backend-common + profiles: ["development"] + container_name: backend + command: ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] + ports: + - "8000:8000" + volumes: + # Mount source for hot reload; keep venv inside image + - ./backend:/app + depends_on: + postgres: + condition: service_healthy + livekit: + condition: service_started + networks: + app_internal: + aliases: ["backend"] + + backend-prod: + <<: *backend-common + profiles: ["production"] + container_name: backend + command: + - gunicorn + - app.main:app + - -k + - uvicorn.workers.UvicornWorker + - --bind + - 0.0.0.0:8000 + - --workers + - "4" + - --timeout + - "120" + expose: + - "8000" + depends_on: + postgres-prod: + condition: service_healthy + livekit-prod: + condition: service_started + networks: + app_internal: + aliases: ["backend"] + proxy: + aliases: ["backend"] + + frontend-dev: + <<: *frontend-common + build: + context: ./frontend + dockerfile: Dockerfile + target: dev + profiles: ["development"] + container_name: frontend + command: ["npm", "run", "dev"] + ports: + - "3000:3000" + volumes: + - ./frontend:/app + - frontend_node_modules:/app/node_modules + depends_on: + backend: + condition: service_started + networks: + app_internal: + aliases: ["frontend"] + + frontend-prod: + <<: *frontend-common + build: + context: ./frontend + dockerfile: Dockerfile + target: runner + profiles: ["production"] + container_name: frontend + # Uses the standalone Next.js output from the Dockerfile + command: ["node", "server.js"] + expose: + - "3000" + depends_on: + backend-prod: + condition: service_started + networks: + app_internal: + aliases: ["frontend"] + proxy: + aliases: ["frontend"] + + postgres-dev: + <<: *postgres-common + profiles: ["development"] + container_name: postgres + ports: + - "5432:5432" + networks: + app_internal: + aliases: ["postgres"] + + postgres-prod: + <<: *postgres-common + profiles: ["production"] + container_name: postgres + networks: + app_internal: + aliases: ["postgres"] + + livekit-dev: + <<: *livekit-common + profiles: ["development"] + container_name: livekit + ports: + - "7880:7880" + - "50000-60000:50000-60000/udp" + networks: + app_internal: + aliases: ["livekit"] + + livekit-prod: + <<: *livekit-common + profiles: ["production"] + container_name: livekit + # UDP media must be published even in production; signaling stays internal. + ports: + - "50000-60000:50000-60000/udp" + networks: + app_internal: + aliases: ["livekit"] + proxy: + aliases: ["livekit"] + +volumes: + postgres_data: + frontend_node_modules: + +networks: + app_internal: + # Private app network for service-to-service traffic + driver: bridge + proxy: + # External network provided by the infra stack (Caddy attaches here) + external: true diff --git a/app/frontend/Dockerfile b/app/frontend/Dockerfile new file mode 100644 index 0000000..c380435 --- /dev/null +++ b/app/frontend/Dockerfile @@ -0,0 +1,44 @@ +# Frontend image for Next.js (dev server + standalone production runner) + +# Dependency + build stage for production +FROM node:22-bookworm-slim AS builder +WORKDIR /app +ENV NODE_ENV=production \ + NEXT_TELEMETRY_DISABLED=1 + +# Install dependencies first for better Docker layer caching +COPY package*.json ./ +RUN npm ci --ignore-scripts + +# Copy full source and build standalone output +COPY . . +RUN npm run build + +# Dev image keeps the toolchain for next dev +FROM node:22-bookworm-slim AS dev +WORKDIR /app +ENV NODE_ENV=development \ + NEXT_TELEMETRY_DISABLED=1 +COPY package*.json ./ +RUN npm install +COPY . . +CMD ["npm", "run", "dev"] + +# Production runtime: minimal Node image serving the standalone build +FROM node:22-slim AS runner +WORKDIR /app +ENV NODE_ENV=production \ + NEXT_TELEMETRY_DISABLED=1 \ + PORT=3000 + +# Copy only the files required to serve the built app +COPY --from=builder /app/.next/standalone ./ +COPY --from=builder /app/public ./public +COPY --from=builder /app/.next/static ./.next/static + +# Drop privileges to the bundled node user for safety +USER node +EXPOSE 3000 + +# Next.js standalone exposes server.js at the root of the standalone output +CMD ["node", "server.js"] diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..3093035 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,613 @@ +# System Architecture + +Below is a summary of the **Production VPS** and **Development Laptop** architectures. Both environments use Docker containers for consistency, with near-identical stacks where practical. + +```mermaid +flowchart LR +%% Client + A(Browser / PWA) + Y(iOS App / Android App) + + subgraph User + A + Y + end + +%% LLM / Realtime + B(OpenAI Realtime API) + Z(Gemini Live API) + + subgraph Large Language Model + B + Z + end + +%% Server-side + C(Caddy) + I(Gitea + Actions + Repositories) + J(Gitea Runner) + + D(Next.js Frontend) + E(FastAPI Backend + Agent Runtime) + G(LiveKit Server) + H[(PostgreSQL + pgvector)] + +%% Client ↔ VPS + A <-- https://www.avaaz.ai --> C + A <-- https://app.avaaz.ai --> C + A & Y <-- https://api.avaaz.ai --> C + A & Y <-- wss://rtc.avaaz.ai --> C + A & Y <-- "udp://rtc.avaaz.ai:50000-60000 (WebRTC Media)" --> G + +%% Caddy ↔ App + C <-- "http://frontend:3000 (app)" --> D + C <-- "http://backend:8000 (api)" --> E + C <-- "ws://livekit:7880 (WebRTC signaling)" --> G + C <-- "http://gitea:3000 (git)" --> I + +%% App internal + D <-- "http://backend:8000" --> E + E <-- "postgresql://postgres:5432" --> H + E <-- "http://livekit:7880 (control)" --> G + E <-- "Agent joins via WebRTC" --> G + +%% Agent ↔ LLM + E <-- "WSS/WebRTC (realtime)" --> B + E <-- "WSS (streaming)" --> Z + +%% CI/CD + I <-- "CI/CD triggers" --> J + + subgraph VPS + subgraph Infra + C + I + J + end + subgraph App + D + E + G + H + end + end + +%% Development Environment + L(VS Code + Git + Docker) + M(Local Docker Compose) + N(Local Browser) + O(Local Frontend) + P(Local Backend) + Q[(Local Postgres)] + R(Local LiveKit) + + L <-- "https://git.avaaz.ai/...git" --> C + L <-- "ssh://git@git.avaaz.ai:2222/..." --> I + L -- "docker compose up" --> M + + M -- "Build & Run" --> O & P & Q & R + + N <-- HTTP --> O & P + N <-- WebRTC --> R + + O <-- HTTP --> P + P <-- SQL --> Q + P <-- HTTP/WebRTC --> R + P <-- WSS/WebRTC --> B + P <-- WSS --> Z + + subgraph Development Laptop + L + M + N + subgraph Local App + O + P + Q + R + end + end +``` + +## 1. Production VPS + +### 1.1 Components + +#### Infra Stack + +Docker Compose: `./infra/docker-compose.yml`. + +| Container | Description | +| -------------- | ----------------------------------------------------------------------------------- | +| `caddy` | **Caddy** – Reverse proxy with automatic HTTPS (TLS termination via Let’s Encrypt). | +| `gitea` | **Gitea + Actions** – Git server using SQLite. Automated CI/CD workflows. | +| `gitea-runner` | **Gitea Runner** – Executes CI/CD jobs defined in Gitea Actions workflows. | + +#### App Stack + +Docker Compose: `./app/docker-compose.yml`. + +| Container | Description | +| ---------- | ----------------------------------------------------------------------------------------- | +| `frontend` | **Next.js Frontend** – SPA/PWA interface served from a Node.js-based Next.js server. | +| `backend` | **FastAPI + Uvicorn Backend** – API, auth, business logic, LiveKit orchestration, agent. | +| `postgres` | **PostgreSQL + pgvector** – Persistent relational database with vector search. | +| `livekit` | **LiveKit Server** – WebRTC signaling plus UDP media for real-time audio and data. | + +The `backend` uses several Python packages such as UV, Ruff, FastAPI, FastAPI Users, FastAPI-pagination, FastStream, FastMCP, Pydantic, PydanticAI, Pydantic-settings, LiveKit Agent, Google Gemini Live API, OpenAI Realtime API, SQLAlchemy, Alembic, docling, Gunicorn, Uvicorn[standard], Pyright, Pytest, Hypothesis, and Httpx to deliver the services. + +### 1.2 Network + +- All containers join a shared `proxy` Docker network. +- Caddy can route to any service by container name. +- App services communicate internally: + - Frontend ↔ Backend + - Backend ↔ Postgres + - Backend ↔ LiveKit + - Backend (agent) ↔ LiveKit & external LLM realtime APIs + +### 1.3 Public DNS Records + +| Hostname | Record Type | Target | Purpose | +| -------------------- | :---------: | -------------- | -------------------------------- | +| **www\.avaaz\.ai** | CNAME | avaaz.ai | Marketing / landing site | +| **avaaz.ai** | A | 217.154.51.242 | Root domain | +| **app.avaaz.ai** | A | 217.154.51.242 | Next.js frontend (SPA/PWA) | +| **api.avaaz.ai** | A | 217.154.51.242 | FastAPI backend | +| **rtc.avaaz.ai** | A | 217.154.51.242 | LiveKit signaling + media | +| **git.avaaz.ai** | A | 217.154.51.242 | Gitea (HTTPS + SSH) | + +### 1.4 Public Inbound Firewall Ports & Protocols + +| Port | Protocol | Purpose | +| -------------: | :------: | --------------------------------------- | +| **80** | TCP | HTTP, ACME HTTP-01 challenge | +| **443** | TCP | HTTPS, WSS (frontend, backend, LiveKit) | +| **2222** | TCP | Git SSH via Gitea | +| **2885** | TCP | VPS SSH access | +| **3478** | UDP | STUN/TURN | +| **5349** | TCP | TURN over TLS | +| **7881** | TCP | LiveKit TCP fallback | +| **50000–60000**| UDP | LiveKit WebRTC media | + +### 1.5 Routing + +#### Caddy + +Caddy routes traffic from public ports 80 and 443 to internal services. + +- `https://www.avaaz.ai` → `http://frontend:3000` +- `https://app.avaaz.ai` → `http://frontend:3000` +- `https://api.avaaz.ai` → `http://backend:8000` +- `wss://rtc.avaaz.ai` → `ws://livekit:7880` +- `https://git.avaaz.ai` → `http://gitea:3000` + +#### Internal Container Network + +- `frontend` → `http://backend:8000` +- `backend` → `postgres://postgres:5432` +- `backend` → `http://livekit:7880` (control) +- `backend` → `ws://livekit:7880` (signaling) +- `backend` → `udp://livekit:50000-60000` (media) +- `gitea-runner` → `/var/run/docker.sock` (Docker API on host) + +#### Outgoing + +- `backend` → `https://api.openai.com/v1/realtime/sessions` +- `backend` → `wss://api.openai.com/v1/realtime?model=gpt-realtime` +- `backend` → `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent` + +### 1.6 Functional Layers + +#### Data Layer + +**Infra:** + +- **SQLite (Gitea)** + - Gitea stores Git metadata (users, repos, issues, Actions metadata) in `/data/gitea/gitea.db`. + - This is a file-backed SQLite database inside a persistent Docker volume. + - Repository contents are stored under `/data/git/`, also volume-backed. + +- **Gitea Runner State** + - Gitea Actions runner stores its registration information and job metadata under `/data/.runner`. + +**App:** + +- **PostgreSQL with pgvector** + - Primary relational database for users, lessons, transcripts, embeddings, and conversational context. + - Hosted in the `postgres` container with a persistent Docker volume. + - Managed via SQLAlchemy and Alembic migrations in the backend. + +- **LiveKit Ephemeral State** + - Room metadata, participant states, and signaling information persist in memory within the `livekit` container. + - LiveKit’s SFU media buffers and room state are **not** persisted across restarts. + +#### Control Layer + +**Infra:** + +- **Caddy** + - TLS termination (Let’s Encrypt). + - Reverse proxy and routing for all public domains. + - ACME certificate renewal. + +- **Gitea** + - Git hosting, pull/clone over SSH and HTTPS. + - CI/CD orchestration via Actions and internal APIs. + +- **Gitea Runner** + - Executes workflows and controls the Docker engine via `/var/run/docker.sock`. + +**App:** + +- **FastAPI Backend** + - Authentication and authorization (`/auth/login`, `/auth/refresh`, `/auth/me`). + - REST APIs for lessons, progress, documents, and file handling. + - LiveKit session management (room mapping `/sessions/default`, token minting `/sessions/default/token`, agent configuration). + - Calls out to OpenAI Realtime and Gemini Live APIs for AI-driven conversational behavior. + +- **LiveKit Server** + - Manages room signaling, participant permissions, and session state. + - Exposes HTTP control endpoint for room and participant management. + +#### Media Layer + +**App:** + +- **User Audio Path** + - Browser/mobile → LiveKit: + - WSS signaling via `rtc.avaaz.ai` → Caddy → `livekit:7880`. + - UDP audio and data channels via `rtc.avaaz.ai:50000–60000` directly to LiveKit on the VPS. + - WebRTC handles ICE, STUN/TURN, jitter buffers, and Opus audio encoding. + +- **AI Agent Audio Path** + - The agent logic inside the backend uses LiveKit Agent SDK to join rooms as a participant. + - Agent → LiveKit: + - WS signaling over the internal Docker network (`ws://livekit:7880`). + - UDP audio transport as part of its WebRTC session. + - Agent → LLM realtime API: + - Secure WSS/WebRTC connection to OpenAI Realtime or Gemini Live. + - The agent transcribes, processes, and generates audio responses, publishing them into the LiveKit room so the user hears natural speech. + +### 1.7 CI/CD Pipeline + +Production CI/CD is handled by **Gitea Actions** running on the VPS. The `gitea-runner` container has access to the host Docker daemon and is responsible for both validation and deployment: + +- `.gitea/workflows/ci.yml` – **Continuous Integration** (branch/PR validation, no deployment). +- `.gitea/workflows/cd.yml` – **Continuous Deployment** (tag-based releases to production). + +#### Build Phase (CI Workflow: `ci.yml`) + +**Triggers** + +- `push` to: + - `feature/**` + - `bugfix/**` +- `pull_request` targeting `main`. + +**Runner & Environment** + +- Runs on the self-hosted runner labeled `linux_amd64`. +- Checks out the relevant branch or PR commit from the `avaaz-app` repository into the runner’s workspace. + +**Steps** + +1. **Checkout code** + Uses `actions/checkout@v4` to fetch the branch or PR head commit. + +2. **Report triggering context** + Logs the event type (`push` or `pull_request`) and branches: + - For `push`: the source branch (e.g., `feature/foo`). + - For `pull_request`: source and target (`main`). + +3. **Static analysis & tests** + - Run linters, type checkers, and unit tests for backend and frontend. + - Ensure the application code compiles/builds. + +4. **Build Docker images for CI** + - Build images (e.g., `frontend:ci` and `backend:ci`) to validate Dockerfiles and build chain. + - These images are tagged for CI only and not used for production. + +5. **Cleanup CI images** + - Remove CI-tagged images at the end of the job (even on failure) to prevent disk usage from accumulating. + +**Outcome** + +- A green CI result on a branch/PR signals that: + - The code compiles/builds. + - Static checks and tests pass. + - Docker images can be built successfully. +- CI does **not** modify the production stack and does **not** depend on tags. + +#### Deploy Phase (CD Workflow: `cd.yml`) + +**Triggers** + +- Creation of a Git tag matching `v*` that points to a commit on the `main` branch in the `avaaz-app` repository. + +**Runner & Environment** + +- Runs on the same `linux_amd64` self-hosted runner. +- Checks out the exact commit referenced by the tag. + +**Steps** + +1. **Checkout tagged commit** + - Uses `actions/checkout@v4` with `ref: ${{ gitea.ref }}` to check out the tagged commit. + +2. **Tag validation** + - Fetches `origin/main`. + - Verifies that the tag commit is an ancestor of `origin/main` (i.e., the tag points to code that has been merged into `main`). + - Fails the deployment if the commit is not in `main`’s history. + +3. **Build & publish release** + - Builds production Docker images for frontend, backend, LiveKit, etc., tagged with the version (e.g., `v0.1.0`). + - Applies database migrations (e.g., via Alembic) if required. + +4. **Restart production stack** + - Restarts or recreates the app stack containers using the newly built/tagged images (e.g., via `docker compose -f docker-compose.yml up -d`). + +5. **Health & readiness checks** + - Probes key endpoints with `curl -f`, such as: + - `https://app.avaaz.ai` + - `https://api.avaaz.ai/health` + - `wss://rtc.avaaz.ai` (signaling-level check) + - If checks fail, marks the deployment as failed and automatically rolls back to previous images. + +**Outcome** + +- Only tagged releases whose commits are on the `main` branch are deployed. +- Deployment is explicit (tag-based), separated from CI validation. + +### 1.8 Typical Workflows + +#### User Login + +1. Browser loads the frontend from `https://app.avaaz.ai`. +2. Frontend submits credentials to `POST https://api.avaaz.ai/auth/login`. +3. Backend validates credentials and returns: + - A short-lived JWT **access token** + - A long-lived opaque **refresh token** + - A minimal user profile for immediate UI hydration +4. Frontend stores tokens appropriately (access token in memory; refresh token in secure storage or an httpOnly cookie). + +#### Load Persistent Session + +1. Frontend calls `GET https://api.avaaz.ai/sessions/default`. +2. Backend retrieves or creates the user’s **persistent conversational session**, which encapsulates: + - Long-running conversation state + - Lesson and progress context + - Historical summary for LLM context initialization +3. Backend prepares the session’s LLM context so that the agent can join with continuity. + +#### Join the Live Conversation Session + +1. Frontend requests a LiveKit access token via `POST https://api.avaaz.ai/sessions/default/token`. +2. Backend generates a **new LiveKit token** (short-lived, room-scoped), containing: + - Identity + - Publish/subscribe permissions + - Expiration (affecting initial join) + - Room ID corresponding to the session +3. Frontend connects to the LiveKit server: + - WSS for signaling + - UDP/SCTP for low-latency audio and file transfer +4. If the user disconnects, the frontend requests a new LiveKit token before rejoining, ensuring seamless continuity. + +#### Conversation with AI Agent + +1. Backend configures the session’s **AI agent** using: + - Historical summary + - Current lesson state + - Language settings and mode (lesson, mock exam, free talk) +2. The agent joins the same LiveKit room as a participant. +3. All media flows through LiveKit: + - User → audio → LiveKit → Agent + - Agent → LLM realtime API → synthesized audio → LiveKit → User +4. The agent guides the user verbally: continuing lessons, revisiting material, running mock exams, or free conversation. + +The user experiences this as a **continuous, ongoing session** with seamless reconnection and state persistence. + +### 1.9 Hardware + +| Class | Description | +|----------------|-------------------------------------------| +| system | Standard PC (i440FX + PIIX, 1996) | +| bus | Motherboard | +| memory | 96KiB BIOS | +| processor | AMD EPYC-Milan Processor | +| memory | 8GiB System Memory | +| bridge | 440FX - 82441FX PMC [Natoma] | +| bridge | 82371SB PIIX3 ISA [Natoma/Triton II] | +| communication | PnP device PNP0501 | +| input | PnP device PNP0303 | +| input | PnP device PNP0f13 | +| storage | PnP device PNP0700 | +| system | PnP device PNP0b00 | +| storage | 82371SB PIIX3 IDE [Natoma/Triton II] | +| bus | 82371SB PIIX3 USB [Natoma/Triton II] | +| bus | UHCI Host Controller | +| input | QEMU USB Tablet | +| bridge | 82371AB/EB/MB PIIX4 ACPI | +| display | QXL paravirtual graphic card | +| generic | Virtio RNG | +| storage | Virtio block device | +| disk | 257GB Virtual I/O device | +| volume | 238GiB EXT4 volume | +| volume | 4095KiB BIOS Boot partition | +| volume | 105MiB Windows FAT volume | +| volume | 913MiB EXT4 volume | +| network | Virtio network device | +| network | Ethernet interface | +| input | Power Button | +| input | AT Translated Set 2 keyboard | +| input | VirtualPS/2 VMware VMMouse | + +## 2. Development Laptop + +### 2.1 Components + +#### App Stack (local Docker) + +- `frontend` (Next.js SPA) +- `backend` (FastAPI) +- `postgres` (PostgreSQL + pgvector) +- `livekit` (local LiveKit Server) + +No Caddy is deployed locally; the browser talks directly to the mapped container ports on `localhost`. + +### 2.2 Network + +- All services run as Docker containers on a shared Docker network. +- Selected ports are published to `localhost` for direct access from the browser and local tools. +- No public domains are used in development; everything is addressed via `http://localhost/...`. + +### 2.3 Domains & IP Addresses + +Local development uses: + +- `http://localhost:3000` → frontend (Next.js dev/server container) +- `http://localhost:8000` → backend API (FastAPI) + - Example auth/session endpoints: + - `POST http://localhost:8000/auth/login` + - `GET http://localhost:8000/sessions/default` + - `POST http://localhost:8000/sessions/default/token` +- `ws://localhost:7880` → LiveKit signaling (local LiveKit server) +- `udp://localhost:50000–60000` → LiveKit/WebRTC media + +No `/etc/hosts` changes or TLS certificates are required; `localhost` acts as a secure origin for WebRTC. + +### 2.4 Ports & Protocols + +| Port | Protocol | Purpose | +|-------------:|:--------:|------------------------------------| +| 3000 | TCP | Frontend (Next.js) | +| 8000 | TCP | Backend API (FastAPI) | +| 5432 | TCP | Postgres + pgvector | +| 7880 | TCP | LiveKit HTTP + WS signaling | +| 50000–60000 | UDP | LiveKit WebRTC media (audio, data) | + +### 2.5 Routing + +No local Caddy or reverse proxy layer is used; routing is direct via published ports. + +#### Internal Container Routing (Docker network) + +- Backend → Postgres: `postgres://postgres:5432` +- Backend → LiveKit: `http://livekit:7880` +- Frontend (server-side) → Backend: `http://backend:8000` + +#### Browser → Containers (via localhost) + +- Browser → Frontend: `http://localhost:3000` +- Browser → Backend API: `http://localhost:8000` + +#### Outgoing (from Backend) + +- `backend` → `https://api.openai.com/v1/realtime/sessions` +- `backend` → `wss://api.openai.com/v1/realtime?model=gpt-realtime` +- `backend` → `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent` + +These calls mirror production agent behavior while pointing to the same cloud LLM realtime endpoints. + +### 2.6 Functional Layers + +#### Data Layer + +- Local Postgres instance mirrors the production schema (including pgvector). +- Database migrations are applied via backend tooling (e.g., Alembic) to keep schema in sync. + +#### Control Layer + +- Backend runs full application logic locally: + - Authentication and authorization + - Lesson and progress APIs + - LiveKit session management (`/sessions/default`, `/sessions/default/token`) and agent control +- Frontend integrates against the same API surface as production, only with `localhost` URLs. + +#### Media Layer + +- Local LiveKit instance handles: + - WS/HTTP signaling on port 7880 + - WebRTC media (audio + data channels) on UDP `50000–60000` +- Agent traffic mirrors production logic: + - LiveKit ↔ Backend ↔ LLM realtime APIs (OpenAI / Gemini). + +### 2.7 Typical Workflows + +#### Developer Pushes Code + +1. Developer pushes to `git.avaaz.ai` over HTTPS/SSL or SSH. +2. CI runs automatically (linting, tests, build validation). No deployment occurs. +3. When a release is ready, the developer creates a version tag (`v*`) on a commit in `main`. +4. CD triggers: validates the tag, rebuilds from the tagged commit, deploys updated containers, then performs post-deploy health checks. + +#### App Development + +- Start the stack: `docker compose -f docker-compose.dev.yml up -d` +- Open the app in the browser: `http://localhost:3000` +- Frontend calls the local backend for: + - `POST http://localhost:8000/auth/login` + - `GET http://localhost:8000//sessions/default` + - `POST http://localhost:8000//sessions/default/token` + +#### API Testing + +- Health check: `curl http://localhost:8000/health` +- Auth and session testing: + + ```bash + curl -X POST http://localhost:8000/auth/login \ + -H "Content-Type: application/json" \ + -d '{"email": "user@example.com", "password": "password"}' + + curl http://localhost:8000/sessions/default \ + -H "Authorization: Bearer " + ``` + +#### LiveKit Testing + +- Frontend connects to LiveKit via: + - Signaling: `ws://localhost:7880` + - WebRTC media: `udp://localhost:50000–60000` +- Backend issues local LiveKit tokens via `POST http://localhost:8000//sessions/default/token`, then connects the AI agent to the local room. + +### 2.8 Hardware + +| Class | Description | +|----------------|--------------------------------------------| +| system | HP Laptop 14-em0xxx | +| bus | 8B27 motherboard bus | +| memory | 128KiB BIOS | +| processor | AMD Ryzen 3 7320U | +| memory | 256KiB L1 cache | +| memory | 2MiB L2 cache | +| memory | 4MiB L3 cache | +| memory | 8GiB System Memory | +| bridge | Family 17h-19h PCIe Root Complex | +| generic | Family 17h-19h IOMMU | +| storage | SK hynix BC901 HFS256GE SSD | +| disk | 256GB NVMe disk | +| volume | 299MiB Windows FAT volume | +| volume | 238GiB EXT4 volume | +| network | RTL8852BE PCIe 802.11ax Wi-Fi | +| display | Mendocino integrated graphics | +| multimedia | Rembrandt Radeon High Definition Audio | +| generic | Family 19h PSP/CCP | +| bus | AMD xHCI Host Controller | +| input | Logitech M705 Mouse | +| input | Logitech K370s/K375s Keyboard | +| multimedia | Jabra SPEAK 510 USB | +| multimedia | Logitech Webcam C925e | +| communication | Bluetooth Radio | +| multimedia | HP True Vision HD Camera | +| bus | FCH SMBus Controller | +| bridge | FCH LPC Bridge | +| power | AE03041 Battery | +| input | Power Button | +| input | Lid Switch | +| input | HP WMI Hotkeys | +| input | AT Translated Set 2 Keyboard | +| input | Video Bus | +| input | SYNA32D9:00 06CB:CE17 Mouse | +| input | SYNA32D9:00 06CB:CE17 Touchpad | +| network | Ethernet Interface | diff --git a/img/favicon.png b/img/favicon.png new file mode 100644 index 0000000..99c4274 Binary files /dev/null and b/img/favicon.png differ diff --git a/img/logo.png b/img/logo.png new file mode 100644 index 0000000..a2753dc Binary files /dev/null and b/img/logo.png differ diff --git a/infra/Caddyfile b/infra/Caddyfile new file mode 100644 index 0000000..e5d7c09 --- /dev/null +++ b/infra/Caddyfile @@ -0,0 +1,99 @@ +{ + # Global Caddy options. + # + # auto_https on + # - Caddy listens on port 80 for every host (ACME + redirect). + # - Automatically issues HTTPS certificates. + # - Automatically redirects HTTP → HTTPS unless disabled. + # +} + +# ------------------------------------------------------------ +# Redirect www → root domain +# ------------------------------------------------------------ +www.avaaz.ai { + # Permanent redirect to naked domain + redir https://avaaz.ai{uri} permanent +} + +# ------------------------------------------------------------ +# Marketing site (optional — if frontend handles it, remove this) +# Redirect root → app +# ------------------------------------------------------------ +avaaz.ai { + # If you have a static marketing page, serve it here. + # If not, redirect visitors to the app. + redir https://app.avaaz.ai{uri} +} + +# ------------------------------------------------------------ +# Frontend (Next.js) +# Public URL: https://app.avaaz.ai +# Internal target: frontend:3000 +# ------------------------------------------------------------ +app.avaaz.ai { + # Reverse-proxy HTTPS traffic to the frontend container + reverse_proxy frontend:3000 + + # Access log for debugging frontend activity + log { + output file /data/app-access.log + } + + # Compression for faster delivery of JS, HTML, etc. + encode gzip zstd +} + +# ------------------------------------------------------------ +# Backend (FastAPI) +# Public URL: https://api.avaaz.ai +# Internal target: backend:8000 +# ------------------------------------------------------------ +api.avaaz.ai { + # Reverse-proxy all API traffic to FastAPI + reverse_proxy backend:8000 + + # Access log — useful for monitoring API traffic and debugging issues + log { + output file /data/api-access.log + } + + # Enable response compression (JSON, text, etc.) + encode gzip zstd +} + +# ------------------------------------------------------------ +# LiveKit (signaling only — media uses direct UDP) +# Public URL: wss://rtc.avaaz.ai +# Internal target: livekit:7880 +# ------------------------------------------------------------ +rtc.avaaz.ai { + # LiveKit uses WebSocket signaling, so we reverse-proxy WS → WS + reverse_proxy livekit:7880 + + # Access log — helps diagnose WebRTC connection failures + log { + output file /data/rtc-access.log + } + + # Compression not needed for WS traffic, but harmless + encode gzip zstd +} + +# ------------------------------------------------------------ +# Gitea (Git server UI + HTTPS + SSH clone) +# Public URL: https://git.avaaz.ai +# Internal target: gitea:3000 +# ------------------------------------------------------------ +git.avaaz.ai { + # Route all HTTPS traffic to Gitea’s web UI + reverse_proxy gitea:3000 + + # Log all Git UI requests and API access + log { + output file /data/git-access.log + } + + # Compress UI responses + encode gzip zstd +} diff --git a/infra/README.md b/infra/README.md new file mode 100644 index 0000000..6b55375 --- /dev/null +++ b/infra/README.md @@ -0,0 +1,679 @@ +# Configuration + +## 1. Configure the firewall at the VPS host + +| Public IP | +| :------------: | +| 217.154.51.242 | + +| Action | Allowed IP | Protocol | Port(s) | Description | +| :-----: | :--------: | :------: | ----------: | :------------ | +| Allow | Any | TCP | 80 | HTTP | +| Allow | Any | TCP | 443 | HTTPS | +| Allow | Any | TCP | 2222 | Git SSH | +| Allow | Any | TCP | 2885 | VPS SSH | +| Allow | Any | UDP | 3478 | STUN/TURN | +| Allow | Any | TCP | 5349 | TURN/TLS | +| Allow | Any | TCP | 7881 | LiveKit TCP | +| Allow | Any | UDP | 50000-60000 | LiveKit Media | + +## 2. Configure the DNS settings at domain registrar + +| Host (avaaz.ai) | Type | Value | +| :-------------: | :---: | :------------: | +| @ | A | 217.154.51.242 | +| www | CNAME | avaaz.ai | +| app | A | 217.154.51.242 | +| api | A | 217.154.51.242 | +| rtc | A | 217.154.51.242 | +| git | A | 217.154.51.242 | + +## 3. Change the SSH port from 22 to 2885 + +1. Connect to the server. + + ```bash + ssh username@avaaz.ai + ``` + +2. Edit the SSH configuration file. + + ```bash + sudo nano /etc/ssh/sshd_config + ``` + +3. Add port 2885 to the file and comment out port 22. + + ```text + #Port 22 + Port 2885 + ``` + +4. Save the file and exit the editor. + + - Press `Ctrl+O`, then `Enter` to save, and `Ctrl+X` to exit. + +5. Restart the SSH service. + + ```bash + sudo systemctl daemon-reload && sudo systemctl restart ssh.socket && sudo systemctl restart ssh.service + ``` + +6. **Before closing the current session**, open a new terminal window and connect to the server to verify the changes work correctly. + + ```bash + ssh username@avaaz.ai # ssh: connect to host avaaz.ai port 22: Connection timed out + ssh username@avaaz.ai -p 2885 + ``` + +7. Once the connection is successful, close the original session safely. + +## 4. Build and deploy the infrastructure + +1. Check with `dig git.avaaz.ai +short` wether the DNS settings have been propagated. + +2. SSH into the VPS to install Docker & docker compose. + + ```bash + ssh username@avaaz.ai -p 2885 + ``` + +3. Update system packages. + + ```bash + sudo apt update && sudo apt upgrade -y + ``` + +4. Install dependencies for Docker’s official repo + + ```bash + sudo apt install -y \ + ca-certificates \ + curl \ + gnupg \ + lsb-release + ``` + +5. Add Docker’s official APT repo. + + ```bash + sudo install -m 0755 -d /etc/apt/keyrings + + curl -fsSL https://download.docker.com/linux/ubuntu/gpg \ + sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + + echo \ + "deb [arch=$(dpkg --print-architecture) \ + signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + + sudo apt update + ``` + +6. Install Docker Engine + compose plugin. + + ```bash + sudo apt install -y \ + docker-ce \ + docker-ce-cli \ + containerd.io \ + docker-buildx-plugin \ + docker-compose-plugin + ``` + +7. Verify the installation. + + ```bash + sudo docker --version + sudo docker compose version + ``` + +8. Create the `/etc/docker/daemon.json` file to avoid issues with overusing disk for log data. + + ```bash + sudo nano /etc/docker/daemon.json + ``` + +9. Paste the following. + + ```json + { + "log-driver": "local", + "log-opts": { + "max-size": "10m", + "max-file": "3" + } + } + ``` + +10. Save the file and exit the editor. + + - Press `Ctrl+O`, then `Enter` to save, and `Ctrl+X` to exit. + +11. Restart the docker service to apply changes. + + ```bash + sudo systemctl daemon-reload + sudo systemctl restart docker + ``` + +12. Create directory for infra stack in `/srv/infra`. + + ```bash + sudo mkdir -p /srv/infra + sudo chown -R $USER:$USER /srv/infra + cd /srv/infra + ``` + +13. Create directories for Gitea (repos, config, etc.) and Runner persistent data. Gitea runs as UID/GID 1000 by default. + + ```bash + mkdir -p gitea-data gitea-runner-data + ``` + +14. Create the `/srv/infra/docker-compose.yml` (Caddy + Gitea + Runner) file. + + ```bash + nano docker-compose.yml + ``` + +15. Paste the following. + + ```yaml + services: + caddy: + # Use the latest official Caddy image + image: caddy:latest + # Docker Compose automatically generates container names: __ + container_name: caddy # Fixed name used by Docker engine + # Automatically restart unless manually stopped + restart: unless-stopped + ports: + # Expose HTTP (ACME + redirect) + - "80:80" + # Expose HTTPS/WSS (frontend, backend, LiveKit) + - "443:443" + volumes: + # Mount the Caddy config file read-only + - ./Caddyfile:/etc/caddy/Caddyfile:ro + # Caddy TLS certs (persistent Docker volume) + - caddy_data:/data + # Internal Caddy state/config + - caddy_config:/config + networks: + # Attach to the shared "proxy" network + - proxy + + gitea: + # Official Gitea image with built-in Actions + image: gitea/gitea:latest + container_name: gitea # Fixed name used by Docker engine + # Auto-restart service + restart: unless-stopped + environment: + # Run Gitea as host user 1000 (prevents permission issues) + - USER_UID=1000 + # Same for group + - USER_GID=1000 + # Use SQLite (stored inside /data) + - GITEA__database__DB_TYPE=sqlite3 + # Location of the SQLite DB + - GITEA__database__PATH=/data/gitea/gitea.db + # Custom config directory + - GITEA_CUSTOM=/data/gitea + volumes: + # Bind mount instead of Docker volume because: + # - We want repos, configs, SSH keys, and SQLite DB **visible and editable** on host + # - Easy backups (just copy `./gitea-data`) + # - Easy migration + # - Avoids losing data if Docker volumes are pruned + - ./gitea-data:/data + networks: + - proxy + ports: + # SSH for Git operations mapped to host 2222 + - "2222:22" + + gitea-runner: + # Official Gitea Actions Runner + image: gitea/act_runner:latest + container_name: gitea-runner # Fixed name used by Docker engine + restart: unless-stopped + depends_on: + # Runner requires Gitea to be available + - gitea + volumes: + # Runner uses host Docker daemon to spin up job containers (Docker-out-of-Docker) + - /var/run/docker.sock:/var/run/docker.sock + # Bind mount instead of volume because: + # - Runner identity is stored in /data/.runner + # - Must persist across container recreations + # - Prevents duplicated runner registrations in Gitea + # - Easy to inspect/reset via `./gitea-runner-data/.runner` + - ./gitea-runner-data:/data + environment: + # Base URL of your Gitea instance + - GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL} + # One-time registration token + - GITEA_RUNNER_REGISTRATION_TOKEN=${GITEA_RUNNER_REGISTRATION_TOKEN} + # Human-readable name for the runner + - GITEA_RUNNER_NAME=${GITEA_RUNNER_NAME} + # Runner labels (e.g., ubuntu-latest) + - GITEA_RUNNER_LABELS=${GITEA_RUNNER_LABELS} + # Set container timezone to UTC for consistent logs + - TZ=Etc/UTC + networks: + - proxy + # Start runner using persisted config + command: ["act_runner", "daemon", "--config", "/data/.runner"] + + networks: + proxy: + # Shared network for Caddy + Gitea (+ later app stack) + name: proxy + # Default Docker bridge network + driver: bridge + + volumes: + # Docker volume for Caddy TLS data (safe to keep inside Docker) + caddy_data: + name: caddy_data + # Docker volume for internal Caddy configs/state + caddy_config: + name: caddy_config + ``` + +16. Save the file and exit the editor. + + - Press `Ctrl+O`, then `Enter` to save, and `Ctrl+X` to exit. + +17. Create the `/srv/infra/.env` file with environment variables. + + ```bash + nano .env + ``` + +18. Paste the following: + + ```env + # Base URL of your Gitea instance (used by the runner to register itself + # and to send/receive workflow job information). + GITEA_INSTANCE_URL=https://git.avaaz.ai + + # One-time registration token generated in: + # Gitea → Site Administration → Actions → Runners → "Generate Token" + # This MUST be filled in once, so the runner can register. + # After registration, the runner stores its identity inside ./gitea-runner-data/.runner + # and this value is no longer needed (can be left blank). + GITEA_RUNNER_REGISTRATION_TOKEN= + + # Human-readable name for this runner. + # This is shown in the Gitea UI so you can distinguish multiple runners: + # Example: "vps-runner", "staging-runner", "gpu-runner" + GITEA_RUNNER_NAME=gitea-runner + + # Runner labels allow workflows to choose specific runners. + # The label format is: label[:schema[:args]] + # - "ubuntu-latest" is the