diff --git a/.github/workflows/ci-quick.yml b/.github/workflows/ci-quick.yml index 580d3976..e3cfebf5 100644 --- a/.github/workflows/ci-quick.yml +++ b/.github/workflows/ci-quick.yml @@ -18,6 +18,11 @@ env: NODE_VERSION: '20.x' PYTHON_VERSION: '3.10' +permissions: + contents: read + pull-requests: write + issues: write + jobs: # ============================================ # Quick validation (lint, type check, fast tests) @@ -106,8 +111,6 @@ jobs: "RESILIENTDB_GRAPHQL_URI=https://cloud.resilientdb.com/graphql" \ "JWT_SECRET=test-secret-key-do-not-use-in-production" > .env - - name: Run backend unit tests only (fast) - # wait for mongodb to be ready on localhost:27017 - name: Wait for MongoDB run: | for i in {1..30}; do @@ -136,12 +139,14 @@ jobs: --ci - name: Build check (frontend) + env: + CI: false run: | cd frontend npm run build - name: Comment PR with quick results - if: github.event_name == 'pull_request' && always() + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false && always() uses: actions/github-script@v7 with: script: | diff --git a/.gitignore b/.gitignore index 28fa9abd..c3997f09 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ coverage/ *.coverage backend/.coverage node_modules +backend/qdrant_storage/ frontend/build/ # Exclude all .env files diff --git a/README.md b/README.md index f602f01f..aacadbec 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,120 @@ +# About ResCanvas + +## The Existing Problem +Drawing is an important aspect of art and free expression within a variety of domains. It has been used to express new ideas and works of art. Tools such as MS Paint allow for drawing to be achievable on the computer, with online tools extending that functionality over the cloud where users can share and collaborate on drawings and other digital works of art. For instance, both Google's Drawing and Canva's Draw application have a sharable canvas page between registered users to perform their drawings. + +However, such online platforms store the drawing and user data in a centralized manner, making personal data easily trackable by their respective companies, and easily sharable to other third parties such as advertisers. Furthermore, the drawings can be censored by both private and public entities, such as governments and tracking agencies. Privacy is important, yet online collaboration is an essential part of many user's daily workflow. Thus, it is necessary to decentralize the data storage aspect of these online applications. It might seem that the closest working example of this is Reddit's pixel platform since all users can make edits on the same page. However, users' data are still stored centrally on their servers. Furthermore, the scope is limited to just putting one pixel at a time for each user on a rate limited basis. + +## Overview of ResCanvas +Introducing ResCanvas, a breakthrough in web-based drawing platforms that utilizes ResilientDB to ensure that user's drawings are securely stored, allowing for multiple users to collaborate and create new works of art and express ideas freely without any limits, tracking, or censorship. The canvas drawing board is the core feature of ResCanvas, designed to allow users to perform drawings using their mouse or touchscreen interface. It is simple to use, yet it allows for infinite possibilities. **To the best of our knowledge, ResCanvas is the first ResilientDB application that combines the key breakthroughs in database decentralization brought forth by ResilientDB, with the power of expression that comes with art, bridging the gap between the arts and the sciences.** + +ResCanvas is designed to seamlessly integrate drawings with the familiarity of online contribution between users using effective synchronization of each user's canvas drawing page. This allows for error-free consistency even when multiple users are drawing all at the same time. Multiple users are able to collaborate on a single canvas, and just as many things in life have strength in numbers, so too are the users on a canvas. One user could be working on one part of the art drawing while other users can finish another component of the drawing in a collaborative manner. + +The key feature of ResCanvas is defined by having all drawings stored persistently within ResilientDB in a stroke by stroke manner. Each stroke is individually cached via in-memory data store using Redis serving as the frontend cache. This ensures that the end user is able to receive all the strokes from the other users regardless of the response latency of ResilientDB, greatly enhancing the performance for the end user. Furthermore, all users will be able to see each other's strokes under a decentralized context and without any use of a centralized server or system for processing requests and storing data. + +## Key Features +* Multiple user concurrent drawing and viewable editing history on a per user, per room basis with custom room and user access controls and permissions for each room +* Drawing data and edit history is synchronized efficiently and consistently across all users within the canvas drawing room +* Fast, efficient loading of data from backend by leveraging the caching capabilities of the Redis frontend data storage framework +* Color and thickness selection tools to customize your drawings +* Persistent, secure storage of drawing data in ResilientDB allowing for censorship free expression +* No sharing of data to third parties, advertisers, government entities, .etc with decentralized storage, all user account information and data is stored in ResilientDB +* Responsive, intuitive UI inspired by Google's Material design theme used throughout the app, without the tracking and privacy issues of existing web applications +* Clear canvas ensures that data is erased for all users in the same room +* Server-side JWT authentication and authorization with robust backend middleware (`backend/middleware/auth.py`) that validates all tokens, enforces access controls, and prevents client-side bypasses +* Backend enforces security with all the authentication, verification, and authorization logic running on the server (clients cannot manipulate or circumvent security checks) +* Real time collaboration using Socket.IO for low latency stroke broadcasting, user notifications, and user activity communication with JWT-protected Socket.IO connections + +### AI Features and API Key +ResCanvas includes optional AI-powered features for sketch generation, beautification, recognition, and style transfer. The backend reads the OpenAI API key from `OPENAI_API_KEY` via `backend/config.py`. If this key is not defined, AI endpoints will return an error and the UI will report failures. + +To enable AI features, set the environment variable before starting the backend (macOS/zsh example): + +``` +export OPENAI_API_KEY="sk-...your-key..." +``` + +Then restart the backend. You may alternatively load it through your process manager or `.env` so that `backend/config.py` picks it up at startup. The drawing model used is `gpt-4o-mini`; if OpenAI is unavailable, the service attempts an Ollama fallback (`llama3:8b`) when installed locally. + +## Detailed Architecture and Concepts +Our application consists of several major components. + +The first one is the **frontend (React)**, which handles drawing input, local smoothing/coalescing of strokes, UI state (tools, color, thickness), optimistic local rendering, and Socket.IO for real-time updates. Thus the frontend handles the user facing side of ResCanvas and ensures a smooth UX while ensuring communication between this frontend layer and the backend. This layer also handles the storage of auth tokens in `localStorage` and its API wrappers (like `frontend/src/api/`) automatically attach JWT access tokens to all protected requests as well. The most important aspect of this layer in terms of security is that the frontend does not perform authentication or authorization logic - it simply presents credentials and tokens to the backend. + +This brings us to the **backend (Flask + Flask-SocketIO)**, which serves as the authoritative security boundary and data handler for the application. The backend validates all JWT tokens server-side using middleware (`backend/middleware/auth.py`), enforces room membership and permissions, verifies client-side signatures for secure rooms, encrypts/decrypts strokes for private/secure rooms, commits transactions to ResilientDB via GraphQL, and also retrieves data as needed according to the frontend's request. All protected API routes and Socket.IO connections require valid JWT access tokens sent via `Authorization: Bearer ` header. Since the backend performs all security checks, clients cannot bypass authentication or authorization and must go through the backend for all sensitive requests. Furthermore, when working with private or secure rooms, the backend handles encryption/decryption and room key management as well (`backend/services/crypto_service.py` and `submit_room_line.py`). + +Going into deeper detail with regards to the backend, it interacts with several other key components. The first one is **ResilientDB**, the persistent, decentralized, immutable transaction log where strokes are ultimately stored, and the core essence of this application. Strokes are written as transactions so the full history is auditable and censorship-resistant. Through the `resilient-python-cache` library, ResilientDB synchronizes data blocks with **MongoDB (canvasCache)**. MongoDB is a warm persistent cache and queryable replica of strokes so the backend can serve reads without contacting ResilientDB directly for every request. This essentially serves as a sync bridge that mirrors ResilientDB into MongoDB. + +From MongoDB, our backend handles the syncronization of data with **Redis**, which is a short-lived, in-memory store keyed by room for fast read/write and undo/redo operations. Redis is intentionally ephemeral as it allows quick synchronization of live sessions while ResilientDB acts as the long-term durable store. Furthermore, this Redis cache can be hosted by any trusted node, and thus ensures that the security and privacy guarantees of ResilientDB are still preserved while ensuring fast performance. It can be said that our backend is also another sync bridge layer, that of between MongoDB and Redis. + +### Data Model and Stroke Format +ResCanvas uses a simple, compact stroke model that is friendly for network transport and decentralized commits. A typical base stroke data payload (JSON) contains the following data: + + - drawingId: unique per-user or per-drawing session + - userId: author id (when available/allowed) + - color: hex or named value + - lineWidth: numeric stroke thickness + - pathData: an array of (x,y) points, optionally compressed (delta-encoded) + - timestamp: client-side timestamp for ordering and replay + - metadata: optional fields for signing, encryption info, transform/offsets + +Note that the path data should be kept compact. The frontend coalesces mouse/touch events and optionally delta-encodes paths before sending to the backend to reduce network bandwidth. For secure rooms, each stroke includes an on-chain/verifiable signature and any necessary auxiliary data to perform signature verification. Custom strokes, stamps, and other types of data will have additional data fields that are relevant to them, such as the stroke style, stamp size, among others. + +### Undo/redo and Edit History +Undo/redo is implemented through per-room, per-user stacks stored in Redis (ephemeral). Each user action that mutates the canvas pushes an entry to the user's undo stack and updates the live room state in Redis. Redo pops from a redo stack and applies the strokes again via the same commit flow (including related signing and encryption rules). Because ResilientDB is immutable, undo/redo on the client is implemented as additional strokes that semantically represent an "undo" (for example a delta or a tombstone stroke) by using a separate metadata layer that signals removal in replay. The visible client behavior is immediate, while the authoritative history in ResilientDB preserves the full append only log. + +### Consistency, Concurrency and Ordering +Multiple users can draw simultaneously since the system is designed for eventual consistency with low-latency broadcast, where each stroke is broadcast immediately via Socket.IO to all connected room participants. This allows the application to provide near real-time feedback. The backend attempts to persist strokes to ResilientDB and caches (Redis/MongoDB). If ResilientDB write is delayed, clients still see strokes from the Socket.IO broadcast and from Redis while waiting for the backend to finishing writing the stroke data. Ordering is primarily guided by timestamps and the sequence of commits in ResilientDB. When replaying history, the authoritative order comes from the ResilientDB transaction log so all data and their ordering is preserved even if the canvas itself is cleared, strokes are undone, or even if the entire canvas room is deleted by the user. + +### ResilientDB Theory +ResilientDB is used as an immutable, decentralized transaction log. There are several key properties that we rely on. + +The first one is that of **immutability**, where once a stroke is committed, it cannot be altered silently. This increases trust and accountability. The second one is **decentralization**, since no single host controls the persistent copy of strokes, reducing censorship and central data harvesting. The third key property that we rely on is **auditability** as the entire canvas history can be inspected and verified against the ResilientDB ledger. This ensures transparency as anyone can view and verify the user's drawing history and actions taken on the application, and serves as a key deterrent against malicious activity on the canvas. + +By treating each stroke as a transaction, we now achieve a chronological, tamper-evident history of canvas changes. Anyone can verify and review the changes to the canvas as the ground truth source of information. The sync bridge mirrors transactions into MongoDB so read queries don't need to hit ResilientDB for every request, while Redis caching further enhances the performance from the UX standpoint by caching the data from MongoDB on an in-memory basis within a trusted node. Removal operations such as undo and redo, as well as clear canvas/room deletions are simulated using time stamp markers to achieve the same effects without altering the historical backend data as well. This hierarchical relationship between ResilientDB, MongoDB, and Redis essentially serves as an unique balance between user experience, security, and privacy. + +### Private Rooms vs Secure Rooms +Recall that public rooms allow anyone to access and draw in them, and so all the data is publically accessible by all registered users without needing to perform decryption and obtaining an access key to the room. In private rooms, access is restricted as only invited users or those with the room key can join such rooms. Strokes may be encrypted so only members with the room key can decrypt. The backend participates in wrapping/unwrapping room keys using `ROOM_MASTER_KEY_B64`. This allows users who want additional privacy while drawing contents containing sensitive or personal information to be able to do so without the risk of exposing all their raw drawings to the general public. + +Secure rooms go further and expand upon the protections of private rooms by requiring client-side signing of strokes with a cryptographic wallet (such as [ResVault](https://chromewebstore.google.com/detail/resvault/ejlihnefafcgfajaomeeogdhdhhajamf?pli=1)). Each stroke is signed by the user's wallet private key and the signature is stored with the stroke. This enables verifiable authorship since anyone can confirm a stroke was created by the owner of a given wallet address. See `WALLET_TESTING_GUIDE.md` for details about ResVault and how to use it with ResCanvas, of which the backend validates signatures for secure rooms in `backend/routes/submit_room_line.py`. + +#### Wallet Integration and Signature Flow for Secure Rooms +1. User connects wallet (such as ResVault) via the frontend UI and grants signing permissions. +2. When drawing in a secure room, the frontend prepares the stroke payload and asks ResVault to sign the serialized stroke or a deterministic hash of it. +3. The signed payload (signature + public key or address) is sent to the backend along with the stroke. +4. The backend verifies the signature before accepting and committing the stroke to persistent storage. + +### Security, Privacy and Threat Model +ResCanvas aims to improve user privacy and resist centralized censorship, and so we mitigated several threats in our application. One of the most significant threats that many web based applications face today is **Central server compromise**, since many existing applications are based on a centralized sever and store important data there. However, in ResCanvas, persistent data is stored on ResilientDB and mirrored to MongoDB, which reduces a single point of failure due to the decentralized nature of ResilientDB. + +We also prevent **data harvesting by a platform operator** from occurring in the first place as decentralized storage and client-side signing for secure rooms reduce linkability and provide verifiability. This ensures that user's data is not being collected for third party usage, for instance, which could result in data being leveraged for commercial purposes and malicious intent. This assurance in user's data security is also guaranteed through our prevention of **client-side authentication bypasses**. All authentication, authorization, and access control logic runs server-side. The backend middleware validates tokens, checks permissions, and enforces room access rules. Even the most malicious of clients cannot manipulate or circumvent security checks because of this middleware. + +Other security protections that ResCanvas offers includes handling the situation where there could be **token theft via XSS**. We manage this risk by having refresh tokens be stored in HttpOnly cookies that cannot be accessed by JavaScript, protecting long-lived sessions from cross-site scripting attacks. Additionally, we prevent **CSRF attacks** by having refresh cookies use `SameSite` attribute. Using this kind of attribute prevents cross-site request forgery from occurring. We also prevent **signature forgery** for secure rooms since the backend verifies cryptographic signatures server-side. This protection ensures that strokes cannot be attributed to users who didn't create them in the first place. + +Despite these significant protections that ResCanvas offers, there are several key trade-offs and assumptions that are worth mentioning. + +For instance, there is still a risk that the application can suffer from **frontend device compromise**. While the backend enforces all security decisions, if a user's device or browser is compromised, attackers could steal access tokens from `localStorage` or wallet keys before signing. So access tokens are short-lived (15 minutes by default) to minimize exposure and refresh tokens in HttpOnly cookies are protected from JavaScript access. + +The core functionality of ResCanvas still depends on the **availability of ResilientDB**. ResilientDB endpoints and GraphQL commit endpoints used by the backend must remain available and trusted by backend operators. If those services are compromised, ledger inclusion or availability may be affected. However, the probability of this occurring is extremely low due to the decentralized, blockchain nature of ResilientDB. + +Furthermore, certain backend layers and services, such as Redis and MongoDB, rely on the user's level of **backend trust**. Users must trust the backend operators to correctly implement and enforce security policies, and also ensure that those backend layers and services are running on trusted nodes. Having nodes that are trustworthy to the user is essential as the backend has access to certain data such as unencrypted strokes for public rooms. + +#### JWT-Based Authentication +- **Access Tokens**: Short-lived JWTs signed with `JWT_SECRET` (default: 15 minutes). Clients must include the token in the `Authorization: Bearer ` header for all protected API calls and Socket.IO connections. +- **Refresh Tokens**: Long-lived tokens (default: 7 days) stored in secure, HttpOnly cookies (`SameSite=Lax` or `Strict`). These cannot be accessed by JavaScript, protecting against XSS attacks. +- **Token Refresh**: When an access token expires, clients call `/auth/refresh` to obtain a new access token using the refresh cookie. The backend validates the refresh token server-side. + +#### Backend Middleware (`backend/middleware/auth.py`) +All protected routes and Socket.IO handlers use the following decorators: +- **`@require_auth`**: Validates JWT signature, checks expiration, and loads the authenticated user into `g.current_user`. Rejects invalid or expired tokens. +- **`@require_auth_optional`**: Allows both authenticated and anonymous access. Authenticated users get enhanced features (e.g., membership-scoped data). +- **`@require_room_access`**: Enforces room-level permissions. Verifies that the authenticated user has appropriate access (owner, editor, viewer) to the requested room. + +--- + +# ResCanvas Setup Guide +ResCanvas is a decentralized collaborative drawing platform that integrates **ResilientDB**, **MongoDB**, and **Redis** for data consistency, caching, and persistence. +

image

@@ -60,6 +177,7 @@ A lightweight synchronization service (`resilient-python-cache`) listens to Resi Because ResilientDB is append-only, certain operations, such as undo and redo actions, are represented as semantic overlays rather than destructive modifications. The backend records these actions into the Redis caching layer and, when required, writes reversible markers to the ledger. Clients then replay strokes based on the authoritative ordering in ResilientDB. # ResCanvas Setup Guide + This guide provides complete instructions to deploy ResCanvas locally, including setup for the cache layer, backend, and frontend. ## Prerequisites @@ -111,7 +229,6 @@ MONGO_URL = "[URI_COPIED_FROM_MONGODB_CONNECTION]" MONGO_DB = "canvasCache" MONGO_COLLECTION = "strokes" ``` - ### Start the Cache Service ```bash diff --git a/backend/NEXT_STEPS.md b/backend/NEXT_STEPS.md new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/backend/NEXT_STEPS.md @@ -0,0 +1 @@ + diff --git a/backend/SETUP_VECTOR_SEARCH.md b/backend/SETUP_VECTOR_SEARCH.md new file mode 100644 index 00000000..6d19d789 --- /dev/null +++ b/backend/SETUP_VECTOR_SEARCH.md @@ -0,0 +1,66 @@ +# Vector Search Setup Guide + +## Quick Start + +### Option 1: Using Docker Compose (Recommended) + +```bash +cd backend + +# Start Qdrant and Redis +docker-compose up -d qdrant redis + +# Verify Qdrant is running +curl http://localhost:6333/healthz +# Should return: {"title":"healthz","version":"1.x.x"} + +# Install Python dependencies (if not already done) +pip install -r requirements.txt + +# Run backend +python app.py + +# Seperate terminal +python worker/embedding_service.py +``` + +### Option 2: Local Qdrant Installation + +```bash +# macOS with Homebrew +brew install qdrant + +# Or using Docker standalone +docker run -p 6333:6333 -p 6334:6334 \ + -v $(pwd)/qdrant_storage:/qdrant/storage \ + qdrant/qdrant + +# Install Python dependencies +cd backend +pip install -r requirements.txt + +# Run backend +python app.py + +# Seperate terminal +python worker/embedding_service.py +``` + +--- + +## 🔧 Configuration + +The following environment variables can be set in `config.py`: + +```bash +# Qdrant Vector Database +QDRANT_HOST=localhost +QDRANT_PORT=6333 +QDRANT_GRPC_PORT=6334 +QDRANT_COLLECTION_NAME=rescanvas_embeddings + +# These are already in your config: +# REDIS_HOST=localhost +# REDIS_PORT=6379 +# MONGO_ATLAS_URI=... +``` diff --git a/backend/app.py b/backend/app.py index 724b7109..0f680de7 100644 --- a/backend/app.py +++ b/backend/app.py @@ -39,6 +39,9 @@ def filter(self, record): app = Flask(__name__) +# Allow large request bodies for thumbnail uploads (up to 20MB) +app.config['MAX_CONTENT_LENGTH'] = 20 * 1024 * 1024 + # Initialize rate limiting BEFORE importing routes (routes use limiter decorators) from middleware.rate_limit import init_limiter, rate_limit_error_handler limiter = init_limiter(app) @@ -56,6 +59,9 @@ def filter(self, record): from routes.frontend import frontend_bp from routes.analytics import analytics_bp from routes.export import export_bp +from routes.ai_assistant import ai_assistant_bp +from routes.search_ai import search_ai_bp +from routes.chatbot import chatbot_bp from services.db import redis_client from services.canvas_counter import get_canvas_draw_count from services.graphql_service import commit_transaction_via_graphql @@ -215,6 +221,8 @@ def handle_all_exceptions(e): app.register_blueprint(submit_room_line_bp) app.register_blueprint(admin_bp) app.register_blueprint(export_bp) +app.register_blueprint(ai_assistant_bp) +app.register_blueprint(chatbot_bp) # Register versioned API v1 blueprints for external applications from api_v1.auth import auth_v1_bp @@ -232,6 +240,7 @@ def handle_all_exceptions(e): app.register_blueprint(users_v1_bp) app.register_blueprint(stamps_bp, url_prefix='/api') app.register_blueprint(templates_v1_bp) +app.register_blueprint(search_ai_bp) # Frontend serving must be last to avoid route conflicts app.register_blueprint(frontend_bp) diff --git a/backend/config.py b/backend/config.py index 35ff246a..af514bf6 100644 --- a/backend/config.py +++ b/backend/config.py @@ -46,6 +46,13 @@ REDIS_HOST = os.getenv("REDIS_HOST", "localhost") REDIS_PORT = int(os.getenv("REDIS_PORT", "6379")) +# Qdrant Vector Database Configuration +QDRANT_HOST = os.getenv("QDRANT_HOST", "localhost") +QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333")) +QDRANT_COLLECTION_NAME = os.getenv("QDRANT_COLLECTION_NAME", "rescanvas_embeddings") +EMBEDDING_DIMENSION = 512 # OpenCLIP ViT-B-32 output dimension +QDRANT_GRPC_PORT = int(os.getenv("QDRANT_GRPC_PORT", "6334")) + # Rate Limiting Configuration RATE_LIMIT_STORAGE_URI = f"redis://{REDIS_HOST}:{REDIS_PORT}" RATE_LIMIT_ENABLED = os.getenv("RATE_LIMIT_ENABLED", "True") == "True" diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index 458867a2..5185f02f 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -18,6 +18,25 @@ services: networks: - rescanvas-network + qdrant: + image: qdrant/qdrant:latest + container_name: rescanvas-qdrant + ports: + - "6333:6333" # HTTP API + - "6334:6334" # gRPC API + volumes: + - qdrant_data:/qdrant/storage + environment: + - QDRANT__SERVICE__GRPC_PORT=6334 + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:6333/healthz"] + interval: 10s + timeout: 5s + retries: 5 + networks: + - rescanvas-network + backend: build: context: . @@ -34,6 +53,9 @@ services: - RATE_LIMIT_STORAGE_URI=redis://redis:6379 - REDIS_HOST=redis - REDIS_PORT=6379 + - QDRANT_HOST=qdrant + - QDRANT_PORT=6333 + - QDRANT_GRPC_PORT=6334 - JWT_SECRET=${JWT_SECRET:-dev-insecure-change-me} - OPENAI_API_KEY=${OPENAI_API_KEY} - ANALYTICS_ENABLED=${ANALYTICS_ENABLED:-True} @@ -46,6 +68,8 @@ services: depends_on: redis: condition: service_healthy + qdrant: + condition: service_healthy restart: unless-stopped healthcheck: test: ["CMD", "curl", "-f", "http://localhost:10010/api/analytics/health"] @@ -59,6 +83,8 @@ services: volumes: redis_data: driver: local + qdrant_data: + driver: local networks: rescanvas-network: diff --git a/backend/requirements.txt b/backend/requirements.txt index 06649a5b..c64780e1 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -51,8 +51,9 @@ python-engineio==4.12.3 python-socketio==5.14.1 redis==6.2.0 requests==2.32.4 -resilient-python-cache==0.1.1 +# resilient-python-cache==0.1.1 rich==13.9.4 +openai>=1.0.0 simple-websocket==1.1.0 simplejson==3.19.3 six==1.17.0 @@ -64,3 +65,9 @@ websockets==10.4 Werkzeug==3.1.3 wrapt==2.0.0 wsproto==1.2.0 +# AI/ML dependencies for semantic search +torch>=2.0.0 +open_clip_torch>=2.20.0 +pillow>=10.0.0 +qdrant-client>=1.7.0 +numpy>=1.24.0 diff --git a/backend/routes/ai_assistant.py b/backend/routes/ai_assistant.py new file mode 100644 index 00000000..40c2189e --- /dev/null +++ b/backend/routes/ai_assistant.py @@ -0,0 +1,208 @@ +from flask import Blueprint, request, jsonify +# Import style transfer function as well +from services.llm_service import ( + prompt_to_drawings, + complete_shape_from_canvas, + beautify_canvas_state, + style_transfer_canvas, +) +from services.llm_service import recognize_objects_in_box +# from services.image_generation_service import ( +# text_to_image as img_text_to_image, +# ) +import logging +import base64 +import io + +ai_assistant_bp = Blueprint('ai_assistant', __name__) +logger = logging.getLogger(__name__) + + +@ai_assistant_bp.route('/api/ai_assistant/drawing', methods=['POST']) +def text_to_drawings(): + """ + Body: { "prompt": "", canvasState: {json object} } + Returns: Parsed drawing JSON (shape/color/size/position/...) or an error payload. + """ + try: + payload = request.get_json(silent=True) or {} + prompt = payload.get("prompt") + canvasState = payload.get("canvasState") or {} + + if not isinstance(prompt, str) or not prompt.strip(): + return jsonify({"error": "bad_request", "detail": "Missing or invalid 'prompt' (string)."}), 400 + + logger.info("AI drawing requested") + result = prompt_to_drawings(prompt.strip(), canvasState) + + print(f"\n\nModel result: {result}\n\n") + + # If services returned an error, surface it with 502 (bad upstream) + if isinstance(result, dict) and "error" in result: + logger.warning("AI drawing failed: %s", result) + return jsonify({"error": "upstream_model_error", "detail": result}), 502 + + return jsonify(result), 200 + except Exception as e: + logger.exception("Unhandled error in /drawing") + return jsonify({"error": "server_error", "detail": str(e)}), 500 + + +@ai_assistant_bp.route('/api/ai_assistant/complete', methods=['POST']) +def shape_completion(): + """ + Body: { "canvasState": { ... } } + Returns: { complete, confidence, object{ color, lineWidth, pathData{...} } } or an error payload. + """ + try: + payload = request.get_json(silent=True) or {} + canvas_state = payload.get("canvasState") + if not isinstance(canvas_state, dict): + return jsonify({"error": "bad_request", "detail": "Missing or invalid 'canvas_state' (object)."}), 400 + + logger.info("AI shape completion requested") + suggestion = complete_shape_from_canvas(canvas_state) + + if not isinstance(canvas_state, dict): + return jsonify({ + "error": "bad_request", + "detail": "Missing or invalid 'canvasState' (object)." + }), 400 + + return jsonify(suggestion), 200 + except Exception as e: + logger.exception("Unhandled error in /complete") + return jsonify({"error": "server_error", "detail": str(e)}), 500 + + +@ai_assistant_bp.route('/api/ai_assistant/image', methods=['POST']) +def text_to_image(): + """ + TODO: To be implemented + Body: { "prompt": "", "width"?: int, "height"?: int, "style"?: str } + Returns: { "imageDataUrl": "data:image/png;base64,..." } + """ + try: + payload = request.get_json(silent=True) or {} + prompt = payload.get("prompt", "") + width = payload.get("width") or 512 + height = payload.get("height") or 512 + style = payload.get("style") or "default" + + if not isinstance(prompt, str) or not prompt.strip(): + return jsonify({ + "error": "bad_request", + "detail": "Missing or invalid 'prompt' (string)." + }), 400 + + logger.info("AI text-to-image requested") + + # Try to generate via image_generation_service + try: + from services.image_generation_service import text_to_image as img_text_to_image + pil_image = img_text_to_image(prompt.strip(), width=width, height=height, style=style) + except Exception as e: + logger.exception("Image generation failed: %s", e) + return jsonify({"error": "image_generation_failed", "detail": str(e)}), 502 + + buf = io.BytesIO() + pil_image.save(buf, format="PNG") + buf.seek(0) + encoded = base64.b64encode(buf.read()).decode("utf-8") + data_url = f"data:image/png;base64,{encoded}" + + return jsonify({"imageDataUrl": data_url}), 200 + + except Exception as e: + logger.exception("Unhandled error in /image") + return jsonify({"error": "server_error", "detail": str(e)}), 500 + + +@ai_assistant_bp.route("/api/ai_assistant/beautify", methods=["POST"]) +def beautify_sketch(): + try: + payload = request.get_json(silent=True) or {} + canvas_state = payload.get("canvasState") + + if not isinstance(canvas_state, dict): + return jsonify({ + "error": "bad_request", + "detail": "Missing or invalid 'canvasState' (object)." + }), 400 + + result = beautify_canvas_state(canvas_state) + # print("\n\ncanvas_state!!!", canvas_state, "\n\n") + # print("\n\nResult!!!", result, "\n\n") + + if not isinstance(result, dict) or "objects" not in result: + logger.warning("Beautify returned invalid payload: %r", result) + return jsonify({ + "error": "upstream_model_error", + "detail": "Beautify model returned invalid payload." + }), 502 + + return jsonify(result), 200 + + except Exception as e: + logger.exception("Unhandled error in /beautify") + return jsonify({"error": "server_error", "detail": str(e)}), 500 + + +@ai_assistant_bp.route('/api/ai_assistant/style', methods=['POST']) +def style_transfer(): + """ + Body: { "canvasState": {...}, "stylePrompt": "/chatbot/message', methods=['POST']) +@limiter.limit("10/minute") +@require_auth +def post_chat_message(roomId): + """ + Handle chatbot messages in a room. + + Args: + roomId: The room ID where the message is sent + + Request body: + message (str): The user's message to the bot + + Returns: + JSON response with the bot's reply + """ + try: + # Get the authenticated user ID + user = g.current_user + claims = g.token_claims + user_id = claims["sub"] + + # Get the message from request body + data = request.get_json() + if not data or 'message' not in data: + return jsonify({"error": "Missing 'message' field in request body"}), 400 + + message = data.get('message') + canvas_context = data.get('canvas_context', {}) + + # Call the chatbot service + reply_text = get_bot_reply(message, roomId, user_id, canvas_context) + + # Return the bot's reply + return jsonify({"reply": reply_text}) + + except Exception as e: + logger.exception("Error processing chatbot message") + return jsonify({"error": "Internal server error"}), 500 + + +@chatbot_bp.route('/rooms//chatbot/history', methods=['GET']) +@require_auth +def get_room_chat_history(roomId): + """ + Retrieve chat history for a room. + + Args: + roomId: The room ID to get history for + + Query params: + limit (int): Maximum number of messages to retrieve (default 50, max 100) + + Returns: + JSON response with chat history + """ + try: + # Get limit from query params, default to 50, max 100 + limit = request.args.get('limit', 50, type=int) + limit = min(limit, 100) # Cap at 100 messages + + # Get chat history from service + history = get_chat_history(roomId, limit) + + return jsonify({"history": history}) + + except Exception as e: + logger.exception("Error retrieving chat history") + return jsonify({"error": "Internal server error"}), 500 diff --git a/backend/routes/rooms.py b/backend/routes/rooms.py index 13e68fde..0354cb28 100644 --- a/backend/routes/rooms.py +++ b/backend/routes/rooms.py @@ -3277,3 +3277,82 @@ def notification_preferences(): except Exception: return jsonify({"status":"error","message":"Failed to persist preferences"}), 500 return jsonify({"status":"ok","preferences": clean}) + + +@rooms_bp.route("/rooms//thumbnail", methods=["POST"]) +@require_auth +@require_room_access(room_id_param='roomId') +def upload_room_thumbnail(roomId): + try: + data = request.get_json() + if not data: + return jsonify({"error": "Request body required"}), 400 + + thumbnail_data = data.get('thumbnail') + if not thumbnail_data: + return jsonify({"error": "thumbnail field required"}), 400 + + # Strip data URL prefix if present + # Format: data:image/png;base64,iVBORw0KG... + if thumbnail_data.startswith('data:'): + if ',' in thumbnail_data: + thumbnail_data = thumbnail_data.split(',', 1)[1] + else: + return jsonify({"error": "Invalid data URL format"}), 400 + + # Decode base64 to binary + import base64 + try: + thumbnail_bytes = base64.b64decode(thumbnail_data) + except Exception as e: + logger.error(f"Failed to decode thumbnail base64 for room {roomId}: {e}") + return jsonify({"error": "Invalid base64 encoding"}), 400 + + # Validate minimum size (at least 100 bytes for a valid image) + if len(thumbnail_bytes) < 100: + return jsonify({"error": "Thumbnail too small, likely invalid"}), 400 + + # Validate maximum size (10MB limit) + if len(thumbnail_bytes) > 10 * 1024 * 1024: + return jsonify({"error": "Thumbnail too large (max 10MB)"}), 400 + + # Optional: Validate it's actually a PNG/JPEG using magic bytes + # PNG magic bytes: 89 50 4E 47 + # JPEG magic bytes: FF D8 FF + is_png = thumbnail_bytes[:4] == b'\x89PNG' + is_jpeg = thumbnail_bytes[:3] == b'\xff\xd8\xff' + + if not (is_png or is_jpeg): + logger.warning(f"Thumbnail for room {roomId} doesn't appear to be PNG or JPEG") + # Don't reject, just log warning + + # Store thumbnail in room document + updated_at = datetime.utcnow() + result = rooms_coll.update_one( + {'_id': ObjectId(roomId)}, + { + '$set': { + 'thumbnail': thumbnail_bytes, # Binary data + 'thumbnailUpdatedAt': updated_at, + 'updatedAt': updated_at # Also update room's main timestamp + } + } + ) + + if result.matched_count == 0: + return jsonify({"error": "Room not found"}), 404 + + logger.info(f"Stored thumbnail for room {roomId}: {len(thumbnail_bytes)} bytes " + f"(format: {'PNG' if is_png else 'JPEG' if is_jpeg else 'unknown'})") + + return jsonify({ + "status": "success", + "roomId": roomId, + "thumbnailSize": len(thumbnail_bytes), + "format": "PNG" if is_png else "JPEG" if is_jpeg else "unknown", + "updatedAt": updated_at.isoformat() + }), 200 + + except Exception as e: + logger.exception(f"Failed to upload thumbnail for room {roomId}: {e}") + return jsonify({"error": "Internal server error", "details": str(e)}), 500 diff --git a/backend/routes/search_ai.py b/backend/routes/search_ai.py new file mode 100644 index 00000000..c8da5808 --- /dev/null +++ b/backend/routes/search_ai.py @@ -0,0 +1,129 @@ +from flask import Blueprint, request, jsonify, g +from middleware.auth import require_auth_optional +from services.db import rooms_coll, shares_coll +from bson import ObjectId +from services.search_algorithms import text_search, image_search +import logging + +search_ai_bp = Blueprint('search_ai', __name__) +logger = logging.getLogger(__name__) + + + + +@search_ai_bp.route('/api/v1/search/ai', methods=['POST']) +@require_auth_optional +def search_ai(): + payload = request.get_json(silent=True) or {} + q = (payload.get('q') or '').strip() + image_b64 = payload.get('image_b64') + user = g.current_user + claims = getattr(g, 'token_claims', None) + + # ---- Visibility: public OR owner OR shared; always exclude archived ---- + vis_or = [{"type": "public"}] + if user and claims and claims.get('sub'): + # collect shared room ObjectIds + try: + shared_cursor = shares_coll.find( + {"$or": [{"userId": claims['sub']}, {"username": claims['sub']}]}, + {"roomId": 1} + ) + oids_obj = [] + oids_str = [] + for doc in shared_cursor: + rid = doc.get("roomId") + # roomId may be stored as a string (hex) or as an ObjectId already + if isinstance(rid, str): + oids_str.append(rid) + try: + oids_obj.append(ObjectId(rid)) + except Exception: + pass + else: + # assume it's an ObjectId or similar + try: + oids_obj.append(ObjectId(rid)) + except Exception: + try: + # fallback: convert to str + oids_str.append(str(rid)) + except Exception: + pass + except Exception: + oids_obj = [] + oids_str = [] + + # Match ownerId stored as string or as ObjectId (legacy/varied schemas) + try: + oid_owner = ObjectId(claims['sub']) + vis_or.append({"ownerId": claims['sub']}) + vis_or.append({"ownerId": oid_owner}) + except Exception: + vis_or.append({"ownerId": claims['sub']}) + # Also match by ownerName to handle legacy documents that store owner + # as a username or when ownerId formats vary. + if claims.get('username'): + vis_or.append({"ownerName": claims.get('username')}) + # include shared rooms by _id; support both ObjectId and string representations + if oids_obj: + vis_or.append({"_id": {"$in": oids_obj}}) + if oids_str: + vis_or.append({"_id": {"$in": oids_str}}) + logger.debug("search_ai: visibility OR clauses count: owners=%s, shared_obj=%s, shared_str=%s", len([c for c in vis_or if 'ownerId' in c or 'ownerName' in c]), len(oids_obj), len(oids_str)) + + # If a text query is provided, search across all non-archived rooms (public+private) + if q: + match = {"archived": {"$ne": True}} + else: + match = {"$and": [{"archived": {"$ne": True}}, {"$or": vis_or}]} + + # ---- Limit / fields ---- + LIMIT = min(int(payload.get("limit", 50)), 100) + fields = {"name": 1, "type": 1, "ownerName": 1, "description": 1, "createdAt": 1, "updatedAt": 1} + + # ---- Fetch candidates (visibility-filtered) ---- + try: + candidates = [] + logger.debug("search_ai: using match=%s", match) + for r in rooms_coll.find(match, fields).limit(LIMIT * 5): # oversample; rank later + candidates.append({ + "id": str(r.get("_id")), + "name": r.get("name"), + "type": r.get("type"), + "ownerName": r.get("ownerName"), + "description": r.get("description"), + "createdAt": r.get("createdAt"), + "updatedAt": r.get("updatedAt"), + }) + logger.info("search_ai: fetched %d candidate rooms (limit=%s)", len(candidates), LIMIT * 5) + except Exception as e: + logger.exception("Search candidate fetch failed: %s", e) + return jsonify({"status": "ok", "results": []}), 200 + + # ---- Match + rank using your stubs ---- + try: + if image_b64: + ranked = image_search(image_b64=image_b64, rooms=candidates, q=q or None, top_n=LIMIT) + elif q: + ranked = text_search(query=q, rooms=candidates, top_n=LIMIT) + else: + # No signals → recency fallback with default score + ranked = sorted( + candidates, + key=lambda x: x.get("updatedAt") or x.get("createdAt"), + reverse=True + )[:LIMIT] + for r in ranked: + r["score"] = 1.0 + except Exception as e: + logger.exception("Search ranking failed: %s", e) + ranked = candidates[:LIMIT] # fail soft + + # ---- Presentation fields ---- + for r in ranked: + r["snippet"] = (r.get("description") or "")[:300] + if "score" not in r: + r["score"] = 1.0 + + return jsonify({"status": "ok", "results": ranked}), 200 diff --git a/backend/services/canvas_service.py b/backend/services/canvas_service.py new file mode 100644 index 00000000..b62d0616 --- /dev/null +++ b/backend/services/canvas_service.py @@ -0,0 +1,144 @@ +""" +Canvas service functions for ResCanvas. +""" + +import logging +from .db import strokes_coll, redis_client +from .canvas_counter import get_canvas_draw_count +from .graphql_service import commit_transaction_via_graphql +from config import SIGNER_PUBLIC_KEY, SIGNER_PRIVATE_KEY, RECIPIENT_PUBLIC_KEY +import time +import json + +logger = logging.getLogger(__name__) + + +def _now_ms(): + """Get current timestamp in milliseconds.""" + return int(time.time() * 1000) + + +def _persist_marker(id_value: str, value_field: str, value): + """Persist a small marker object (id + value) into ResDB so it survives Redis flush.""" + payload = { + "operation": "CREATE", + "amount": 1, + "signerPublicKey": SIGNER_PUBLIC_KEY, + "signerPrivateKey": SIGNER_PRIVATE_KEY, + "recipientPublicKey": RECIPIENT_PUBLIC_KEY, + "asset": { + "data": { + "id": id_value, + value_field: value + } + } + } + try: + commit_transaction_via_graphql(payload) + except Exception: + logger.exception("Failed to persist marker %s", id_value) + + +def clear_canvas(room_id: str) -> dict: + """ + Clear all strokes from a canvas/room. + + This function: + 1. Deletes all strokes from MongoDB for the room + 2. Sets a clear timestamp marker in Redis and ResDB + 3. Clears undo/redo stacks for the room + + Args: + room_id: The room ID to clear + + Returns: + dict: Status dictionary with success flag and metadata + """ + try: + # Get current timestamp and draw count + ts = _now_ms() + try: + res_draw_count = int(get_canvas_draw_count()) + except Exception: + logger.exception("Failed reading canvas draw count; defaulting to 0") + res_draw_count = 0 + + # Delete all strokes from MongoDB for this room + delete_result = strokes_coll.delete_many({"roomId": room_id}) + deleted_count = delete_result.deleted_count + logger.info(f"Deleted {deleted_count} strokes from room {room_id}") + + # Set clear timestamp markers in Redis + redis_ts_cache_key = f"last-clear-ts:{room_id}" + redis_ts_legacy = f"clear-canvas-timestamp:{room_id}" + resdb_ts_id = f"clear-canvas-timestamp:{room_id}" + redis_count_key = f"res-canvas-draw-count:{room_id}" + redis_count_legacy = f"draw_count_clear_canvas:{room_id}" + resdb_count_id = f"res-canvas-draw-count:{room_id}" + + try: + redis_client.set(redis_ts_cache_key, ts) + redis_client.set(redis_ts_legacy, ts) + redis_client.set(redis_count_key, res_draw_count) + redis_client.set(redis_count_legacy, res_draw_count) + except Exception: + logger.exception("Failed setting Redis keys for clear markers") + + # Persist markers to ResDB + try: + _persist_marker(resdb_count_id, "value", res_draw_count) + _persist_marker(resdb_ts_id, "ts", ts) + _persist_marker(redis_count_legacy, "value", res_draw_count) + except Exception: + logger.exception("Failed persisting clear markers to ResDB") + + # Clear undo/redo stacks for this room + try: + # Clear room-specific undo/redo keys + for pattern in (f"{room_id}:*:undo", f"{room_id}:*:redo"): + for key in redis_client.scan_iter(pattern): + try: + redis_client.delete(key) + except Exception: + pass + + # Clear generic undo/redo keys for this room + for key in redis_client.scan_iter("undo-*"): + try: + data = redis_client.get(key) + if not data: + continue + rec = json.loads(data) + if rec.get("roomId") == room_id: + redis_client.delete(key) + except Exception: + pass + + for key in redis_client.scan_iter("redo-*"): + try: + data = redis_client.get(key) + if not data: + continue + rec = json.loads(data) + if rec.get("roomId") == room_id: + redis_client.delete(key) + except Exception: + pass + except Exception: + logger.exception("Failed clearing undo/redo stacks for room %s", room_id) + + return { + "success": True, + "room_id": room_id, + "deleted_count": deleted_count, + "timestamp": ts, + "draw_count": res_draw_count + } + + except Exception as e: + logger.exception("Failed to clear canvas for room %s", room_id) + return { + "success": False, + "error": str(e), + "room_id": room_id + } diff --git a/backend/services/chatbot_service.py b/backend/services/chatbot_service.py new file mode 100644 index 00000000..9e3e0c2f --- /dev/null +++ b/backend/services/chatbot_service.py @@ -0,0 +1,179 @@ +""" +AI-powered chatbot service for ResCanvas using OpenAI's GPT API. +""" + +import openai +import os +import json +import logging +from datetime import datetime +from .prompt_templates import RESCANVAS_SYSTEM_PROMPT +from .canvas_service import clear_canvas +from .db import chat_history_coll + +logger = logging.getLogger(__name__) + + +def get_chat_history(room_id: str, limit: int = 50): + """ + Retrieve chat history for a room from MongoDB. + + Args: + room_id: The room ID to get history for + limit: Maximum number of messages to retrieve (default 50) + + Returns: + List of chat messages sorted by timestamp (oldest first) + """ + try: + # Query chat history, sorted by timestamp descending, limit results + messages = list(chat_history_coll.find( + {"roomId": room_id}, + {"_id": 0, "sender": 1, "message": 1, "timestamp": 1} + ).sort("timestamp", -1).limit(limit)) + + # Reverse to get chronological order (oldest first) + messages.reverse() + + return messages + except Exception as e: + logger.exception(f"Error retrieving chat history for room {room_id}") + return [] + + +def get_bot_reply(message: str, room_id: str, user_id: str, canvas_context: dict = None): + """ + Generate a bot reply using OpenAI's GPT API with tool calling support. + + Args: + message: The user's message to the bot + room_id: The room ID where the message was sent + user_id: The ID of the user sending the message + canvas_context: Current canvas state (object count, active users, etc.) + + Returns: + A string reply from the bot + """ + # Authentication: Check for OpenAI API key + api_key = os.environ.get('OPENAI_API_KEY') + if not api_key: + return 'AI service is not configured.' + + # Context Construction: Use real canvas context or fallback to defaults + if canvas_context is None: + canvas_context = { + 'room_id': room_id, + 'active_users': [], + 'object_count': 0 + } + + # Message Construction: Build the messages list + messages = [ + { + "role": "system", + "content": RESCANVAS_SYSTEM_PROMPT + }, + { + "role": "system", + "content": f"Current Canvas Context: {json.dumps(canvas_context)}" + }, + { + "role": "user", + "content": message + } + ] + + # Tool Definitions: Define available tools for the AI + tools = [ + { + "type": "function", + "function": { + "name": "clear_canvas", + "description": "Clears all drawings and strokes from the current canvas. Use this when the user explicitly asks to clear or delete everything.", + "parameters": { + "type": "object", + "properties": {}, + "required": [] + } + } + } + ] + + try: + # Save user message to database + timestamp = datetime.utcnow() + try: + chat_history_coll.insert_one({ + "roomId": room_id, + "userId": user_id, + "sender": "user", + "message": message, + "timestamp": timestamp + }) + except Exception as e: + logger.exception(f"Error saving user message to chat history: {e}") + + # API Call: Initialize OpenAI client and make the request with tools + client = openai.Client(api_key=api_key) + response = client.chat.completions.create( + model="gpt-4o-mini", + messages=messages, + tools=tools + ) + + # Tool Call Handling: Check if the AI wants to call a tool + message_response = response.choices[0].message + + bot_reply = None + + if message_response.tool_calls: + # Process each tool call + for tool_call in message_response.tool_calls: + if tool_call.function.name == "clear_canvas": + # Execute the clear canvas function + result = clear_canvas(room_id) + + if result.get("success"): + bot_reply = f"I have cleared the canvas for you. All {result.get('deleted_count', 0)} strokes have been removed." + else: + bot_reply = f"I attempted to clear the canvas, but encountered an error: {result.get('error', 'Unknown error')}" + + # If we processed tool calls but didn't return, fall back to a generic message + if bot_reply is None: + bot_reply = "I've processed your request." + else: + # Extract response content if no tool calls + bot_reply = message_response.content + + # Save bot reply to database + try: + chat_history_coll.insert_one({ + "roomId": room_id, + "userId": "bot", + "sender": "bot", + "message": bot_reply, + "timestamp": datetime.utcnow() + }) + except Exception as e: + logger.exception(f"Error saving bot reply to chat history: {e}") + + return bot_reply + + except Exception as e: + # Graceful error handling + logger.exception(f"Error in get_bot_reply: {e}") + bot_reply = f"I'm having trouble processing your request right now. Please try again later." + + # Try to save error response to history + try: + chat_history_coll.insert_one({ + "roomId": room_id, + "userId": "bot", + "sender": "bot", + "message": bot_reply, + "timestamp": datetime.utcnow() + }) + except Exception: + pass + + return bot_reply diff --git a/backend/services/db.py b/backend/services/db.py index d079a74a..167e10da 100644 --- a/backend/services/db.py +++ b/backend/services/db.py @@ -39,6 +39,7 @@ invites_coll = mongo_client[DB_NAME]["room_invites"] notifications_coll = mongo_client[DB_NAME]["notifications"] stamps_coll = mongo_client[DB_NAME]["stamps"] +chat_history_coll = mongo_client[DB_NAME]["chat_history"] # Analytics collections try: @@ -74,6 +75,7 @@ shares_coll.create_index([("roomId", 1), ("userId", 1)], unique=True) strokes_coll.create_index([("roomId", 1), ("ts", 1)]) stamps_coll.create_index([("user_id", 1), ("deleted", 1)]) +chat_history_coll.create_index([("roomId", 1), ("timestamp", -1)]) def get_db(): """Get database connection""" diff --git a/backend/services/embedding_service.py b/backend/services/embedding_service.py new file mode 100644 index 00000000..aa43855b --- /dev/null +++ b/backend/services/embedding_service.py @@ -0,0 +1,29 @@ +import torch, numpy as np, open_clip +from PIL import Image + +# Check Device +device = "cuda" if torch.cuda.is_available() else "cpu" +# Load model and tokenizer +model, preprocess, _ = open_clip.create_model_and_transforms( + 'ViT-B-32', pretrained='laion2b_s34b_b79k', device=device +) +tokenizer = open_clip.get_tokenizer('ViT-B-32') + +# Set model to evaluation mode +model.eval() + +# Function to generate text embeddings +def embed_text(texts: list[str]) -> np.ndarray: + with torch.no_grad(): + tok = tokenizer(texts) + feats = model.encode_text(tok.to(device)) + feats = feats / feats.norm(dim=-1, keepdim=True) + return feats.cpu().numpy().astype(np.float32) # e.g., (N, 512) + +# Function to generate image embeddings +def embed_image(png_path: str) -> np.ndarray: + img = preprocess(Image.open(png_path)).unsqueeze(0).to(device) + with torch.no_grad(): + feats = model.encode_image(img) + feats = feats / feats.norm(dim=-1, keepdim=True) + return feats.cpu().numpy().astype(np.float32) # (1, 512) diff --git a/backend/services/image_generation_service.py b/backend/services/image_generation_service.py new file mode 100644 index 00000000..01957519 --- /dev/null +++ b/backend/services/image_generation_service.py @@ -0,0 +1,61 @@ + +from config import OPENAI_API_KEY + + +def text_to_image(prompt: str, width: int = 512, height: int = 512, style: str = "default"): + """ + Generate an image for the given prompt. Returns a PIL.Image object on success. + This function attempts to use the OpenAI Images API if available; otherwise, + it falls back to generating a very small placeholder image so the endpoint + returns something useful for UI development. + """ + # Try OpenAI Images (if package available) + try: + from openai import OpenAI + from PIL import Image + import base64 + import io + + client = OpenAI(api_key=OPENAI_API_KEY) + + # Attempt to use images.generate if available on the client + try: + resp = client.images.generate( + model="gpt-image-1", + prompt=prompt, + size=f"{width}x{height}" + ) + # The response may contain base64 data depending on the SDK version + b64 = None + if isinstance(resp, dict) and resp.get("data") and isinstance(resp["data"], list): + item = resp["data"][0] + if isinstance(item, dict) and item.get("b64_json"): + b64 = item.get("b64_json") + if b64: + img_bytes = base64.b64decode(b64) + return Image.open(io.BytesIO(img_bytes)) + except Exception: + # Fall back to other approaches below + pass + + except Exception: + # openai or PIL not available - fall through to placeholder + pass + + # Placeholder fallback: create a simple blank image (Pillow may be missing) + try: + from PIL import Image, ImageDraw, ImageFont + img = Image.new("RGBA", (width, height), (255, 255, 255, 255)) + draw = ImageDraw.Draw(img) + # Draw a small placeholder label in the center if fonts available + try: + f = ImageFont.load_default() + text = "AI\nImage" + w, h = draw.multiline_textsize(text, font=f) + draw.multiline_text(((width - w) / 2, (height - h) / 2), text, fill=(120, 120, 120), font=f, align="center") + except Exception: + pass + return img + except Exception as e: + raise RuntimeError("No image generation backend available: " + str(e)) + diff --git a/backend/services/llm_service.py b/backend/services/llm_service.py new file mode 100644 index 00000000..6a1f5475 --- /dev/null +++ b/backend/services/llm_service.py @@ -0,0 +1,1656 @@ +# pip install openai ollama +import json +import typing + +# === text to drawings ========================================================= +# System prompt +SYSTEM_PROMPT = """ +You are a drawing-command generator for a canvas app. + +Inputs you will be given: +- CanvasState: { "drawings": [ ... ], "bounds": { "width": number, "height": number } } +- UserPrompt: a natural-language scene description + +Goal: +Return a SINGLE JSON object with an "objects" array. Each item is a canvas-ready drawing command that our app can render directly. + +Output (JSON ONLY, no comments, no markdown): +{ + "objects": [ + { + "color": "#RRGGBB", + "lineWidth": number, + "pathData": { + "tool": "shape|freehand", + "type": "rectangle|circle|line|polygon|text|stroke", + // Use one of these geometry encodings (no others): + // For circle/rectangle/line: + "start": {"x": number, "y": number}, + "end": {"x": number, "y": number}, + // For polygon (including triangles): + "points": [ {"x": number, "y": number}, ... ], + // For text: + "text": "string" + // For freehand strokes (preferred for smooth lines): + // use "tool": "freehand", "type": "stroke", + // and provide "points" as an ordered list along the stroke path. + } + } + ] +} + +Rules & Defaults (match our canvas code): +- Use ABSOLUTE pixel coordinates with (0,0) at top-left; all points MUST lie within [0, bounds.width] × [0, bounds.height]. +- Color words → hex (e.g., "red"→"#FF0000", "blue"→"#0000FF"). +- Sizes: tiny=20, small=40, medium=80, large=140, huge=220. For circles, represent size by the distance between start and end (radius as line length). +- Relative positions from the prompt (e.g., "center", "top-right") must be converted to absolute: + center=(W/2,H/2), top-left=(0,0), top=(W/2,0), top-right=(W,0), + left=(0,H/2), right=(W,H/2), bottom-left=(0,H), bottom=(W/2,H), bottom-right=(W,H). + +Style & tool selection: +- Prefer smooth, natural drawings using the freehand brush by default: + - Use "tool": "freehand" and "type": "stroke" with a "points" array that traces the stroke. +- Match the existing canvas style from CanvasState.drawings: + - If drawings are mostly geometric shapes (rectangles, circles, polygons, straight lines), + then also use mostly "shape" commands. + - If drawings are mostly strokes (drawingType === "stroke" or freehand-like paths), + then use mostly freehand strokes. + - If both are present, combine both: + - Use shapes for rigid objects (buildings, cars, roads, UI panels, etc.). + - Use freehand strokes for organic forms (trees, people, animals, clouds) and fine details. +- When following the user’s style, keep lineWidth and overall complexity visually consistent + with the existing drawings. + +Detail & realism: +- Treat each named object as something that should look like it was drawn by an expert. +- Avoid simple, undetailed blocks. Examples: + - A "city" must not just be a few plain rectangles. Use multiple buildings and add windows, + doors, and varied roof lines. + - A "building" should have at least windows and a door, plus simple roof or edge details. + - A "car" should at least show body, wheels, windows, and a hint of lights or motion. +- For complex or important objects (cities, buildings, cars, trees, faces, + people): + - Break them into several shapes and/or strokes (roughly 3–8 primitives per main object). + - Add visible details using either small shapes or short freehand strokes. +- Keep the total number of objects modest: enough to look like a clean expert sketch, + not hundreds of tiny primitives. + +When CanvasState is provided: +- Avoid obvious overlaps with existing content unless the prompt demands it (e.g., “on top of…”). +- Keep new objects visually distinct (slight offsets are OK when crowded). +- Respect the existing composition: do not cover up important existing drawings unless + the prompt says to replace or draw over something. + +Content fidelity: +- Include EVERY explicitly mentioned object; respect counts, colors, sizes, and spatial relations. +- If motion/action is described, suggest simple visual cues (e.g., angled line, small polygon “arrow”, or secondary object) using primitives or strokes. +- If ambiguous, choose a common-sense default and continue. + +Constraints: +- Output MUST be valid JSON matching the schema above. Do not include IDs (the app assigns them). +- Keep a modest number of objects (clear but not cluttered). +""" + +# Few-shot to stabilize canvas-native formatting +FEWSHOT_USER_1 = """ +CanvasState: {"drawings":[],"bounds":{"width":1800,"height":800}} +UserPrompt: draw a small blue circle at the top-right +""" + +FEWSHOT_ASSISTANT_JSON_1 = { + "objects": [ + { + "color": "#0000FF", + "lineWidth": 2, + "pathData": { + "tool": "shape", + "type": "circle", + "start": {"x": 2900, "y": 100}, + "end": {"x": 2940, "y": 100}, + }, + } + ] +} + +FEWSHOT_USER_2 = """ +CanvasState: +{"drawings":[],"bounds":{"width":1800,"height":800}} +UserPrompt: +"draw a red car driving in the woods" +""" + +FEWSHOT_ASSISTANT_JSON_2 = { + "objects": [ + { + "color": "#228B22", + "lineWidth": 2, + "pathData": { + "tool": "shape", + "type": "polygon", + "points": [ + {"x": 600, "y": 1050}, + {"x": 650, "y": 950}, + {"x": 700, "y": 1050}, + ], + }, + }, + { + "color": "#8B4513", + "lineWidth": 2, + "pathData": { + "tool": "shape", + "type": "rectangle", + "start": {"x": 645, "y": 1050}, + "end": {"x": 655, "y": 1100}, + }, + }, + { + "color": "#228B22", + "lineWidth": 2, + "pathData": { + "tool": "shape", + "type": "polygon", + "points": [ + {"x": 2300, "y": 1000}, + {"x": 2350, "y": 900}, + {"x": 2400, "y": 1000}, + ], + }, + }, + { + "color": "#8B4513", + "lineWidth": 2, + "pathData": { + "tool": "shape", + "type": "rectangle", + "start": {"x": 2345, "y": 1000}, + "end": {"x": 2355, "y": 1050}, + }, + }, + { + "color": "#555555", + "lineWidth": 6, + "pathData": { + "tool": "shape", + "type": "line", + "start": {"x": 400, "y": 1400}, + "end": {"x": 2600, "y": 1500}, + }, + }, + { + "color": "#FF0000", + "lineWidth": 2, + "pathData": { + "tool": "shape", + "type": "rectangle", + "start": {"x": 1450, "y": 1380}, + "end": {"x": 1650, "y": 1450}, + }, + }, + { + "color": "#FF0000", + "lineWidth": 2, + "pathData": { + "tool": "shape", + "type": "polygon", + "points": [ + {"x": 1500, "y": 1380}, + {"x": 1600, "y": 1380}, + {"x": 1550, "y": 1340}, + ], + }, + }, + { + "color": "#000000", + "lineWidth": 2, + "pathData": { + "tool": "shape", + "type": "circle", + "start": {"x": 1500, "y": 1450}, + "end": {"x": 1520, "y": 1450}, + }, + }, + { + "color": "#000000", + "lineWidth": 2, + "pathData": { + "tool": "shape", + "type": "circle", + "start": {"x": 1600, "y": 1450}, + "end": {"x": 1620, "y": 1450}, + }, + }, + { + "color": "#000000", + "lineWidth": 2, + "pathData": { + "tool": "shape", + "type": "line", + "start": {"x": 1420, "y": 1415}, + "end": {"x": 1450, "y": 1400}, + }, + }, + # Example freehand stroke for grass/detail under the car + { + "color": "#006400", + "lineWidth": 3, + "pathData": { + "tool": "freehand", + "type": "stroke", + "points": [ + {"x": 1400, "y": 1505}, + {"x": 1450, "y": 1498}, + {"x": 1500, "y": 1502}, + {"x": 1550, "y": 1496}, + {"x": 1600, "y": 1500}, + ], + }, + }, + ] +} + +FEWSHOT_USER_3 = """ +CanvasState: +{ + "drawings": [ + {"color":"#8B4513","lineWidth":2,"pathData":{"tool":"shape","type":"rectangle","start":{"x":1400,"y":1200},"end":{"x":1600,"y":1270}}}, + {"color":"#FF0000","lineWidth":2,"pathData":{"tool":"shape","type":"polygon","points":[{"x":1400,"y":1200},{"x":1500,"y":1120},{"x":1600,"y":1200}]}} + ], + "bounds":{"width":1800,"height":800} +} +UserPrompt: +"add a blue window to the right of the house" +""" + +FEWSHOT_ASSISTANT_JSON_3 = { + "objects": [ + { + "color": "#0000FF", + "lineWidth": 2, + "pathData": { + "tool": "shape", + "type": "rectangle", + "start": {"x": 1650, "y": 1210}, + "end": {"x": 1690, "y": 1245}, + }, + } + ] +} + + +def _get_text_to_drawings_initial_message( + prompt: str, canvasState: dict[str, typing.Any] +) -> list[dict]: + """ + Build the minimal, few-shot seeded chat message list for the + text→drawing JSON parser. + + Args: + prompt: The end-user natural language description (e.g., "draw a small + blue circle"). + canvasState (dict[str, Any]): + A Python dictionary representing the current state of the canvas. + + Returns: + A list of role/content dicts suitable for OpenAI/Ollama chat APIs: + [system, user(few-shot), assistant(few-shot), user(actual prompt)]. + """ + canvas_json = json.dumps(canvasState, separators=(",", ":")) + + # Combine into a single message for the model + user_prompt = ( + f"CanvasState:\n{canvas_json}\n" + f"UserPrompt:\nDescribe all drawing commands (shapes and freehand strokes) " + f"needed to draw this scene: {prompt}" + ) + + return [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": FEWSHOT_USER_1}, + {"role": "assistant", "content": json.dumps(FEWSHOT_ASSISTANT_JSON_1)}, + {"role": "user", "content": FEWSHOT_USER_2}, + {"role": "assistant", "content": json.dumps(FEWSHOT_ASSISTANT_JSON_2)}, + {"role": "user", "content": FEWSHOT_USER_3}, + {"role": "assistant", "content": json.dumps(FEWSHOT_ASSISTANT_JSON_3)}, + {"role": "user", "content": user_prompt}, + ] + + +def openai_prompt_to_json(prompt: str, canvasState: dict[str, typing.Any]) -> dict: + """ + Convert a natural-language drawing prompt into structured JSON + using the OpenAI GPT-4o-mini model. + + Args: + prompt: The user's text prompt describing the drawing. + + Returns: + Dict containing parsed drawing attributes or an error payload. + """ + try: + from config import OPENAI_API_KEY + from openai import OpenAI + + if not OPENAI_API_KEY: + return {"error": "openai_not_configured", "detail": "OPENAI_API_KEY is not set in environment"} + + client = OpenAI(api_key=OPENAI_API_KEY) + + resp = client.chat.completions.create( + model="gpt-4o-mini", + response_format={"type": "json_object"}, # forces valid JSON + temperature=0.1, + messages=_get_text_to_drawings_initial_message(prompt, canvasState), + max_tokens=5000, + ) + content = resp.choices[0].message.content + return json.loads(content) + except Exception as e: + return {"error": "openai_failed", "detail": str(e)} + + +def ollama_prompt_to_json(prompt: str, canvasState: dict[str, typing.Any]) -> dict: + """ + Convert a natural-language drawing prompt into structured JSON + using a locally hosted Ollama model as a fallback. + + Args: + prompt: The user's text prompt describing the drawing. + + Returns: + Dict containing parsed drawing attributes or an error payload. + """ + try: + import ollama + + response = ollama.chat( + model="llama3:8b", + messages=_get_text_to_drawings_initial_message(prompt, canvasState), + ) + + return json.loads(response["message"]["content"]) + except Exception as e: + return {"error": "ollama_failed", "detail": str(e)} + + +def prompt_to_drawings(prompt: str, canvasState: dict[str, typing.Any]) -> dict: + """ + Route a drawing prompt to OpenAI first, then fall back to Ollama + if the cloud model fails. Guarantees a dictionary response. + + Args: + prompt: The user's text prompt describing the drawing. + + Returns: + Dict containing parsed drawing attributes or an error payload. + """ + model_output = openai_prompt_to_json(prompt, canvasState) + + # If user setup openai API's properly and no errors + # occured, return the model's output + if "error" not in model_output: + return model_output + + # Fallback + fallback_model_output = ollama_prompt_to_json(prompt, canvasState) + return fallback_model_output + + +# === Shape Completion ========================================================= +SHAPE_COMPLETION_SYSTEM = """ +You are a drawing intent and completion engine for a canvas app. + +You receive a CanvasState JSON object with: +- bounds: { "width": number, "height": number } +- drawings: array of drawing objects; the last one(s) are often the user's most recent strokes. + Each drawing has fields like: + - color: "#RRGGBB" + - lineWidth: number + - pathData: + For freehand strokes: + { "tool": "freehand", "type": "stroke", + "points": [ { "x": number, "y": number }, ... ] } + For geometric shapes: + { "tool": "shape", "type": "line|rectangle|circle|polygon|text", + "start": { "x": number, "y": number }, + "end": { "x": number, "y": number }, + "points": [ { "x": number, "y": number }, ... ], + "text": "optional" } + +GOAL +1. First, infer what the user is trying to draw at a higher level: + - Are they sketching a recognizable object (e.g., tree, house, car, plane, star, person, cloud)? + - Or are they just drawing an abstract or standalone geometric shape (line, rectangle, circle, polygon)? +2. Then, infer the SINGLE most likely next primitive that would continue or complete that intent, + matching the user's current drawing style: + - If their recent drawings are mainly freehand strokes: + → Predict the next stroke as a freehand stroke (tool = "freehand", type = "stroke"). + - If their recent drawings are mainly shapes: + → Predict the next geometric shape (tool = "shape"). +3. Always output ONE object that can be used as a "ghost" suggestion of what to draw next. + +OUTPUT FORMAT (JSON ONLY, no comments, no markdown): +{ + "complete": true|false, + "confidence": number, // 0.0–1.0 + "object": { + "color": "#RRGGBB", + "lineWidth": number, + "pathData": { + "tool": "shape|freehand", + "type": "line|circle|rectangle|polygon|stroke|text", + "start": { "x": number, "y": number }, + "end": { "x": number, "y": number }, + "points": [ { "x": number, "y": number }, ... ], + "text": "string" + } + } +} + +STYLE MATCHING +- Look at the LAST few drawings in CanvasState.drawings. +- If most of them use { "tool": "freehand", "type": "stroke" }, + then your suggestion must also be a freehand stroke with a "points" array. +- If most of them use { "tool": "shape", ... }, + then your suggestion must be a geometric shape (line, rectangle, circle, polygon, or text). +- Preserve the approximate lineWidth and color of the user's most recent drawing. + +SCALE & EXTENT (VERY IMPORTANT) +- Your suggestion should be a VISIBLE continuation, not a tiny jitter. +- Estimate the size of the user's most recent stroke or shape (its bounding box or start–end distance). +- For freehand strokes: + - Make the new stroke span a similar scale (roughly 50%–150% of the last stroke's span). + - Avoid strokes whose bounding box width AND height are both very small (e.g., less than ~20 pixels) + unless ALL of the user's recent strokes are that small. + - Prefer 8–30 points for a typical suggested stroke so it feels like a substantial continuation, + not just a tiny segment. +- For shapes: + - Suggested lines, rectangles, circles, or polygons should have a meaningful size as well, + comparable to the existing elements they are extending. + - Do NOT suggest micro-lines or tiny shapes unless the entire drawing is made of such tiny elements. + +SEMANTIC INTENT +- Try to recognize common objects from the partial sketch: tree, car, house, plane, star, cloud, person, etc. +- If you can infer a likely object: + - For a tree: you might add more foliage strokes, the trunk, or branches. + - For a house: you might add the roof, door, or window. + - For a car: you might add wheels, windows, or body details. +- If the sketch is too ambiguous or looks abstract: + - Focus on geometric completion: straightening or extending a line, + closing a polygon, or completing a circle/rectangle. + +GEOMETRY AND BOUNDS +- Use ABSOLUTE pixel coordinates within [0, bounds.width] × [0, bounds.height], + with (0,0) at the top-left. +- For shapes: + - line/rectangle/circle must include "start" and "end". + - polygon must include "points". +- For freehand strokes: + - Provide a "points" array with an ordered path for the stroke. + - Points should form a smooth, coherent segment that clearly continues the drawing. + +CONFIDENCE AND COMPLETENESS +- Use "confidence" to express how sure you are about the user's intent. +- If you are very unsure (confidence < 0.4): + - Set "complete": false. + - Still return your best-effort next primitive so the UI can show a light ghost suggestion. +- If the suggestion would clearly complete a part of the object (e.g., final wheel, final edge, roof line): + - You may set "complete": true for that part, even if the whole scene is not finished. + +COLOR AND WIDTH +- Default color: use the color of the user's last drawing if available; otherwise "#000000". +- Default lineWidth: match the user's last drawing's lineWidth, or use 2 if missing. + +CONSTRAINTS +- Output MUST be valid JSON and MUST match the schema above. +- Do NOT output explanations, natural language, or multiple objects. +- Always return a single best "object" that predicts the next stroke or shape. +""" + + +SHAPE_COMPLETION_FEWSHOT_USER_1 = """ +CanvasState: +{ + "drawings": [ + { + "color": "#228B22", + "lineWidth": 3, + "pathData": { + "tool": "freehand", + "type": "stroke", + "points": [ + {"x": 300, "y": 200}, + {"x": 340, "y": 180}, + {"x": 380, "y": 210}, + {"x": 360, "y": 240}, + {"x": 320, "y": 230}, + {"x": 300, "y": 200} + ] + } + } + ], + "bounds": {"width":1200,"height":800} +} +""" + +SHAPE_COMPLETION_FEWSHOT_ASSISTANT_JSON_1 = { + "complete": False, + "confidence": 0.78, + "object": { + "color": "#228B22", + "lineWidth": 3, + "pathData": { + "tool": "freehand", + "type": "stroke", + "points": [ + {"x": 340, "y": 220}, + {"x": 380, "y": 230}, + {"x": 410, "y": 210}, + {"x": 400, "y": 180}, + {"x": 370, "y": 170}, + {"x": 340, "y": 180} + ] + }, + }, +} + + +SHAPE_COMPLETION_FEWSHOT_USER_2 = """ +CanvasState: +{ + "drawings": [ + { + "color": "#8B4513", + "lineWidth": 2, + "pathData": { + "tool": "shape", + "type": "rectangle", + "start": {"x": 400, "y": 300}, + "end": {"x": 600, "y": 450} + } + }, + { + "color": "#8B0000", + "lineWidth": 2, + "pathData": { + "tool": "shape", + "type": "polygon", + "points": [ + {"x": 400, "y": 300}, + {"x": 500, "y": 220}, + {"x": 600, "y": 300} + ] + } + } + ], + "bounds": {"width":1200,"height":800} +} +""" + +SHAPE_COMPLETION_FEWSHOT_ASSISTANT_JSON_2 = { + "complete": False, + "confidence": 0.85, + "object": { + "color": "#654321", + "lineWidth": 2, + "pathData": { + "tool": "shape", + "type": "rectangle", + "start": {"x": 470, "y": 360}, + "end": {"x": 530, "y": 450} + }, + }, +} + +SHAPE_COMPLETION_FEWSHOT_USER_3 = """ +CanvasState: +{ + "drawings": [ + { + "color": "#FF0000", + "lineWidth": 3, + "pathData": { + "tool": "freehand", + "type": "stroke", + "points": [ + {"x": 600, "y": 500}, + {"x": 650, "y": 480}, + {"x": 720, "y": 460}, + {"x": 800, "y": 460}, + {"x": 880, "y": 480}, + {"x": 930, "y": 510} + ] + } + } + ], + "bounds": {"width":1800,"height":800} +} +""" + +SHAPE_COMPLETION_FEWSHOT_ASSISTANT_JSON_3 = { + "complete": False, + "confidence": 0.70, + "object": { + "color": "#000000", + "lineWidth": 3, + "pathData": { + "tool": "freehand", + "type": "stroke", + "points": [ + {"x": 680, "y": 510}, + {"x": 700, "y": 540}, + {"x": 730, "y": 550}, + {"x": 760, "y": 540}, + {"x": 780, "y": 510} + ] + }, + }, +} + +def _get_shape_completion_initial_message( + canvas_state: dict[str, typing.Any] +) -> list[dict]: + """ + Build the few-shot seeded chat messages for shape completion. + + Args: + canvas_state (dict[str, Any]): + The current canvas state. Expected keys: + - "drawings": list of existing drawings (color, lineWidth, pathData, etc.) + - "bounds": { "width": number, "height": number } + + Returns: + list[dict]: Chat messages for OpenAI/Ollama APIs: + [system, user(few-shot), assistant(few-shot), user(few-shot), assistant(few-shot), user(actual)] + """ + canvas_json = json.dumps(canvas_state, separators=(",", ":")) + user_msg = f"CanvasState:\n{canvas_json}" + + return [ + {"role": "system", "content": SHAPE_COMPLETION_SYSTEM}, + {"role": "user", "content": SHAPE_COMPLETION_FEWSHOT_USER_1}, + {"role": "assistant", "content": json.dumps(SHAPE_COMPLETION_FEWSHOT_ASSISTANT_JSON_1)}, + {"role": "user", "content": SHAPE_COMPLETION_FEWSHOT_USER_2}, + {"role": "assistant", "content": json.dumps(SHAPE_COMPLETION_FEWSHOT_ASSISTANT_JSON_2)}, + {"role": "user", "content": SHAPE_COMPLETION_FEWSHOT_USER_3}, + {"role": "assistant", "content": json.dumps(SHAPE_COMPLETION_FEWSHOT_ASSISTANT_JSON_3)}, + {"role": "user", "content": user_msg}, + ] + + +def openai_complete_shape(canvas_state: dict) -> dict: + """ + Infer and complete a likely shape from the current partial input using OpenAI. + + Args: + canvas_state (dict): Current canvas (drawings + bounds). + + Returns: + dict: { complete, confidence, object{ color, lineWidth, pathData{...} } } or error payload. + """ + try: + from config import OPENAI_API_KEY + from openai import OpenAI + + client = OpenAI(api_key=OPENAI_API_KEY) + + resp = client.chat.completions.create( + model="gpt-4.1-mini", + response_format={"type": "json_object"}, + temperature=0.1, + messages=_get_shape_completion_initial_message(canvas_state), + max_tokens=220, + ) + return json.loads(resp.choices[0].message.content) + except Exception as e: + return {"error": "openai_completion_failed", "detail": str(e)} + + +def ollama_complete_shape(canvas_state: dict) -> dict: + """ + Infer and complete a likely shape from the current partial input using Ollama. + + Args: + canvas_state (dict): Current canvas (drawings + bounds). + + Returns: + dict: { complete, confidence, object{ color, lineWidth, pathData{...} } } or error payload. + """ + try: + import ollama + + response = ollama.chat( + model="llama3:8b", + messages=_get_shape_completion_initial_message(canvas_state), + ) + return json.loads(response["message"]["content"]) + except Exception as e: + return {"error": "ollama_completion_failed", "detail": str(e)} + + +def complete_shape_from_canvas(canvas_state: dict) -> dict: + """ + Perform AI-based shape completion using OpenAI first, then Ollama. + + Args: + canvas_state (dict): Current canvas (drawings + bounds). + + Returns: + dict: Inferred shape completion result. + """ + model_output = openai_complete_shape(canvas_state) + if "error" not in model_output: + return model_output + return ollama_complete_shape(canvas_state) + + +# === Canvas Beautification (canvas-state) ====================== +BEAUTIFY_SYSTEM_PROMPT = """ +You are a sketch beautifier for a canvas drawing app. + +You receive a CanvasState JSON object with: +- width: number +- height: number +- objects: array of drawing objects, each with: + - id: string + - color: "#RRGGBB" + - lineWidth: number + - pathData: + For freehand strokes: + { + "tool": "freehand", + "type": "stroke", + "points": [ { "x": number, "y": number }, ... ] + } + For geometric shapes: + { + "tool": "shape", + "type": "line|rectangle|circle|polygon|text", + "start": { "x": number, "y": number }, + "end": { "x": number, "y": number }, + "points": [ { "x": number, "y": number }, ... ], + "text": "optional" + } + +GOAL +Transform the input CanvasState into a BEAUTIFIED version of the same drawing. +- Keep the overall composition, layout, and intent the same. +- Make the drawing look smoother, cleaner, and more deliberate. +- Always return your highest-quality beautification. + +OUTPUT FORMAT (JSON ONLY, no comments, no markdown): +{ + "objects": [ + { + "id": "string", + "color": "#RRGGBB", + "lineWidth": number, + "pathData": { + "tool": "shape|freehand", + "type": "line|rectangle|circle|polygon|stroke|text", + "start": { "x": number, "y": number }, + "end": { "x": number, "y": number }, + "points": [ { "x": number, "y": number }, ... ], + "text": "string" + } + }, + ... + ] +} + +BEAUTIFICATION RULES + +PRESERVE INTENT +- Do NOT change what the user is drawing: a tree must remain a tree, a car remains a car, a house remains a house, etc. +- Do NOT radically move objects: positions should remain similar; small adjustments to align or straighten are allowed. +- Keep overall proportions and relative sizes of parts (e.g., door vs house, wheels vs car body). + +STROKE SMOOTHING (FREEHAND) +- For freehand strokes (tool = "freehand", type = "stroke"): + - Remove jitter and noise; smooth the path into more confident curves and lines. + - Use a reasonable number of points: not too sparse and not excessively dense. + In general, 16–64 points per long stroke is enough. + - Ensure the stroke flows smoothly with consistent direction and curvature. + - Preserve the approximate start and end positions and overall shape of the stroke. + +GEOMETRIC CLEANUP (SHAPES) +- For lines, rectangles, circles, and polygons (tool = "shape"): + - Straighten almost-straight lines. + - Regularize rectangles so opposite sides are parallel and corners are clean. + - Regularize circles or ellipses to look smooth and round. + - Clean polygon vertices so angles look intentional, not wobbly. +- You MAY, when appropriate, upgrade a clearly intended shape drawn as a messy stroke + into a cleaner geometric shape (e.g., a wobbly "shape" polygon into a neat rectangle), + as long as the user's intent is obvious and the style of the rest of the drawing is respected. + +STYLE PRESERVATION +- Maintain the existing color palette and lineWidth relationships. +- Do NOT randomly change colors. +- Line widths can be slightly adjusted for consistency, but must feel similar to the original. +- If the whole drawing is sketchy and loose, keep a sketchy-but-clean look rather than making it fully technical or CAD-like. + +GLOBAL CONSISTENCY +- Objects that belong together (e.g., house and roof, car body and wheels, tree trunk and foliage) + should remain visually aligned and coherent after beautification. +- You may slightly align related parts (e.g., windows in a row, wheels centered vertically) if it improves cleanliness without changing the composition. + +CONSTRAINTS +- You must return a JSON object with an "objects" array using the same schema as above. +- The number of objects should usually be similar to the input; you may split or merge strokes when it clearly improves the visual quality, but do not randomly add or remove important elements. +- Do NOT output explanations, natural language, or extra fields. +- Do NOT leave the drawing partially processed: every object should be beautified as needed. +""" + +BEAUTIFY_FEWSHOT_USER_1 = """ +CanvasState: +{ + "width": 800, + "height": 600, + "objects": [ + { + "id": "stroke1", + "color": "#000000", + "lineWidth": 3, + "pathData": { + "tool": "freehand", + "type": "stroke", + "points": [ + {"x": 100, "y": 300}, + {"x": 130, "y": 295}, + {"x": 160, "y": 290}, + {"x": 190, "y": 292}, + {"x": 220, "y": 300}, + {"x": 250, "y": 310}, + {"x": 280, "y": 315} + ] + } + } + ] +} +""" + +BEAUTIFY_FEWSHOT_ASSISTANT_JSON_1 = { + "objects": [ + { + "id": "stroke1", + "color": "#000000", + "lineWidth": 3, + "pathData": { + "tool": "freehand", + "type": "stroke", + "points": [ + {"x": 100, "y": 300}, + {"x": 130, "y": 295}, + {"x": 160, "y": 292}, + {"x": 190, "y": 295}, + {"x": 220, "y": 302}, + {"x": 250, "y": 310}, + {"x": 280, "y": 315} + ] + } + } + ] +} + + +BEAUTIFY_FEWSHOT_USER_2 = """ +CanvasState: +{ + "width": 800, + "height": 600, + "objects": [ + { + "id": "rect1", + "color": "#333333", + "lineWidth": 2, + "pathData": { + "tool": "shape", + "type": "rectangle", + "start": {"x": 200, "y": 200}, + "end": {"x": 400, "y": 320} + } + } + ] +} +""" + +BEAUTIFY_FEWSHOT_ASSISTANT_JSON_2 = { + "objects": [ + { + "id": "rect1", + "color": "#333333", + "lineWidth": 2, + "pathData": { + "tool": "shape", + "type": "rectangle", + "start": {"x": 200, "y": 200}, + "end": {"x": 400, "y": 320} + } + } + ] +} + + +def _get_beautify_canvas_initial_message( + canvas_state: dict[str, typing.Any] +) -> list[dict]: + """ + Build few-shot seeded messages for beautification. + """ + canvas_json = json.dumps(canvas_state, ensure_ascii=False) + + return [ + {"role": "system", "content": BEAUTIFY_SYSTEM_PROMPT}, + {"role": "user", "content": BEAUTIFY_FEWSHOT_USER_1}, + {"role": "assistant", "content": json.dumps(BEAUTIFY_FEWSHOT_ASSISTANT_JSON_1)}, + {"role": "user", "content": BEAUTIFY_FEWSHOT_USER_2}, + {"role": "assistant", "content": json.dumps(BEAUTIFY_FEWSHOT_ASSISTANT_JSON_2)}, + {"role": "user", "content": f"CanvasState:\n{canvas_json}"}, + ] + +def openai_beautify_canvas( + canvas_state: dict[str, typing.Any], +) -> dict: + """ + Beautify the canvas using OpenAI. Returns either: + { "objects": [...] } + or + { "error": "...", "detail": "..." } + """ + try: + from config import OPENAI_API_KEY + from openai import OpenAI + + client = OpenAI(api_key=OPENAI_API_KEY) + + resp = client.chat.completions.create( + model="gpt-4.1-mini", + response_format={"type": "json_object"}, # forces JSON + temperature=0.1, + messages=_get_beautify_canvas_initial_message(canvas_state), + max_tokens=10000, + ) + + content = resp.choices[0].message.content + + print(f"\n\n{content}\n\n") + parsed = json.loads(content) + print(f"\n\n{parsed}\n\n") + + return parsed + + except Exception as e: + return {"error": "openai_beautify_failed", "detail": str(e)} + + +def ollama_beautify_canvas( + canvas_state: dict[str, typing.Any] +) -> dict: + """ + Beautify the canvas using a local Ollama model. Same contract as + openai_beautify_canvas: either { "objects": [...] } or { "error": ... }. + """ + try: + import ollama + + response = ollama.chat( + model="llama3:8b", + messages=_get_beautify_canvas_initial_message(canvas_state), + ) + + parsed = json.loads(response["message"]["content"]) + + if not isinstance(parsed, dict) or "objects" not in parsed: + return { + "error": "ollama_beautify_invalid_output", + "detail": "Missing 'objects' field in model response.", + } + + if not isinstance(parsed["objects"], list): + return { + "error": "ollama_beautify_invalid_output", + "detail": "'objects' is not a list in model response.", + } + + return parsed + + except Exception as e: + return {"error": "ollama_beautify_failed", "detail": str(e)} + + +def beautify_canvas_state( + canvas_state: dict[str, typing.Any] +) -> dict: + """ + Perform AI-based canvas beautification with rollback, following the + same pattern as prompt_to_drawings and complete_shape_from_canvas. + + Order: + 1) Try OpenAI. + 2) If that fails or returns invalid output, try Ollama. + 3) If both fail, ROLLBACK to the ORIGINAL drawings and return: + { "objects": canvas_state.drawings } + + Returns: + dict with at least: + { "objects": [...] } + """ + # Primary: OpenAI + model_output = openai_beautify_canvas(canvas_state) + if "error" not in model_output and "objects" in model_output: + return model_output + + print(f"\n\nFAILED OPENAI API!: {model_output} \n\n") + + # Fallback: Ollama + fallback_output = ollama_beautify_canvas(canvas_state) + if "error" not in fallback_output and "objects" in fallback_output: + return fallback_output + + # Rollback: both failed => return original drawings as objects + original_drawings = canvas_state.get("objects", []) + return {"objects": original_drawings} +# === Style Transfer (apply an artistic style to an existing canvas) ========== +STYLE_TRANSFER_SYSTEM = """ +You are an artistic style transfer engine for a canvas app. + +Inputs: +- CanvasState: { "width": number, "height": number, "objects": [ { id, color, lineWidth, pathData, ... } ] } +- StylePrompt: short natural language description of the style to apply (e.g. "Van Gogh oil painting", "watercolor sketch", "8-bit pixel art"). + +Goal: +Return a JSON object with an "objects" array representing the same scene but restyled to match the StylePrompt. +You may output rasterized image objects by returning objects with { "drawingType":"image", "imageDataUrl": "data:image/png;base64,...", "x":0, "y":0, "width":W, "height":H }. +Prefer returning vector-like modifications (colors, stroke styles, simplified geometry) when possible. + +Output (JSON ONLY): +{ + "objects": [ ... ] +} + +Constraints: +- Keep the same composition and relative positions. Do not invent new major scene elements. +- Output valid JSON. The app will accept either vector objects (shape/freehand) or image objects with data URLs. + +Renderer-capabilities and metadata guidance for the model: + +When producing vector strokes/objects, you SHOULD (when appropriate) include an optional `metadata` object on each returned object describing how the canvas renderer should display the primitive using ResCanvas features. Allowed metadata fields and helper functions: + +- `drawingType`: "stroke" | "image" | "stamp" (default: "stroke") +- `brushType`: string (one of: "normal", "wacky", "drip", "scatter", "neon", "chalk", "spray", "mixed") +- `brushParams`: object (tool-specific parameters, e.g. { "scatterAmount": 0.3, "texture":"thick", "mixColors": ["#FFCC33","#FF9900"] }) +- `stampData`: object (for stamps/images: { "imageDataUrl": string, "x": number, "y": number, "width": number, "height": number }) + +Additionally, you may think in terms of small renderer functions the frontend provides; include which function you would like to use by setting `metadata.brushType` (or `drawingType:"stamp"` + `stampData`). Example functions available to you: + +- Brush(brushType, brushParams): draws strokes with the named brush and parameters. +- MixedColor(colors[]): blends several palette colors for richer strokes. +- Stamp(imageDataUrl, x, y, width, height): places a raster/stamp element. + +If the model cannot produce a full vector restyling, it may return a single image object with a data URL (drawingType: "image"). Prefer vector output when possible. If you include `brushType` and `brushParams`, the frontend will attempt to render strokes using the project's brush implementations. +""" + +# Few-shot example to show how to emit metadata for a Van Gogh oil-painting style +FEWSHOT_STYLE_USER_1 = """ +CanvasState: +{ + "objects": [ + {"color":"#FFD700","lineWidth":4,"pathData":{"tool":"shape","type":"circle","start":{"x":1600,"y":80},"end":{"x":1640,"y":80}}} + ], + "width":1800, + "height":800 +} +StylePrompt: +Van Gogh oil painting +""" + +FEWSHOT_STYLE_ASSISTANT_JSON_1 = { + "objects": [ + { + "color": "#FFCC33", + "lineWidth": 5, + "pathData": { + "tool": "freehand", + "type": "stroke", + "points": [ + {"x":1590, "y":70}, + {"x":1605, "y":60}, + {"x":1620, "y":70}, + {"x":1635, "y":90}, + {"x":1645, "y":85} + ] + }, + "metadata": { + "drawingType": "stroke", + "brushType": "wacky", + "brushParams": { + "texture": "thick", + "mixColors": ["#FFCC33", "#FF9900", "#FFFF66"], + "opacity": 0.9 + } + } + } + ] +} + +# Few-shot 2: watercolor / wash example that prefers soft scatter spray brush +FEWSHOT_STYLE_USER_2 = """ +CanvasState: +{ + "objects": [ + {"color":"#228B22","lineWidth":3,"pathData":{"tool":"freehand","type":"stroke","points":[{"x":200,"y":300},{"x":260,"y":280}]}} + ], + "width":1200, + "height":800 +} +StylePrompt: +watercolor wash +""" + +FEWSHOT_STYLE_ASSISTANT_JSON_2 = { + "objects": [ + { + "color": "#2E8B57", + "lineWidth": 4, + "pathData": { + "tool": "freehand", + "type": "stroke", + "points": [ + {"x":195, "y":295}, + {"x":225, "y":285}, + {"x":255, "y":290} + ] + }, + "metadata": { + "drawingType": "stroke", + "brushType": "spray", + "brushParams": {"opacity": 0.55, "scatterAmount": 0.25} + } + } + ] +} + +# Few-shot 3: stamp example (rasterized element) demonstrating stampData +FEWSHOT_STYLE_USER_3 = """ +CanvasState: +{ + "objects": [ + {"color":"#8B4513","lineWidth":2,"pathData":{"tool":"shape","type":"rectangle","start":{"x":400,"y":300},"end":{"x":600,"y":450}}} + ], + "width":1200, + "height":800 +} +StylePrompt: +children sticker stamps +""" + +FEWSHOT_STYLE_ASSISTANT_JSON_3 = { + "objects": [ + { + "drawingType": "image", + "imageDataUrl": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA", + "x": 420, + "y": 320, + "width": 80, + "height": 80, + "metadata": { + "drawingType": "stamp", + "stampData": { + "imageDataUrl": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA", + "x": 420, + "y": 320, + "width": 80, + "height": 80 + } + } + } + ] +} + + +def _get_style_transfer_message(canvas_state: dict, style_prompt: str) -> list[dict]: + canvas_json = json.dumps(canvas_state, ensure_ascii=False) + user_msg = f"CanvasState:\n{canvas_json}\nStylePrompt:\n{style_prompt}" + return [ + {"role": "system", "content": STYLE_TRANSFER_SYSTEM}, + {"role": "user", "content": FEWSHOT_STYLE_USER_1}, + {"role": "assistant", "content": json.dumps(FEWSHOT_STYLE_ASSISTANT_JSON_1)}, + {"role": "user", "content": user_msg}, + ] + + +def openai_style_transfer(canvas_state: dict, style_prompt: str) -> dict: + try: + from config import OPENAI_API_KEY + from openai import OpenAI + + client = OpenAI(api_key=OPENAI_API_KEY) + + resp = client.chat.completions.create( + model="gpt-4.1-mini", + response_format={"type": "json_object"}, + temperature=0.2, + messages=_get_style_transfer_message(canvas_state, style_prompt), + max_tokens=8000, + ) + + content = resp.choices[0].message.content + return json.loads(content) + except Exception as e: + return {"error": "openai_style_failed", "detail": str(e)} + + +def ollama_style_transfer(canvas_state: dict, style_prompt: str) -> dict: + try: + import ollama + + response = ollama.chat( + model="llama3:8b", + messages=_get_style_transfer_message(canvas_state, style_prompt), + ) + + return json.loads(response["message"]["content"]) + except Exception as e: + return {"error": "ollama_style_failed", "detail": str(e)} + + +def _map_style_to_brush(style_prompt: str) -> tuple[str, dict]: + """Map style prompts to brush types and parameters.""" + s = (style_prompt or "").lower() + brush = "normal" + params: dict = {} + + if "watercolor" in s or "wash" in s: + brush = "spray" + params = {"opacity": 0.6, "scatterAmount": 0.2} + elif "van gogh" in s or "oil" in s or "impasto" in s: + # Van Gogh / oil styles benefit from mixed, textured strokes. Use + # a `mixed` brushType with a base `wacky` texture and a small + # palette to blend for thicker impasto-like strokes. + brush = "mixed" + params = { + "base": "wacky", + "texture": "thick", + "mixColors": ["#FFCC33", "#FF9900", "#FFFF66"], + "opacity": 0.9, + "mixAmount": 0.6, + } + elif "neon" in s or "glow" in s: + brush = "neon" + params = {"glow": True, "intensity": 0.9} + elif "chalk" in s or "pastel" in s: + brush = "chalk" + params = {"grain": 0.6} + elif "spray" in s or "splatter" in s: + brush = "spray" + params = {"scatterAmount": 0.5} + elif "drip" in s: + brush = "drip" + params = {"dripRate": 0.4} + elif "scatter" in s: + brush = "scatter" + params = {"scatterAmount": 0.4} + elif "mixed" in s: + brush = "mixed" + params = {"mixColors": ["#FFFFFF", "#000000"], "mixAmount": 0.5} + elif "stamp" in s or "sticker" in s or "collage" in s: + brush = "normal" + params = {"preferStamp": True} + + return brush, params + + +def _postprocess_style_objects(objects: list, style_prompt: str) -> list: + """ + Ensure objects include metadata dict with brushType/brushParams or stampData. + """ + if not isinstance(objects, list): + return objects + + default_brush, default_params = _map_style_to_brush(style_prompt) + + processed = [] + had_explicit_metadata = False + + for obj in objects: + if not isinstance(obj, dict): + processed.append(obj) + continue + + had_meta = "metadata" in obj + meta = obj.get("metadata", {}) or {} + + if obj.get("drawingType") == "image" or obj.get("imageDataUrl"): + meta.setdefault("drawingType", "image") + meta.setdefault("stampData", { + "imageDataUrl": obj.get("imageDataUrl") or obj.get("image"), + "x": obj.get("x", 0), + "y": obj.get("y", 0), + "width": obj.get("width"), + "height": obj.get("height"), + }) + else: + meta.setdefault("drawingType", "stroke") + meta.setdefault("brushType", meta.get("brushType", default_brush)) + meta.setdefault("brushParams", meta.get("brushParams", default_params)) + + obj["metadata"] = meta + processed.append(obj) + + if had_meta and isinstance(meta, dict) and meta.get("brushType"): + had_explicit_metadata = True + + s = (style_prompt or "").lower() + if ("van gogh" in s or "oil" in s or "impasto" in s) and not had_explicit_metadata: + overlays = [] + for idx, obj in enumerate(processed): + if not isinstance(obj, dict): + continue + if obj.get("metadata", {}).get("drawingType") == "image": + continue + + bbox = _bbox_from_path(obj.get("pathData", {})) + if bbox is None: + continue + + overlay_objs = _create_impasto_overlays(obj, bbox, default_params, idx) + overlays.extend(overlay_objs) + + processed.extend(overlays) + + return processed + + +def _bbox_from_path(pathData: dict) -> typing.Optional[dict]: + """ + Compute a simple bounding box from a pathData dict. Returns None if + no usable geometry is present. + """ + if not isinstance(pathData, dict): + return None + + xs = [] + ys = [] + + if "points" in pathData and isinstance(pathData["points"], list): + for p in pathData["points"]: + try: + xs.append(float(p.get("x", 0))) + ys.append(float(p.get("y", 0))) + except Exception: + continue + else: + # Try start/end + start = pathData.get("start") + end = pathData.get("end") + if isinstance(start, dict) and isinstance(end, dict): + try: + xs.extend([float(start.get("x", 0)), float(end.get("x", 0))]) + ys.extend([float(start.get("y", 0)), float(end.get("y", 0))]) + except Exception: + pass + + if not xs or not ys: + return None + + return {"min_x": min(xs), "max_x": max(xs), "min_y": min(ys), "max_y": max(ys)} + + +def _create_impasto_overlays(obj: dict, bbox: dict, default_params: dict, idx: int) -> list: + """ + Create overlay freehand strokes to emulate impasto texture. + """ + overlays = [] + + color = obj.get("metadata", {}).get("brushParams", {}).get("mixColors", []) + if color and isinstance(color, list): + overlay_color = color[0] + else: + overlay_color = obj.get("color", "#000000") + + width = bbox["max_x"] - bbox["min_x"] + height = bbox["max_y"] - bbox["min_y"] + cx = (bbox["min_x"] + bbox["max_x"]) / 2 + cy = (bbox["min_y"] + bbox["max_y"]) / 2 + + offs = ((idx % 3) - 1) * 4 + + def make_stroke(rel_points, lw_mult): + pts = [] + for rx, ry in rel_points: + pts.append({"x": cx + rx * width + offs, "y": cy + ry * height + offs}) + return { + "color": overlay_color, + "lineWidth": max(2, int(obj.get("lineWidth", 2) * lw_mult)), + "pathData": {"tool": "freehand", "type": "stroke", "points": pts}, + "metadata": { + "drawingType": "stroke", + "brushType": default_params.get("base", "wacky") if isinstance(default_params, dict) else "wacky", + "brushParams": dict(default_params or {}, **{"opacity": default_params.get("opacity", 0.9)}) + } + } + + stroke1 = make_stroke([( -0.3, -0.2), ( -0.1, -0.25), (0.1, -0.15)], 1.0) + stroke2 = make_stroke([( -0.4, 0.1), (0.0, 0.15), (0.35, 0.05)], 1.3) + + overlays.append(stroke1) + overlays.append(stroke2) + + return overlays + + +def style_transfer_canvas(canvas_state: dict, style_prompt: str) -> dict: + """Apply style transfer to canvas using OpenAI or Ollama.""" + model_output = openai_style_transfer(canvas_state, style_prompt) + if isinstance(model_output, dict) and "error" not in model_output and "objects" in model_output: + model_output["objects"] = _postprocess_style_objects(model_output.get("objects", []), style_prompt) + return model_output + + fallback_output = ollama_style_transfer(canvas_state, style_prompt) + if isinstance(fallback_output, dict) and "error" not in fallback_output and "objects" in fallback_output: + fallback_output["objects"] = _postprocess_style_objects(fallback_output.get("objects", []), style_prompt) + return fallback_output + + original_objects = canvas_state.get("objects", []) + return {"objects": original_objects} + + +# === Simple Vector-based Object Recognition ================================= +RECOGNITION_SYSTEM = """ +You are an object recognizer for a vector canvas. IMPORTANT: the inputs you +receive are vector primitives (shapes and freehand strokes) encoded as JSON +geometry (points, start/end for shapes, line widths, and colors). These are +NOT raster images — do not assume photographic textures or pixels. Use the +geometric cues (circle-like points, grouped strokes, polygons, repeated +small circles for wheels, trunk+foliage strokes for trees, etc.) to form your +label. + +You will be given a small JSON payload describing the subset of canvas +objects that intersect the user's selection box and the bounding box itself. +Return a single JSON object containing a short `label` describing the primary +object or scene contained in the selection, a `confidence` score between 0.0 +and 1.0, and an optional short `explanation` that states which geometric cues +led to the label. + +OUTPUT (JSON ONLY): +{ + "label": "string", + "confidence": number, // 0.0 - 1.0 + "explanation": "string (optional)" +} + +Rules: +- Prefer concise common-sense labels (e.g., "tree", "car", "house", "face", + "circle", "text: 'Hello'", "unknown"). If unsure, return "unknown" with + a low confidence (e.g., 0.2). +- Use confidence to reflect certainty; 0.6+ for reasonable guesses, 0.85+ for + strong matches. +- Do not invent objects not supported by the provided geometry; prefer + conservative labels when ambiguous. +""" + +# Few-shot examples to anchor vector-domain recognition expectations +FEWSHOT_RECO_USER_1 = ''' +SelectionBox: +{"x":100,"y":50,"width":60,"height":60} +CanvasObjects: +{"objects":[{"color":"#000000","lineWidth":2,"pathData":{"tool":"shape","type":"circle","start":{"x":130,"y":80},"end":{"x":150,"y":80}}}],"bounds":{"width":400,"height":300}} +''' + +FEWSHOT_RECO_ASSISTANT_1 = {"label": "circle", "confidence": 0.95, "explanation": "Single circular shape primitive (start/end) within selection."} + +FEWSHOT_RECO_USER_2 = ''' +SelectionBox: +{"x":200,"y":200,"width":120,"height":150} +CanvasObjects: +{"objects":[{"color":"#8B4513","lineWidth":3,"pathData":{"tool":"freehand","type":"stroke","points":[{"x":240,"y":230},{"x":245,"y":270},{"x":250,"y":310}]}},{"color":"#228B22","lineWidth":2,"pathData":{"tool":"freehand","type":"stroke","points":[{"x":220,"y":210},{"x":230,"y":190},{"x":250,"y":200},{"x":270,"y":210}]}}],"bounds":{"width":800,"height":600}} +''' + +FEWSHOT_RECO_ASSISTANT_2 = {"label": "tree", "confidence": 0.88, "explanation": "Brown trunk stroke plus clustered green freehand strokes resembling foliage."} + +FEWSHOT_RECO_USER_3 = ''' +SelectionBox: +{"x":140,"y":120,"width":220,"height":120} +CanvasObjects: +{"objects":[{"color":"#FF0000","lineWidth":2,"pathData":{"tool":"shape","type":"rectangle","start":{"x":150,"y":160},"end":{"x":320,"y":210}}},{"color":"#000000","lineWidth":2,"pathData":{"tool":"shape","type":"circle","start":{"x":180,"y":210},"end":{"x":200,"y":210}}},{"color":"#000000","lineWidth":2,"pathData":{"tool":"shape","type":"circle","start":{"x":270,"y":210},"end":{"x":290,"y":210}}}],"bounds":{"width":800,"height":600}} +''' + +FEWSHOT_RECO_ASSISTANT_3 = {"label": "car", "confidence": 0.92, "explanation": "Rectangular body plus two circular wheel primitives along its bottom edge."} + +FEWSHOT_RECO_USER_4 = ''' +SelectionBox: +{"x":100,"y":100,"width":220,"height":200} +CanvasObjects: +{"objects":[{"color":"#8B4513","lineWidth":2,"pathData":{"tool":"shape","type":"rectangle","start":{"x":120,"y":180},"end":{"x":260,"y":260}}},{"color":"#FF0000","lineWidth":2,"pathData":{"tool":"shape","type":"polygon","points":[{"x":120,"y":180},{"x":190,"y":120},{"x":260,"y":180}]}}],"bounds":{"width":800,"height":600}} +''' + +FEWSHOT_RECO_ASSISTANT_4 = {"label": "house", "confidence": 0.9, "explanation": "Rectangle base plus triangular roof polygon — typical house geometry."} + +FEWSHOT_RECO_USER_5 = ''' +SelectionBox: +{"x":50,"y":50,"width":200,"height":80} +CanvasObjects: +{"objects":[{"color":"#000000","lineWidth":2,"pathData":{"tool":"shape","type":"text","text":"Hello"}}],"bounds":{"width":400,"height":200}} +''' + +FEWSHOT_RECO_ASSISTANT_5 = {"label": "text: 'Hello'", "confidence": 0.98, "explanation": "A text primitive with the exact string 'Hello' present in the selection."} + +def _get_recognition_message(canvas_objects: list, box: dict, bounds: dict) -> list[dict]: + objs_json = json.dumps({"objects": canvas_objects, "bounds": bounds}, ensure_ascii=False) + user_msg = f"SelectionBox:\n{json.dumps(box)}\nCanvasObjects:\n{objs_json}\n\nPlease identify the primary object or scene contained within the selection box and return JSON as specified." + return [ + {"role": "system", "content": RECOGNITION_SYSTEM}, + {"role": "user", "content": user_msg}, + ] + + +def openai_recognize_objects(canvas_objects: list, box: dict, bounds: dict) -> dict: + try: + from config import OPENAI_API_KEY + from openai import OpenAI + + client = OpenAI(api_key=OPENAI_API_KEY) + + resp = client.chat.completions.create( + model="gpt-4.1-mini", + response_format={"type": "json_object"}, + temperature=0.0, + messages=_get_recognition_message(canvas_objects, box, bounds), + max_tokens=300, + ) + content = resp.choices[0].message.content + return json.loads(content) + except Exception as e: + return {"error": "openai_recognition_failed", "detail": str(e)} + + +def ollama_recognize_objects(canvas_objects: list, box: dict, bounds: dict) -> dict: + try: + import ollama + + response = ollama.chat( + model="llama3:8b", + messages=_get_recognition_message(canvas_objects, box, bounds), + ) + + return json.loads(response["message"]["content"]) + except Exception as e: + return {"error": "ollama_recognition_failed", "detail": str(e)} + + +def _rule_based_recognize(canvas_objects: list, box: dict) -> typing.Optional[dict]: + """Lightweight rule-based recognizer for obvious geometric cases.""" + try: + def is_circle(obj): + pd = obj.get("pathData", {}) + return pd.get("type") == "circle" + + def is_text(obj): + pd = obj.get("pathData", {}) + return pd.get("type") == "text" and isinstance(pd.get("text"), str) + + def count_type(t): + return sum(1 for o in canvas_objects if isinstance(o.get("pathData"), dict) and o.get("pathData", {}).get("type") == t) + + circle_count = count_type("circle") + if circle_count == 1 and len(canvas_objects) == 1: + return {"label": "circle", "confidence": 0.95, "explanation": "Single circular shape primitive within selection."} + + for o in canvas_objects: + if is_text(o): + txt = o.get("pathData", {}).get("text", "") + return {"label": f"text: '{txt}'", "confidence": 0.98, "explanation": "A text primitive with an explicit string was found."} + + rect_count = count_type("rectangle") + poly_count = count_type("polygon") + wheel_count = circle_count + if (rect_count + poly_count) >= 1 and wheel_count >= 2: + return {"label": "car", "confidence": 0.9, "explanation": "Rectangular/polygonal body plus multiple circular wheel primitives."} + + def is_triangle(o): + pd = o.get("pathData", {}) + pts = pd.get("points") if isinstance(pd.get("points"), list) else [] + return pd.get("type") == "polygon" and len(pts) == 3 + + tri_count = sum(1 for o in canvas_objects if is_triangle(o)) + if rect_count >= 1 and tri_count >= 1: + return {"label": "house", "confidence": 0.9, "explanation": "Rectangular base plus triangular roof polygon detected."} + + def hex_to_rgb(h): + try: + h = h.lstrip("#") + return tuple(int(h[i:i+2], 16) for i in (0, 2, 4)) + except Exception: + return (0, 0, 0) + + def color_close(hexcolor, target_rgb, tol=100): + r, g, b = hex_to_rgb(hexcolor or "#000000") + tr, tg, tb = target_rgb + return ((r-tr)**2 + (g-tg)**2 + (b-tb)**2) <= (tol**2) + + brown_rgb = (139, 69, 19) + green_rgb = (34, 139, 34) + trunk = any(o for o in canvas_objects if o.get("pathData", {}).get("tool") == "freehand" and color_close(o.get("color", "#000000"), brown_rgb, tol=120)) + foliage = any(o for o in canvas_objects if o.get("pathData", {}).get("tool") == "freehand" and color_close(o.get("color", "#000000"), green_rgb, tol=120)) + if trunk and foliage: + return {"label": "tree", "confidence": 0.88, "explanation": "Brown trunk-like stroke plus clustered green freehand strokes resembling foliage."} + + except Exception: + pass + + return None + + +def recognize_objects_in_box(canvas_objects: list, box: dict, bounds: dict) -> dict: + """Lightweight recognition using OpenAI with Ollama fallback.""" + try: + rule_out = _rule_based_recognize(canvas_objects, box) + if isinstance(rule_out, dict): + return rule_out + except Exception: + pass + + model_output = openai_recognize_objects(canvas_objects, box, bounds) + if isinstance(model_output, dict) and "error" not in model_output: + return model_output + return ollama_recognize_objects(canvas_objects, box, bounds) diff --git a/backend/services/prompt_templates.py b/backend/services/prompt_templates.py new file mode 100644 index 00000000..f53480a7 --- /dev/null +++ b/backend/services/prompt_templates.py @@ -0,0 +1,21 @@ +""" +Prompt templates for the ResCanvas AI assistant. +""" + +RESCANVAS_SYSTEM_PROMPT = """You are ResCanvas Bot, a helpful assistant for a collaborative drawing application. + +Your role and capabilities: +- You are friendly, concise, and helpful +- You have access to the current canvas context (provided in JSON format) +- You can assist users with questions about the canvas, drawing tools, and collaboration features +- If a user asks you to perform an action (like clearing the canvas, undoing strokes, or managing rooms), you should attempt to do so using your available tools +- You provide clear, actionable guidance for using ResCanvas features + +Guidelines: +- Keep responses brief and to the point +- Focus on canvas-related topics and drawing collaboration +- If a user asks for something unrelated to ResCanvas or drawing, politely decline and redirect them to canvas-related topics +- When providing instructions, be specific about which tools or features to use +- If you cannot help with a request, clearly explain why and suggest alternatives if possible + +Remember: You are here to enhance the collaborative drawing experience, not for general conversation.""" diff --git a/backend/services/search_algorithms.py b/backend/services/search_algorithms.py new file mode 100644 index 00000000..73be15dd --- /dev/null +++ b/backend/services/search_algorithms.py @@ -0,0 +1,217 @@ +"""Search algorithm implementations for ResCanvas. + +Provides text and image-based semantic search using CLIP embeddings and Qdrant vector search. +""" +from typing import List, Dict, Any +import logging +import base64 +import io +from PIL import Image +import tempfile +import os +import numpy as np + +logger = logging.getLogger(__name__) + +DEFAULT_TOP_N = 50 + +# Lazy imports to avoid startup failures if dependencies aren't installed +_embedding_service = None +_vector_search_service = None + + +def _get_services(): + # Load embedding and vector search services. + global _embedding_service, _vector_search_service + + if _embedding_service is None or _vector_search_service is None: + try: + from services import embedding_service, vector_search_service + _embedding_service = embedding_service + _vector_search_service = vector_search_service + logger.info("Loaded embedding and vector search services") + except Exception as e: + logger.error(f"Failed to load AI services: {e}") + raise + + return _embedding_service, _vector_search_service + + +def text_search(query: str, rooms: List[Dict[str, Any]], top_n: int = DEFAULT_TOP_N, seed: int | None = None) -> List[Dict[str, Any]]: + """ + Semantic text search using CLIP embeddings. + + Args: + query: Natural language search query (e.g., "rooms with trees") + rooms: List of candidate room dicts (pre-filtered by visibility) + top_n: Maximum number of results to return + seed: Unused (kept for backward compatibility) + + Returns: + List of rooms ranked by semantic similarity with score field added + """ + try: + embed_svc, vector_svc = _get_services() + + # Generate embedding for the query text + query_embedding = embed_svc.embed_text([query]) # Returns (1, 512) + + if query_embedding is None or query_embedding.size == 0: + logger.warning("Failed to generate query embedding, falling back to random") + return _fallback_random_search(rooms, top_n, seed) + + # Search vector database for similar canvases + vector_results = vector_svc.search_by_embedding( + query_embedding=query_embedding, + top_k=top_n, # Get extra results to filter by visibility + score_threshold=0.0 # (Optional) Minimum score for results + ) + + # Create a map of room_id -> score from vector results + score_map = {str(r['room_id']): r['score'] for r in vector_results} + + # Match vector results with provided rooms (visibility-filtered) + # and add scores + scored_rooms = [] + for room in rooms: + room_id = room.get('id') or str(room.get('_id', '')) + if room_id in score_map: + room_copy = {**room, 'score': score_map[room_id]} + scored_rooms.append(room_copy) + else: + # Room not in vector DB yet, give low score + room_copy = {**room, 'score': 0.1} + scored_rooms.append(room_copy) + + # Sort by score descending + scored_rooms.sort(key=lambda x: x['score'], reverse=True) + + logger.info(f"Text search for '{query}' returned {len(scored_rooms[:top_n])} results") + return scored_rooms[:top_n] + + except Exception as e: + logger.exception(f"Text search failed: {e}") + return _fallback_random_search(rooms, top_n, seed) + + +def image_search(image_b64: str, rooms: List[Dict[str, Any]], q: str | None = None, top_n: int = DEFAULT_TOP_N, seed: int | None = None) -> List[Dict[str, Any]]: + """ + Semantic image search using CLIP embeddings. + + Args: + image_b64: Base64-encoded image (without data URI prefix) + rooms: List of candidate room dicts (pre-filtered by visibility) + q: Optional text query to combine with image (future enhancement) + top_n: Maximum number of results to return + seed: Unused (kept for backward compatibility) + + Returns: + List of rooms ranked by visual similarity with score field added + """ + try: + embed_svc, vector_svc = _get_services() + + # Decode base64 image and save to temporary file + try: + image_data = base64.b64decode(image_b64) + image = Image.open(io.BytesIO(image_data)) + + # Save to temp file (embedding service expects file path) + with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as tmp: + tmp_path = tmp.name + image.save(tmp_path, 'PNG') + + # Generate embedding for the image + image_embedding = embed_svc.embed_image(tmp_path) # Returns (1, 512) + + # Clean up temp file + os.unlink(tmp_path) + + # If a text query is provided, generate text embedding and combine + if q: + try: + text_embedding = embed_svc.embed_text([q]) # (1, 512) + except Exception as e: + logger.warning(f"Failed to generate text embedding for hybrid search: {e}") + text_embedding = None + + if text_embedding is not None and getattr(text_embedding, 'size', 0) > 0: + # Flatten to 1D arrays + img_vec = np.asarray(image_embedding).reshape(-1).astype(np.float32) + txt_vec = np.asarray(text_embedding).reshape(-1).astype(np.float32) + + # L2-normalize + # img_norm = np.linalg.norm(img_vec) + # txt_norm = np.linalg.norm(txt_vec) + # if img_norm > 0: + # img_vec = img_vec / img_norm + # if txt_norm > 0: + # txt_vec = txt_vec / txt_norm + + # Weighted combination (image-heavy by default) + weight_image = 0.6 + weight_text = 0.4 + combined = weight_image * img_vec + weight_text * txt_vec + + # Re-normalize combined vector + comb_norm = np.linalg.norm(combined) + if comb_norm > 0: + combined = (combined / comb_norm).astype(np.float32) + + query_embedding = combined.reshape(1, -1) + logger.info("Performed hybrid (image+text) embedding combination (image_weight=%s,text_weight=%s)", weight_image, weight_text) + else: + # fall back to image-only embedding + query_embedding = image_embedding + else: + query_embedding = image_embedding + + except Exception as e: + logger.error(f"Failed to process image: {e}") + return _fallback_random_search(rooms, top_n, seed) + + if query_embedding is None or query_embedding.size == 0: + logger.warning("Failed to generate image embedding, falling back to random") + return _fallback_random_search(rooms, top_n, seed) + + # Search vector database for similar canvases + vector_results = vector_svc.search_by_embedding( + query_embedding=query_embedding, + top_k=top_n, # Get extra results to filter by visibility + score_threshold=0.0 # (Optional) Minimum score for results + ) + + # Create a map of room_id -> score from vector results + score_map = {str(r['room_id']): r['score'] for r in vector_results} + + # Match vector results with provided rooms (visibility-filtered) + scored_rooms = [] + for room in rooms: + room_id = room.get('id') or str(room.get('_id', '')) + if room_id in score_map: + room_copy = {**room, 'score': score_map[room_id]} + scored_rooms.append(room_copy) + else: + # Room not in vector DB yet, give low score + room_copy = {**room, 'score': 0.1} + scored_rooms.append(room_copy) + + # Sort by score descending + scored_rooms.sort(key=lambda x: x['score'], reverse=True) + + logger.info(f"Image search returned {len(scored_rooms[:top_n])} results") + return scored_rooms[:top_n] + + except Exception as e: + logger.exception(f"Image search failed: {e}") + return _fallback_random_search(rooms, top_n, seed) + + +def _fallback_random_search(rooms: List[Dict[str, Any]], top_n: int, seed: int | None = None) -> List[Dict[str, Any]]: + """Fallback to random ranking if embedding search fails.""" + import random + logger.warning("Using fallback random search") + rng = random.Random(seed) if seed is not None else random + scored = [{**r, "score": rng.random()} for r in rooms] + scored.sort(key=lambda x: x["score"], reverse=True) + return scored[:top_n] diff --git a/backend/services/vector_search_service.py b/backend/services/vector_search_service.py new file mode 100644 index 00000000..ca82d3db --- /dev/null +++ b/backend/services/vector_search_service.py @@ -0,0 +1,273 @@ +""" +Vector search service using Qdrant for semantic canvas search. + +This service manages storage and retrieval of canvas embeddings in Qdrant, +enabling semantic similarity search across canvases. +""" +import logging +import numpy as np +from typing import List, Dict, Any, Optional +from qdrant_client import QdrantClient +from qdrant_client.models import Distance, VectorParams, PointStruct, Filter, FieldCondition, MatchValue +from qdrant_client.http.exceptions import UnexpectedResponse +from config import QDRANT_HOST, QDRANT_PORT, QDRANT_COLLECTION_NAME, EMBEDDING_DIMENSION + +logger = logging.getLogger(__name__) + +# Global Qdrant client +_qdrant_client: Optional[QdrantClient] = None + + +def get_qdrant_client() -> QdrantClient: + # Get Qdrant client + global _qdrant_client + + # If client not initialized, create it + if _qdrant_client is None: + try: + _qdrant_client = QdrantClient(host=QDRANT_HOST, port=QDRANT_PORT) + logger.info(f"Connected to Qdrant at {QDRANT_HOST}:{QDRANT_PORT}") + _ensure_collection_exists() + except Exception as e: + logger.error(f"Failed to connect to Qdrant: {e}") + raise + return _qdrant_client + + +def _ensure_collection_exists(): + # Create collection if it doesn't exist. + client = _qdrant_client + try: + # Check if collection exists (should be a single collection name) + collections = client.get_collections().collections + collection_names = [c.name for c in collections] + + # If it doesnt exist create Qdrant collection + if QDRANT_COLLECTION_NAME not in collection_names: + logger.info(f"Creating Qdrant collection: {QDRANT_COLLECTION_NAME}") + client.create_collection( + collection_name=QDRANT_COLLECTION_NAME, + vectors_config=VectorParams( + size=EMBEDDING_DIMENSION, # e.g., 512 + distance=Distance.COSINE # Cosine similarity for normalized embeddings + ) + ) + logger.info(f"Collection {QDRANT_COLLECTION_NAME} created successfully") + else: + logger.debug(f"Collection {QDRANT_COLLECTION_NAME} already exists") + except Exception as e: + logger.error(f"Error ensuring collection exists: {e}") + raise + + +def store_canvas_embedding( + room_id: str, + embedding: np.ndarray, + metadata: Optional[Dict[str, Any]] = None +) -> bool: + # Store or update a canvas embedding in Qdrant.x + try: + # Get Qdrant client + client = get_qdrant_client() + + # Ensure embedding is a numpy array + if not isinstance(embedding, np.ndarray): + embedding = np.array(embedding, dtype=np.float32) + + # Ensure embedding is the right shape and type + if embedding.ndim == 2: + embedding = embedding.flatten() + + if len(embedding) != EMBEDDING_DIMENSION: + logger.error(f"Embedding dimension mismatch: expected {EMBEDDING_DIMENSION}, got {len(embedding)}") + return False + + # Prepare payload with metadata + payload = metadata or {} + payload['room_id'] = room_id + + # Use hashlib for consistent point IDs across worker restarts + import hashlib + point_id = int(hashlib.md5(room_id.encode()).hexdigest()[:15], 16) + + # Update the room (will update if exists, insert if new) + client.upsert( + collection_name=QDRANT_COLLECTION_NAME, + points=[ + PointStruct( + id=point_id, + vector=embedding.tolist(), # Convert to list for Qdrant + payload=payload + ) + ] + ) + + logger.info(f"Stored embedding for room_id={room_id}") + return True + + except Exception as e: + logger.exception(f"Failed to store embedding for room_id={room_id}: {e}") + return False + + +def search_by_embedding( + query_embedding: np.ndarray, + top_k: int = 50, + filters: Optional[Dict[str, Any]] = None, + score_threshold: float = 0.0 +) -> List[Dict[str, Any]]: + # Search for similar canvases using vector similarity. + try: + # Get Qdrant client + client = get_qdrant_client() + + # Ensure embedding is the right shape + if query_embedding.ndim == 2: + query_embedding = query_embedding.flatten() + + if len(query_embedding) != EMBEDDING_DIMENSION: + logger.error(f"Query embedding dimension mismatch: expected {EMBEDDING_DIMENSION}, got {len(query_embedding)}") + return [] + + # Build Qdrant filter if provided + qdrant_filter = None + if filters: + conditions = [] + for key, value in filters.items(): + conditions.append( + FieldCondition(key=key, match=MatchValue(value=value)) + ) + if conditions: + qdrant_filter = Filter(must=conditions) + + # Perform vector search + search_result = client.search( + collection_name=QDRANT_COLLECTION_NAME, + query_vector=query_embedding.tolist(), # Convert to list for Qdrant + limit=top_k, + query_filter=qdrant_filter, + score_threshold=score_threshold + ) + + # Format results + results = [] + for hit in search_result: + result = { + 'room_id': hit.payload.get('room_id'), + 'score': float(hit.score), + **hit.payload # Include all metadata + } + results.append(result) + + logger.info(f"Vector search returned {len(results)} results (top_k={top_k})") + return results + + except Exception as e: + logger.exception(f"Vector search failed: {e}") + return [] + + +def update_canvas_embedding( + room_id: str, + new_embedding: np.ndarray, + metadata: Optional[Dict[str, Any]] = None +) -> bool: + # Update an existing canvas embedding. + return store_canvas_embedding(room_id, new_embedding, metadata) + + +def delete_canvas_embedding(room_id: str) -> bool: + # Delete a canvas embedding from Qdrant. + try: + # Get Qdrant client + client = get_qdrant_client() + + import hashlib + point_id = int(hashlib.md5(room_id.encode()).hexdigest()[:15], 16) + + # Delete the room + client.delete( + collection_name=QDRANT_COLLECTION_NAME, + points_selector=[point_id] + ) + + logger.info(f"Deleted embedding for room_id={room_id}") + return True + + except Exception as e: + logger.exception(f"Failed to delete embedding for room_id={room_id}: {e}") + return False + + +def batch_store_embeddings(embeddings: List[Dict[str, Any]]) -> int: + # Store multiple embeddings in batch (more efficient). + try: + # Get Qdrant client + client = get_qdrant_client() + points = [] + + for item in embeddings: + room_id = item['room_id'] + embedding = item['embedding'] + metadata = item.get('metadata', {}) + + # Prepare embedding + if embedding.ndim == 2: + embedding = embedding.flatten() + + if len(embedding) != EMBEDDING_DIMENSION: + logger.warning(f"Skipping room_id={room_id} due to dimension mismatch") + continue + + # Prepare payload + payload = metadata.copy() + payload['room_id'] = room_id + + #point_id = hash(room_id) & 0x7FFFFFFFFFFFFFFF + # Use hashlib for consistent point IDs across worker restarts + import hashlib + point_id = int(hashlib.md5(room_id.encode()).hexdigest()[:15], 16) + + points.append( + PointStruct( + id=point_id, + vector=embedding.tolist(), + payload=payload + ) + ) + + if points: + client.upsert( + collection_name=QDRANT_COLLECTION_NAME, + points=points + ) + logger.info(f"Batch stored {len(points)} embeddings") + return len(points) + + return 0 + + except Exception as e: + logger.exception(f"Batch store failed: {e}") + return 0 + + +def get_collection_stats() -> Dict[str, Any]: + # Get statistics about the vector collection. + try: + # Get Qdrant client + client = get_qdrant_client() + collection_info = client.get_collection(collection_name=QDRANT_COLLECTION_NAME) + + return { + 'collection_name': QDRANT_COLLECTION_NAME, + 'vectors_count': collection_info.vectors_count, + 'points_count': collection_info.points_count, + 'status': collection_info.status, + 'config': { + 'dimension': EMBEDDING_DIMENSION, + 'distance': 'COSINE' + } + } + except Exception as e: + logger.exception(f"Failed to get collection stats: {e}") + return {'error': str(e)} diff --git a/backend/workers/embedding_worker.py b/backend/workers/embedding_worker.py new file mode 100644 index 00000000..661fb36b --- /dev/null +++ b/backend/workers/embedding_worker.py @@ -0,0 +1,482 @@ +""" +Embedding Worker - Incremental Canvas Embedding Generation + +Automatically generates and updates vector embeddings for canvases in the background. +This worker monitors for new/modified rooms and updates their embeddings without +requiring manual re-population. + +Usage: + python -m workers.embedding_worker [--interval SECONDS] [--batch-size N] +""" + +import sys +import os +import time +import logging +import argparse +from datetime import datetime, timedelta +from typing import Dict, List, Optional, Set +import threading +import signal + +# Add parent directory to path +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from services.db import rooms_coll, strokes_coll, mongo_client +from services.embedding_service import embed_text, embed_image +from services.vector_search_service import store_canvas_embedding, get_collection_stats +from bson import ObjectId +import numpy as np + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s:%(lineno)d – %(message)s", + datefmt="%Y-%m-%d %H:%M:%S" +) +logger = logging.getLogger(__name__) + +# Configuration +DEFAULT_UPDATE_INTERVAL = 300 # 5 minutes - how often to check for updates +DEFAULT_DEBOUNCE_PERIOD = 180 # 3 minutes - minimum time since last room update before embedding +DEFAULT_BATCH_SIZE = 10 # Process this many rooms per batch +DEFAULT_THUMBNAIL_SIZE = (512, 512) # Thumbnail dimensions for visual embeddings + +# Global state +_shutdown_requested = False +_last_processed_times: Dict[str, datetime] = {} # room_id -> last embedding time + + +class CanvasRenderer: + """ + Renders canvas strokes into a PIL Image for visual embedding generation. + """ + + @staticmethod + def render_room_thumbnail(room_id: str, size: tuple = DEFAULT_THUMBNAIL_SIZE) -> Optional[bytes]: + """ + Retrieve stored canvas thumbnail from MongoDB. + + Frontend uploads thumbnails via POST /api/rooms//thumbnail using canvas.toDataURL(). + This method retrieves the stored thumbnail bytes for embedding generation. + + Args: + room_id: Room ID to get thumbnail for + size: Unused (kept for API compatibility) + + Returns: + PNG/JPEG image bytes, or None if no thumbnail available + """ + try: + room = rooms_coll.find_one( + {'_id': ObjectId(room_id)}, + {'thumbnail': 1, 'thumbnailUpdatedAt': 1} + ) + + if not room: + logger.debug(f"Room {room_id} not found") + return None + + if 'thumbnail' not in room: + logger.debug(f"No thumbnail stored for room {room_id}") + return None + + thumbnail_bytes = room['thumbnail'] + + # Validate it's actually binary image data + if not isinstance(thumbnail_bytes, bytes): + logger.warning(f"Invalid thumbnail type for room {room_id}: {type(thumbnail_bytes)}") + return None + + if len(thumbnail_bytes) < 100: + logger.warning(f"Thumbnail too small for room {room_id}: {len(thumbnail_bytes)} bytes") + return None + + # Log when thumbnail was last updated (for debugging staleness) + updated_at = room.get('thumbnailUpdatedAt') + if updated_at: + logger.debug(f"Retrieved thumbnail for room {room_id}: {len(thumbnail_bytes)} bytes " + f"(updated {updated_at})") + else: + logger.debug(f"Retrieved thumbnail for room {room_id}: {len(thumbnail_bytes)} bytes") + + return thumbnail_bytes + + except Exception as e: + logger.exception(f"Failed to retrieve thumbnail for room {room_id}: {e}") + return None + + +class EmbeddingWorker: + """Background worker for incremental embedding updates.""" + + def __init__(self, + update_interval: int = DEFAULT_UPDATE_INTERVAL, + debounce_period: int = DEFAULT_DEBOUNCE_PERIOD, + batch_size: int = DEFAULT_BATCH_SIZE): + """ + Initialize the embedding worker. + + Args: + update_interval: How often (seconds) to check for room updates + debounce_period: Minimum time (seconds) since room update before embedding + batch_size: Maximum rooms to process per iteration + """ + self.update_interval = update_interval + self.debounce_period = debounce_period + self.batch_size = batch_size + self.renderer = CanvasRenderer() + + # Track which rooms we've already processed + self.processed_rooms: Set[str] = set() + + logger.info(f"Initialized EmbeddingWorker: update_interval={update_interval}s, " + f"debounce_period={debounce_period}s, batch_size={batch_size}") + + def should_process_room(self, room: Dict) -> bool: + """ + Determine if a room should have its embedding updated. + + Conditions for processing: + 1. Room was created/updated recently + 2. Sufficient time has passed since last update (debouncing) + 3. Room doesn't already have current embedding + 4. Room is not archived + """ + room_id = str(room['_id']) + + # Skip archived rooms + if room.get('archived'): + return False + + # Check when room was last updated + updated_at = room.get('updatedAt') or room.get('createdAt') + if not updated_at: + return False + + # Debounce: Don't embed if room was updated very recently + # (might still be actively being edited) + time_since_update = datetime.utcnow() - updated_at + if time_since_update.total_seconds() < self.debounce_period: + logger.debug(f"Room {room_id} updated {time_since_update.total_seconds()}s ago, " + f"waiting for debounce period ({self.debounce_period}s)") + return False + + # Check if we've already processed this room recently + last_processed = _last_processed_times.get(room_id) + if last_processed: + # Only re-process if room was updated after our last embedding + if updated_at <= last_processed: + return False + + return True + + def find_rooms_to_update(self) -> List[Dict]: + """ + Find rooms that need embedding updates. + + Strategy: + 1. Query rooms updated in the last (update_interval + debounce_period) + 2. Filter to those that meet processing criteria + 3. Limit to batch_size + """ + try: + # Look for rooms updated since our last check (with some overlap) + lookback_window = self.update_interval + self.debounce_period + cutoff_time = datetime.utcnow() - timedelta(seconds=lookback_window) + + # Query rooms that have been updated but not too recently + query = { + 'archived': {'$ne': True}, + 'updatedAt': {'$gte': cutoff_time} + } + + rooms = list(rooms_coll.find(query).sort('updatedAt', 1).limit(self.batch_size * 2)) + + # Filter using our processing logic + to_process = [room for room in rooms if self.should_process_room(room)] + + # Limit batch size + to_process = to_process[:self.batch_size] + + logger.info(f"Found {len(to_process)} rooms to process (from {len(rooms)} candidates)") + return to_process + + except Exception as e: + logger.exception(f"Failed to find rooms to update: {e}") + return [] + + def generate_embedding_for_room(self, room: Dict) -> bool: + """ + Generate and store embedding for a single room. + + Process: + 1. Extract text metadata (name, description) + 2. Attempt to render canvas thumbnail + 3. Generate text embedding (always) + 4. Generate image embedding (if thumbnail available) + 5. Combine embeddings if both available + 6. Store in Qdrant + + Args: + room: Room document from MongoDB + + Returns: + True if embedding was successfully generated and stored + """ + room_id = str(room['_id']) + room_name = room.get('name', '') + room_desc = room.get('description', '') + room_type = room.get('type', 'public') + room_owner = room.get('ownerName', '') + + try: + logger.info(f"Processing room '{room_name}' (id={room_id})") + + # 1. Generate text embedding (from name + description) + text = f"{room_name}. {room_desc}" if room_desc else room_name + text_embedding = None + + if text.strip(): + text_embedding = embed_text([text]) + logger.debug(f" Generated text embedding: shape={text_embedding.shape}") + + # 2. Attempt to render canvas thumbnail + thumbnail_bytes = self.renderer.render_room_thumbnail(room_id) + image_embedding = None + + if thumbnail_bytes: + # Save thumbnail temporarily and generate embedding + import tempfile + with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as tmp: + tmp.write(thumbnail_bytes) + tmp_path = tmp.name + + try: + image_embedding = embed_image(tmp_path) + logger.debug(f" Generated image embedding: shape={image_embedding.shape}") + finally: + os.unlink(tmp_path) + else: + logger.debug(f" No thumbnail available, using text-only embedding") + + # 3. Combine embeddings if we have both + if text_embedding is not None and image_embedding is not None: + # Hybrid embedding: weighted combination + # Flatten to 1D + text_vec = np.asarray(text_embedding).reshape(-1).astype(np.float32) + img_vec = np.asarray(image_embedding).reshape(-1).astype(np.float32) + + # L2-normalize + # text_norm = np.linalg.norm(text_vec) + # img_norm = np.linalg.norm(img_vec) + # if text_norm > 0: + # text_vec = text_vec / text_norm + # if img_norm > 0: + # img_vec = img_vec / img_norm + + # Weighted combination (image-heavy since visual search is primary) + weight_text = 0.4 + weight_image = 0.6 + combined = weight_text * text_vec + weight_image * img_vec + + # Re-normalize + comb_norm = np.linalg.norm(combined) + if comb_norm > 0: + combined = combined / comb_norm + + final_embedding = combined.reshape(1, -1) + logger.info(f" Combined text+image embedding (weights: {weight_text}/{weight_image})") + + elif text_embedding is not None: + final_embedding = text_embedding + logger.info(f" Using text-only embedding") + + elif image_embedding is not None: + final_embedding = image_embedding + logger.info(f" Using image-only embedding") + + else: + logger.warning(f" No embedding could be generated for room {room_id}") + return False + + # 4. Store in Qdrant + success = store_canvas_embedding( + room_id=room_id, + embedding=final_embedding, + metadata={ + 'name': room_name, + 'description': room_desc, + 'type': room_type, + 'ownerName': room_owner, + 'updated_at': room.get('updatedAt', datetime.utcnow()).isoformat(), + 'has_visual': thumbnail_bytes is not None + } + ) + + if success: + logger.info(f"Successfully stored embedding for '{room_name}'") + _last_processed_times[room_id] = datetime.utcnow() + return True + else: + logger.error(f"Failed to store embedding for '{room_name}'") + return False + + except Exception as e: + logger.exception(f"Failed to generate embedding for room {room_id}: {e}") + return False + + def run_iteration(self) -> Dict: + """ + Run one iteration of the worker loop. + """ + start_time = time.time() + + # Find rooms that need updates + rooms_to_process = self.find_rooms_to_update() + + if not rooms_to_process: + logger.debug("No rooms to process in this iteration") + return { + 'processed': 0, + 'success': 0, + 'failed': 0, + 'duration_seconds': time.time() - start_time + } + + # Process each room + success_count = 0 + failed_count = 0 + + for room in rooms_to_process: + if _shutdown_requested: + logger.info("Shutdown requested, stopping iteration") + break + + try: + if self.generate_embedding_for_room(room): + success_count += 1 + else: + failed_count += 1 + except Exception as e: + logger.exception(f"Error processing room {room.get('_id')}: {e}") + failed_count += 1 + + duration = time.time() - start_time + + stats = { + 'processed': len(rooms_to_process), + 'success': success_count, + 'failed': failed_count, + 'duration_seconds': duration + } + + logger.info(f"Iteration complete: {stats}") + return stats + + def run(self): + """ + Main worker loop. Runs indefinitely until shutdown. + """ + logger.info("Embedding worker started") + logger.info(f"Configuration: update_interval={self.update_interval}s, " + f"debounce={self.debounce_period}s, batch_size={self.batch_size}") + + # Print initial Qdrant stats + try: + stats = get_collection_stats() + logger.info(f"Qdrant collection: {stats.get('collection_name')}, " + f"points: {stats.get('points_count')}") + except Exception as e: + logger.warning(f"Could not fetch Qdrant stats: {e}") + + iteration = 0 + + while not _shutdown_requested: + iteration += 1 + logger.info(f"--- Iteration {iteration} ---") + + try: + stats = self.run_iteration() + + # Log summary + if stats['processed'] > 0: + logger.info(f"Processed {stats['processed']} rooms " + f"({stats['success']} success, {stats['failed']} failed) " + f"in {stats['duration_seconds']:.1f}s") + + except Exception as e: + logger.exception(f"Error in worker iteration: {e}") + + # Sleep until next iteration + if not _shutdown_requested: + logger.debug(f"Sleeping for {self.update_interval}s") + for _ in range(self.update_interval): + if _shutdown_requested: + break + time.sleep(1) + + logger.info("Embedding worker shutting down gracefully") + + +def signal_handler(signum, frame): + """Handle shutdown signals gracefully.""" + global _shutdown_requested + logger.info(f"Received signal {signum}, initiating shutdown...") + _shutdown_requested = True + + +def main(): + """Main entry point for the embedding worker.""" + parser = argparse.ArgumentParser( + description='Background worker for incremental canvas embedding generation' + ) + parser.add_argument( + '--interval', + type=int, + default=DEFAULT_UPDATE_INTERVAL, + help=f'Update check interval in seconds (default: {DEFAULT_UPDATE_INTERVAL})' + ) + parser.add_argument( + '--debounce', + type=int, + default=DEFAULT_DEBOUNCE_PERIOD, + help=f'Debounce period in seconds (default: {DEFAULT_DEBOUNCE_PERIOD})' + ) + parser.add_argument( + '--batch-size', + type=int, + default=DEFAULT_BATCH_SIZE, + help=f'Maximum rooms per batch (default: {DEFAULT_BATCH_SIZE})' + ) + parser.add_argument( + '--once', + action='store_true', + help='Run once and exit (for testing)' + ) + + args = parser.parse_args() + + # Set up signal handlers for graceful shutdown + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + # Create and run worker + worker = EmbeddingWorker( + update_interval=args.interval, + debounce_period=args.debounce, + batch_size=args.batch_size + ) + + if args.once: + logger.info("Running in single-iteration mode") + stats = worker.run_iteration() + logger.info(f"Single iteration complete: {stats}") + else: + worker.run() + + logger.info("Embedding worker exited") + + +if __name__ == '__main__': + main() diff --git a/frontend/build/README.md b/frontend/build/README.md new file mode 100644 index 00000000..02e23f3e --- /dev/null +++ b/frontend/build/README.md @@ -0,0 +1,353 @@ +# About ResCanvas + +## The existing problem +Drawing is an important aspect of art and free expression within a variety of domains. It has been used to express new ideas and works of art. Tools such as MS Paint allow for drawing to be achievable on the computer, with online tools extending that functionality over the cloud where users can share and collaborate on drawings and other digital works of art. For instance, both Google's Drawing and Canva's Draw application have a sharable canvas page between registered users to perform their drawings. + +However, such online platforms store the drawing and user data in a centralized manner, making personal data easily trackable by their respective companies, and easily sharable to other third parties such as advertisers. Furthermore, the drawings can be censored by both private and public entities, such as governments and tracking agencies. Privacy is important, yet online collaboration is an essential part of many user's daily workflow. Thus, it is necessary to decentralize the data storage aspect of these online applications. It might seem that the closest working example of this is Reddit's pixel platform since all users can make edits on the same page. However, users' data are still stored centrally on their servers. Furthermore, the scope is limited to just putting one pixel at a time for each user on a rate limited basis. + +## Overview of ResCanvas +Introducing ResCanvas, a breakthrough in web-based drawing platforms that utilizes ResilientDB to ensure that user's drawings are securely stored, allowing for multiple users to collaborate and create new works of art and express ideas freely without any limits, tracking, or censorship. The canvas drawing board is the core feature of ResCanvas, designed to allow users to perform drawings using their mouse or touchscreen interface. It is simple to use, yet it allows for infinite possibilities. **To the best of our knowledge, ResCanvas is the first ResilientDB application that combines the key breakthroughs in database decentralization brought forth by ResilientDB, with the power of expression that comes with art, bridging the gap between the arts and the sciences.** + +ResCanvas is designed to seamlessly integrate drawings with the familiarity of online contribution between users using effective synchronization of each user's canvas drawing page. This allows for error-free consistency even when multiple users are drawing all at the same time. Multiple users are able to collaborate on a single canvas, and just as many things in life have strength in numbers, so too are the users on a canvas. One user could be working on one part of the art drawing while other users can finish another component of the drawing in a collaborative manner. + +The key feature of ResCanvas is defined by having all drawings stored persistently within ResilientDB in a stroke by stroke manner. Each stroke is individually cached via in-memory data store using Redis serving as the frontend cache. This ensures that the end user is able to receive all the strokes from the other users regardless of the response latency of ResilientDB, greatly enhancing the performance for the end user. Furthermore, all users will be able to see each other's strokes under a decentralized context and without any use of a centralized server or system for processing requests and storing data. + +## Key Features +* Multiple user concurrent drawing and viewable editing history on a per user, per room basis with custom room and user access controls and permissions for each room +* Drawing data and edit history is synchronized efficiently and consistently across all users within the canvas drawing room +* Fast, efficient loading of data from backend by leveraging the caching capabilities of the Redis frontend data storage framework +* Color and thickness selection tools to customize your drawings +* Persistent, secure storage of drawing data in ResilientDB allowing for censorship free expression +* No sharing of data to third parties, advertisers, government entities, .etc with decentralized storage, all user account information and data is stored in ResilientDB +* Responsive, intuitive UI inspired by Google's Material design theme used throughout the app, without the tracking and privacy issues of existing web applications +* Clear canvas ensures that data is erased for all users in the same room +* JWT-based authentication for API and Socket.IO access (login via `/auth/login`, include `Authorization: Bearer `) +* Real-time collaboration using Socket.IO for low-latency stroke broadcasting, user notifications, and user activity communication + +## At a glance +A room-based, JWT-authenticated collaborative drawing application with a React frontend and a Flask backend. This provides real-time collaborative canvases (rooms) with low-latency stroke broadcasting (Socket.IO) and persistent stroke storage. +- Backend: Flask + Flask-SocketIO. Routes live under `backend/routes/` (notably `auth.py`, `rooms.py`, `submit_room_line.py`). +- Frontend: React (create-react-app) in `frontend/` — uses `socket.io-client` and stores auth in `localStorage`. +- Storage & cache: Redis for fast room-scoped caching and undo/redo. MongoDB is used as a mirror/persistent cache collection (`canvasCache.strokes`). + +## Key files and locations +- `backend/app.py` — Flask entrypoint and Socket.IO initialization +- `backend/routes/auth.py` — login/refresh/logout and `@require_auth` middleware usage +- `backend/routes/rooms.py` — all room CRUD and stroke endpoints +- `backend/routes/submit_room_line.py` — detailed stroke handling, encryption for private/secure rooms, signature verification +- `backend/services/` — DB, GraphQL commit helper, Socket.IO helpers, crypto utilities +- `frontend/src/` — React app, API clients under `frontend/src/api/`, `frontend/src/services/` contains socket and canvas helpers + +## Authentication +Authentication uses JWT access tokens plus an HttpOnly refresh token cookie, and this process happens server side for enhanced security. Clients can't bypass this protection since authorization and verification all happens in the backend. +- Login/refresh/logout endpoints: `/auth/login`, `/auth/refresh`, `/auth/logout` (see `backend/routes/auth.py`). +- Access tokens: JWTs signed with `JWT_SECRET`. Send as `Authorization: Bearer ` for protected REST and Socket.IO connections. +- Refresh tokens: HttpOnly cookie (server-managed). The backend middleware enforces token validation and injects `g.current_user` for handlers. + +## Important endpoints +- Create/list rooms: POST/GET `/rooms` +- Room details: GET `/rooms/` +- Post stroke: POST `/rooms//strokes` (requires auth and room access) +- Get strokes: GET `/rooms//strokes` (works with or without auth but returns membership-scoped data when authenticated) +- Undo/redo/clear: `/rooms//undo`, `/rooms//redo`, `/rooms//clear` + +For secure rooms (type `secure`) strokes must be signed client-side; the backend validates signatures in `submit_room_line.py`. + +## API for External Applications + +ResCanvas provides a **versioned REST API** (`/api/v1/*`) and an **official JavaScript SDK** (`@rescanvas/client`) for external applications to integrate collaborative drawing functionality. This generalized API layer allows developers to build third-party apps, mobile clients, integrations, and automation tools on top of ResCanvas. + +### Versioned API Endpoints + +All API v1 endpoints are prefixed with `/api/v1` and maintain backward compatibility. The legacy endpoints (without `/api/v1` prefix) continue to work for existing applications. + +**Authentication** (`/api/v1/auth/*`): +- `POST /api/v1/auth/register` — Register new user +- `POST /api/v1/auth/login` — Login and obtain JWT token +- `POST /api/v1/auth/refresh` — Refresh access token +- `POST /api/v1/auth/logout` — Logout and invalidate tokens +- `GET /api/v1/auth/me` — Get current user info +- `PUT /api/v1/auth/password` — Change password + +**Rooms** (`/api/v1/rooms/*`): +- `POST /api/v1/rooms` — Create new room +- `GET /api/v1/rooms` — List accessible rooms +- `GET /api/v1/rooms/` — Get room details +- `PUT /api/v1/rooms/` — Update room settings +- `DELETE /api/v1/rooms/` — Delete room +- `POST /api/v1/rooms//strokes` — Add stroke to room +- `GET /api/v1/rooms//strokes` — Get all room strokes +- `POST /api/v1/rooms//undo` — Undo last stroke +- `POST /api/v1/rooms//redo` — Redo undone stroke +- `POST /api/v1/rooms//clear` — Clear entire canvas +- `POST /api/v1/rooms//share` — Share room with users +- `GET /api/v1/rooms//members` — Get room members +- `PUT /api/v1/rooms//members/` — Update member permissions +- `DELETE /api/v1/rooms//members/` — Remove member +- `POST /api/v1/rooms//leave` — Leave shared room + +**Invitations** (`/api/v1/invites/*`): +- `GET /api/v1/invites` — List pending invitations +- `POST /api/v1/invites//accept` — Accept invitation +- `POST /api/v1/invites//decline` — Decline invitation + +**Notifications** (`/api/v1/notifications/*`): +- `GET /api/v1/notifications` — List notifications +- `PUT /api/v1/notifications/` — Mark as read +- `DELETE /api/v1/notifications/` — Delete notification +- `DELETE /api/v1/notifications` — Clear all notifications +- `GET /api/v1/notifications/preferences` — Get preferences +- `PUT /api/v1/notifications/preferences` — Update preferences + +**Users** (`/api/v1/users/*`): +- `GET /api/v1/users/search?q=` — Search users +- `GET /api/v1/users/suggest` — Get user suggestions + +### JavaScript SDK + +Install the official SDK via npm: + +```bash +npm install @rescanvas/client +``` + +**Quick Start:** + +```javascript +import ResCanvasClient from '@rescanvas/client'; + +const client = new ResCanvasClient({ + baseURL: 'http://localhost:10010', + socketURL: 'http://localhost:10010' +}); + +// Authenticate +await client.auth.login('username', 'password'); + +// Create a room +const room = await client.rooms.create({ + name: 'My Drawing', + type: 'public' +}); + +// Add a stroke +await client.rooms.addStroke(room.id, { + points: [[10, 20], [30, 40], [50, 60]], + color: '#000000', + width: 2 +}); + +// Real-time updates +client.socket.connect(); +client.socket.onStroke((data) => { + console.log('New stroke received:', data); +}); +``` + +**SDK Features:** +- Automatic JWT token management and refresh +- Retry logic with exponential backoff +- Type-safe API methods +- Real-time Socket.IO integration +- Comprehensive error handling +- Promise-based async/await API + +**Documentation:** +- SDK README: `sdk/javascript/README.md` + +### Testing the API + +Comprehensive test suites are available: + +```bash +# Backend API v1 tests +cd backend +pytest tests/test_api_v1.py -v + +# SDK tests (when available) +cd sdk/javascript +npm test +``` + +### Contributing to the API + +We welcome contributions! The API layer is designed to be extended with new endpoints while maintaining backward compatibility. Please see `CONTRIBUTING.md` for guidelines. + +## Key configuration (backend) +See `backend/config.py` and set the following environment variables as appropriate (examples shown in the repository's `.env` usage): +- `MONGO_ATLAS_URI` / `MONGO_URI` — MongoDB connection string +- `JWT_SECRET` — HMAC secret for signing access tokens +- `ACCESS_TOKEN_EXPIRES_SECS`, `REFRESH_TOKEN_EXPIRES_SECS` — token lifetimes +- `REFRESH_TOKEN_COOKIE_NAME`, `REFRESH_TOKEN_COOKIE_SECURE`, `REFRESH_TOKEN_COOKIE_SAMESITE` +- `ROOM_MASTER_KEY_B64` — used to (re)wrap room keys for private/secure rooms +- `SIGNER_PUBLIC_KEY`, `SIGNER_PRIVATE_KEY`, `RECIPIENT_PUBLIC_KEY` — used when committing strokes via the GraphQL service + +The code loads environment variables via `python-dotenv` in `backend/config.py`. + +## Getting started: set up instructions +0. **Resilient Python Cache (first terminal window)** + - Navigate to `backend/incubator-resilientdb-resilient-python-cache/` + - Run `pip install resilient-python-cache` + - Create an `.env` file in this directory with the following contents (**replacing everything that is between brakets with your actual values**): + ``` + MONGO_URL = "[mongodb+srv://:@cluster0-saugt.mongodb.net/test?retryWrites=true&w=majority]" + MONGO_DB = "canvasCache" + MONGO_COLLECTION = "strokes" + ``` + - Then run `python example.py`. + - This will take in these three env parameters and start the MongoDB caching service with ResilientDB. The `example.py` file contains the `resilientdb://crow.resilientdb.com` endpoint which allows MongoDB to interface and stay in sync with ResilientDB in `cache.py`. +1. **Backend (second terminal window)** + - Create and activate a Python venv inside `backend/` and install requirements: + - `python3 -m venv venv` + - `source venv/bin/activate` + - `pip install -r requirements.txt` + - Then, run `python gen_keys.py` to generate a public-private key pair which will be pasted into the `.env` file in the next step + - Create an `.env` file directly under this `backend/` folder with the following contents (**replacing everything that is between brakets with your actual values**): + ``` + MONGO_ATLAS_URI=[mongodb+srv://:@cluster0-saugt.mongodb.net/test?retryWrites=true&w=majority] + SIGNER_PUBLIC_KEY=[PUBLIC_KEY_COPIED_FROM_GEN_KEYS_PY] + SIGNER_PRIVATE_KEY=[PRIVATE_KEY_COPIED_FROM_GEN_KEYS_PY] + RESILIENTDB_BASE_URI=https://crow.resilientdb.com + RESILIENTDB_GRAPHQL_URI=https://cloud.resilientdb.com/graphql + - Ensure Redis and MongoDB are accessible to the backend. + - Install the redis caching server: + - `sudo apt install redis-server` + - `sudo nano /etc/redis/redis.conf` + - Start the redis server: + - `sudo systemctl restart redis.service` + - Run the backend: + - `python app.py` + +2. **Frontend (third terminal window)** + - From the `frontend/` directory, run: + - `npm install` + - `npm start` + - **The ResCanvas application should now start up** + +## Authentication examples (curl): + - Login to obtain access token (also sets refresh cookie): + ``` + curl -X POST http://127.0.0.1:10010/auth/login -H "Content-Type: application/json" -d '{"username":"testuser","password":"testpass"}' + - Post a stroke (replace `` and ``): + ``` + curl -X POST http://127.0.0.1:10010/rooms//strokes -H "Content-Type: application/json" -H "Authorization: Bearer " -d '{"drawingId":"d1","color":"#000","lineWidth":3,"pathData":[],"timestamp": 1696940000000}' + +## Detailed architecture and concepts + +This section expands on the high-level overview and documents the key design concepts, data model, and important theory behind ResCanvas. + +### Architectural components +- Frontend (React): handles drawing input, local smoothing/coalescing of strokes, UI state (tools, color, thickness), optimistic local rendering, and Socket.IO for real-time updates. The app stores auth state in `localStorage` and the frontend API wrappers attach JWT access tokens for protected calls. +- Backend (Flask + Flask-SocketIO): receives stroke writes, performs room membership checks, optionally verifies client-side signatures (secure rooms), encrypts strokes for private/secure rooms, commits transactions to ResilientDB via GraphQL and writes to Redis and MongoDB caches for low latency reads. All protected API routes and Socket.IO connections expect JWT access tokens via `Authorization: Bearer ` as the backend enforces token signature and expiry server-side. When working with private or secure rooms, the backend will encrypt/decrypt strokes and may require a room wrapped key (`backend/services/crypto_service.py` and `submit_room_line.py`). +- ResilientDB: the persistent, decentralized, immutable transaction log where strokes are ultimately stored. Strokes are written as transactions so the full history is auditable and censorship-resistant. +- Redis: short-lived, in-memory store keyed by room for fast read/write and undo/redo operations. Redis is intentionally ephemeral: it allows quick synchronization of live sessions while ResilientDB acts as the long-term durable store. +- MongoDB (canvasCache): a warm persistent cache and queryable replica of strokes so the backend can serve reads without contacting ResilientDB directly for every request. A sync bridge mirrors ResilientDB into MongoDB. + +### Data model and stroke format +ResCanvas uses a simple, compact stroke model that is friendly for network transport and decentralized commits. A typical stroke (JSON) contains the following data: + +- drawingId: unique per-user or per-drawing session +- userId: author id (when available/allowed) +- color: hex or named value +- lineWidth: numeric stroke thickness +- pathData: an array of (x,y) points, optionally compressed (delta-encoded) +- timestamp: client-side timestamp for ordering and replay +- metadata: optional fields for signing, encryption info, transform/offsets + +Note that the path data should be kept compact. The frontend coalesces mouse/touch events and optionally delta-encodes paths before sending to the backend to reduce network bandwidth. For secure rooms, each stroke includes an on-chain/verifiable signature and any necessary auxiliary data to perform signature verification. + +### Consistency, concurrency and ordering +Multiple users can draw simultaneously since the system is designed for eventual consistency with low-latency broadcast, where each stroke is broadcast immediately via Socket.IO to all connected room participants. This allows the application to provide near real-time feedback. The backend attempts to persist strokes to ResilientDB and caches (Redis/MongoDB). If ResilientDB write is delayed, clients still see strokes from the Socket.IO broadcast and from Redis while waiting for the backend to finishing writing the stroke data. Ordering is primarily guided by timestamps and the sequence of commits in ResilientDB. When replaying history, the authoritative order comes from the ResilientDB transaction log. + +### ResilientDB theory: why and how we use it here +ResilientDB is used as an immutable, decentralized transaction log. Key properties we rely on: + +- Immutability: once a stroke is committed, it cannot be altered silently. This increases trust and accountability. +- Decentralization: no single host controls the persistent copy of strokes, reducing censorship and central data harvesting. +- Auditability: the entire canvas history can be inspected and verified against the ResilientDB ledger. + +By treating each stroke as a transaction, we now achieve a chronological, tamper-evident history of canvas changes and so anyone can verfy and review the changes to the canvas as the ground truth source of information. The sync bridge mirrors transactions into MongoDB so read queries don't need to hit ResilientDB for every request, while redis caching further enhances the perforance from the UX standpoint by caching the data from MongoDB on an in-memory basis within a trusted node. Removal operations such as undo and redo, as well as clear canvas/room deletions are simulated using time stamp markers to achieve the same effects without altering the historical backend data as well. + +### Private rooms vs Secure rooms +In private rooms, access is restricted as only invited users or those with the room key can join such rooms. Strokes may be encrypted so only members with the room key can decrypt. The backend participates in wrapping/unwrapping room keys using `ROOM_MASTER_KEY_B64`. + +Secure rooms go further and expand upon the protections of private rooms by requiring client-side signing of strokes with a cryptographic wallet (such as ResVault). Each stroke is signed by the user's wallet private key and the signature is stored with the stroke. This enables verifiable authorship — anyone can confirm a stroke was created by the owner of a given wallet address. See `WALLET_TESTING_GUIDE.md` for details about ResVault and how to use it with ResCanvas, of which the backend validates signatures for secure rooms in `backend/routes/submit_room_line.py`. + +### Security, privacy and threat model +ResCanvas aims to improve user privacy and resist centralized censorship, but there are trade-offs and responsibilities as shown below. + +- Threats mitigated: + - Central server compromise: persistent data is stored on ResilientDB and mirrored to MongoDB, reducing a single point of failure. + - Data harvesting by a platform operator: decentralized storage and client-side signing for secure rooms reduce linkability and provide verifiability. + +- Limitations and assumptions: + - Frontend devices can still execute untrusted code (browser). Compromise of a user's device or browser extension can leak keys or manipulate strokes before signing. + - ResilientDB endpoints and graph commit endpoints used by the backend must remain available and trusted by the backend operators. If those services are compromised, ledger inclusion or availability may be affected. + +Security practices implemented: +- JWT authentication and HttpOnly refresh cookies to protect session tokens from XSS. +- Signature verification on server-side for secure rooms. +- Optional encryption of strokes for private rooms using per-room wrapped keys. + +### Wallet integration and signature flow for secure rooms +1. User connects wallet (such as ResVault) via the frontend UI and grants signing permissions. +2. When drawing in a secure room, the frontend prepares the stroke payload and asks ResVault to sign the serialized stroke or a deterministic hash of it. +3. The signed payload (signature + public key or address) is sent to the backend along with the stroke. +4. The backend verifies the signature before accepting and committing the stroke to persistent storage. + +### Undo/redo and edit history +Undo/redo is implemented through per-room, per-user stacks stored in Redis (ephemeral). Each user action that mutates the canvas pushes an entry to the user's undo stack and updates the live room state in Redis. Redo pops from a redo stack and applies the strokes again via the same commit flow (including related signing and encryption rules). Because ResilientDB is immutable, undo/redo on the client is implemented as additional strokes that semantically represent an "undo" (for example a delta or a tombstone stroke) by using a separate metadata layer that signals removal in replay. The visible client behavior is immediate, while the authoritative history in ResilientDB preserves the full append only log. + +### Developer workflows and testing + +#### Quick Start Testing + +ResCanvas has a comprehensive test suite with tests that are covering both the backend and frontend: + +```bash +# Fast optimized testing which is recommended for development +./scripts/run_all_tests_parallel.sh --fast + +# Full test suite with coverage +./scripts/run_all_tests_parallel.sh + +# Tests with sequential script +./scripts/run_all_tests_unified.sh +``` + +#### Test Suite Breakdown + +- **Backend Tests** (99 tests): + - Unit tests: `pytest tests/unit/ -v` + - Integration tests: `pytest tests/integration/ -v` + - E2E tests: `pytest tests/test_*.py -v` + +- **Frontend Unit Tests** (139 tests): + - Run with Jest: `cd frontend && npm test` + - Parallel by default (4 workers) + +- **Frontend E2E Tests** (56 tests): + - Playwright: `cd frontend && npx playwright test` + - Tests auth, rooms, collaboration, drawing, error handling + +#### CI/CD Integration + +**GitHub Actions workflows** automatically test every push and PR: + +- **Full Test Suite** (`ci-tests.yml`): Matrix testing across Python 3.10/3.11 and Node 20.x/22.x +- **Quick Check** (`ci-quick.yml`): Fast feedback loop for PRs (~5-8 min) + +**CI Setup Notes:** + +- **Key Generation**: The workflows automatically run `gen_keys.py` to generate signing keys as per the README setup instructions. This requires `PyNaCl` and `base58` packages (now included in requirements.txt). +- **MongoDB Connection**: In GitHub Actions, the MongoDB service is accessible at hostname `mongodb` (not `localhost`). The workflows use `mongodb://testuser:testpass@mongodb:27017/?authSource=admin`. +- **Environment variable names**: The backend expects `JWT_SECRET` (not `JWT_SECRET_KEY`) and `RES_DB_BASE_URI`/`RESILIENTDB_BASE_URI` for ResilientDB endpoint configuration. +- **Codecov uploads**: If the repository is private, set a `CODECOV_TOKEN` secret in Settings → Secrets → Actions. The workflows skip Codecov upload if the token is not defined. +- **Manual trigger**: Actions → CI - Full Test Suite → Run workflow + +## Contributors +* Henry Chou - Team Leader and Full Stack Developer +* Varun Ringnekar - Full Stack Developer +* Chris Ruan - Frontend Developer +* Shaokang Xie - Backend Developer +* Yubo Bai - Frontend Developer \ No newline at end of file diff --git a/frontend/package-lock.json b/frontend/package-lock.json index f1b6a622..f64749d3 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -96,6 +96,7 @@ "version": "7.26.0", "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.26.0.tgz", "integrity": "sha512-i1SLeK+DzNnQ3LL/CswPCa/E5u4lh1k6IAEphON8F+cXt0t9euTshDru0q7/IqMa1PMPz5RnHuHscF8/ZJsStg==", + "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.26.0", @@ -715,6 +716,7 @@ "version": "7.26.0", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.26.0.tgz", "integrity": "sha512-B+O2DnPc0iG+YXFqOxv2WNuNU97ToWjOomUQ78DouOENWUaM5sVrmet9mcomUGQFwpJd//gvUagXBSdzO1fRKg==", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.25.9" }, @@ -1524,6 +1526,7 @@ "version": "7.25.9", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.25.9.tgz", "integrity": "sha512-s5XwpQYCqGerXl+Pu6VDL3x0j2d82eiV77UJ8a2mDHAW7j9SWRqQ2y1fNo1Z74CdcYipl5Z41zvjj4Nfzq36rw==", + "peer": true, "dependencies": { "@babel/helper-annotate-as-pure": "^7.25.9", "@babel/helper-module-imports": "^7.25.9", @@ -2327,6 +2330,7 @@ "resolved": "https://registry.npmjs.org/@emotion/react/-/react-11.13.5.tgz", "integrity": "sha512-6zeCUxUH+EPF1s+YF/2hPVODeV/7V07YU5x+2tfuRL8MdW6rv5vb2+CBEGTGwBdux0OIERcOS+RzxeK80k2DsQ==", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.18.3", "@emotion/babel-plugin": "^11.13.5", @@ -2370,6 +2374,7 @@ "resolved": "https://registry.npmjs.org/@emotion/styled/-/styled-11.13.5.tgz", "integrity": "sha512-gnOQ+nGLPvDXgIx119JqGalys64lhMdnNQA9TMxhDA4K0Hq5+++OE20Zs5GxiCV9r814xQ2K5WmtofSpHVW6BQ==", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.18.3", "@emotion/babel-plugin": "^11.13.5", @@ -3050,6 +3055,7 @@ "version": "6.4.7", "resolved": "https://registry.npmjs.org/@mui/material/-/material-6.4.7.tgz", "integrity": "sha512-K65StXUeGAtFJ4ikvHKtmDCO5Ab7g0FZUu2J5VpoKD+O6Y3CjLYzRi+TMlI3kaL4CL158+FccMoOd/eaddmeRQ==", + "peer": true, "dependencies": { "@babel/runtime": "^7.26.0", "@mui/core-downloads-tracker": "^6.4.7", @@ -3719,7 +3725,6 @@ "version": "10.4.0", "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.0.tgz", "integrity": "sha512-pemlzrSESWbdAloYml3bAJMEfNh1Z7EduzqPKprCH5S341frlpYnUEW0H72dLxa6IsYr+mPno20GiSm+h9dEdQ==", - "peer": true, "dependencies": { "@babel/code-frame": "^7.10.4", "@babel/runtime": "^7.12.5", @@ -3738,7 +3743,6 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", "integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==", - "peer": true, "dependencies": { "dequal": "^2.0.3" } @@ -4314,6 +4318,7 @@ "version": "18.3.12", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.12.tgz", "integrity": "sha512-D2wOSq/d6Agt28q7rSI3jhU7G6aiuzljDGZ2hTZHIkrTLUI+AF3WMeKkEZ9nN2fkBAlcktT6vcZjDFiIhMYEQw==", + "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.0.2" @@ -4437,6 +4442,7 @@ "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.62.0.tgz", "integrity": "sha512-TiZzBSJja/LbhNPvk6yc0JrX9XqhQ0hdh6M2svYfsHGejaKFIAGd9MQ+ERIMzLGlN/kZoYIgdxFV0PuljTKXag==", + "peer": true, "dependencies": { "@eslint-community/regexpp": "^4.4.0", "@typescript-eslint/scope-manager": "5.62.0", @@ -4488,6 +4494,7 @@ "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.62.0.tgz", "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", @@ -4827,6 +4834,7 @@ "version": "8.14.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4905,6 +4913,7 @@ "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -5713,6 +5722,7 @@ "url": "https://github.com/sponsors/ai" } ], + "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001669", "electron-to-chromium": "^1.5.41", @@ -7057,6 +7067,7 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -7988,6 +7999,7 @@ "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -10816,6 +10828,7 @@ "version": "27.5.1", "resolved": "https://registry.npmjs.org/jest/-/jest-27.5.1.tgz", "integrity": "sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==", + "peer": true, "dependencies": { "@jest/core": "^27.5.1", "import-local": "^3.0.2", @@ -13861,6 +13874,7 @@ "url": "https://github.com/sponsors/ai" } ], + "peer": true, "dependencies": { "nanoid": "^3.3.7", "picocolors": "^1.1.0", @@ -14953,6 +14967,7 @@ "version": "6.1.2", "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -15296,6 +15311,7 @@ "version": "18.3.1", "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -15458,6 +15474,7 @@ "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -15506,6 +15523,7 @@ "version": "0.11.0", "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.11.0.tgz", "integrity": "sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A==", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -16296,6 +16314,7 @@ "version": "2.79.2", "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.79.2.tgz", "integrity": "sha512-fS6iqSPZDs3dr/y7Od6y5nha8dW1YnbgtsyotCVvoFGKbERG++CVRFv1meyGDE1SNItQA8BrnCw7ScdAhRJ3XQ==", + "peer": true, "bin": { "rollup": "dist/bin/rollup" }, @@ -16515,6 +16534,7 @@ "version": "8.17.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -18058,6 +18078,7 @@ "version": "0.21.3", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "peer": true, "engines": { "node": ">=10" }, @@ -18597,6 +18618,7 @@ "version": "5.96.1", "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.96.1.tgz", "integrity": "sha512-l2LlBSvVZGhL4ZrPwyr8+37AunkcYj5qh8o6u2/2rzoPc8gxFJkLj1WxNgooi9pnoc06jh0BjuXnamM4qlujZA==", + "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.6", @@ -18664,6 +18686,7 @@ "version": "4.15.2", "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-4.15.2.tgz", "integrity": "sha512-0XavAZbNJ5sDrCbkpWL8mia0o5WPOd2YGtxrEiZkBK9FjLppIUK2TgxK6qGD2P3hUXTJNNPVibrerKcx5WkR1g==", + "peer": true, "dependencies": { "@types/bonjour": "^3.5.9", "@types/connect-history-api-fallback": "^1.3.5", @@ -19062,6 +19085,7 @@ "version": "8.17.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -19477,6 +19501,7 @@ "version": "7.26.0", "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.26.0.tgz", "integrity": "sha512-i1SLeK+DzNnQ3LL/CswPCa/E5u4lh1k6IAEphON8F+cXt0t9euTshDru0q7/IqMa1PMPz5RnHuHscF8/ZJsStg==", + "peer": true, "requires": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.26.0", @@ -19893,6 +19918,7 @@ "version": "7.26.0", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.26.0.tgz", "integrity": "sha512-B+O2DnPc0iG+YXFqOxv2WNuNU97ToWjOomUQ78DouOENWUaM5sVrmet9mcomUGQFwpJd//gvUagXBSdzO1fRKg==", + "peer": true, "requires": { "@babel/helper-plugin-utils": "^7.25.9" } @@ -20384,6 +20410,7 @@ "version": "7.25.9", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.25.9.tgz", "integrity": "sha512-s5XwpQYCqGerXl+Pu6VDL3x0j2d82eiV77UJ8a2mDHAW7j9SWRqQ2y1fNo1Z74CdcYipl5Z41zvjj4Nfzq36rw==", + "peer": true, "requires": { "@babel/helper-annotate-as-pure": "^7.25.9", "@babel/helper-module-imports": "^7.25.9", @@ -20897,6 +20924,7 @@ "version": "11.13.5", "resolved": "https://registry.npmjs.org/@emotion/react/-/react-11.13.5.tgz", "integrity": "sha512-6zeCUxUH+EPF1s+YF/2hPVODeV/7V07YU5x+2tfuRL8MdW6rv5vb2+CBEGTGwBdux0OIERcOS+RzxeK80k2DsQ==", + "peer": true, "requires": { "@babel/runtime": "^7.18.3", "@emotion/babel-plugin": "^11.13.5", @@ -20929,6 +20957,7 @@ "version": "11.13.5", "resolved": "https://registry.npmjs.org/@emotion/styled/-/styled-11.13.5.tgz", "integrity": "sha512-gnOQ+nGLPvDXgIx119JqGalys64lhMdnNQA9TMxhDA4K0Hq5+++OE20Zs5GxiCV9r814xQ2K5WmtofSpHVW6BQ==", + "peer": true, "requires": { "@babel/runtime": "^7.18.3", "@emotion/babel-plugin": "^11.13.5", @@ -21418,6 +21447,7 @@ "version": "6.4.7", "resolved": "https://registry.npmjs.org/@mui/material/-/material-6.4.7.tgz", "integrity": "sha512-K65StXUeGAtFJ4ikvHKtmDCO5Ab7g0FZUu2J5VpoKD+O6Y3CjLYzRi+TMlI3kaL4CL158+FccMoOd/eaddmeRQ==", + "peer": true, "requires": { "@babel/runtime": "^7.26.0", "@mui/core-downloads-tracker": "^6.4.7", @@ -21793,7 +21823,6 @@ "version": "10.4.0", "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.0.tgz", "integrity": "sha512-pemlzrSESWbdAloYml3bAJMEfNh1Z7EduzqPKprCH5S341frlpYnUEW0H72dLxa6IsYr+mPno20GiSm+h9dEdQ==", - "peer": true, "requires": { "@babel/code-frame": "^7.10.4", "@babel/runtime": "^7.12.5", @@ -21809,7 +21838,6 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", "integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==", - "peer": true, "requires": { "dequal": "^2.0.3" } @@ -22322,6 +22350,7 @@ "version": "18.3.12", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.12.tgz", "integrity": "sha512-D2wOSq/d6Agt28q7rSI3jhU7G6aiuzljDGZ2hTZHIkrTLUI+AF3WMeKkEZ9nN2fkBAlcktT6vcZjDFiIhMYEQw==", + "peer": true, "requires": { "@types/prop-types": "*", "csstype": "^3.0.2" @@ -22442,6 +22471,7 @@ "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.62.0.tgz", "integrity": "sha512-TiZzBSJja/LbhNPvk6yc0JrX9XqhQ0hdh6M2svYfsHGejaKFIAGd9MQ+ERIMzLGlN/kZoYIgdxFV0PuljTKXag==", + "peer": true, "requires": { "@eslint-community/regexpp": "^4.4.0", "@typescript-eslint/scope-manager": "5.62.0", @@ -22467,6 +22497,7 @@ "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.62.0.tgz", "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==", + "peer": true, "requires": { "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", @@ -22723,7 +22754,8 @@ "acorn": { "version": "8.14.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", - "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==" + "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==", + "peer": true }, "acorn-globals": { "version": "6.0.0", @@ -22778,6 +22810,7 @@ "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "peer": true, "requires": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -23356,6 +23389,7 @@ "version": "4.24.2", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.2.tgz", "integrity": "sha512-ZIc+Q62revdMcqC6aChtW4jz3My3klmCO1fEmINZY/8J3EpBg5/A/D0AKmBveUh6pgoeycoMkVMko84tuYS+Gg==", + "peer": true, "requires": { "caniuse-lite": "^1.0.30001669", "electron-to-chromium": "^1.5.41", @@ -24267,7 +24301,8 @@ "d3-selection": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", - "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==" + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", + "peer": true }, "d3-shape": { "version": "3.2.0", @@ -24950,6 +24985,7 @@ "version": "8.57.1", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", + "peer": true, "requires": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -26905,6 +26941,7 @@ "version": "27.5.1", "resolved": "https://registry.npmjs.org/jest/-/jest-27.5.1.tgz", "integrity": "sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==", + "peer": true, "requires": { "@jest/core": "^27.5.1", "import-local": "^3.0.2", @@ -29020,6 +29057,7 @@ "version": "8.4.47", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.47.tgz", "integrity": "sha512-56rxCq7G/XfB4EkXq9Egn5GCqugWvDFjafDOThIdMBsI15iqPqR5r15TfSr1YPYeEI19YeaXMCbY6u88Y76GLQ==", + "peer": true, "requires": { "nanoid": "^3.3.7", "picocolors": "^1.1.0", @@ -29615,6 +29653,7 @@ "version": "6.1.2", "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "peer": true, "requires": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -29865,6 +29904,7 @@ "version": "18.3.1", "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "peer": true, "requires": { "loose-envify": "^1.1.0" } @@ -29987,6 +30027,7 @@ "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "peer": true, "requires": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -30022,7 +30063,8 @@ "react-refresh": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.11.0.tgz", - "integrity": "sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A==" + "integrity": "sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A==", + "peer": true }, "react-router": { "version": "7.0.2", @@ -30565,6 +30607,7 @@ "version": "2.79.2", "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.79.2.tgz", "integrity": "sha512-fS6iqSPZDs3dr/y7Od6y5nha8dW1YnbgtsyotCVvoFGKbERG++CVRFv1meyGDE1SNItQA8BrnCw7ScdAhRJ3XQ==", + "peer": true, "requires": { "fsevents": "~2.3.2" } @@ -30694,6 +30737,7 @@ "version": "8.17.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "peer": true, "requires": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -31869,7 +31913,8 @@ "type-fest": { "version": "0.21.3", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", - "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==" + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "peer": true }, "type-is": { "version": "1.6.18", @@ -32245,6 +32290,7 @@ "version": "5.96.1", "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.96.1.tgz", "integrity": "sha512-l2LlBSvVZGhL4ZrPwyr8+37AunkcYj5qh8o6u2/2rzoPc8gxFJkLj1WxNgooi9pnoc06jh0BjuXnamM4qlujZA==", + "peer": true, "requires": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.6", @@ -32313,6 +32359,7 @@ "version": "4.15.2", "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-4.15.2.tgz", "integrity": "sha512-0XavAZbNJ5sDrCbkpWL8mia0o5WPOd2YGtxrEiZkBK9FjLppIUK2TgxK6qGD2P3hUXTJNNPVibrerKcx5WkR1g==", + "peer": true, "requires": { "@types/bonjour": "^3.5.9", "@types/connect-history-api-fallback": "^1.3.5", @@ -32579,6 +32626,7 @@ "version": "8.17.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "peer": true, "requires": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", diff --git a/frontend/public/README.md b/frontend/public/README.md new file mode 100644 index 00000000..02e23f3e --- /dev/null +++ b/frontend/public/README.md @@ -0,0 +1,353 @@ +# About ResCanvas + +## The existing problem +Drawing is an important aspect of art and free expression within a variety of domains. It has been used to express new ideas and works of art. Tools such as MS Paint allow for drawing to be achievable on the computer, with online tools extending that functionality over the cloud where users can share and collaborate on drawings and other digital works of art. For instance, both Google's Drawing and Canva's Draw application have a sharable canvas page between registered users to perform their drawings. + +However, such online platforms store the drawing and user data in a centralized manner, making personal data easily trackable by their respective companies, and easily sharable to other third parties such as advertisers. Furthermore, the drawings can be censored by both private and public entities, such as governments and tracking agencies. Privacy is important, yet online collaboration is an essential part of many user's daily workflow. Thus, it is necessary to decentralize the data storage aspect of these online applications. It might seem that the closest working example of this is Reddit's pixel platform since all users can make edits on the same page. However, users' data are still stored centrally on their servers. Furthermore, the scope is limited to just putting one pixel at a time for each user on a rate limited basis. + +## Overview of ResCanvas +Introducing ResCanvas, a breakthrough in web-based drawing platforms that utilizes ResilientDB to ensure that user's drawings are securely stored, allowing for multiple users to collaborate and create new works of art and express ideas freely without any limits, tracking, or censorship. The canvas drawing board is the core feature of ResCanvas, designed to allow users to perform drawings using their mouse or touchscreen interface. It is simple to use, yet it allows for infinite possibilities. **To the best of our knowledge, ResCanvas is the first ResilientDB application that combines the key breakthroughs in database decentralization brought forth by ResilientDB, with the power of expression that comes with art, bridging the gap between the arts and the sciences.** + +ResCanvas is designed to seamlessly integrate drawings with the familiarity of online contribution between users using effective synchronization of each user's canvas drawing page. This allows for error-free consistency even when multiple users are drawing all at the same time. Multiple users are able to collaborate on a single canvas, and just as many things in life have strength in numbers, so too are the users on a canvas. One user could be working on one part of the art drawing while other users can finish another component of the drawing in a collaborative manner. + +The key feature of ResCanvas is defined by having all drawings stored persistently within ResilientDB in a stroke by stroke manner. Each stroke is individually cached via in-memory data store using Redis serving as the frontend cache. This ensures that the end user is able to receive all the strokes from the other users regardless of the response latency of ResilientDB, greatly enhancing the performance for the end user. Furthermore, all users will be able to see each other's strokes under a decentralized context and without any use of a centralized server or system for processing requests and storing data. + +## Key Features +* Multiple user concurrent drawing and viewable editing history on a per user, per room basis with custom room and user access controls and permissions for each room +* Drawing data and edit history is synchronized efficiently and consistently across all users within the canvas drawing room +* Fast, efficient loading of data from backend by leveraging the caching capabilities of the Redis frontend data storage framework +* Color and thickness selection tools to customize your drawings +* Persistent, secure storage of drawing data in ResilientDB allowing for censorship free expression +* No sharing of data to third parties, advertisers, government entities, .etc with decentralized storage, all user account information and data is stored in ResilientDB +* Responsive, intuitive UI inspired by Google's Material design theme used throughout the app, without the tracking and privacy issues of existing web applications +* Clear canvas ensures that data is erased for all users in the same room +* JWT-based authentication for API and Socket.IO access (login via `/auth/login`, include `Authorization: Bearer `) +* Real-time collaboration using Socket.IO for low-latency stroke broadcasting, user notifications, and user activity communication + +## At a glance +A room-based, JWT-authenticated collaborative drawing application with a React frontend and a Flask backend. This provides real-time collaborative canvases (rooms) with low-latency stroke broadcasting (Socket.IO) and persistent stroke storage. +- Backend: Flask + Flask-SocketIO. Routes live under `backend/routes/` (notably `auth.py`, `rooms.py`, `submit_room_line.py`). +- Frontend: React (create-react-app) in `frontend/` — uses `socket.io-client` and stores auth in `localStorage`. +- Storage & cache: Redis for fast room-scoped caching and undo/redo. MongoDB is used as a mirror/persistent cache collection (`canvasCache.strokes`). + +## Key files and locations +- `backend/app.py` — Flask entrypoint and Socket.IO initialization +- `backend/routes/auth.py` — login/refresh/logout and `@require_auth` middleware usage +- `backend/routes/rooms.py` — all room CRUD and stroke endpoints +- `backend/routes/submit_room_line.py` — detailed stroke handling, encryption for private/secure rooms, signature verification +- `backend/services/` — DB, GraphQL commit helper, Socket.IO helpers, crypto utilities +- `frontend/src/` — React app, API clients under `frontend/src/api/`, `frontend/src/services/` contains socket and canvas helpers + +## Authentication +Authentication uses JWT access tokens plus an HttpOnly refresh token cookie, and this process happens server side for enhanced security. Clients can't bypass this protection since authorization and verification all happens in the backend. +- Login/refresh/logout endpoints: `/auth/login`, `/auth/refresh`, `/auth/logout` (see `backend/routes/auth.py`). +- Access tokens: JWTs signed with `JWT_SECRET`. Send as `Authorization: Bearer ` for protected REST and Socket.IO connections. +- Refresh tokens: HttpOnly cookie (server-managed). The backend middleware enforces token validation and injects `g.current_user` for handlers. + +## Important endpoints +- Create/list rooms: POST/GET `/rooms` +- Room details: GET `/rooms/` +- Post stroke: POST `/rooms//strokes` (requires auth and room access) +- Get strokes: GET `/rooms//strokes` (works with or without auth but returns membership-scoped data when authenticated) +- Undo/redo/clear: `/rooms//undo`, `/rooms//redo`, `/rooms//clear` + +For secure rooms (type `secure`) strokes must be signed client-side; the backend validates signatures in `submit_room_line.py`. + +## API for External Applications + +ResCanvas provides a **versioned REST API** (`/api/v1/*`) and an **official JavaScript SDK** (`@rescanvas/client`) for external applications to integrate collaborative drawing functionality. This generalized API layer allows developers to build third-party apps, mobile clients, integrations, and automation tools on top of ResCanvas. + +### Versioned API Endpoints + +All API v1 endpoints are prefixed with `/api/v1` and maintain backward compatibility. The legacy endpoints (without `/api/v1` prefix) continue to work for existing applications. + +**Authentication** (`/api/v1/auth/*`): +- `POST /api/v1/auth/register` — Register new user +- `POST /api/v1/auth/login` — Login and obtain JWT token +- `POST /api/v1/auth/refresh` — Refresh access token +- `POST /api/v1/auth/logout` — Logout and invalidate tokens +- `GET /api/v1/auth/me` — Get current user info +- `PUT /api/v1/auth/password` — Change password + +**Rooms** (`/api/v1/rooms/*`): +- `POST /api/v1/rooms` — Create new room +- `GET /api/v1/rooms` — List accessible rooms +- `GET /api/v1/rooms/` — Get room details +- `PUT /api/v1/rooms/` — Update room settings +- `DELETE /api/v1/rooms/` — Delete room +- `POST /api/v1/rooms//strokes` — Add stroke to room +- `GET /api/v1/rooms//strokes` — Get all room strokes +- `POST /api/v1/rooms//undo` — Undo last stroke +- `POST /api/v1/rooms//redo` — Redo undone stroke +- `POST /api/v1/rooms//clear` — Clear entire canvas +- `POST /api/v1/rooms//share` — Share room with users +- `GET /api/v1/rooms//members` — Get room members +- `PUT /api/v1/rooms//members/` — Update member permissions +- `DELETE /api/v1/rooms//members/` — Remove member +- `POST /api/v1/rooms//leave` — Leave shared room + +**Invitations** (`/api/v1/invites/*`): +- `GET /api/v1/invites` — List pending invitations +- `POST /api/v1/invites//accept` — Accept invitation +- `POST /api/v1/invites//decline` — Decline invitation + +**Notifications** (`/api/v1/notifications/*`): +- `GET /api/v1/notifications` — List notifications +- `PUT /api/v1/notifications/` — Mark as read +- `DELETE /api/v1/notifications/` — Delete notification +- `DELETE /api/v1/notifications` — Clear all notifications +- `GET /api/v1/notifications/preferences` — Get preferences +- `PUT /api/v1/notifications/preferences` — Update preferences + +**Users** (`/api/v1/users/*`): +- `GET /api/v1/users/search?q=` — Search users +- `GET /api/v1/users/suggest` — Get user suggestions + +### JavaScript SDK + +Install the official SDK via npm: + +```bash +npm install @rescanvas/client +``` + +**Quick Start:** + +```javascript +import ResCanvasClient from '@rescanvas/client'; + +const client = new ResCanvasClient({ + baseURL: 'http://localhost:10010', + socketURL: 'http://localhost:10010' +}); + +// Authenticate +await client.auth.login('username', 'password'); + +// Create a room +const room = await client.rooms.create({ + name: 'My Drawing', + type: 'public' +}); + +// Add a stroke +await client.rooms.addStroke(room.id, { + points: [[10, 20], [30, 40], [50, 60]], + color: '#000000', + width: 2 +}); + +// Real-time updates +client.socket.connect(); +client.socket.onStroke((data) => { + console.log('New stroke received:', data); +}); +``` + +**SDK Features:** +- Automatic JWT token management and refresh +- Retry logic with exponential backoff +- Type-safe API methods +- Real-time Socket.IO integration +- Comprehensive error handling +- Promise-based async/await API + +**Documentation:** +- SDK README: `sdk/javascript/README.md` + +### Testing the API + +Comprehensive test suites are available: + +```bash +# Backend API v1 tests +cd backend +pytest tests/test_api_v1.py -v + +# SDK tests (when available) +cd sdk/javascript +npm test +``` + +### Contributing to the API + +We welcome contributions! The API layer is designed to be extended with new endpoints while maintaining backward compatibility. Please see `CONTRIBUTING.md` for guidelines. + +## Key configuration (backend) +See `backend/config.py` and set the following environment variables as appropriate (examples shown in the repository's `.env` usage): +- `MONGO_ATLAS_URI` / `MONGO_URI` — MongoDB connection string +- `JWT_SECRET` — HMAC secret for signing access tokens +- `ACCESS_TOKEN_EXPIRES_SECS`, `REFRESH_TOKEN_EXPIRES_SECS` — token lifetimes +- `REFRESH_TOKEN_COOKIE_NAME`, `REFRESH_TOKEN_COOKIE_SECURE`, `REFRESH_TOKEN_COOKIE_SAMESITE` +- `ROOM_MASTER_KEY_B64` — used to (re)wrap room keys for private/secure rooms +- `SIGNER_PUBLIC_KEY`, `SIGNER_PRIVATE_KEY`, `RECIPIENT_PUBLIC_KEY` — used when committing strokes via the GraphQL service + +The code loads environment variables via `python-dotenv` in `backend/config.py`. + +## Getting started: set up instructions +0. **Resilient Python Cache (first terminal window)** + - Navigate to `backend/incubator-resilientdb-resilient-python-cache/` + - Run `pip install resilient-python-cache` + - Create an `.env` file in this directory with the following contents (**replacing everything that is between brakets with your actual values**): + ``` + MONGO_URL = "[mongodb+srv://:@cluster0-saugt.mongodb.net/test?retryWrites=true&w=majority]" + MONGO_DB = "canvasCache" + MONGO_COLLECTION = "strokes" + ``` + - Then run `python example.py`. + - This will take in these three env parameters and start the MongoDB caching service with ResilientDB. The `example.py` file contains the `resilientdb://crow.resilientdb.com` endpoint which allows MongoDB to interface and stay in sync with ResilientDB in `cache.py`. +1. **Backend (second terminal window)** + - Create and activate a Python venv inside `backend/` and install requirements: + - `python3 -m venv venv` + - `source venv/bin/activate` + - `pip install -r requirements.txt` + - Then, run `python gen_keys.py` to generate a public-private key pair which will be pasted into the `.env` file in the next step + - Create an `.env` file directly under this `backend/` folder with the following contents (**replacing everything that is between brakets with your actual values**): + ``` + MONGO_ATLAS_URI=[mongodb+srv://:@cluster0-saugt.mongodb.net/test?retryWrites=true&w=majority] + SIGNER_PUBLIC_KEY=[PUBLIC_KEY_COPIED_FROM_GEN_KEYS_PY] + SIGNER_PRIVATE_KEY=[PRIVATE_KEY_COPIED_FROM_GEN_KEYS_PY] + RESILIENTDB_BASE_URI=https://crow.resilientdb.com + RESILIENTDB_GRAPHQL_URI=https://cloud.resilientdb.com/graphql + - Ensure Redis and MongoDB are accessible to the backend. + - Install the redis caching server: + - `sudo apt install redis-server` + - `sudo nano /etc/redis/redis.conf` + - Start the redis server: + - `sudo systemctl restart redis.service` + - Run the backend: + - `python app.py` + +2. **Frontend (third terminal window)** + - From the `frontend/` directory, run: + - `npm install` + - `npm start` + - **The ResCanvas application should now start up** + +## Authentication examples (curl): + - Login to obtain access token (also sets refresh cookie): + ``` + curl -X POST http://127.0.0.1:10010/auth/login -H "Content-Type: application/json" -d '{"username":"testuser","password":"testpass"}' + - Post a stroke (replace `` and ``): + ``` + curl -X POST http://127.0.0.1:10010/rooms//strokes -H "Content-Type: application/json" -H "Authorization: Bearer " -d '{"drawingId":"d1","color":"#000","lineWidth":3,"pathData":[],"timestamp": 1696940000000}' + +## Detailed architecture and concepts + +This section expands on the high-level overview and documents the key design concepts, data model, and important theory behind ResCanvas. + +### Architectural components +- Frontend (React): handles drawing input, local smoothing/coalescing of strokes, UI state (tools, color, thickness), optimistic local rendering, and Socket.IO for real-time updates. The app stores auth state in `localStorage` and the frontend API wrappers attach JWT access tokens for protected calls. +- Backend (Flask + Flask-SocketIO): receives stroke writes, performs room membership checks, optionally verifies client-side signatures (secure rooms), encrypts strokes for private/secure rooms, commits transactions to ResilientDB via GraphQL and writes to Redis and MongoDB caches for low latency reads. All protected API routes and Socket.IO connections expect JWT access tokens via `Authorization: Bearer ` as the backend enforces token signature and expiry server-side. When working with private or secure rooms, the backend will encrypt/decrypt strokes and may require a room wrapped key (`backend/services/crypto_service.py` and `submit_room_line.py`). +- ResilientDB: the persistent, decentralized, immutable transaction log where strokes are ultimately stored. Strokes are written as transactions so the full history is auditable and censorship-resistant. +- Redis: short-lived, in-memory store keyed by room for fast read/write and undo/redo operations. Redis is intentionally ephemeral: it allows quick synchronization of live sessions while ResilientDB acts as the long-term durable store. +- MongoDB (canvasCache): a warm persistent cache and queryable replica of strokes so the backend can serve reads without contacting ResilientDB directly for every request. A sync bridge mirrors ResilientDB into MongoDB. + +### Data model and stroke format +ResCanvas uses a simple, compact stroke model that is friendly for network transport and decentralized commits. A typical stroke (JSON) contains the following data: + +- drawingId: unique per-user or per-drawing session +- userId: author id (when available/allowed) +- color: hex or named value +- lineWidth: numeric stroke thickness +- pathData: an array of (x,y) points, optionally compressed (delta-encoded) +- timestamp: client-side timestamp for ordering and replay +- metadata: optional fields for signing, encryption info, transform/offsets + +Note that the path data should be kept compact. The frontend coalesces mouse/touch events and optionally delta-encodes paths before sending to the backend to reduce network bandwidth. For secure rooms, each stroke includes an on-chain/verifiable signature and any necessary auxiliary data to perform signature verification. + +### Consistency, concurrency and ordering +Multiple users can draw simultaneously since the system is designed for eventual consistency with low-latency broadcast, where each stroke is broadcast immediately via Socket.IO to all connected room participants. This allows the application to provide near real-time feedback. The backend attempts to persist strokes to ResilientDB and caches (Redis/MongoDB). If ResilientDB write is delayed, clients still see strokes from the Socket.IO broadcast and from Redis while waiting for the backend to finishing writing the stroke data. Ordering is primarily guided by timestamps and the sequence of commits in ResilientDB. When replaying history, the authoritative order comes from the ResilientDB transaction log. + +### ResilientDB theory: why and how we use it here +ResilientDB is used as an immutable, decentralized transaction log. Key properties we rely on: + +- Immutability: once a stroke is committed, it cannot be altered silently. This increases trust and accountability. +- Decentralization: no single host controls the persistent copy of strokes, reducing censorship and central data harvesting. +- Auditability: the entire canvas history can be inspected and verified against the ResilientDB ledger. + +By treating each stroke as a transaction, we now achieve a chronological, tamper-evident history of canvas changes and so anyone can verfy and review the changes to the canvas as the ground truth source of information. The sync bridge mirrors transactions into MongoDB so read queries don't need to hit ResilientDB for every request, while redis caching further enhances the perforance from the UX standpoint by caching the data from MongoDB on an in-memory basis within a trusted node. Removal operations such as undo and redo, as well as clear canvas/room deletions are simulated using time stamp markers to achieve the same effects without altering the historical backend data as well. + +### Private rooms vs Secure rooms +In private rooms, access is restricted as only invited users or those with the room key can join such rooms. Strokes may be encrypted so only members with the room key can decrypt. The backend participates in wrapping/unwrapping room keys using `ROOM_MASTER_KEY_B64`. + +Secure rooms go further and expand upon the protections of private rooms by requiring client-side signing of strokes with a cryptographic wallet (such as ResVault). Each stroke is signed by the user's wallet private key and the signature is stored with the stroke. This enables verifiable authorship — anyone can confirm a stroke was created by the owner of a given wallet address. See `WALLET_TESTING_GUIDE.md` for details about ResVault and how to use it with ResCanvas, of which the backend validates signatures for secure rooms in `backend/routes/submit_room_line.py`. + +### Security, privacy and threat model +ResCanvas aims to improve user privacy and resist centralized censorship, but there are trade-offs and responsibilities as shown below. + +- Threats mitigated: + - Central server compromise: persistent data is stored on ResilientDB and mirrored to MongoDB, reducing a single point of failure. + - Data harvesting by a platform operator: decentralized storage and client-side signing for secure rooms reduce linkability and provide verifiability. + +- Limitations and assumptions: + - Frontend devices can still execute untrusted code (browser). Compromise of a user's device or browser extension can leak keys or manipulate strokes before signing. + - ResilientDB endpoints and graph commit endpoints used by the backend must remain available and trusted by the backend operators. If those services are compromised, ledger inclusion or availability may be affected. + +Security practices implemented: +- JWT authentication and HttpOnly refresh cookies to protect session tokens from XSS. +- Signature verification on server-side for secure rooms. +- Optional encryption of strokes for private rooms using per-room wrapped keys. + +### Wallet integration and signature flow for secure rooms +1. User connects wallet (such as ResVault) via the frontend UI and grants signing permissions. +2. When drawing in a secure room, the frontend prepares the stroke payload and asks ResVault to sign the serialized stroke or a deterministic hash of it. +3. The signed payload (signature + public key or address) is sent to the backend along with the stroke. +4. The backend verifies the signature before accepting and committing the stroke to persistent storage. + +### Undo/redo and edit history +Undo/redo is implemented through per-room, per-user stacks stored in Redis (ephemeral). Each user action that mutates the canvas pushes an entry to the user's undo stack and updates the live room state in Redis. Redo pops from a redo stack and applies the strokes again via the same commit flow (including related signing and encryption rules). Because ResilientDB is immutable, undo/redo on the client is implemented as additional strokes that semantically represent an "undo" (for example a delta or a tombstone stroke) by using a separate metadata layer that signals removal in replay. The visible client behavior is immediate, while the authoritative history in ResilientDB preserves the full append only log. + +### Developer workflows and testing + +#### Quick Start Testing + +ResCanvas has a comprehensive test suite with tests that are covering both the backend and frontend: + +```bash +# Fast optimized testing which is recommended for development +./scripts/run_all_tests_parallel.sh --fast + +# Full test suite with coverage +./scripts/run_all_tests_parallel.sh + +# Tests with sequential script +./scripts/run_all_tests_unified.sh +``` + +#### Test Suite Breakdown + +- **Backend Tests** (99 tests): + - Unit tests: `pytest tests/unit/ -v` + - Integration tests: `pytest tests/integration/ -v` + - E2E tests: `pytest tests/test_*.py -v` + +- **Frontend Unit Tests** (139 tests): + - Run with Jest: `cd frontend && npm test` + - Parallel by default (4 workers) + +- **Frontend E2E Tests** (56 tests): + - Playwright: `cd frontend && npx playwright test` + - Tests auth, rooms, collaboration, drawing, error handling + +#### CI/CD Integration + +**GitHub Actions workflows** automatically test every push and PR: + +- **Full Test Suite** (`ci-tests.yml`): Matrix testing across Python 3.10/3.11 and Node 20.x/22.x +- **Quick Check** (`ci-quick.yml`): Fast feedback loop for PRs (~5-8 min) + +**CI Setup Notes:** + +- **Key Generation**: The workflows automatically run `gen_keys.py` to generate signing keys as per the README setup instructions. This requires `PyNaCl` and `base58` packages (now included in requirements.txt). +- **MongoDB Connection**: In GitHub Actions, the MongoDB service is accessible at hostname `mongodb` (not `localhost`). The workflows use `mongodb://testuser:testpass@mongodb:27017/?authSource=admin`. +- **Environment variable names**: The backend expects `JWT_SECRET` (not `JWT_SECRET_KEY`) and `RES_DB_BASE_URI`/`RESILIENTDB_BASE_URI` for ResilientDB endpoint configuration. +- **Codecov uploads**: If the repository is private, set a `CODECOV_TOKEN` secret in Settings → Secrets → Actions. The workflows skip Codecov upload if the token is not defined. +- **Manual trigger**: Actions → CI - Full Test Suite → Run workflow + +## Contributors +* Henry Chou - Team Leader and Full Stack Developer +* Varun Ringnekar - Full Stack Developer +* Chris Ruan - Frontend Developer +* Shaokang Xie - Backend Developer +* Yubo Bai - Frontend Developer \ No newline at end of file diff --git a/frontend/src/api/apiClient.js b/frontend/src/api/apiClient.js index 6a7d454d..0f30676f 100644 --- a/frontend/src/api/apiClient.js +++ b/frontend/src/api/apiClient.js @@ -17,18 +17,14 @@ import { showRateLimitNotification, globalRateLimitMonitor, } from '../utils/rateLimitHandler'; +import { getAuthToken } from '../utils/authUtils'; -const API_BASE = process.env.REACT_APP_API_BASE; +// Fallback to backend default (used in local development) when REACT_APP_API_BASE +// is not provided. This prevents malformed requests like "undefined/api/..." +const API_BASE = process.env.REACT_APP_API_BASE || 'http://localhost:10010'; console.log("API Base URL", API_BASE) -/** - * Get auth token from localStorage - */ -function getAuthToken() { - return localStorage.getItem('token'); -} - /** * Build headers for API request */ diff --git a/frontend/src/api/chatbotApi.js b/frontend/src/api/chatbotApi.js new file mode 100644 index 00000000..bffa6fed --- /dev/null +++ b/frontend/src/api/chatbotApi.js @@ -0,0 +1,37 @@ +/** + * Chatbot API - Endpoints for interacting with the AI chatbot + * + * Authentication: All endpoints require valid JWT token in Authorization header + * Rate Limiting: 10 requests per minute per user + */ + +import apiClient from './apiClient'; + +/** + * Send a message to the chatbot + * Backend: POST /rooms/{roomId}/chatbot/message + * Middleware: @require_auth + @limiter.limit("10/minute") + * + * @param {string} roomId - The room ID where the message is sent + * @param {string} message - The user's message to the bot + * @param {Array} history - Optional conversation history for context + * @param {Object} canvasContext - Current canvas state (object count, active users, etc.) + * @returns {Promise<{reply: string}>} - The bot's reply + */ +export const postChatMessage = async (roomId, message, history = [], canvasContext = null) => { + const body = { message, history, canvas_context: canvasContext }; + return apiClient.post(`/rooms/${roomId}/chatbot/message`, body); +}; + +/** + * Get chat history for a room + * Backend: GET /rooms/{roomId}/chatbot/history + * Middleware: @require_auth + * + * @param {string} roomId - The room ID to get history for + * @param {number} limit - Maximum number of messages to retrieve (default 50, max 100) + * @returns {Promise<{history: Array}>} - Array of chat messages + */ +export const getChatHistory = async (roomId, limit = 50) => { + return apiClient.get(`/rooms/${roomId}/chatbot/history?limit=${limit}`); +}; diff --git a/frontend/src/components/AI/AIAssistantPanel.jsx b/frontend/src/components/AI/AIAssistantPanel.jsx new file mode 100644 index 00000000..8031d0e9 --- /dev/null +++ b/frontend/src/components/AI/AIAssistantPanel.jsx @@ -0,0 +1,160 @@ +import React, { useState } from "react"; +import '../../styles/ai-assistant.css'; +import { Tooltip, IconButton } from "@mui/material"; +import AutoAwesomeIcon from '@mui/icons-material/AutoAwesome'; +import ImageIcon from '@mui/icons-material/Image'; +import RoundedCornerIcon from '@mui/icons-material/RoundedCorner'; +import AutoFixHighIcon from '@mui/icons-material/AutoFixHigh'; +import FormatPaintIcon from '@mui/icons-material/FormatPaint'; +import SearchIcon from '@mui/icons-material/Search'; + +export default function AIAssistantPanel({ + open, + onClose, + isBusy, + error, + showPromptInput, + onShapeCompletionToggle, + onBeautify, + onRecognizeToggle, +}) { + const [activeButton, setActiveButton] = useState(""); + + const handlePanelItemClick = (itemTitle) => { + // If it's the beautify button, do NOT toggle + if (itemTitle === "Beautify sketch") { + if (typeof onBeautify === "function" && !isBusy) { + onBeautify(); + } + return; + } + + const next = activeButton === itemTitle ? "" : itemTitle; + setActiveButton(next); + + if (itemTitle === "Shape auto completion") { + if (typeof onShapeCompletionToggle === "function") { + onShapeCompletionToggle(next === "Shape auto completion"); + } + return; + } + + if (itemTitle === "Recognize selection") { + if (typeof onRecognizeToggle === "function") { + onRecognizeToggle(next === "Recognize selection"); + } + return; + } + + if (next === "Generate sketch") { + showPromptInput(true, { + type: 'drawing', + placeholder: "Describe what to draw…" + }); + } else if (next === "Generate image") { + showPromptInput(true, { + type: 'image', + placeholder: "Describe the image to generate…" + }); + } else if (next === "Style transfer") { + showPromptInput(true, { + type: 'style', + placeholder: "Describe the style (e.g. 'Van Gogh oil painting')…" + }); + } else { + showPromptInput(false, { + type: '', + placeholder: "" + }); + } + }; + + const renderStyleClass = (itemTitle) => { + // Beautify sketch should never be active + if (itemTitle === "Beautify sketch") { + return "ai-asisstant-panel-item"; + } + + return itemTitle === activeButton + ? "ai-asisstant-panel-item ai-asisstant-panel-item-active" + : "ai-asisstant-panel-item"; + }; + + return ( +
+ +
handlePanelItemClick("Generate sketch")} + aria-pressed={activeButton === "Generate sketch"} + > + + + + + +
+ +
handlePanelItemClick("Generate image")} + aria-pressed={activeButton === "Generate image"} + > + + + + + +
+ +
handlePanelItemClick("Style transfer")} + aria-pressed={activeButton === "Style transfer"} + > + + + + + +
+ +
handlePanelItemClick("Recognize selection")} + aria-pressed={activeButton === "Recognize selection"} + > + + + + + +
+ +
handlePanelItemClick("Shape auto completion")} + aria-pressed={activeButton === "Shape auto completion"} + > + + + + + +
+ +
handlePanelItemClick("Beautify sketch")} + aria-pressed={false} + > + + + + + +
+ +
+ ); +} diff --git a/frontend/src/components/AI/PromptInput.jsx b/frontend/src/components/AI/PromptInput.jsx new file mode 100644 index 00000000..c6fbf848 --- /dev/null +++ b/frontend/src/components/AI/PromptInput.jsx @@ -0,0 +1,89 @@ +import React, { useState } from 'react'; +import PropTypes from 'prop-types'; +import { Button, TextareaAutosize, Box, CircularProgress } from '@mui/material'; + +export default function PromptInput({ + show=false, + loading=false, + onSubmit=()=>{}, + placeholder = 'Describe what to draw…', +}) { + const [text, setText] = useState(''); + + const handleSubmit = async () => { + if (!text.trim() || loading) return; + onSubmit?.(text.trim()); + }; + + const handleKeyDownPress = (e) => { + if (e.key === 'Enter' && !e.shiftKey) { + e.preventDefault(); + handleSubmit(); + } else if (e.key === 'Escape') { + e.preventDefault(); + setText(''); + } + }; + + return ( + + setText(e.target.value)} + onKeyDown={handleKeyDownPress} + disabled={loading} + minRows={1} + style={{ + width: 'auto', + minWidth: 450, + resize: 'none', + border: 'none', + outline: 'none', + fontSize: '14px', + padding: '8px', + background: 'transparent', + fontFamily: 'inherit', + }} + /> + + + ); +} diff --git a/frontend/src/components/AI/ShapeCompletionOverlay.jsx b/frontend/src/components/AI/ShapeCompletionOverlay.jsx new file mode 100644 index 00000000..79572f52 --- /dev/null +++ b/frontend/src/components/AI/ShapeCompletionOverlay.jsx @@ -0,0 +1,198 @@ +import { Box, IconButton } from '@mui/material'; +import CheckIcon from '@mui/icons-material/Check'; +import CloseIcon from '@mui/icons-material/Close'; + +export default function ShapeCompletionOverlay({ + open = false, + suggestion = null, + anchor = null, + panOffset = { x: 0, y: 0 }, + canvasWidth, + canvasHeight, + onAccept = () => {}, + onReject = () => {}, +}) { + if (!open || !suggestion || !suggestion.object || !suggestion.object.pathData) { + return null; + } + + const { object } = suggestion; + const { pathData } = object; + + const strokeColor = object.color || '#00A0FF'; + const strokeWidth = object.lineWidth || 2; + const ghostOpacity = 0.25; + + const ax = (anchor?.x ?? canvasWidth / 2) + panOffset.x; + const ay = (anchor?.y ?? canvasHeight / 2) + panOffset.y; + + const renderShape = () => { + const t = pathData.type; + const tool = pathData.tool || 'shape'; + + if ( + tool === 'freehand' && + t === 'stroke' && + Array.isArray(pathData.points) && + pathData.points.length > 1 + ) { + const pointsAttr = pathData.points.map(p => `${p.x},${p.y}`).join(' '); + return ( + + ); + } + + if (['line', 'circle', 'rectangle'].includes(t) && pathData.start && pathData.end) { + const { start, end } = pathData; + + if (t === 'line') { + return ( + + ); + } + + if (t === 'circle') { + const cx = (start.x + end.x) / 2; + const cy = (start.y + end.y) / 2; + const dx = end.x - start.x; + const dy = end.y - start.y; + const r = Math.sqrt(dx * dx + dy * dy); + + return ( + + ); + } + + if (t === 'rectangle') { + const x = Math.min(start.x, end.x); + const y = Math.min(start.y, end.y); + const w = Math.abs(end.x - start.x); + const h = Math.abs(end.y - start.y); + + return ( + + ); + } + } + + if (t === 'polygon' && Array.isArray(pathData.points) && pathData.points.length > 1) { + const pointsAttr = pathData.points.map(p => `${p.x},${p.y}`).join(' '); + return ( + + ); + } + + if (t === 'text' && typeof pathData.text === 'string' && pathData.start) { + return ( + + {pathData.text} + + ); + } + + return null; + }; + + return ( + <> + + {renderShape()} + + + + + + + + + + + + ); +} \ No newline at end of file diff --git a/frontend/src/components/Blog.js b/frontend/src/components/Blog.js index f2b487ff..fbb411d8 100644 --- a/frontend/src/components/Blog.js +++ b/frontend/src/components/Blog.js @@ -72,14 +72,19 @@ function Blog() { sx={{ minHeight: '100vh', backgroundColor: '#f9f9f9', - paddingLeft: '2rem', - paddingRight: '2rem', - paddingTop: '1rem', - paddingBottom: '1rem', - boxSizing: 'border-box', - }} + padding: '2rem', + overflowY: 'auto', + overflowY: 'auto', + overflowX: 'hidden', + WebkitOverflowScrolling: 'touch', + '&::-webkit-scrollbar': { width: '12px' }, + '&::-webkit-scrollbar-track': { background: 'transparent' }, + '&::-webkit-scrollbar-thumb': { backgroundColor: '#c1c1c1', borderRadius: '6px' }, + scrollbarWidth: 'auto', + scrollbarColor: '#c1c1c1 transparent', + }} > - { }, - onOpenSettings = null, - viewOnly = false, - isOwner = false, - roomType = "public", - walletConnected = false, - templateId = null, -}) { - const canvasRef = useRef(null); - const snapshotRef = useRef(null); - const tempPathRef = useRef([]); - const clamp = (value, min, max) => Math.min(max, Math.max(min, value)); - - const currentUserRef = useRef(null); - if (currentUserRef.current === null) { - try { - const uname = - getUsername(auth) || - `anon_${Date.now()}_${Math.random().toString(36).substr(2, 5)}`; - currentUserRef.current = `${uname}|${Date.now()}`; - } catch (e) { - currentUserRef.current = `anon_${Date.now()}_${Math.random() - .toString(36) - .substr(2, 5)}`; - } - } - const currentUser = currentUserRef.current; - - const [drawing, setDrawing] = useState(false); - const [color, setColor] = useState("#000000"); - const [lineWidth, setLineWidth] = useState(5); - const [drawMode, setDrawMode] = useState("freehand"); - const [shapeType, setShapeType] = useState("circle"); - const [brushStyle] = useState("round"); - const [shapeStart, setShapeStart] = useState(null); - - const brushEngine = useBrushEngine(); - const [currentBrushType, setCurrentBrushType] = useState("normal"); - const [brushParams, setBrushParams] = useState({}); - const [selectedStamp, setSelectedStamp] = useState(null); - const [stampSettings, setStampSettings] = useState({ - size: 50, - rotation: 0, - opacity: 100, - }); - const [backendStamps, setBackendStamps] = useState([]); - const [stampPreview, setStampPreview] = useState(null); // { x, y, stamp, settings } - const stampPreviewRef = useRef(null); - const [activeFilter, setActiveFilter] = useState(null); - const [filterParams, setFilterParams] = useState({}); - const [isFilterPreview, setIsFilterPreview] = useState(false); - const filterCanvasRef = useRef(null); - const originalCanvasDataRef = useRef(null); // For preview mode undo - const preFilterCanvasStateRef = useRef(null); - - const [showColorPicker, setShowColorPicker] = useState(false); - const [isRefreshing, setIsRefreshing] = useState(false); - const [previousColor, setPreviousColor] = useState(null); - const [clearDialogOpen, setClearDialogOpen] = useState(false); - const [undoStack, setUndoStack] = useState([]); - const [redoStack, setRedoStack] = useState([]); - const [undoAvailable, setUndoAvailable] = useState(false); - const [redoAvailable, setRedoAvailable] = useState(false); - const [hasFilters, setHasFilters] = useState(false); // Track if filters exist for UI updates - - const [templateObjects, setTemplateObjects] = useState([]); - const templateObjectsRef = useRef([]); - - useEffect(() => { - templateObjectsRef.current = templateObjects; - }, [templateObjects]); - - const canvasWidth = DEFAULT_CANVAS_WIDTH; - const canvasHeight = DEFAULT_CANVAS_HEIGHT; - - const [panOffset, setPanOffset] = useState({ x: 0, y: 0 }); - const [isPanning, setIsPanning] = useState(false); - const panStartRef = useRef({ x: 0, y: 0 }); - const panOriginRef = useRef({ x: 0, y: 0 }); - const PAN_REFRESH_COOLDOWN_MS = 2000; - const panLastRefreshRef = useRef(0); - const panRefreshSkippedRef = useRef(false); - const panEndRefreshTimerRef = useRef(null); - const pendingPanRefreshRef = useRef(false); - const [pendingDrawings, setPendingDrawings] = useState([]); - const refreshTimerRef = useRef(null); - const submissionQueueRef = useRef([]); - const isSubmittingRef = useRef(false); - const confirmedStrokesRef = useRef(new Set()); - const lastDrawnStateRef = useRef(null); // Track last drawn state to avoid redundant redraws - const isDrawingInProgressRef = useRef(false); // Prevent concurrent drawing operations - const offscreenCanvasRef = useRef(null); // Offscreen canvas for flicker free rendering - const cachedCanvasRef = useRef(null); // Cached canvas for incremental rendering - const cachedDrawingIdsRef = useRef(new Set()); // Track which drawings are in the cache - const forceNextRedrawRef = useRef(false); // Force next redraw even if signature matches for undo redo - const templateCanvasRef = useRef(null); - const cachedTemplateIdRef = useRef(null); - const [historyMode, setHistoryMode] = useState(false); - const [historyRange, setHistoryRange] = useState(null); // {start, end} in epoch ms - const [historyDialogOpen, setHistoryDialogOpen] = useState(false); - const [historyStartInput, setHistoryStartInput] = useState(""); - const [historyEndInput, setHistoryEndInput] = useState(""); - const [isLoading, setIsLoading] = useState(false); - - const [localSnack, setLocalSnack] = useState({ - open: false, - message: "", - duration: 4000, - }); - const [confirmDestructiveOpen, setConfirmDestructiveOpen] = useState(false); - const [destructiveConfirmText, setDestructiveConfirmText] = useState(""); - const showLocalSnack = (msg, duration = 4000) => - setLocalSnack({ open: true, message: String(msg), duration }); - - // editingEnabled controls whether the user can perform mutating actions. - // When historyMode is active, a specific user is selected for replay, or - // when viewOnly is true (room is archived or user is a viewer), editing - // should be disabled. - // For secure rooms, wallet must be connected to allow editing. - const editingEnabled = !( - historyMode || - (selectedUser && selectedUser !== "") || - viewOnly || - (roomType === "secure" && !walletConnected) - ); - const closeLocalSnack = () => - setLocalSnack({ open: false, message: "", duration: 4000 }); - - // Keyboard shortcuts state - const [commandPaletteOpen, setCommandPaletteOpen] = useState(false); - const [shortcutsHelpOpen, setShortcutsHelpOpen] = useState(false); - const shortcutManagerRef = useRef(null); - - // ResilientDB health monitoring state - const [resilientDBHealthy, setResilientDBHealthy] = useState(true); - const [resilientDBQueueSize, setResilientDBQueueSize] = useState(0); - - const roomUiRef = useRef({}); - const previousSelectedUserRef = useRef(null); // Track previous selectedUser to detect changes - const isRefreshingSelectedUserRef = useRef(false); // Prevent concurrent refreshes - const selectedUserRefreshQueueRef = useRef(null); // Queue the next refresh target - const selectedUserAbortControllerRef = useRef(null); // Cancel pending operations - const roomStacksRef = useRef({}); - const roomClipboardRef = useRef({}); - const roomClearedAtRef = useRef({}); - const drawAllDrawingsRef = useRef(null); // Store reference to drawAllDrawings function - - useEffect(() => { - if (!currentRoomId) return; - const ui = - roomUiRef.current[currentRoomId] || - JSON.parse( - localStorage.getItem(`rescanvas:toolbar:${currentRoomId}`) || "null" - ) || - {}; - if (ui.color) setColor(ui.color); - if (ui.lineWidth) setLineWidth(ui.lineWidth); - if (ui.drawMode) setDrawMode(ui.drawMode); - if (ui.shapeType) setShapeType(ui.shapeType); - if (ui.previousColor !== undefined) setPreviousColor(ui.previousColor); - if (ui.selectedStamp) setSelectedStamp(ui.selectedStamp); - if (ui.stampSettings) setStampSettings(ui.stampSettings); - if (ui.currentBrushType) { - setCurrentBrushType(ui.currentBrushType); - brushEngine.setBrushType(ui.currentBrushType); - } - if (ui.brushParams) { - setBrushParams(ui.brushParams); - brushEngine.setBrushParams(ui.brushParams); - } - roomUiRef.current[currentRoomId] = { - color: ui.color ?? color, - lineWidth: ui.lineWidth ?? lineWidth, - drawMode: ui.drawMode ?? drawMode, - shapeType: ui.shapeType ?? shapeType, - previousColor: ui.previousColor ?? previousColor, - selectedStamp: ui.selectedStamp ?? selectedStamp, - stampSettings: ui.stampSettings ?? stampSettings, - currentBrushType: ui.currentBrushType ?? currentBrushType, - brushParams: ui.brushParams ?? brushParams, - }; - const stacks = roomStacksRef.current[currentRoomId] || { - undo: [], - redo: [], - }; - setUndoStack(stacks.undo); - setRedoStack(stacks.redo); - const clip = roomClipboardRef.current[currentRoomId] || null; - if (setCutImageData) setCutImageData(clip); - }, [currentRoomId]); - - // Load template objects when templateId changes - useEffect(() => { - - if (!templateId) { - setTemplateObjects([]); - templateCanvasRef.current = null; - cachedTemplateIdRef.current = null; - return; - } - - const template = TEMPLATE_LIBRARY.find(t => t.id === templateId); - - if (template && template.canvas && template.canvas.objects) { - setTemplateObjects(template.canvas.objects); - templateCanvasRef.current = null; - cachedTemplateIdRef.current = null; - } else { - setTemplateObjects([]); - templateCanvasRef.current = null; - cachedTemplateIdRef.current = null; - } - }, [templateId, currentRoomId]); - - // Force redraw whenever templateObjects change (ensures templates appear immediately) - useEffect(() => { - if (!templateObjects || templateObjects.length === 0) return; - - const timer = setTimeout(() => { - if (drawAllDrawingsRef.current) { - lastDrawnStateRef.current = null; // Force redraw by clearing cache - drawAllDrawingsRef.current(); - } else { - console.warn('drawAllDrawingsRef not ready yet'); - } - }, 100); - - return () => clearTimeout(timer); - }, [templateObjects]); - - // ResilientDB health monitoring - useEffect(() => { - startMonitoring(); - - const unsubscribe = onHealthChange(({ isHealthy, queueSize }) => { - setResilientDBHealthy(isHealthy); - setResilientDBQueueSize(queueSize || 0); - if (!isHealthy) { - console.warn(`[Canvas] ResilientDB is down - ${queueSize} strokes queued for sync`); - } else { - console.log('[Canvas] ResilientDB is healthy - blockchain persistence active'); - } - }); - - return () => { - stopMonitoring(); - unsubscribe(); - }; - }, []); - - useEffect(() => { - if (!currentRoomId) return; - const ui = { color, lineWidth, drawMode, shapeType, previousColor, selectedStamp, stampSettings, currentBrushType, brushParams }; - roomUiRef.current[currentRoomId] = ui; - try { - localStorage.setItem( - `rescanvas:toolbar:${currentRoomId}`, - JSON.stringify(ui) - ); - } catch { } - }, [currentRoomId, color, lineWidth, drawMode, shapeType, previousColor, selectedStamp, stampSettings, currentBrushType, brushParams]); - - useEffect(() => { - if (!currentRoomId) return; - const cur = roomStacksRef.current[currentRoomId] || { undo: [], redo: [] }; - cur.undo = undoStack; - roomStacksRef.current[currentRoomId] = cur; - }, [currentRoomId, undoStack]); - useEffect(() => { - if (!currentRoomId) return; - const cur = roomStacksRef.current[currentRoomId] || { undo: [], redo: [] }; - cur.redo = redoStack; - roomStacksRef.current[currentRoomId] = cur; - }, [currentRoomId, redoStack]); - - useEffect(() => { - const handleMouseUp = () => { - setIsPanning(false); - panOriginRef.current = { ...panOffset }; - try { - if (panEndRefreshTimerRef.current) { - clearTimeout(panEndRefreshTimerRef.current); - panEndRefreshTimerRef.current = null; - } - if (panRefreshSkippedRef.current) { - panRefreshSkippedRef.current = false; - mergedRefreshCanvas("pan-mouseup-skipped").finally(() => { - try { - setIsLoading(false); - } catch (e) { } - }); - } - if (pendingPanRefreshRef.current) { - pendingPanRefreshRef.current = false; - mergedRefreshCanvas("pan-mouseup-pending").finally(() => { - try { - setIsLoading(false); - } catch (e) { } - }); - } - } catch (e) { } - }; - document.addEventListener("mouseup", handleMouseUp); - return () => document.removeEventListener("mouseup", handleMouseUp); - }, [panOffset]); - - // Process submission queue to ensure strokes are submitted sequentially - const processSubmissionQueue = async () => { - if (isSubmittingRef.current || submissionQueueRef.current.length === 0) { - return; - } - - isSubmittingRef.current = true; - - while (submissionQueueRef.current.length > 0) { - const submission = submissionQueueRef.current.shift(); - try { - await submission(); - } catch (error) { - console.error("Error processing queued submission:", error); - } - } - - isSubmittingRef.current = false; - - // After processing all queued submissions, schedule a refresh to sync with backend - if (refreshTimerRef.current) clearTimeout(refreshTimerRef.current); - refreshTimerRef.current = setTimeout(() => { - mergedRefreshCanvas("post-queue").catch((e) => - console.error("Error during post-queue refresh:", e) - ); - refreshTimerRef.current = null; - }, 500); - }; - - useEffect(() => { - if (!auth?.token || !currentRoomId) return; - try { - setSocketToken(auth.token); - } catch (e) { } - - const socket = getSocket(auth.token); - - try { - socket.emit("join_room", { roomId: currentRoomId, token: auth?.token }); - } catch (e) { - socket.emit("join_room", { roomId: currentRoomId }); - } - - const scheduleRefresh = (delay = 300) => { - try { - if (refreshTimerRef.current) clearTimeout(refreshTimerRef.current); - } catch (e) { } - refreshTimerRef.current = setTimeout(() => { - mergedRefreshCanvas().catch((e) => - console.error("Error during scheduled refresh:", e) - ); - refreshTimerRef.current = null; - }, delay); - }; - - const strokeBatch = []; - let batchTimer = null; - - const flushStrokeBatch = () => { - if (strokeBatch.length === 0) return; - - const batch = [...strokeBatch]; - strokeBatch.length = 0; - - setPendingDrawings((prev) => [...prev, ...batch]); - - requestAnimationFrame(() => { - drawAllDrawings(); - }); - - scheduleRefresh(350); - }; - - const handleNewStroke = (data) => { - try { - const myName = getUsername(auth); - if (data.user === myName) { - // This is confirmation of our own stroke - const stroke = data.stroke; - if (stroke && stroke.drawingId) { - confirmedStrokesRef.current.add(stroke.drawingId); - } - return; - } - } catch (e) { - try { - const user = getAuthUser(auth) || {}; - if (data.user === user.username) { - // This is confirmation of our own stroke - const stroke = data.stroke; - if (stroke && stroke.drawingId) { - confirmedStrokesRef.current.add(stroke.drawingId); - } - return; - } - } catch (e2) { } - } - - const stroke = data.stroke; - - // Extract metadata for advanced features (stamps, brushes, filters) - const metadata = { - brushStyle: stroke.brushStyle, - brushType: stroke.brushType, - brushParams: stroke.brushParams, - drawingType: stroke.drawingType, - stampData: stroke.stampData, - stampSettings: stroke.stampSettings, - filterType: stroke.filterType, - filterParams: stroke.filterParams, - }; - - const drawing = new Drawing( - stroke.drawingId || - `remote_${Date.now()}_${Math.random().toString(36).substr(2, 5)}`, - stroke.color || "#000000", - stroke.lineWidth || 5, - stroke.pathData || [], - stroke.ts || stroke.timestamp || Date.now(), - stroke.user || "Unknown", - metadata - ); - - try { - const clearedAt = roomClearedAtRef.current[currentRoomId]; - if ( - clearedAt && - (drawing.timestamp || drawing.ts || Date.now()) < clearedAt - ) { - return; - } - } catch (e) { } - - strokeBatch.push(drawing); - - if (drawing.drawingType === "stamp" && drawing.stampData && drawing.stampData.image) { - setBackendStamps((prevStamps) => { - const imageKey = drawing.stampData.image.substring(0, 100); - const alreadyExists = prevStamps.some(s => - s.image && s.image.substring(0, 100) === imageKey - ); - - if (!alreadyExists) { - console.log('Adding new custom stamp from Socket.IO:', drawing.stampData.name || 'Custom Stamp'); - return [...prevStamps, { - id: `stamp-${Date.now()}-${prevStamps.length}`, - name: drawing.stampData.name || 'Custom Stamp', - category: drawing.stampData.category || 'custom', - image: drawing.stampData.image, - emoji: drawing.stampData.emoji - }]; - } - return prevStamps; - }); - } - - clearTimeout(batchTimer); - batchTimer = setTimeout(flushStrokeBatch, 50); - }; - - const handleUserJoined = (data) => { - try { - if (!data) return; - if (data.roomId !== currentRoomId) return; - console.debug("socket user_joined event", data); - if (data.username) { - showLocalSnack(`${data.username} joined the canvas.`); - } - } catch (e) { } - }; - - const handleUserLeft = (data) => { - try { - if (!data) return; - if (data.roomId !== currentRoomId) return; - console.debug("socket user_left event", data); - if (data.username) { - showLocalSnack(`${data.username} left the canvas.`); - } - } catch (e) { } - }; - - const handleStrokeUndone = (data) => { - console.log("Stroke undone event received:", data); - - forceNextRedrawRef.current = true; - lastDrawnStateRef.current = null; - - // Schedule refresh instead of immediate refresh to avoid flicker - if (refreshTimerRef.current) clearTimeout(refreshTimerRef.current); - refreshTimerRef.current = setTimeout(() => { - mergedRefreshCanvas("undo-event"); - refreshTimerRef.current = null; - }, 100); - - if (currentRoomId) { - checkUndoRedoAvailability( - auth, - setUndoAvailable, - setRedoAvailable, - currentRoomId - ); - } - }; - - const handleCanvasCleared = (data) => { - console.log("Canvas cleared event received:", data); - const clearedAt = data && data.clearedAt ? data.clearedAt : Date.now(); - if (currentRoomId) roomClearedAtRef.current[currentRoomId] = clearedAt; - - // Clear local authoritative drawings and pending drawings that predate the clear - try { - userData.clearDrawings(); - } catch (e) { } - setPendingDrawings([]); - serverCountRef.current = 0; - - setUndoStack([]); - setRedoStack([]); - setUndoAvailable(false); - setRedoAvailable(false); - try { - if (currentRoomId) { - roomStacksRef.current[currentRoomId] = { undo: [], redo: [] }; - roomClipboardRef.current[currentRoomId] = null; - } - } catch (e) { } - - clearCanvasForRefresh(); - drawAllDrawings(); - - if (currentRoomId) { - checkUndoRedoAvailability( - auth, - setUndoAvailable, - setRedoAvailable, - currentRoomId - ); - } - }; - - socket.on("new_stroke", handleNewStroke); - socket.on("stroke_undone", handleStrokeUndone); - socket.on("canvas_cleared", handleCanvasCleared); - socket.on("user_joined", handleUserJoined); - socket.on("user_left", handleUserLeft); - socket.on("user_joined_debug", (d) => { - console.debug("socket user_joined_debug", d); - }); - - return () => { - socket.off("new_stroke", handleNewStroke); - socket.off("stroke_undone", handleStrokeUndone); - socket.off("canvas_cleared", handleCanvasCleared); - socket.off("user_joined", handleUserJoined); - socket.off("user_left", handleUserLeft); - try { - socket.emit("leave_room", { - roomId: currentRoomId, - token: auth?.token, - }); - } catch (e) { - socket.emit("leave_room", { roomId: currentRoomId }); - } - try { - if (refreshTimerRef.current) clearTimeout(refreshTimerRef.current); - } catch (e) { } - try { - if (batchTimer) clearTimeout(batchTimer); - } catch (e) { } - }; - }, [auth?.token, currentRoomId, auth?.user?.username]); - - useEffect(() => { - (async () => { - try { - setUndoStack([]); - setRedoStack([]); - setUndoAvailable(false); - setRedoAvailable(false); - if (currentRoomId) { - roomStacksRef.current[currentRoomId] = { undo: [], redo: [] }; - } - - // Reset selectedUser tracking when room changes - previousSelectedUserRef.current = null; - isRefreshingSelectedUserRef.current = false; - selectedUserRefreshQueueRef.current = null; - - if (auth?.token && currentRoomId) { - try { - await resetMyStacks(auth.token, currentRoomId); - } catch (e) { } - - // Restore undo/redo stacks from backend after page refresh - try { - const stacks = await restoreUndoRedoStacks( - auth, - currentRoomId, - setUndoStack, - setRedoStack, - setUndoAvailable, - setRedoAvailable - ); - - // Update room stacks ref for room switching - if (currentRoomId && stacks) { - roomStacksRef.current[currentRoomId] = { - undo: stacks.undo || [], - redo: stacks.redo || [] - }; - } - - console.log(`Restored stacks for room ${currentRoomId}:`, { - undoCount: stacks.undo?.length || 0, - redoCount: stacks.redo?.length || 0 - }); - } catch (e) { - console.warn("Failed to restore undo/redo stacks:", e); - // Fallback to just checking availability - try { - await checkUndoRedoAvailability( - auth, - setUndoAvailable, - setRedoAvailable, - currentRoomId - ); - } catch (e2) { } - } - } - } catch (e) { } - })(); - }, [auth?.token, currentRoomId]); - - // Force full refresh when selectedUser changes (drawing history selection/deselection) - useEffect(() => { - if (!currentRoomId || !auth?.token) return; - - // Serialize selectedUser for comparison (handles both string and object) - const serializeSelectedUser = (user) => { - if (!user || user === "") return ""; - if (typeof user === "string") return user; - if (typeof user === "object") - return JSON.stringify({ - user: user.user, - periodStart: user.periodStart, - }); - return String(user); - }; - - const currentSerialized = serializeSelectedUser(selectedUser); - const previousSerialized = previousSelectedUserRef.current; - - // Only refresh if selectedUser actually changed - if (currentSerialized === previousSerialized) { - return; - } - - // If a refresh is in progress, queue this change for execution after current one completes - if (isRefreshingSelectedUserRef.current) { - console.debug( - "[selectedUser] Refresh in progress, queuing new selection:", - currentSerialized - ); - selectedUserRefreshQueueRef.current = currentSerialized; - return; - } - - const performRefresh = async (targetSerialized) => { - isRefreshingSelectedUserRef.current = true; - - try { - setIsLoading(true); - - // Update the ref to mark this as the last processed value - previousSelectedUserRef.current = targetSerialized; - - // Force complete refresh from backend - userData.drawings = []; - setPendingDrawings([]); - serverCountRef.current = 0; - lastDrawnStateRef.current = null; - - const isDeselect = !selectedUser || selectedUser === ""; - const logLabel = isDeselect - ? "selectedUser-deselect" - : "selectedUser-select"; - console.debug(`[selectedUser] Performing full refresh: ${logLabel}`, { - to: targetSerialized, - }); - - await clearCanvasForRefresh(); - await mergedRefreshCanvas(logLabel); - await drawAllDrawings(); - } catch (error) { - console.error("Error refreshing on selectedUser change:", error); - } finally { - setIsLoading(false); - isRefreshingSelectedUserRef.current = false; - - // Check if there's a queued refresh waiting - if (selectedUserRefreshQueueRef.current !== null) { - const queuedTarget = selectedUserRefreshQueueRef.current; - selectedUserRefreshQueueRef.current = null; - - // Only process queued refresh if it's different from what we just processed - if (queuedTarget !== targetSerialized) { - console.debug( - "[selectedUser] Processing queued selection:", - queuedTarget - ); - // Use setTimeout to break out of the current call stack - setTimeout(() => performRefresh(queuedTarget), 0); - } - } - } - }; - - // Start the refresh - performRefresh(currentSerialized); - }, [selectedUser, currentRoomId]); - - const clearCanvasForRefresh = async () => { - const canvas = canvasRef.current; - if (!canvas) return; // Guard against null ref during tests - - const context = canvas.getContext("2d"); - if (!context) return; // Guard against null context during tests - - context.clearRect(0, 0, canvasWidth, canvasHeight); - setUserData(initializeUserData()); - setPendingDrawings([]); - serverCountRef.current = 0; - - // Clear selection overlay artifacts - setSelectionRect(null); - setSelectionStart(null); - - // Reset draw mode to freehand if in select mode - if (drawMode === "select") { - setDrawMode("freehand"); - } - }; - - const refreshCanvasButtonHandler = async () => { - if (isRefreshing) return; - setIsRefreshing(true); - setIsLoading(true); - try { - // Force full refresh from backend by clearing local state - userData.drawings = []; - setPendingDrawings([]); - serverCountRef.current = 0; - lastDrawnStateRef.current = null; - - await clearCanvasForRefresh(); - await mergedRefreshCanvas("refresh-button"); - await drawAllDrawings(); - updateFilterState(); // Update filter state after refresh - } catch (error) { - console.error("Error during canvas refresh:", error); - handleAuthError(error); - } finally { - setIsRefreshing(false); - setIsLoading(false); - } - }; - - const initializeUserData = () => { - const uniqueUserId = - auth?.user?.id || - `user_${Date.now()}_${Math.random().toString(36).substr(2, 5)}`; - const username = auth?.user?.username || "MainUser"; - return new UserData(uniqueUserId, username); - }; - const [userData, setUserData] = useState(() => initializeUserData()); - const generateId = () => - `drawing_${Date.now()}_${Math.random().toString(36).substr(2, 5)}`; - const serverCountRef = useRef(0); - - // Helper function to update filter state - const updateFilterState = () => { - // Use setUserData callback to read current state accurately - setUserData((currentUserData) => { - const filterExists = currentUserData.drawings.some((d) => d.drawingType === "filter"); - const filterCount = currentUserData.drawings.filter((d) => d.drawingType === "filter").length; - console.log(`[updateFilterState] filterExists=${filterExists}, filterCount=${filterCount}`); - setHasFilters(filterExists); - return currentUserData; - }); - }; - - // Advanced Brush/Stamp/Filter Functions - const handleBrushSelect = (brushType) => { - console.log("handleBrushSelect called with:", brushType); - setCurrentBrushType(brushType); - brushEngine.setBrushType(brushType); - setDrawMode("freehand"); - console.log("Current brush type set to:", brushType); - }; - - const handleBrushParamsChange = (params) => { - setBrushParams(params); - brushEngine.setBrushParams(params); - }; - - const placeStamp = async (x, y, stamp, settings) => { - const canvas = canvasRef.current; - if (!canvas || !stamp) return; - - const context = canvas.getContext("2d"); - - // Render stamp immediately using proper context management - if (stamp.emoji) { - context.save(); - context.globalAlpha = settings.opacity / 100; - context.translate(x, y); - context.rotate((settings.rotation * Math.PI) / 180); - - const size = settings.size; - context.font = `${size}px serif`; - context.textAlign = "center"; - context.textBaseline = "middle"; - context.fillText(stamp.emoji, 0, 0); - - context.restore(); - } else if (stamp.image) { - // For image stamps, load and render synchronously using async/await - try { - const img = await new Promise((resolve, reject) => { - const image = new Image(); - image.onload = () => resolve(image); - image.onerror = () => reject(new Error("Failed to load image")); - image.src = stamp.image; - }); - - context.save(); - context.globalAlpha = settings.opacity / 100; - context.translate(x, y); - context.rotate((settings.rotation * Math.PI) / 180); - - const size = settings.size; - context.drawImage(img, -size / 2, -size / 2, size, size); - - context.restore(); - } catch (error) { - console.error("Failed to load stamp image:", stamp.image?.substring(0, 100), error); - } - } - - // Create drawing record for stamp - const stampDrawing = new Drawing( - generateId(), - color, - lineWidth, - [{ x, y }], - Date.now(), - currentUser, - { - drawingType: "stamp", - stampData: stamp, - stampSettings: settings, - } - ); - - stampDrawing.roomId = currentRoomId; - - userData.addDrawing(stampDrawing); - setPendingDrawings((prev) => [...prev, stampDrawing]); - - // Add to undo stack - setUndoStack((prev) => [...prev, stampDrawing]); - setRedoStack([]); - - // Use submission queue to ensure stamps are submitted in order - try { - const submitTask = async () => { - try { - console.log("Submitting queued stamp:", { - drawingId: stampDrawing.drawingId, - stampData: stampDrawing.stampData, - }); - - await submitToDatabase( - stampDrawing, - auth, - { roomId: currentRoomId, roomType }, - setUndoAvailable, - setRedoAvailable - ); - - console.log("Stamp submitted successfully:", stampDrawing.drawingId); - - // Mark stamp as confirmed - confirmedStrokesRef.current.add(stampDrawing.drawingId); - - if (currentRoomId) { - checkUndoRedoAvailability( - auth, - setUndoAvailable, - setRedoAvailable, - currentRoomId - ); - } - } catch (error) { - console.error("Error during queued stamp submission:", error); - setPendingDrawings((prev) => - prev.filter((d) => d.drawingId !== stampDrawing.drawingId) - ); - handleAuthError(error); - showLocalSnack("Failed to save stamp. Please try again."); - } - }; - - submissionQueueRef.current.push(submitTask); - processSubmissionQueue(); - } catch (error) { - console.error("Error preparing stamp submission:", error); - handleAuthError(error); - showLocalSnack("Failed to prepare stamp. Please try again."); - } - }; - - const handleStampSelect = (stamp, settings) => { - setSelectedStamp(stamp); - setStampSettings(settings); - setDrawMode("stamp"); - }; - - const handleStampChange = (stamp, settings) => { - setSelectedStamp(stamp); - setStampSettings(settings); - }; - - const applyFilter = async (filterType, params) => { - if (!canvasRef.current) return; - - // Always cancel preview mode first and clean up state - if (isFilterPreview) { - setIsFilterPreview(false); - } - preFilterCanvasStateRef.current = null; - originalCanvasDataRef.current = null; - - // Check if we already have a filter of this type applied - const existingFilterIndex = userData.drawings.findIndex( - (d) => d.drawingType === "filter" && d.filterType === filterType - ); - - let filterDrawing; - let isReplacement = existingFilterIndex !== -1; - - if (isReplacement) { - const existingFilter = userData.drawings[existingFilterIndex]; - existingFilter.filterParams = { ...params }; // Clone params - existingFilter.timestamp = Date.now(); - filterDrawing = existingFilter; - - // Update React state to reflect the filter parameter change - const newUserData = new UserData(userData.userId, userData.username); - newUserData.drawings = [...userData.drawings]; // Clone the array to trigger state update - setUserData(newUserData); - - // Force a complete redraw with the updated filter parameters - // This will redraw all strokes first, then apply the filter - lastDrawnStateRef.current = null; - forceNextRedrawRef.current = true; - await drawAllDrawings(); - - showLocalSnack(`Updated ${filterType} filter`); - updateFilterState(); - - // For filter updates, we need to submit the UPDATE to backend - // The backend should handle this as an update, not a new drawing - try { - await submitToDatabase( - filterDrawing, - auth, - { - roomId: currentRoomId, - roomType, - }, - setUndoAvailable, - setRedoAvailable - ); - - if (currentRoomId) { - checkUndoRedoAvailability( - auth, - setUndoAvailable, - setRedoAvailable, - currentRoomId - ); - } - } catch (error) { - console.error("Error submitting filter update:", error); - handleAuthError(error); - } - - return; // Exit early for updates - } - - // Create NEW filter record for new filter type - filterDrawing = new Drawing( - generateId(), - "#000000", - 0, - [], - Date.now(), - currentUser, - { - drawingType: "filter", - filterType, - filterParams: { ...params }, // Clone params - } - ); - - // Set filter properties directly on the drawing object - filterDrawing.drawingType = "filter"; - filterDrawing.filterType = filterType; - filterDrawing.filterParams = { ...params }; - filterDrawing.roomId = currentRoomId; - - userData.addDrawing(filterDrawing); - - // Update React state so components know about the new filter - const newUserData = new UserData(userData.userId, userData.username); - newUserData.drawings = [...userData.drawings]; // Clone array with new filter - setUserData(newUserData); - - setPendingDrawings((prev) => [...prev, filterDrawing]); - - setUndoStack((prev) => [...prev, filterDrawing]); - setRedoStack([]); - - // Force complete redraw this will render all strokes THEN apply filter - lastDrawnStateRef.current = null; - forceNextRedrawRef.current = true; - await drawAllDrawings(); - - showLocalSnack(`Applied ${filterType} filter`); - updateFilterState(); - - try { - await submitToDatabase( - filterDrawing, - auth, - { - roomId: currentRoomId, - roomType, - }, - setUndoAvailable, - setRedoAvailable - ); - - // Check undo/redo availability after filter submission - if (currentRoomId) { - checkUndoRedoAvailability( - auth, - setUndoAvailable, - setRedoAvailable, - currentRoomId - ); - } - } catch (error) { - console.error("Error submitting filter:", error); - // On error, remove the failed filter from pending - setPendingDrawings((prev) => - prev.filter((d) => d.drawingId !== filterDrawing.drawingId) - ); - handleAuthError(error); - } - }; - - const previewFilter = async (filterType, params) => { - const canvas = canvasRef.current; - if (!canvas) return; - - // If already in preview mode, first restore to base state - if (isFilterPreview && preFilterCanvasStateRef.current) { - const img = new Image(); - img.onload = async () => { - const context = canvas.getContext("2d"); - context.clearRect(0, 0, canvas.width, canvas.height); - context.drawImage(img, 0, 0); - - // Now apply the new preview - await applyPreviewFilter(canvas, filterType, params); - }; - img.src = preFilterCanvasStateRef.current; - return; - } - - // Store the current canvas state before preview (only once) - if (!preFilterCanvasStateRef.current) { - preFilterCanvasStateRef.current = canvas.toDataURL(); - } - - await applyPreviewFilter(canvas, filterType, params); - }; - - const applyPreviewFilter = async (canvas, filterType, params) => { - // Check if this filter type already exists in the drawings - const existingFilterIndex = userData.drawings.findIndex( - (d) => d.drawingType === "filter" && d.filterType === filterType - ); - - if (existingFilterIndex !== -1) { - // Temporarily remove this filter, redraw, then apply preview - const originalDrawings = [...userData.drawings]; - userData.drawings = userData.drawings.filter((d, i) => i !== existingFilterIndex); - - lastDrawnStateRef.current = null; - forceNextRedrawRef.current = true; - await drawAllDrawings(); - - // Restore drawings array - userData.drawings = originalDrawings; - } - - // Apply the preview filter on top of current canvas - const context = canvas.getContext("2d"); - const imageData = context.getImageData(0, 0, canvas.width, canvas.height); - const filteredImageData = applyImageFilter(imageData, filterType, params); - context.putImageData(filteredImageData, 0, 0); - - setIsFilterPreview(true); - }; - - const undoFilter = async () => { - // If in preview mode, restore from saved canvas state - if (isFilterPreview && preFilterCanvasStateRef.current) { - const canvas = canvasRef.current; - if (!canvas) return; - - const context = canvas.getContext("2d"); - const img = new Image(); - img.onload = async () => { - context.clearRect(0, 0, canvas.width, canvas.height); - context.drawImage(img, 0, 0); - setIsFilterPreview(false); - preFilterCanvasStateRef.current = null; - originalCanvasDataRef.current = null; - }; - img.src = preFilterCanvasStateRef.current; - return; - } - - // If not in preview mode, use regular undo (which properly syncs with backend) - if (!editingEnabled) { - showLocalSnack("Undo is disabled in view-only mode."); - return; - } - - if (undoStack.length === 0) { - showLocalSnack("No actions to undo."); - return; - } - - // Simply call the regular undo function, which will undo the last action - // This properly coordinates with the backend's undo system - await undo(); - }; - - const clearAllFilters = async () => { - // Clear preview state if active - if (isFilterPreview) { - setIsFilterPreview(false); - preFilterCanvasStateRef.current = null; - originalCanvasDataRef.current = null; - } - - if (!editingEnabled) { - showLocalSnack("Clear filters is disabled in view-only mode."); - return; - } - - // Find all filter drawings in userData (not just undo stack) - // Use setUserData callback to get the latest state - let filterDrawings = []; - setUserData((currentUserData) => { - const allDrawings = currentUserData.drawings || []; - filterDrawings = allDrawings.filter( - (drawing) => drawing.drawingType === "filter" - ); - console.log(`[clearAllFilters] Found ${filterDrawings.length} filters to clear`, filterDrawings); - return currentUserData; - }); - - if (filterDrawings.length === 0) { - showLocalSnack("No filters to clear."); - return; - } - - if (isRefreshing) { - showLocalSnack("Please wait for the canvas to refresh."); - return; - } - - try { - showLocalSnack(`Clearing ${filterDrawings.length} filter(s)...`); - - // Get filter IDs before removing from local state - const filterIds = filterDrawings.map(f => f.drawingId).filter(id => id); - - // Remove all filter drawings from local state immediately using proper state update - setUserData((currentUserData) => { - const newUserData = new UserData(currentUserData.userId, currentUserData.username); - newUserData.drawings = currentUserData.drawings.filter( - (d) => d.drawingType !== "filter" - ); - console.log(`[clearAllFilters] Removed ${filterDrawings.length} filters, ${newUserData.drawings.length} drawings remain`); - return newUserData; - }); - - // Remove from pendingDrawings - setPendingDrawings((prev) => - prev.filter((d) => d.drawingType !== "filter") - ); - - // Remove from undo stack (if present) - setUndoStack((prev) => - prev.filter((d) => d.drawingType !== "filter") - ); - - // Force a complete redraw immediately to show filters are gone - lastDrawnStateRef.current = null; - forceNextRedrawRef.current = true; - await drawAllDrawings(); - - showLocalSnack(`Cleared ${filterDrawings.length} filter(s).`); - updateFilterState(); // Update filter state for UI - - // Now sync with backend - create undo markers for each filter - if (filterIds.length > 0) { - try { - // Import the API function - const { markStrokesAsUndone } = await import('../api/rooms'); - - try { - await markStrokesAsUndone(auth.token, currentRoomId, filterIds); - console.log(`Marked ${filterIds.length} filters as undone in backend`); - } catch (apiError) { - // If the API doesn't exist, fall back to calling undo multiple times - console.warn("markStrokesAsUndone API not available, using fallback"); - - // Fallback: call regular undo endpoint for each filter - const { undoRoomAction } = await import('../api/rooms'); - for (let i = 0; i < Math.min(filterDrawings.length, 10); i++) { - try { - const result = await undoRoomAction(auth.token, currentRoomId); - if (result.status === "noop") break; - await new Promise(resolve => setTimeout(resolve, 50)); - } catch (e) { - console.warn("Error calling undoRoomAction:", e); - break; - } - } - } - - await checkUndoRedoAvailability( - auth, - setUndoAvailable, - setRedoAvailable, - currentRoomId - ); - } catch (e) { - console.error("Error syncing filter removal with backend:", e); - } - } - } catch (error) { - console.error("Error clearing all filters:", error); - showLocalSnack("Failed to clear all filters. Refreshing canvas..."); - // Refresh to restore state - await refreshCanvasButtonHandler(); - } - }; - - const applyImageFilter = (imageData, filterType, params) => { - const data = imageData.data; - const filtered = new ImageData( - new Uint8ClampedArray(data), - imageData.width, - imageData.height - ); - - switch (filterType) { - case "blur": - return applyBlurFilter(filtered, params.intensity || 5); - case "hueShift": - return applyHueShiftFilter( - filtered, - params.hue || 0, - params.saturation || 0 - ); - case "chalk": - return applyChalkFilter( - filtered, - params.roughness || 50, - params.opacity || 80 - ); - case "fade": - return applyFadeFilter(filtered, params.amount || 30); - case "vintage": - return applyVintageFilter( - filtered, - params.sepia || 60, - params.vignette || 40 - ); - case "neon": - return applyNeonFilter( - filtered, - params.intensity || 15, - params.color || 180 - ); - default: - return filtered; - } - }; - - const applyBlurFilter = (imageData, intensity) => { - // Optimized separable box blur - O(n) instead of O(n²) - // This is much faster and won't crash even with higher intensity values - const data = imageData.data; - const width = imageData.width; - const height = imageData.height; - - const radius = Math.max(1, Math.floor(intensity)); - const temp = new Uint8ClampedArray(data); - const result = new Uint8ClampedArray(data); - - // Horizontal pass - for (let y = 0; y < height; y++) { - let r = 0, g = 0, b = 0, a = 0; - let count = 0; - - // Initialize window - for (let x = -radius; x <= radius; x++) { - if (x >= 0 && x < width) { - const idx = (y * width + x) * 4; - r += data[idx]; - g += data[idx + 1]; - b += data[idx + 2]; - a += data[idx + 3]; - count++; - } - } - - // Slide window across row - for (let x = 0; x < width; x++) { - const idx = (y * width + x) * 4; - temp[idx] = r / count; - temp[idx + 1] = g / count; - temp[idx + 2] = b / count; - temp[idx + 3] = a / count; - - // Remove left pixel - const leftX = x - radius; - if (leftX >= 0) { - const leftIdx = (y * width + leftX) * 4; - r -= data[leftIdx]; - g -= data[leftIdx + 1]; - b -= data[leftIdx + 2]; - a -= data[leftIdx + 3]; - count--; - } - - // Add right pixel - const rightX = x + radius + 1; - if (rightX < width) { - const rightIdx = (y * width + rightX) * 4; - r += data[rightIdx]; - g += data[rightIdx + 1]; - b += data[rightIdx + 2]; - a += data[rightIdx + 3]; - count++; - } - } - } - - // Vertical pass - for (let x = 0; x < width; x++) { - let r = 0, g = 0, b = 0, a = 0; - let count = 0; - - // Initialize window - for (let y = -radius; y <= radius; y++) { - if (y >= 0 && y < height) { - const idx = (y * width + x) * 4; - r += temp[idx]; - g += temp[idx + 1]; - b += temp[idx + 2]; - a += temp[idx + 3]; - count++; - } - } - - // Slide window down column - for (let y = 0; y < height; y++) { - const idx = (y * width + x) * 4; - result[idx] = r / count; - result[idx + 1] = g / count; - result[idx + 2] = b / count; - result[idx + 3] = a / count; - - // Remove top pixel - const topY = y - radius; - if (topY >= 0) { - const topIdx = (topY * width + x) * 4; - r -= temp[topIdx]; - g -= temp[topIdx + 1]; - b -= temp[topIdx + 2]; - a -= temp[topIdx + 3]; - count--; - } - - // Add bottom pixel - const bottomY = y + radius + 1; - if (bottomY < height) { - const bottomIdx = (bottomY * width + x) * 4; - r += temp[bottomIdx]; - g += temp[bottomIdx + 1]; - b += temp[bottomIdx + 2]; - a += temp[bottomIdx + 3]; - count++; - } - } - } - - return new ImageData(result, width, height); - }; - - const applyHueShiftFilter = (imageData, hueShift, saturationShift) => { - const data = imageData.data; - - for (let i = 0; i < data.length; i += 4) { - const r = data[i]; - const g = data[i + 1]; - const b = data[i + 2]; - - // Convert RGB to HSL - const max = Math.max(r, g, b) / 255; - const min = Math.min(r, g, b) / 255; - const diff = max - min; - const sum = max + min; - - let h = 0; - const l = sum / 2; - const s = diff === 0 ? 0 : l > 0.5 ? diff / (2 - sum) : diff / sum; - - if (diff !== 0) { - switch (max) { - case r / 255: - h = (g - b) / 255 / diff + (g < b ? 6 : 0); - break; - case g / 255: - h = (b - r) / 255 / diff + 2; - break; - case b / 255: - h = (r - g) / 255 / diff + 4; - break; - } - h /= 6; - } - - // Apply shifts - h = (h + hueShift / 360) % 1; - const newS = Math.max(0, Math.min(1, s + saturationShift / 100)); - - // Convert back to RGB - const hue2rgb = (p, q, t) => { - if (t < 0) t += 1; - if (t > 1) t -= 1; - if (t < 1 / 6) return p + (q - p) * 6 * t; - if (t < 1 / 2) return q; - if (t < 2 / 3) return p + (q - p) * (2 / 3 - t) * 6; - return p; - }; - - let newR, newG, newB; - - if (newS === 0) { - newR = newG = newB = l; - } else { - const q = l < 0.5 ? l * (1 + newS) : l + newS - l * newS; - const p = 2 * l - q; - newR = hue2rgb(p, q, h + 1 / 3); - newG = hue2rgb(p, q, h); - newB = hue2rgb(p, q, h - 1 / 3); - } - - data[i] = Math.round(newR * 255); - data[i + 1] = Math.round(newG * 255); - data[i + 2] = Math.round(newB * 255); - } - - return imageData; - }; - - const applyChalkFilter = (imageData, roughness, opacity) => { - const data = imageData.data; - - for (let i = 0; i < data.length; i += 4) { - const noise = (Math.random() - 0.5) * (roughness / 100) * 255; - const opacityFactor = opacity / 100; - - data[i] = Math.max(0, Math.min(255, data[i] + noise)) * opacityFactor; - data[i + 1] = - Math.max(0, Math.min(255, data[i + 1] + noise)) * opacityFactor; - data[i + 2] = - Math.max(0, Math.min(255, data[i + 2] + noise)) * opacityFactor; - data[i + 3] = data[i + 3] * opacityFactor; - } - - return imageData; - }; - - const applyFadeFilter = (imageData, amount) => { - const data = imageData.data; - const fadeAmount = 1 - amount / 100; - - for (let i = 0; i < data.length; i += 4) { - data[i + 3] = data[i + 3] * fadeAmount; - } - - return imageData; - }; - - const applyVintageFilter = (imageData, sepia, vignette) => { - const data = imageData.data; - const width = imageData.width; - const height = imageData.height; - const sepiaAmount = sepia / 100; - - for (let i = 0; i < data.length; i += 4) { - const r = data[i]; - const g = data[i + 1]; - const b = data[i + 2]; - - // Apply sepia - data[i] = Math.min( - 255, - (r * 0.393 + g * 0.769 + b * 0.189) * sepiaAmount + - r * (1 - sepiaAmount) - ); - data[i + 1] = Math.min( - 255, - (r * 0.349 + g * 0.686 + b * 0.168) * sepiaAmount + - g * (1 - sepiaAmount) - ); - data[i + 2] = Math.min( - 255, - (r * 0.272 + g * 0.534 + b * 0.131) * sepiaAmount + - b * (1 - sepiaAmount) - ); - - // Apply vignette - const x = (i / 4) % width; - const y = Math.floor(i / 4 / width); - const centerX = width / 2; - const centerY = height / 2; - const distance = Math.sqrt((x - centerX) ** 2 + (y - centerY) ** 2); - const maxDistance = Math.sqrt(centerX ** 2 + centerY ** 2); - const vignetteAmount = 1 - (distance / maxDistance) * (vignette / 100); - - data[i] *= vignetteAmount; - data[i + 1] *= vignetteAmount; - data[i + 2] *= vignetteAmount; - } - - return imageData; - }; - - const applyNeonFilter = (imageData, intensity, hue) => { - const data = imageData.data; - const width = imageData.width; - const height = imageData.height; - const glowIntensity = intensity / 25; // More aggressive scaling (max 50 -> 2.0) - - // Create a copy for the glow effect - const result = new Uint8ClampedArray(data); - - // Generate neon color based on hue using proper HSL to RGB conversion - const hueNormalized = hue / 360; - const neonR = Math.abs(Math.sin((hueNormalized) * Math.PI * 2)) * 255; - const neonG = Math.abs(Math.sin((hueNormalized + 0.333) * Math.PI * 2)) * 255; - const neonB = Math.abs(Math.sin((hueNormalized + 0.666) * Math.PI * 2)) * 255; - - for (let i = 0; i < data.length; i += 4) { - const alpha = data[i + 3]; - - // Only apply effect to visible pixels (any stroke) - if (alpha > 5) { - const r = data[i]; - const g = data[i + 1]; - const b = data[i + 2]; - - // Calculate brightness - const brightness = (r + g + b) / 3; - - // Apply aggressive neon glow with color tinting - const alphaFactor = alpha / 255; - const colorFactor = glowIntensity * alphaFactor; - - // Mix original color with neon color and boost brightness - const boost = 1 + (glowIntensity * 0.8); - result[i] = Math.min(255, (r * boost) + (neonR * colorFactor * 0.7)); - result[i + 1] = Math.min(255, (g * boost) + (neonG * colorFactor * 0.7)); - result[i + 2] = Math.min(255, (b * boost) + (neonB * colorFactor * 0.7)); - - // Ensure the effect is visible even on dark strokes - const minBrightness = 60 * glowIntensity; - const currentBrightness = (result[i] + result[i + 1] + result[i + 2]) / 3; - if (currentBrightness < minBrightness) { - const brightnessFactor = minBrightness / Math.max(currentBrightness, 1); - result[i] = Math.min(255, result[i] * brightnessFactor); - result[i + 1] = Math.min(255, result[i + 1] * brightnessFactor); - result[i + 2] = Math.min(255, result[i + 2] * brightnessFactor); - } - } - } - - return new ImageData(result, width, height); - }; - - const drawAllDrawings = async () => { - const currentTemplateObjects = templateObjectsRef.current || []; - - if (isDrawingInProgressRef.current) { - console.log('Drawing already in progress, skipping drawAllDrawings call'); - return; - } - - isDrawingInProgressRef.current = true; - - // Save current brush state - const savedBrushType = brushEngine ? brushEngine.brushType : null; - const savedBrushParams = brushEngine ? brushEngine.brushParams : null; - - try { - setIsLoading(true); - const canvas = canvasRef.current; - if (!canvas) { - setIsLoading(false); - isDrawingInProgressRef.current = false; - return; - } - const context = canvas.getContext("2d"); - if (!context) { - setIsLoading(false); - isDrawingInProgressRef.current = false; - return; - } - - // Include any locally-pending drawings (e.g. received via socket but - // not yet reflected by a backend refresh) so they render immediately. - const userDrawingIds = new Set((userData.drawings || []).map(d => d.drawingId)); - const uniquePendingDrawings = (pendingDrawings || []).filter( - pd => !userDrawingIds.has(pd.drawingId) - ); - - const combined = [ - ...(userData.drawings || []), - ...uniquePendingDrawings, - ]; - - // Create a state signature to detect if we need to redraw - const filterSignature = combined - .filter(d => d.drawingType === "filter") - .map(f => `${f.drawingId}:${f.filterType}`) - .join(','); - - const templateSignature = currentTemplateObjects?.length > 0 - ? currentTemplateObjects.map(t => `${t.type}:${t.x || t.x1 || t.cx}:${t.y || t.y1 || t.cy}`).join(',') - : ''; - - const stateSignature = `${combined.length}|${filterSignature}|${pendingDrawings.length}|${currentTemplateObjects?.length || 0}|${templateSignature}`; - - if (lastDrawnStateRef.current === stateSignature) { - console.log('State unchanged, skipping redraw'); - setIsLoading(false); - isDrawingInProgressRef.current = false; - return; - } - - // Check if we can do incremental rendering - let newDrawingsOnly = []; - if (lastDrawnStateRef.current && - cachedCanvasRef.current && - cachedDrawingIdsRef.current.size > 0 && - combined.length > cachedDrawingIdsRef.current.size && - currentTemplateObjects?.length === 0) { - - // Find drawings that aren't in the cache - newDrawingsOnly = combined.filter(d => !cachedDrawingIdsRef.current.has(d.drawingId)); - - const hasNewFiltersOrCuts = newDrawingsOnly.some( - d => d.drawingType === "filter" || (d.pathData && d.pathData.tool === "cut") - ); - - // Verify all cached drawings are still present - const currentIds = new Set(combined.map(d => d.drawingId)); - const allCachedPresent = Array.from(cachedDrawingIdsRef.current).every(id => currentIds.has(id)); - - if (newDrawingsOnly.length > 0 && allCachedPresent && !hasNewFiltersOrCuts && newDrawingsOnly.length <= 5) { - console.log(`[Optimization] Incremental rendering: adding ${newDrawingsOnly.length} new drawings`); - // We can use incremental rendering! - } else { - // Fall back to full redraw - newDrawingsOnly = []; - cachedCanvasRef.current = null; - cachedDrawingIdsRef.current.clear(); - } - } else { - // Clear cache - we need full redraw - cachedCanvasRef.current = null; - cachedDrawingIdsRef.current.clear(); - } - - // Clear force flag after checking it - forceNextRedrawRef.current = false; - lastDrawnStateRef.current = stateSignature; - - // for flicker free rendering - if (!offscreenCanvasRef.current || - offscreenCanvasRef.current.width !== canvasWidth || - offscreenCanvasRef.current.height !== canvasHeight - ) { - offscreenCanvasRef.current = document.createElement("canvas"); - offscreenCanvasRef.current.width = canvasWidth; - offscreenCanvasRef.current.height = canvasHeight; - } - - const offscreenContext = offscreenCanvasRef.current.getContext("2d"); - offscreenContext.imageSmoothingEnabled = false; - - // If we can do incremental rendering, start from cached canvas - if (newDrawingsOnly.length > 0 && cachedCanvasRef.current) { - console.log("[drawAllDrawings] Using incremental rendering - copying from cache"); - offscreenContext.clearRect(0, 0, canvasWidth, canvasHeight); - offscreenContext.drawImage(cachedCanvasRef.current, 0, 0); - } else { - // Full redraw - offscreenContext.clearRect(0, 0, canvasWidth, canvasHeight); - } - - // This avoids async rendering issues with image stamps - const stampsToRender = []; - - // Create and render template layer with caching - let templateCanvas = null; - if (currentTemplateObjects && currentTemplateObjects.length > 0) { - const templateSignature = `${templateId}_${currentTemplateObjects.length}`; - - if (cachedTemplateIdRef.current === templateSignature && templateCanvasRef.current) { - templateCanvas = templateCanvasRef.current; - console.log("[drawAllDrawings] Using cached template layer"); - } else { - templateCanvas = document.createElement('canvas'); - templateCanvas.width = canvasWidth; - templateCanvas.height = canvasHeight; - const templateContext = templateCanvas.getContext('2d'); - templateContext.imageSmoothingEnabled = false; - - templateContext.save(); - templateContext.globalAlpha = 0.5; - - let renderedCount = 0; - for (const obj of currentTemplateObjects) { - try { - if (obj.type === 'line') { - templateContext.beginPath(); - templateContext.moveTo(obj.x1, obj.y1); - templateContext.lineTo(obj.x2, obj.y2); - templateContext.strokeStyle = obj.color || '#333'; - templateContext.lineWidth = obj.lineWidth || 2; - templateContext.stroke(); - renderedCount++; - } else if (obj.type === 'rectangle') { - templateContext.strokeStyle = obj.stroke || '#333'; - templateContext.lineWidth = obj.lineWidth || 2; - if (obj.fill && obj.fill !== 'transparent') { - templateContext.fillStyle = obj.fill; - templateContext.fillRect(obj.x, obj.y, obj.width, obj.height); - } - templateContext.strokeRect(obj.x, obj.y, obj.width, obj.height); - renderedCount++; - } else if (obj.type === 'circle') { - templateContext.beginPath(); - templateContext.arc(obj.cx, obj.cy, obj.radius, 0, Math.PI * 2); - templateContext.strokeStyle = obj.stroke || '#333'; - templateContext.lineWidth = obj.lineWidth || 2; - if (obj.fill && obj.fill !== 'transparent') { - templateContext.fillStyle = obj.fill; - templateContext.fill(); - } - templateContext.stroke(); - renderedCount++; - } else if (obj.type === 'ellipse') { - templateContext.beginPath(); - templateContext.ellipse(obj.cx, obj.cy, obj.rx, obj.ry, 0, 0, Math.PI * 2); - templateContext.strokeStyle = obj.stroke || '#333'; - templateContext.lineWidth = obj.lineWidth || 2; - if (obj.fill && obj.fill !== 'transparent') { - templateContext.fillStyle = obj.fill; - templateContext.fill(); - } - templateContext.stroke(); - renderedCount++; - } else if (obj.type === 'text') { - templateContext.fillStyle = obj.color || '#333'; - templateContext.font = `${obj.bold ? 'bold ' : ''}${obj.fontSize || 16}px Arial`; - templateContext.fillText(obj.text || '', obj.x, obj.y); - renderedCount++; - } else { - console.warn('Unknown template object type:', obj.type); - } - } catch (e) { - console.warn('Failed to render template object:', obj, e); - } - } - templateContext.restore(); - - templateCanvasRef.current = templateCanvas; - cachedTemplateIdRef.current = templateSignature; - console.log("[drawAllDrawings] Cached new template layer"); - } - } else { - console.log('No template objects to render'); - } - - if (templateCanvas) { - offscreenContext.drawImage(templateCanvas, 0, 0); - } - - const cutOriginalIds = new Set(); - try { - combined.forEach((d) => { - if ( - d && - d.pathData && - d.pathData.tool === "cut" && - Array.isArray(d.pathData.originalStrokeIds) - ) { - d.pathData.originalStrokeIds.forEach((id) => - cutOriginalIds.add(id) - ); - } - }); - } catch (e) { } - - const sortedDrawings = combined.sort((a, b) => { - const orderA = - a.order !== undefined ? a.order : a.timestamp || a.ts || 0; - const orderB = - b.order !== undefined ? b.order : b.timestamp || b.ts || 0; - return orderA - orderB; - }); - - // Separate filter drawings from regular drawings - const regularDrawings = []; - const filterDrawings = []; - for (const drawing of sortedDrawings) { - if (drawing.drawingType === "filter") { - filterDrawings.push(drawing); - } else { - regularDrawings.push(drawing); - } - } - - // Optimization: Pre-load all image stamps with timeout to prevent indefinite blocking - const imageStampCache = new Map(); - const imageStampPromises = []; - - for (const drawing of regularDrawings) { - if (drawing.drawingType === "stamp" && drawing.stampData && drawing.stampData.image && !drawing.stampData.emoji) { - const imageUrl = drawing.stampData.image; - if (!imageStampCache.has(imageUrl)) { - const promise = new Promise((resolve) => { - const img = new Image(); - img.onload = () => { - imageStampCache.set(imageUrl, img); - resolve(img); - }; - img.onerror = () => { - console.error("[drawAllDrawings] Failed to pre-load stamp image:", imageUrl.substring(0, 100)); - imageStampCache.set(imageUrl, null); // Mark as failed - resolve(null); - }; - img.src = imageUrl; - }); - imageStampPromises.push(promise); - } - } - } - - // Wait for all stamp images to load before rendering, with 2-second timeout - if (imageStampPromises.length > 0) { - console.log("[drawAllDrawings] Pre-loading", imageStampPromises.length, "stamp images"); - await Promise.race([ - Promise.all(imageStampPromises), - new Promise(resolve => setTimeout(() => { - console.warn("[drawAllDrawings] Stamp image loading timeout after 2s - proceeding anyway"); - resolve(); - }, 2000)) - ]); - console.log("[drawAllDrawings] Stamp images loaded (or timeout reached)"); - } - - // Render drawings in chronological order. When a 'cut' record appears - // we immediately apply a destination-out erase so it removes prior content - // but does not erase strokes that are drawn after the cut. - const maskedOriginals = new Set(); - let seenAnyCut = false; - - // If doing incremental rendering, only draw new drawings - const drawingsToRender = newDrawingsOnly.length > 0 ? newDrawingsOnly : regularDrawings; - console.log(`[drawAllDrawings] Rendering ${drawingsToRender.length} drawings (incremental: ${newDrawingsOnly.length > 0})`); - - for (const drawing of drawingsToRender) { - // If this is a cut record, apply the erase to the canvas now. - if (drawing && drawing.pathData && drawing.pathData.tool === "cut") { - seenAnyCut = true; - try { - if (Array.isArray(drawing.pathData.originalStrokeIds)) { - drawing.pathData.originalStrokeIds.forEach((id) => - maskedOriginals.add(id) - ); - } - } catch (e) { } - - if (drawing.pathData && drawing.pathData.rect) { - const r = drawing.pathData.rect; - offscreenContext.save(); - try { - offscreenContext.globalCompositeOperation = "destination-out"; - offscreenContext.fillStyle = "rgba(0,0,0,1)"; - // Expand rect slightly to avoid hairline due to subpixel antialiasing - offscreenContext.fillRect( - Math.floor(r.x) - 2, - Math.floor(r.y) - 2, - Math.ceil(r.width) + 4, - Math.ceil(r.height) + 4 - ); - } finally { - offscreenContext.restore(); - } - - // Restore template layer in the cut region so templates remain visible - if (templateCanvas) { - offscreenContext.drawImage( - templateCanvas, - Math.floor(r.x) - 2, - Math.floor(r.y) - 2, - Math.ceil(r.width) + 4, - Math.ceil(r.height) + 4, - Math.floor(r.x) - 2, - Math.floor(r.y) - 2, - Math.ceil(r.width) + 4, - Math.ceil(r.height) + 4 - ); - } - } - - continue; - } - - // Skip originals that have been masked by a cut - if ( - drawing && - drawing.drawingId && - (cutOriginalIds.has(drawing.drawingId) || - maskedOriginals.has(drawing.drawingId)) - ) { - continue; - } - - // Skip temporary white "erase" helper strokes when we've seen a cut - // record; destination-out masking is authoritative and drawing white - // strokes can produce hairlines. - try { - if ( - seenAnyCut && - drawing && - drawing.color && - typeof drawing.color === "string" && - drawing.color.toLowerCase() === "#ffffff" - ) { - continue; - } - } catch (e) { } - - // Draw the drawing normally - offscreenContext.globalAlpha = 1.0; - let viewingUser = null; - let viewingPeriodStart = null; - if (selectedUser) { - if (typeof selectedUser === "string") viewingUser = selectedUser; - else if (typeof selectedUser === "object") { - viewingUser = selectedUser.user; - viewingPeriodStart = selectedUser.periodStart; - } - } - if (viewingUser && drawing.user !== viewingUser) { - offscreenContext.globalAlpha = 0.1; - } else if (viewingPeriodStart !== null) { - const ts = drawing.timestamp || drawing.order || 0; - if ( - ts < viewingPeriodStart || - ts >= viewingPeriodStart + 5 * 60 * 1000 - ) { - offscreenContext.globalAlpha = 0.1; - } - } - - // Apply custom opacity if specified - if (drawing.opacity !== undefined && drawing.opacity !== 1.0) { - offscreenContext.globalAlpha *= drawing.opacity; - } - - // Stamps have pathData as array but need special rendering - render inline to preserve z-order - if (drawing.drawingType === "stamp" && drawing.stampData && drawing.stampSettings && Array.isArray(drawing.pathData) && drawing.pathData.length > 0) { - const stamp = drawing.stampData; - const settings = drawing.stampSettings; - const position = drawing.pathData[0]; - - try { - offscreenContext.save(); - offscreenContext.translate(position.x, position.y); - offscreenContext.rotate(((settings.rotation || 0) * Math.PI) / 180); - - const size = settings.size || 50; - - if (stamp.emoji) { - // Render emoji stamp - offscreenContext.font = `${size}px serif`; - offscreenContext.textAlign = "center"; - offscreenContext.textBaseline = "middle"; - offscreenContext.fillText(stamp.emoji, 0, 0); - console.log("[drawAllDrawings] Rendered emoji stamp inline:", stamp.emoji); - } else if (stamp.image) { - // Render image stamp using pre-loaded image - const img = imageStampCache.get(stamp.image); - if (img) { - offscreenContext.globalAlpha = (settings.opacity || 100) / 100 * offscreenContext.globalAlpha; - offscreenContext.drawImage(img, -size / 2, -size / 2, size, size); - console.log("[drawAllDrawings] Rendered image stamp inline"); - } else { - console.warn("[drawAllDrawings] Image stamp not in cache:", stamp.image?.substring(0, 100)); - } - } - - offscreenContext.restore(); - } catch (error) { - offscreenContext.restore(); - console.error("[drawAllDrawings] Error rendering stamp:", error); - } - } else if (drawing.drawingType === "stamp") { - console.warn("[drawAllDrawings] Stamp NOT rendered - missing requirements:", { - drawingId: drawing.drawingId, - drawingType: drawing.drawingType, - hasStampData: !!drawing.stampData, - hasStampSettings: !!drawing.stampSettings, - pathDataIsArray: Array.isArray(drawing.pathData), - pathDataLength: drawing.pathData ? drawing.pathData.length : 0, - pathDataType: typeof drawing.pathData, - pathDataValue: drawing.pathData, - fullDrawing: drawing - }); - } else if (Array.isArray(drawing.pathData)) { - const pts = drawing.pathData; - if (pts.length > 0) { - // Check if this is an advanced brush drawing - if (drawing.brushType && drawing.brushType !== "normal" && brushEngine) { - console.log("Rendering advanced brush in drawAllDrawings:", { - id: drawing.drawingId, - brushType: drawing.brushType, - pointCount: pts.length - }); - - // Use brush engine to render advanced brush strokes - offscreenContext.save(); - brushEngine.updateContext(offscreenContext); - - // Start the stroke at the first point - offscreenContext.beginPath(); - offscreenContext.moveTo(pts[0].x, pts[0].y); - - // Render the stroke using the brush engine with explicit brush type - brushEngine.startStroke(pts[0].x, pts[0].y); - for (let i = 1; i < pts.length; i++) { - // Use drawWithType instead of draw to bypass state dependency - brushEngine.drawWithType( - pts[i].x, - pts[i].y, - drawing.lineWidth, - drawing.color, - drawing.brushType // Pass brush type directly - ); - } - offscreenContext.restore(); - } else { - if (drawing.brushType && drawing.brushType !== "normal") { - console.log("Advanced brush found but no brushEngine:", { - id: drawing.drawingId, - brushType: drawing.brushType, - hasBrushEngine: !!brushEngine - }); - } - // Default rendering for normal brush - offscreenContext.beginPath(); - offscreenContext.moveTo(pts[0].x, pts[0].y); - for (let i = 1; i < pts.length; i++) - offscreenContext.lineTo(pts[i].x, pts[i].y); - offscreenContext.strokeStyle = drawing.color; - offscreenContext.lineWidth = drawing.lineWidth; - offscreenContext.lineCap = drawing.brushStyle || "round"; - offscreenContext.lineJoin = drawing.brushStyle || "round"; - offscreenContext.stroke(); - } - } - } else if (drawing.pathData && drawing.pathData.tool === "shape") { - if (drawing.pathData.points) { - const pts = drawing.pathData.points; - offscreenContext.save(); - offscreenContext.beginPath(); - offscreenContext.moveTo(pts[0].x, pts[0].y); - for (let i = 1; i < pts.length; i++) - offscreenContext.lineTo(pts[i].x, pts[i].y); - offscreenContext.closePath(); - offscreenContext.fillStyle = drawing.color; - offscreenContext.fill(); - offscreenContext.restore(); - } else { - const { - type, - start, - end, - brushStyle: storedBrush, - } = drawing.pathData; - offscreenContext.save(); - offscreenContext.fillStyle = drawing.color; - offscreenContext.lineWidth = drawing.lineWidth; - if (type === "circle") { - const radius = Math.sqrt( - (end.x - start.x) ** 2 + (end.y - start.y) ** 2 - ); - offscreenContext.beginPath(); - offscreenContext.arc(start.x, start.y, radius, 0, Math.PI * 2); - offscreenContext.fill(); - } else if (type === "rectangle") { - offscreenContext.fillRect( - start.x, - start.y, - end.x - start.x, - end.y - start.y - ); - } else if (type === "hexagon") { - const radius = Math.sqrt( - (end.x - start.x) ** 2 + (end.y - start.y) ** 2 - ); - offscreenContext.beginPath(); - for (let i = 0; i < 6; i++) { - const angle = (Math.PI / 3) * i; - const xPoint = start.x + radius * Math.cos(angle); - const yPoint = start.y + radius * Math.sin(angle); - if (i === 0) offscreenContext.moveTo(xPoint, yPoint); - else offscreenContext.lineTo(xPoint, yPoint); - } - offscreenContext.closePath(); - offscreenContext.fill(); - } else if (type === "line") { - offscreenContext.beginPath(); - offscreenContext.moveTo(start.x, start.y); - offscreenContext.lineTo(end.x, end.y); - offscreenContext.strokeStyle = drawing.color; - offscreenContext.lineWidth = drawing.lineWidth; - const cap = storedBrush || drawing.brushStyle || "round"; - offscreenContext.lineCap = cap; - offscreenContext.lineJoin = cap; - offscreenContext.stroke(); - } - offscreenContext.restore(); - } - } else if (drawing.pathData && drawing.pathData.tool === "image") { - const { image, x, y, width, height } = drawing.pathData; - let img = new Image(); - img.src = image; - img.onload = () => { - offscreenContext.drawImage(img, x, y, width, height); - }; - } - } - if (!selectedUser) { - // Group users by 5-minute intervals - // Use both committed drawings and pending drawings so the UI's - // user/time-group list reflects the strokes the user currently sees. - const groupMap = {}; - const groupingSource = [ - ...(userData.drawings || []), - ...(pendingDrawings || []), - ]; - groupingSource.forEach((d) => { - try { - const ts = d.timestamp || d.order || 0; - const periodStart = - Math.floor(ts / (5 * 60 * 1000)) * (5 * 60 * 1000); - if (!groupMap[periodStart]) groupMap[periodStart] = new Set(); - if (d.user) groupMap[periodStart].add(d.user); - } catch (e) { } - }); - const groups = Object.keys(groupMap).map((k) => ({ - periodStart: parseInt(k), - users: Array.from(groupMap[k]), - })); - groups.sort((a, b) => b.periodStart - a.periodStart); - if (selectedUser && selectedUser !== "") { - let stillExists = false; - if (typeof selectedUser === "string") { - for (const g of groups) { - if (g.users.includes(selectedUser)) { - stillExists = true; - break; - } - } - } else if (typeof selectedUser === "object" && selectedUser.user) { - for (const g of groups) { - if ( - g.periodStart === selectedUser.periodStart && - g.users.includes(selectedUser.user) - ) { - stillExists = true; - break; - } - } - } - - if (!stillExists) { - try { - setSelectedUser(""); - } catch (e) { - /* swallow if setter changed */ - } - } - } - - setUserList(groups); - } - - // Apply filters as post-processing after all regular drawings are rendered - if (filterDrawings.length > 0) { - console.log("[drawAllDrawings] Applying", filterDrawings.length, "filter(s)"); - for (const filterDrawing of filterDrawings) { - try { - if (filterDrawing.filterType && filterDrawing.filterParams) { - const imageData = offscreenContext.getImageData(0, 0, canvasWidth, canvasHeight); - const filteredImageData = applyImageFilter( - imageData, - filterDrawing.filterType, - filterDrawing.filterParams - ); - offscreenContext.putImageData(filteredImageData, 0, 0); - console.log("[drawAllDrawings] Applied filter:", filterDrawing.filterType); - } - } catch (e) { - console.error("[drawAllDrawings] Error applying filter:", filterDrawing.filterType, e); - } - } - } - - // Copy offscreen canvas to visible canvas atomically - console.log("[drawAllDrawings] Copying offscreen canvas to visible canvas. Total strokes rendered:", drawingsToRender.length, "filters:", filterDrawings.length); - context.imageSmoothingEnabled = false; - context.clearRect(0, 0, canvasWidth, canvasHeight); - context.drawImage(offscreenCanvasRef.current, 0, 0); - - // Update cache after successful render (only if no filters/cuts and not in incremental mode) - if (filterDrawings.length === 0 && !combined.some(d => d.pathData && d.pathData.tool === "cut")) { - if (!cachedCanvasRef.current || cachedCanvasRef.current.width !== canvasWidth || cachedCanvasRef.current.height !== canvasHeight) { - cachedCanvasRef.current = document.createElement("canvas"); - cachedCanvasRef.current.width = canvasWidth; - cachedCanvasRef.current.height = canvasHeight; - } - const cacheContext = cachedCanvasRef.current.getContext("2d"); - cacheContext.clearRect(0, 0, canvasWidth, canvasHeight); - cacheContext.drawImage(offscreenCanvasRef.current, 0, 0); - - // Update cached drawing IDs - cachedDrawingIdsRef.current = new Set(combined.map(d => d.drawingId)); - console.log(`[drawAllDrawings] Cached ${cachedDrawingIdsRef.current.size} drawings for future incremental rendering`); - } - - console.log("[drawAllDrawings] Canvas update complete"); - } catch (e) { - console.error("Error in drawAllDrawings:", e); - } finally { - // Restore current brush state - if (brushEngine && savedBrushType) { - brushEngine.setBrushType(savedBrushType); - if (savedBrushParams) { - brushEngine.setBrushParams(savedBrushParams); - } - } - setIsLoading(false); - isDrawingInProgressRef.current = false; - } - }; - - drawAllDrawingsRef.current = drawAllDrawings; - - const undo = async () => { - if (!editingEnabled) { - showLocalSnack("Undo is disabled in view-only mode."); - return; - } - if (undoStack.length === 0) return; - if (isRefreshing) { - showLocalSnack( - "Please wait for the canvas to refresh before undoing again." - ); - return; - } - try { - await undoAction({ - auth, - currentUser: auth?.username || "anonymous", - undoStack, - setUndoStack, - setRedoStack, - userData, - drawAllDrawings, - refreshCanvasButtonHandler: refreshCanvasButtonHandler, - roomId: currentRoomId, - }); - // After undo completes, refresh undo/redo availability from server - try { - await checkUndoRedoAvailability( - auth, - setUndoAvailable, - setRedoAvailable, - currentRoomId - ); - } catch (e) { } - updateFilterState(); // Update filter state after undo - } catch (error) { - console.error("Error during undo:", error); - } - }; - - const redo = async () => { - if (!editingEnabled) { - showLocalSnack("Redo is disabled in view-only mode."); - return; - } - if (redoStack.length === 0) return; - if (isRefreshing) { - showLocalSnack( - "Please wait for the canvas to refresh before redoing again." - ); - return; - } - try { - await redoAction({ - auth, - currentUser: auth?.username || "anonymous", - redoStack, - setRedoStack, - setUndoStack, - userData, - drawAllDrawings, - refreshCanvasButtonHandler: refreshCanvasButtonHandler, - roomId: currentRoomId, - }); - // After redo completes, refresh undo/redo availability from server - try { - await checkUndoRedoAvailability( - auth, - setUndoAvailable, - setRedoAvailable, - currentRoomId - ); - } catch (e) { } - updateFilterState(); // Update filter state after redo - } catch (error) { - console.error("Error during redo:", error); - } - }; - - // Register keyboard shortcuts and commands - useEffect(() => { - // Initialize shortcut manager - if (!shortcutManagerRef.current) { - shortcutManagerRef.current = new KeyboardShortcutManager(); - } - - const manager = shortcutManagerRef.current; - - // Register all commands with the command registry - const commands = [ - // Command Palette & Help - { - id: 'commands.palette', - label: 'Open Command Palette', - description: 'Quick access to all commands', - keywords: ['palette', 'search', 'find'], - category: 'Commands', - action: () => setCommandPaletteOpen(true), - shortcut: { key: 'k', modifiers: { ctrl: true } } - }, - { - id: 'commands.shortcuts', - label: 'Show Keyboard Shortcuts', - description: 'View all available keyboard shortcuts', - keywords: ['help', 'shortcuts', 'keys'], - category: 'Commands', - action: () => setShortcutsHelpOpen(true), - shortcut: { key: '/', modifiers: { ctrl: true } } - }, - { - id: 'commands.cancel', - label: 'Cancel / Escape', - description: 'Cancel current action or close dialogs', - keywords: ['cancel', 'escape', 'close'], - category: 'Commands', - action: () => { - if (commandPaletteOpen) setCommandPaletteOpen(false); - else if (shortcutsHelpOpen) setShortcutsHelpOpen(false); - else if (drawing) setDrawing(false); - }, - shortcut: { key: 'Escape', modifiers: {} } - }, - - // Edit Operations - { - id: 'edit.undo', - label: 'Undo', - description: 'Undo the last action', - keywords: ['undo', 'revert'], - category: 'Edit', - action: undo, - shortcut: { key: 'z', modifiers: { ctrl: true } }, - enabled: () => editingEnabled && undoStack.length > 0 - }, - { - id: 'edit.redo', - label: 'Redo', - description: 'Redo the last undone action', - keywords: ['redo', 'repeat'], - category: 'Edit', - action: redo, - shortcut: { key: 'z', modifiers: { ctrl: true, shift: true } }, - enabled: () => editingEnabled && redoStack.length > 0 - }, - - // Canvas Operations - { - id: 'canvas.clear', - label: 'Clear Canvas', - description: 'Remove all strokes from canvas', - keywords: ['clear', 'delete', 'reset'], - category: 'Canvas', - action: () => { - if (editingEnabled) { - setClearDialogOpen(true); - } else { - showLocalSnack('Canvas clearing is disabled in view-only mode'); - } - }, - shortcut: { key: 'k', modifiers: { ctrl: true, shift: true } }, - enabled: () => editingEnabled - }, - { - id: 'canvas.refresh', - label: 'Refresh Canvas', - description: 'Reload canvas from server', - keywords: ['refresh', 'reload'], - category: 'Canvas', - action: refreshCanvasButtonHandler, - shortcut: { key: 'r', modifiers: { ctrl: true } } - }, - { - id: 'canvas.settings', - label: 'Canvas Settings', - description: 'Open canvas settings', - keywords: ['settings', 'preferences'], - category: 'Canvas', - action: () => { - if (onOpenSettings) onOpenSettings(); - }, - shortcut: { key: ',', modifiers: { ctrl: true } }, - visible: () => !!onOpenSettings - }, - - // Tools - { - id: 'tool.pen', - label: 'Select Pen Tool', - description: 'Switch to freehand drawing', - keywords: ['pen', 'draw', 'brush'], - category: 'Tools', - action: () => { - if (editingEnabled) { - setDrawMode('freehand'); - showLocalSnack('Pen tool selected'); - } - }, - shortcut: { key: 'p', modifiers: {} }, - enabled: () => editingEnabled - }, - { - id: 'tool.eraser', - label: 'Select Eraser', - description: 'Switch to eraser mode', - keywords: ['eraser', 'erase', 'remove'], - category: 'Tools', - action: () => { - if (editingEnabled) { - setDrawMode('eraser'); - showLocalSnack('Eraser selected'); - } - }, - shortcut: { key: 'e', modifiers: {} }, - enabled: () => editingEnabled - }, - { - id: 'tool.rectangle', - label: 'Select Rectangle Tool', - description: 'Draw rectangles and squares', - keywords: ['rectangle', 'rect', 'square'], - category: 'Tools', - action: () => { - if (editingEnabled) { - setDrawMode('shape'); - setShapeType('rectangle'); - showLocalSnack('Rectangle tool selected'); - } - }, - shortcut: { key: 'r', modifiers: {} }, - enabled: () => editingEnabled - }, - { - id: 'tool.circle', - label: 'Select Circle Tool', - description: 'Draw circles and ellipses', - keywords: ['circle', 'oval', 'ellipse'], - category: 'Tools', - action: () => { - if (editingEnabled) { - setDrawMode('shape'); - setShapeType('circle'); - showLocalSnack('Circle tool selected'); - } - }, - shortcut: { key: 'c', modifiers: {} }, - enabled: () => editingEnabled - }, - { - id: 'tool.line', - label: 'Select Line Tool', - description: 'Draw straight lines', - keywords: ['line', 'straight'], - category: 'Tools', - action: () => { - if (editingEnabled) { - setDrawMode('shape'); - setShapeType('line'); - showLocalSnack('Line tool selected'); - } - }, - shortcut: { key: 'l', modifiers: {} }, - enabled: () => editingEnabled - } - ]; - - // Register commands with command registry - // Clear first to ensure clean state - commandRegistry.clear(); - - // Register each command (allowOverwrite for React re-renders) - commands.forEach(cmd => { - commandRegistry.register(cmd, { allowOverwrite: true }); - }); - - // Register keyboard shortcuts - manager.clear(); - commands.forEach(cmd => { - if (cmd.shortcut) { - manager.register( - cmd.shortcut.key, - cmd.shortcut.modifiers, - () => { - // Check if command is enabled before executing - if (cmd.enabled && !cmd.enabled()) { - return; - } - cmd.action(); - }, - cmd.label, - cmd.category - ); - } - }); - - // Add global keyboard event listener - const handleKeyDown = (event) => manager.handleKeyDown(event); - document.addEventListener('keydown', handleKeyDown); - - // Cleanup - return () => { - document.removeEventListener('keydown', handleKeyDown); - manager.clear(); - }; - }, [ - editingEnabled, - undoStack, - redoStack, - undo, - redo, - refreshCanvasButtonHandler, - onOpenSettings, - commandPaletteOpen, - shortcutsHelpOpen, - drawing - ]); - - const { - selectionStart, - setSelectionStart, - selectionRect, - setSelectionRect, - cutImageData, - setCutImageData, - handleCutSelection, - } = useCanvasSelection( - canvasRef, - currentUser, - userData, - generateId, - drawAllDrawings, - currentRoomId, - setUndoAvailable, - setRedoAvailable, - auth, - roomType, - showLocalSnack - ); - - // Draw a preview of a shape (for shape mode) - const drawShapePreview = (start, end, shape, color, lineWidth) => { - if (!start || !end) return; - - const canvas = canvasRef.current; - const context = canvas.getContext("2d"); - context.save(); - context.strokeStyle = color; - context.lineWidth = lineWidth; - context.setLineDash([5, 3]); - - if (shape === "circle") { - const radius = Math.sqrt((end.x - start.x) ** 2 + (end.y - start.y) ** 2); - context.beginPath(); - context.arc(start.x, start.y, radius, 0, Math.PI * 2); - context.stroke(); - } else if (shape === "rectangle") { - context.strokeRect(start.x, start.y, end.x - start.x, end.y - start.y); - } else if (shape === "hexagon") { - const radius = Math.sqrt((end.x - start.x) ** 2 + (end.y - start.y) ** 2); - context.beginPath(); - - for (let i = 0; i < 6; i++) { - const angle = (Math.PI / 3) * i; - const xPoint = start.x + radius * Math.cos(angle); - const yPoint = start.y + radius * Math.sin(angle); - - if (i === 0) context.moveTo(xPoint, yPoint); - else context.lineTo(xPoint, yPoint); - } - context.closePath(); - context.stroke(); - } else if (shape === "line") { - context.beginPath(); - context.moveTo(start.x, start.y); - context.lineTo(end.x, end.y); - context.lineCap = brushStyle; - context.lineJoin = brushStyle; - context.stroke(); - } - - context.restore(); - }; - - // Handle paste action for cut selection - const handlePaste = async (e) => { - if (!editingEnabled) { - showLocalSnack("Editing is disabled in view-only mode."); - setDrawMode("freehand"); - return; - } - if ( - !cutImageData || - !Array.isArray(cutImageData) || - cutImageData.length === 0 - ) { - showLocalSnack("No cut selection available to paste."); - setDrawMode("freehand"); - return; - } - - const canvas = canvasRef.current; - const rectCanvas = canvas.getBoundingClientRect(); - const scaleX = canvas.width / rectCanvas.width; - const scaleY = canvas.height / rectCanvas.height; - const pasteX = (e.clientX - rectCanvas.left) * scaleX; - const pasteY = (e.clientY - rectCanvas.top) * scaleY; - - let minX = Infinity, - minY = Infinity; - - cutImageData.forEach((drawing) => { - if (Array.isArray(drawing.pathData)) { - drawing.pathData.forEach((pt) => { - minX = Math.min(minX, pt.x); - minY = Math.min(minY, pt.y); - }); - } else if (drawing.pathData && drawing.pathData.tool === "shape") { - if (drawing.pathData.points && Array.isArray(drawing.pathData.points)) { - drawing.pathData.points.forEach((pt) => { - minX = Math.min(minX, pt.x); - minY = Math.min(minY, pt.y); - }); - } else if (drawing.pathData.type === "line") { - if (drawing.pathData.start) { - minX = Math.min(minX, drawing.pathData.start.x); - minY = Math.min(minY, drawing.pathData.start.y); - } - if (drawing.pathData.end) { - minX = Math.min(minX, drawing.pathData.end.x); - minY = Math.min(minY, drawing.pathData.end.y); - } - } - } - }); - - if (minX === Infinity || minY === Infinity) { - showLocalSnack("Invalid cut data."); - return; - } - - const offsetX = pasteX - minX; - const offsetY = pasteY - minY; - let pastedDrawings = []; - - const newDrawings = cutImageData - .map((originalDrawing) => { - let newPathData; - if (Array.isArray(originalDrawing.pathData)) { - newPathData = originalDrawing.pathData.map((pt) => ({ - x: pt.x + offsetX, - y: pt.y + offsetY, - })); - } else if ( - originalDrawing.pathData && - originalDrawing.pathData.tool === "shape" - ) { - if ( - originalDrawing.pathData.points && - Array.isArray(originalDrawing.pathData.points) - ) { - const newPoints = originalDrawing.pathData.points.map((pt) => ({ - x: pt.x + offsetX, - y: pt.y + offsetY, - })); - newPathData = { ...originalDrawing.pathData, points: newPoints }; - } else if (originalDrawing.pathData.type === "line") { - const newStart = { - x: originalDrawing.pathData.start.x + offsetX, - y: originalDrawing.pathData.start.y + offsetY, - }; - const newEnd = { - x: originalDrawing.pathData.end.x + offsetX, - y: originalDrawing.pathData.end.y + offsetY, - }; - newPathData = { - ...originalDrawing.pathData, - start: newStart, - end: newEnd, - }; - } - } else { - return null; - } - - // Preserve all metadata from original drawing - const metadata = { - brushStyle: originalDrawing.brushStyle, - brushType: originalDrawing.brushType, - brushParams: originalDrawing.brushParams, - drawingType: originalDrawing.drawingType, - stampData: originalDrawing.stampData, - stampSettings: originalDrawing.stampSettings, - filterType: originalDrawing.filterType, - filterParams: originalDrawing.filterParams, - }; - - return new Drawing( - generateId(), - originalDrawing.color, - originalDrawing.lineWidth, - newPathData, - Date.now(), - currentUser, - metadata - ); - }) - .filter(Boolean); - - // Show all pasted items immediately (optimistic UI) - console.log("[handlePaste] Starting optimistic paste operation:", { - drawingCount: newDrawings.length, - drawingTypes: newDrawings.map(d => d.drawingType || "stroke") - }); - - const pasteRecordId = generateId(); - - // Attach parentPasteId to each new drawing - for (const nd of newDrawings) { - nd.roomId = currentRoomId; - nd.parentPasteId = pasteRecordId; - if (!nd.pathData) nd.pathData = {}; - nd.pathData.parentPasteId = pasteRecordId; - - // Add to local canvas immediately - userData.addDrawing(nd); - } - console.log("[handlePaste] Attached parentPasteId to all drawings:", pasteRecordId); - - // Redraw canvas immediately with all pasted items - drawAllDrawings(); - - // Now submit to backend in background (no dialog) - setRedoStack([]); - - try { - // Submit all pasted drawings in batch - const result = await submitBatchToDatabase( - newDrawings, - auth, - { roomId: currentRoomId, roomType, skipUndoStack: true }, - setUndoAvailable, - setRedoAvailable - ); - - console.log("[handlePaste] Batch submission complete:", result); - - // Create and submit paste record - const pastedIds = newDrawings.map((d) => d.drawingId); - const pasteRecord = new Drawing( - pasteRecordId, - "#FFFFFF", - 1, - { tool: "paste", cut: false, pastedDrawingIds: pastedIds }, - Date.now(), - currentUser - ); - - await submitToDatabase( - pasteRecord, - auth, - { roomId: currentRoomId, roomType }, - setUndoAvailable, - setRedoAvailable - ); - - console.log("[handlePaste] Paste record submitted successfully"); - - // Update undo stack - setUndoStack((prev) => [ - ...prev, - { type: "paste", pastedDrawings: newDrawings, backendCount: 1 }, - ]); - - // Update undo/redo availability - if (currentRoomId) { - checkUndoRedoAvailability( - auth, - setUndoAvailable, - setRedoAvailable, - currentRoomId - ); - } - - setCutImageData([]); - setDrawMode("freehand"); - - } catch (error) { - console.error("Failed to complete paste operation:", error); - userData.drawings = userData.drawings.filter( - d => d.parentPasteId !== pasteRecordId - ); - drawAllDrawings(); - handleAuthError(error); - } - - tempPathRef.current = []; - }; - - const mergedRefreshCanvas = async (sourceLabel = undefined) => { - try { - if (sourceLabel) { - console.log('mergedRefreshCanvas called from:', sourceLabel, '==='); - console.debug('mergedRefreshCanvas called from:', sourceLabel); - } else { - console.log('mergedRefreshCanvas called (no label) ==='); - console.debug('mergedRefreshCanvas called'); - } - } catch (e) { } - // If currently panning, defer refresh until pan ends to avoid races and frequent backend calls. - try { - if (isPanning) { - console.debug( - "[mergedRefreshCanvas] deferring because isPanning=true, marking pendingPanRefreshRef" - ); - pendingPanRefreshRef.current = true; - return; - } - } catch (e) { } - - if (sourceLabel === "undo-event" || sourceLabel === "redo-event") { - console.log("[mergedRefreshCanvas] Forcing complete state reset for undo/redo"); - lastDrawnStateRef.current = null; - } - - setIsLoading(true); - const backendCount = await backendRefreshCanvas( - serverCountRef.current, - userData, - drawAllDrawings, - historyRange ? historyRange.start : undefined, - historyRange ? historyRange.end : undefined, - { - roomId: currentRoomId, - auth, - clearLastDrawnState: () => { - console.log("[mergedRefreshCanvas] Clearing lastDrawnStateRef to force redraw"); - lastDrawnStateRef.current = null; - } - } - ); - - const pendingSnapshot = [...pendingDrawings]; - - // Don't clear all pending drawings, only mark confirmed ones for removal - - serverCountRef.current = backendCount; - // Re-append any pending drawings that the backend didn't return. - const drawingMatches = (a, b) => { - if (!a || !b) return false; - if (a.drawingId && b.drawingId && a.drawingId === b.drawingId) - return true; - - try { - const sameUser = a.user === b.user; - const tsA = a.timestamp || a.ts || 0; - const tsB = b.timestamp || b.ts || 0; - const tsClose = Math.abs(tsA - tsB) < 1000; - const lenA = Array.isArray(a.pathData) - ? a.pathData.length - : a.pathData && a.pathData.points - ? a.pathData.points.length - : 0; - const lenB = Array.isArray(b.pathData) - ? b.pathData.length - : b.pathData && b.pathData.points - ? b.pathData.points.length - : 0; - const lenClose = Math.abs(lenA - lenB) <= 1; - return sameUser && tsClose && lenClose; - } catch (e) { - return false; - } - }; - - try { - const cutOriginalIds = new Set(); - (userData.drawings || []).forEach((d) => { - if ( - d.pathData && - d.pathData.tool === "cut" && - Array.isArray(d.pathData.originalStrokeIds) - ) { - d.pathData.originalStrokeIds.forEach((id) => cutOriginalIds.add(id)); - } - }); - - if (cutOriginalIds.size > 0) { - userData.drawings = (userData.drawings || []).filter( - (d) => !cutOriginalIds.has(d.drawingId) - ); - } - } catch (e) { - // best-effort - } - - // Re-append pending drawings that the backend didn't return, but - // skip any pending items older than the authoritative clearedAt timestamp - const clearedAt = currentRoomId - ? roomClearedAtRef.current[currentRoomId] - : null; - const stillPending = []; - - pendingSnapshot.forEach((pd) => { - try { - const pdTs = pd.timestamp || pd.ts || 0; - if (clearedAt && pdTs < clearedAt) { - // This pending drawing was created before a server clear; ignore it - return; - } - } catch (e) { } - - const exists = userData.drawings.find((d) => drawingMatches(d, pd)); - if (!exists) { - // Backend doesn't have it yet, keep it pending - userData.drawings.push(pd); - stillPending.push(pd); - } else { - // If pending drawing has stampData but backend version doesn't, use pending version - if (pd.drawingType === "stamp" && pd.stampData) { - const backendMatch = exists; - if (!backendMatch.stampData || !backendMatch.stampData.image && pd.stampData.image) { - console.warn("Backend stamp missing stampData, using pending version:", { - drawingId: pd.drawingId, - pendingHasStampData: !!pd.stampData, - backendHasStampData: !!backendMatch.stampData, - pendingImageLength: pd.stampData.image ? pd.stampData.image.length : 0, - backendImageLength: backendMatch.stampData && backendMatch.stampData.image ? backendMatch.stampData.image.length : 0 - }); - - // Replace backend version with pending version that has complete data - const idx = userData.drawings.findIndex((d) => drawingMatches(d, pd)); - if (idx !== -1) { - userData.drawings[idx] = pd; - } - } - } - - // Backend has it, mark as confirmed and remove from pending - if (pd.drawingId) { - confirmedStrokesRef.current.add(pd.drawingId); - } - } - }); - - // Update pending drawings to only include those still not confirmed by backend - setPendingDrawings(stillPending); - - // CRITICAL: Deduplicate filters - only keep the LATEST of each filter type - // This prevents stacking when backend returns duplicates - const filtersByType = new Map(); - const nonFilterDrawings = []; - - (userData.drawings || []).forEach((drawing) => { - if (drawing.drawingType === "filter" && drawing.filterType) { - const existing = filtersByType.get(drawing.filterType); - // Keep the one with the latest timestamp - if (!existing || (drawing.timestamp || 0) > (existing.timestamp || 0)) { - filtersByType.set(drawing.filterType, drawing); - } - } else { - nonFilterDrawings.push(drawing); - } - }); - - // Rebuild drawings array with deduplicated filters - const deduplicatedDrawings = [ - ...nonFilterDrawings, - ...Array.from(filtersByType.values()) - ]; - - console.log(`[mergedRefreshCanvas] Deduplicated filters. Filter count: ${filtersByType.size}, Total drawings: ${deduplicatedDrawings.length}`); - - // CRITICAL: Update both the mutable userData object AND React state - // Update userData in place so the closure reference works - userData.drawings = deduplicatedDrawings; - - // Also update React state to trigger re-renders - const newUserData = new UserData(userData.userId, userData.username); - newUserData.drawings = deduplicatedDrawings; - setUserData(newUserData); - - // Extract custom stamps from all drawings and update stamp panel - extractCustomStamps(); - - // Use requestAnimationFrame for smoother rendering - requestAnimationFrame(() => { - drawAllDrawings(); - setIsLoading(false); - updateFilterState(); // Update filter state after loading drawings - }); - }; - - // Extract custom stamps from backend drawings and update StampPanel - const extractCustomStamps = () => { - try { - const customStamps = []; - const seenStamps = new Map(); // Deduplicate by image content or emoji - - (userData.drawings || []).forEach((drawing) => { - if (drawing.drawingType === "stamp" && drawing.stampData) { - const stamp = drawing.stampData; - - // Skip default emoji stamps (they're already in StampPanel) - if (stamp.emoji && !stamp.image) { - return; - } - - // For custom image stamps, create a unique key based on image content - if (stamp.image) { - const imageKey = stamp.image.substring(0, 100); // Use first 100 chars as key - - if (!seenStamps.has(imageKey)) { - seenStamps.set(imageKey, true); - customStamps.push({ - id: `stamp-${Date.now()}-${customStamps.length}`, - name: stamp.name || 'Custom Stamp', - category: stamp.category || 'custom', - image: stamp.image, - emoji: stamp.emoji - }); - } - } - } - }); - - if (customStamps.length > 0) { - console.log('Extracted custom stamps from backend:', customStamps.length); - setBackendStamps(customStamps); - } - } catch (error) { - console.error('Error extracting custom stamps:', error); - } - }; - - const startDrawingHandler = (e) => { - const canvas = canvasRef.current; - const rect = canvas.getBoundingClientRect(); - const x = e.clientX - rect.left; - const y = e.clientY - rect.top; - - if (e.button === 1) { - // Middle mouse button: start panning - setIsPanning(true); - panStartRef.current = { x: e.clientX, y: e.clientY }; - panOriginRef.current = { ...panOffset }; - setIsLoading(true); - // Throttle pan-triggered refreshes: if we recently refreshed, defer until pan end - try { - const now = Date.now(); - const diff = now - panLastRefreshRef.current; - console.debug( - `[pan] now=${now} lastRefresh=${panLastRefreshRef.current} diff=${diff} cooldown=${PAN_REFRESH_COOLDOWN_MS}` - ); - if (diff > PAN_REFRESH_COOLDOWN_MS) { - panLastRefreshRef.current = now; - console.debug("[pan] triggering immediate mergedRefreshCanvas"); - mergedRefreshCanvas("pan-start").finally(() => setIsLoading(false)); - } else { - // Mark that we skipped the immediate refresh and schedule a deferred refresh on mouseup - panRefreshSkippedRef.current = true; - console.debug( - "[pan] skipped immediate refresh; scheduling deferred refresh on mouseup" - ); - if (panEndRefreshTimerRef.current) - clearTimeout(panEndRefreshTimerRef.current); - panEndRefreshTimerRef.current = setTimeout(() => { - if (panRefreshSkippedRef.current) { - panRefreshSkippedRef.current = false; - panLastRefreshRef.current = Date.now(); - console.debug("[pan] deferred timer firing mergedRefreshCanvas"); - mergedRefreshCanvas("pan-deferred").finally(() => - setIsLoading(false) - ); - } - panEndRefreshTimerRef.current = null; - }, Math.max(200, PAN_REFRESH_COOLDOWN_MS - diff)); - setIsLoading(false); - } - } catch (e) { - mergedRefreshCanvas().finally(() => setIsLoading(false)); - } - return; - } - - if (!editingEnabled) return; - - if (drawMode === "eraser" || drawMode === "freehand") { - const context = canvas.getContext("2d"); - context.strokeStyle = color; - context.lineWidth = lineWidth; - context.lineCap = brushStyle; - context.lineJoin = brushStyle; - - // Initialize brush engine for advanced brushes - if (brushEngine) { - brushEngine.updateContext(context); - brushEngine.startStroke(x, y); - - // For normal brush, we still need the standard path setup - if (currentBrushType === "normal") { - context.beginPath(); - context.moveTo(x, y); - } - } else { - // Fallback if no brush engine - context.beginPath(); - context.moveTo(x, y); - } - - tempPathRef.current = [{ x, y }]; - setDrawing(true); - } else if (drawMode === "shape") { - setShapeStart({ x, y }); - setDrawing(true); - - const dataURL = canvas.toDataURL(); - let snapshotImg = new Image(); - - snapshotImg.src = dataURL; - snapshotRef.current = snapshotImg; - } else if (drawMode === "select") { - setSelectionStart({ x, y }); - setSelectionRect(null); - setDrawing(true); - - const dataURL = canvas.toDataURL(); - let snapshotImg = new Image(); - - snapshotImg.src = dataURL; - snapshotRef.current = snapshotImg; - } else if (drawMode === "paste") { - handlePaste(e); - } else if (drawMode === "stamp") { - // Start stamp preview on mousedown (will place on mouseup) - if (selectedStamp && stampSettings) { - setStampPreview({ x, y, stamp: selectedStamp, settings: stampSettings }); - stampPreviewRef.current = { x, y, stamp: selectedStamp, settings: stampSettings }; - setDrawing(true); // Enable dragging - } - } - }; - - const handlePan = (e) => { - if (!isPanning) return; - - // If the middle button is no longer pressed, stop panning. - if (!(e.buttons & 4)) { - setIsPanning(false); - panOriginRef.current = { ...panOffset }; - return; - } - - const deltaX = e.clientX - panStartRef.current.x; - const deltaY = e.clientY - panStartRef.current.y; - let newX = panOriginRef.current.x + deltaX; - let newY = panOriginRef.current.y + deltaY; - const containerWidth = window.innerWidth; - const containerHeight = window.innerHeight; - - // Calculate minimum allowed offsets so that the canvas edge is not exceeded. - // Our canvas is fixed at canvasWidth and canvasHeight. - const minX = containerWidth - canvasWidth; // This will be negative if canvasWidth > containerWidth - const minY = containerHeight - canvasHeight; - - newX = clamp(newX, minX, 0); - newY = clamp(newY, minY, 0); - - setPanOffset({ - x: newX, - y: newY, - }); - }; - - const drawHandler = (e) => { - if (isPanning) { - handlePan(e); - return; - } - if (!editingEnabled) return; // prevent drawing but allow other handlers like panning to proceed - if (!drawing) return; - - const canvas = canvasRef.current; - const rect = canvas.getBoundingClientRect(); - const x = e.clientX - rect.left; - const y = e.clientY - rect.top; - - console.log( - "Drawing with brush type:", - currentBrushType, - "drawMode:", - drawMode - ); - - // Update stamp preview position during drag - if (drawMode === "stamp" && stampPreviewRef.current) { - setStampPreview({ ...stampPreviewRef.current, x, y }); - stampPreviewRef.current = { ...stampPreviewRef.current, x, y }; - return; - } - - if (drawMode === "eraser" || drawMode === "freehand") { - const context = canvas.getContext("2d"); - - // Use advanced brush engine if available - if (brushEngine && currentBrushType !== "normal") { - console.log("Drawing with advanced brush engine:", currentBrushType); - // Ensure context is up to date - brushEngine.updateContext(context); - - // Ensure brush engine has current state - if (brushEngine.brushType !== currentBrushType) { - brushEngine.setBrushType(currentBrushType); - } - if ( - JSON.stringify(brushEngine.brushParams) !== - JSON.stringify(brushParams) - ) { - brushEngine.setBrushParams(brushParams); - } - - brushEngine.draw(x, y, lineWidth, color); - } else { - console.log("Drawing with normal brush"); - // Default drawing behavior - context.lineTo(x, y); - context.stroke(); - context.beginPath(); - context.moveTo(x, y); - } - - tempPathRef.current.push({ x, y }); - } else if (drawMode === "shape" && drawing) { - // update shape preview with adjusted coordinates - if (snapshotRef.current && snapshotRef.current.complete) { - const context = canvas.getContext("2d"); - context.clearRect(0, 0, canvasWidth, canvasHeight); - context.drawImage(snapshotRef.current, 0, 0); - } - - drawShapePreview(shapeStart, { x, y }, shapeType, color, lineWidth); - } else if (drawMode === "select" && drawing) { - setSelectionRect({ start: selectionStart, end: { x, y } }); - - if (snapshotRef.current && snapshotRef.current.complete) { - const context = canvas.getContext("2d"); - context.clearRect(0, 0, canvasWidth, canvasHeight); - context.drawImage(snapshotRef.current, 0, 0); - } - - const context = canvas.getContext("2d"); - context.save(); - context.strokeStyle = "blue"; - context.lineWidth = 1; - context.setLineDash([6, 3]); - - const s = selectionStart; - const selX = Math.min(s.x, x); - const selY = Math.min(s.y, y); - const selWidth = Math.abs(x - s.x); - const selHeight = Math.abs(y - s.y); - - context.strokeRect(selX, selY, selWidth, selHeight); - context.restore(); - } - }; - - const stopDrawingHandler = async (e) => { - if (isPanning && e.button === 1) { - setIsPanning(false); - return; - } - if (!drawing) return; - setDrawing(false); - - if (!editingEnabled) { - tempPathRef.current = []; - return; - } - - snapshotRef.current = null; - const canvas = canvasRef.current; - const rect = canvas.getBoundingClientRect(); - const finalX = e.clientX - rect.left; - const finalY = e.clientY - rect.top; - - if (drawMode === "eraser" || drawMode === "freehand") { - const newDrawing = new Drawing( - `drawing_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`, - color, - lineWidth, - tempPathRef.current, - Date.now(), - currentUser, - { - brushStyle: brushStyle, - brushType: currentBrushType, - brushParams: brushParams, - drawingType: "stroke", - } - ); - newDrawing.roomId = currentRoomId; - newDrawing.brushType = currentBrushType; - newDrawing.brushParams = brushParams; - - setUndoStack((prev) => [...prev, newDrawing]); - setRedoStack([]); - - try { - userData.addDrawing(newDrawing); - // Add to pending drawings for immediate display (optimistic UI) - setPendingDrawings((prev) => [...prev, newDrawing]); - - // Use requestAnimationFrame for immediate, smooth redraw - requestAnimationFrame(() => { - drawAllDrawings(); - }); - - // Queue the submission instead of submitting immediately - const submitTask = async () => { - try { - console.log("Submitting queued stroke:", { - drawingId: newDrawing.drawingId, - pathLength: tempPathRef.current.length, - }); - - await submitToDatabase( - newDrawing, - auth, - { - roomId: currentRoomId, - roomType, - }, - setUndoAvailable, - setRedoAvailable - ); - - // Mark stroke as confirmed - confirmedStrokesRef.current.add(newDrawing.drawingId); - - if (currentRoomId) { - checkUndoRedoAvailability( - auth, - setUndoAvailable, - setRedoAvailable, - currentRoomId - ); - } - } catch (error) { - console.error("Error during queued freehand submission:", error); - // On error, remove the failed stroke from pending - setPendingDrawings((prev) => - prev.filter((d) => d.drawingId !== newDrawing.drawingId) - ); - handleAuthError(error); - } - }; - - submissionQueueRef.current.push(submitTask); - processSubmissionQueue(); - } catch (error) { - console.error("Error preparing freehand stroke:", error); - handleAuthError(error); - } finally { - setIsRefreshing(false); - } - tempPathRef.current = []; - } else if (drawMode === "shape") { - if (!shapeStart) { - return; - } - - const finalEnd = { x: finalX, y: finalY }; - const context = canvas.getContext("2d"); - - context.save(); - context.fillStyle = color; - context.lineWidth = lineWidth; - context.setLineDash([]); - if (shapeType === "circle") { - const radius = Math.sqrt( - (finalEnd.x - shapeStart.x) ** 2 + (finalEnd.y - shapeStart.y) ** 2 - ); - - context.beginPath(); - context.arc(shapeStart.x, shapeStart.y, radius, 0, Math.PI * 2); - context.fill(); - } else if (shapeType === "rectangle") { - context.fillRect( - shapeStart.x, - shapeStart.y, - finalEnd.x - shapeStart.x, - finalEnd.y - shapeStart.y - ); - } else if (shapeType === "hexagon") { - const radius = Math.sqrt( - (finalEnd.x - shapeStart.x) ** 2 + (finalEnd.y - shapeStart.y) ** 2 - ); - context.beginPath(); - for (let i = 0; i < 6; i++) { - const angle = (Math.PI / 3) * i; - const xPoint = shapeStart.x + radius * Math.cos(angle); - const yPoint = shapeStart.y + radius * Math.sin(angle); - - if (i === 0) context.moveTo(xPoint, yPoint); - else context.lineTo(xPoint, yPoint); - } - - context.closePath(); - context.fill(); - } else if (shapeType === "line") { - context.beginPath(); - context.moveTo(shapeStart.x, shapeStart.y); - context.lineTo(finalEnd.x, finalEnd.y); - context.strokeStyle = color; - context.lineWidth = lineWidth; - context.lineCap = brushStyle; - context.lineJoin = brushStyle; - context.stroke(); - } - context.restore(); - - const shapeDrawingData = { - tool: "shape", - type: shapeType, - start: shapeStart, - end: finalEnd, - brushStyle: shapeType === "line" ? brushStyle : undefined, - }; - - const newDrawing = new Drawing( - `drawing_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`, - color, - lineWidth, - shapeDrawingData, - Date.now(), - currentUser, - { - brushStyle: shapeType === "line" ? brushStyle : "round", - brushType: currentBrushType, - brushParams: brushParams, - drawingType: "shape", - } - ); - newDrawing.roomId = currentRoomId; - - userData.addDrawing(newDrawing); - setPendingDrawings((prev) => [...prev, newDrawing]); - - // Use requestAnimationFrame for smooth shape rendering - requestAnimationFrame(() => { - drawAllDrawings(); - }); - - setUndoStack((prev) => [...prev, newDrawing]); - setRedoStack([]); - - // Queue the submission - const submitTask = async () => { - try { - await submitToDatabase( - newDrawing, - auth, - { - roomId: currentRoomId, - roomType, - }, - setUndoAvailable, - setRedoAvailable - ); - - // Mark stroke as confirmed - confirmedStrokesRef.current.add(newDrawing.drawingId); - - // Update undo/redo availability after shape submission - if (currentRoomId) { - checkUndoRedoAvailability( - auth, - setUndoAvailable, - setRedoAvailable, - currentRoomId - ); - } - } catch (error) { - console.error("Error during queued shape submission:", error); - // On error, remove the failed stroke from pending - setPendingDrawings((prev) => - prev.filter((d) => d.drawingId !== newDrawing.drawingId) - ); - handleAuthError(error); - } - }; - - submissionQueueRef.current.push(submitTask); - processSubmissionQueue(); - - setShapeStart(null); - } else if (drawMode === "select") { - setDrawing(false); - - try { - await mergedRefreshCanvas(); - } catch (error) { - console.error("Error during select submission or refresh:", error); - } finally { - setIsRefreshing(false); - } - - mergedRefreshCanvas(); - } else if (drawMode === "stamp" && stampPreviewRef.current) { - // Place stamp at final position on mouseup - const { x, y, stamp, settings } = stampPreviewRef.current; - await placeStamp(x, y, stamp, settings); - - // Clear preview - setStampPreview(null); - stampPreviewRef.current = null; - } - }; - - const openHistoryDialog = () => { - setSelectedUser(""); - setHistoryDialogOpen(true); - }; - - const handleApplyHistory = async (startMs, endMs) => { - // startMs and endMs are epoch ms. If not provided, read from inputs. - const start = - startMs !== undefined - ? startMs - : historyStartInput - ? new Date(historyStartInput).getTime() - : NaN; - const end = - endMs !== undefined - ? endMs - : historyEndInput - ? new Date(historyEndInput).getTime() - : NaN; - - if (isNaN(start) || isNaN(end)) { - showLocalSnack( - "Please select both start and end date/time before applying History Recall." - ); - return; - } - if (start > end) { - showLocalSnack("Invalid time range selected. Make sure start <= end."); - return; - } - - // Deselect any selected user when entering history recall - setSelectedUser(""); - setHistoryRange({ start, end }); - setIsLoading(true); - - // Try to load drawings for the requested time range - await clearCanvasForRefresh(); - // set a temporary historyRange so mergedRefreshCanvas will use it - setHistoryRange({ start, end }); - try { - const backendCount = await backendRefreshCanvas( - serverCountRef.current, - userData, - drawAllDrawings, - start, - end, - { roomId: currentRoomId, auth } - ); - serverCountRef.current = backendCount; - // If no drawings loaded, inform user and rollback historyRange - if (!userData.drawings || userData.drawings.length === 0) { - setHistoryRange(null); - showLocalSnack( - "No drawings were found in that date/time range. Please select another range or exit history recall mode." - ); - return; - } - setHistoryMode(true); - setHistoryDialogOpen(false); - } catch (e) { - console.error("Error applying history range:", e); - setHistoryRange(null); - showLocalSnack( - "An error occurred while loading history. See console for details." - ); - } finally { - setIsLoading(false); - } - }; - - // Auto-refresh when the active room changes - useEffect(() => { - // wipe local cache so we don't flash previous room's strokes - userData.drawings = []; - setIsRefreshing(true); - - // clear what's on screen immediately - try { - if (canvasRef.current) { - const ctx = canvasRef.current.getContext("2d"); - if (ctx) { - ctx.clearRect( - 0, - 0, - canvasRef.current.width, - canvasRef.current.height - ); - } - drawAllDrawings(); - } - } catch { } - - // reload for the new room - (async () => { - try { - await mergedRefreshCanvas(); // already room-aware - } finally { - setIsRefreshing(false); - } - })(); - }, [currentRoomId, canvasRefreshTrigger]); - - const exitHistoryMode = async () => { - // Deselect any selected user when leaving history mode - setSelectedUser(""); - setHistoryMode(false); - setHistoryRange(null); - setIsLoading(true); - try { - await clearCanvasForRefresh(); - serverCountRef.current = await backendRefreshCanvas( - serverCountRef.current, - userData, - drawAllDrawings, - undefined, - undefined, - { roomId: currentRoomId, auth } - ); - } finally { - setIsLoading(false); - } - }; - - const clearCanvas = async () => { - if (!editingEnabled) { - showLocalSnack("Cannot clear canvas in view-only mode."); - return; - } - const canvas = canvasRef.current; - const context = canvas.getContext("2d"); - - context.clearRect(0, 0, canvasWidth, canvasHeight); - - setUserData(initializeUserData()); - setUndoStack([]); - setRedoStack([]); - setPendingDrawings([]); - serverCountRef.current = 0; - }; - - const handleExportCanvas = async () => { - if (!currentRoomId) { - showLocalSnack("Cannot export: not in a room"); - return; - } - - try { - setIsLoading(true); - showLocalSnack("Exporting canvas data..."); - - const { exportRoomCanvas } = await import('../api/rooms'); - console.log('[Export] Calling API with roomId:', currentRoomId); - console.log('[Export] Auth token present:', !!auth?.token); - - const exportData = await exportRoomCanvas(auth?.token, currentRoomId); - - console.log('[Export] Received exportData:', { - exists: !!exportData, - type: typeof exportData, - keys: exportData ? Object.keys(exportData) : [], - hasStrokes: exportData ? !!exportData.strokes : false, - strokeCount: exportData ? exportData.strokeCount : 'N/A' - }); - - if (!exportData) { - console.error('[Export] exportData is null or undefined'); - showLocalSnack("Export failed: no data returned from server"); - return; - } - - if (!exportData.strokes) { - console.error('[Export] exportData.strokes is missing:', exportData); - showLocalSnack(`Export failed: no strokes in response (got ${exportData.strokeCount || 0} count)`); - return; - } - - // Create a downloadable JSON file - const dataStr = JSON.stringify(exportData, null, 2); - const dataBlob = new Blob([dataStr], { type: 'application/json' }); - const url = URL.createObjectURL(dataBlob); - const link = document.createElement('a'); - link.href = url; - link.download = `${exportData.roomName || 'canvas'}_export_${Date.now()}.json`; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - URL.revokeObjectURL(url); - - showLocalSnack(`Exported ${exportData.strokeCount} strokes successfully`); - console.log('[Export] Success - downloaded file'); - } catch (error) { - console.error("[Export] Error caught:", error); - console.error("[Export] Error stack:", error.stack); - showLocalSnack(`Export failed: ${error.message || 'Unknown error'}`); - } finally { - setIsLoading(false); - } - }; - - const handleImportCanvas = async () => { - if (!currentRoomId) { - showLocalSnack("Cannot import: not in a room"); - return; - } - - if (!editingEnabled) { - showLocalSnack("Cannot import in view-only mode"); - return; - } - - // Create a file input element - const input = document.createElement('input'); - input.type = 'file'; - input.accept = 'application/json,.json'; - - input.onchange = async (e) => { - const file = e.target.files[0]; - if (!file) return; - - try { - setIsLoading(true); - showLocalSnack("Reading import file..."); - - const text = await file.text(); - const importData = JSON.parse(text); - - if (!importData.strokes || !Array.isArray(importData.strokes)) { - showLocalSnack("Invalid import file: missing strokes array"); - return; - } - - // Ask user if they want to clear existing canvas - const clearExisting = window.confirm( - `Import ${importData.strokes.length} strokes?\n\n` + - `Click OK to replace current canvas, or Cancel to merge with existing drawings.` - ); - - showLocalSnack(`Importing ${importData.strokes.length} strokes...`); - - const { importRoomCanvas } = await import('../api/rooms'); - const result = await importRoomCanvas(auth?.token, currentRoomId, importData, clearExisting); - - if (result.status === 'success') { - showLocalSnack( - `Import complete: ${result.imported} imported, ${result.failed} failed`, - 6000 - ); - - // Refresh canvas to show imported data - setTimeout(async () => { - try { - await clearCanvasForRefresh(); - await mergedRefreshCanvas("post-import"); - } catch (error) { - console.error("Error refreshing after import:", error); - } - }, 500); - } else { - showLocalSnack(`Import failed: ${result.message || 'Unknown error'}`); - } - } catch (error) { - console.error("Import error:", error); - showLocalSnack(`Import failed: ${error.message || 'Invalid file format'}`); - } finally { - setIsLoading(false); - } - }; - - input.click(); - }; - - const toggleColorPicker = (event) => { - const viewportHeight = window.innerHeight; - const pickerHeight = 350; - const rect = event.target.getBoundingClientRect(); - const pickerElement = document.querySelector(".Canvas-color-picker"); - - setShowColorPicker(!showColorPicker); - - if (rect.bottom + pickerHeight > viewportHeight && pickerElement) { - pickerElement.classList.add("Canvas-color-picker--adjust-bottom"); - } else if (pickerElement) { - pickerElement.classList.remove("Canvas-color-picker--adjust-bottom"); - } - }; - - const closeColorPicker = () => { - setShowColorPicker(false); - }; - - useEffect(() => { - setIsRefreshing(true); - clearCanvasForRefresh(); - - mergedRefreshCanvas().then(() => { - setTimeout(() => { - setIsRefreshing(false); - }, 500); - }); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [selectedUser]); - - useEffect(() => { - setUndoAvailable(undoStack.length > 0); - setRedoAvailable(redoStack.length > 0); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [undoStack, redoStack]); - - const [showToolbar, setShowToolbar] = useState(true); - const [hoverToolbar, setHoverToolbar] = useState(false); - - return ( -
- {/* Top header: room name + optional history range + exit button */} - - - {currentRoomName || "Master (not in a room)"} - - - {historyMode && historyRange && ( - - {new Date(historyRange.start).toLocaleString()} —{" "} - {new Date(historyRange.end).toLocaleString()} - - )} - - {currentRoomId && ( - - )} - - - {/* Archived overlay banner - visible when viewOnly (archived or explicit viewer) */} - {viewOnly && ( - - - - Archived — View Only - - {/* Owner-only destructive delete button placed under the banner */} - {isOwner && ( - - - - )} - - - )} - - {/* Wallet disconnected banner - visible when secure room wallet is not connected */} - {roomType === "secure" && !walletConnected && ( - - - - ⚠ Wallet Not Connected — Canvas Locked - - - - )} - - {/* ResilientDB health status banner */} - - - - - {/* Confirm Destructive Delete dialog (owner-only) */} - { - setConfirmDestructiveOpen(false); - setDestructiveConfirmText(""); - }} - > - Permanently delete room - - - This will permanently delete this room and all its data for every - user. This action is irreversible. - - - To confirm, type DELETE below. - - setDestructiveConfirmText(e.target.value)} - placeholder="Type DELETE to confirm" - sx={{ mt: 1 }} - /> - - - - - - - - - - {/* Stamp preview overlay */} - {stampPreview && ( - - {stampPreview.stamp.emoji ? ( - - {stampPreview.stamp.emoji} - - ) : stampPreview.stamp.image ? ( - Stamp preview - ) : null} - - )} - - setHoverToolbar(true)} - onMouseLeave={() => setHoverToolbar(false)} - > - setShowToolbar((v) => !v)} - sx={{ - position: "absolute", - right: showToolbar ? 0 : -20, - top: "50%", - transform: "translateY(-50%)", - - width: 20, - height: 60, - display: "flex", - alignItems: "center", - justifyContent: "center", - - opacity: hoverToolbar ? 1 : 0, - transition: "opacity 0.2s", - bgcolor: "rgba(0,0,0,0.2)", - cursor: "pointer", - zIndex: 1001, - }} - > - - {showToolbar ? ( - - ) : ( - - )} - - - { - if (!editingEnabled) { - showLocalSnack("Cut is disabled in view-only mode."); - return; - } - showLocalSnack("Cutting selection... This may take a moment."); - try { - const result = await handleCutSelection(); - if (result && result.compositeCutAction) { - setUndoStack((prev) => [...prev, result.compositeCutAction]); - } - setIsRefreshing(true); - showLocalSnack("Syncing cut operation..."); - try { - await mergedRefreshCanvas(); - showLocalSnack("Cut completed successfully!"); - } catch (e) { - console.error("Error syncing cut with server:", e); - showLocalSnack("Cut completed, but sync failed. Try refreshing."); - } finally { - setIsRefreshing(false); - } - } catch (e) { - console.error("Error during cut:", e); - showLocalSnack("Cut operation failed. Please try again."); - } - }} - cutImageData={cutImageData} - setClearDialogOpen={setClearDialogOpen} - /* Export/Import handlers */ - handleExportCanvas={handleExportCanvas} - handleImportCanvas={handleImportCanvas} - /* Advanced brush/stamp/filter props */ - currentBrushType={currentBrushType} - onBrushSelect={handleBrushSelect} - onBrushParamsChange={handleBrushParamsChange} - selectedStamp={selectedStamp} - onStampSelect={handleStampSelect} - onStampChange={handleStampChange} - backendStamps={backendStamps} - onFilterApply={applyFilter} - onFilterPreview={previewFilter} - onFilterUndo={undoFilter} - onClearAllFilters={clearAllFilters} - canUndoFilter={ - !!originalCanvasDataRef.current || - undoStack.some((drawing) => drawing.drawingType === "filter") - } - canClearFilters={hasFilters} - appliedFilters={ - (() => { - const filters = userData.drawings.filter((drawing) => drawing.drawingType === "filter"); - console.log(`[Canvas render] Passing ${filters.length} applied filters to Toolbar`, filters); - return filters; - })() - } - /* History Recall props (required so the toolbar can open/change/exit history mode) */ - openHistoryDialog={openHistoryDialog} - exitHistoryMode={exitHistoryMode} - historyMode={historyMode} - controlsDisabled={!editingEnabled} - onOpenSettings={onOpenSettings} - /> - - - {isRefreshing && ( -
-
-
- )} - - {/* History Recall Dialog */} - setHistoryDialogOpen(false)} - aria-labelledby="history-recall-dialog" - > - - History Recall - Select Date/Time Range - - - - Choose a start and end date/time to recall drawings from - ResilientDB. Only drawings within the selected range will be loaded. - - - setHistoryStartInput(e.target.value)} - InputLabelProps={{ shrink: true }} - /> - setHistoryEndInput(e.target.value)} - InputLabelProps={{ shrink: true }} - /> - - - - - - - - - - - - - {historyMode - ? "History Mode Enabled — Canvas Editing Disabled" - : selectedUser && selectedUser !== "" - ? "Viewing Past Drawing History of Selected User — Canvas Editing Disabled" - : ""} - - - - - {/* Loading overlay: fades in/out while drawings load */} - - - - Loading Drawings... - - - - setClearDialogOpen(false)}> - Clear Canvas - - - Are you sure you want to clear the canvas for everyone? - - - - - - - - - {/* Command Palette - Quick command search and execution */} - setCommandPaletteOpen(false)} - commands={commandRegistry.getAll()} - onExecute={(command) => { - try { - command.action(); - } catch (error) { - console.error('[Canvas] Error executing command:', error); - showLocalSnack('Error executing command'); - } - }} - /> - - {/* Keyboard Shortcuts Help Dialog */} - setShortcutsHelpOpen(false)} - shortcuts={shortcutManagerRef.current?.getAllShortcuts() || []} - /> - - -
- ); -} - -export default Canvas; +import React, { useRef, useState, useEffect } from "react"; +import "../styles/Canvas.css"; +import { + Box, + Fade, + Paper, + Button, + Dialog, + DialogActions, + DialogContent, + DialogContentText, + DialogTitle, + IconButton, + TextField, + Typography, + CircularProgress, +} from '@mui/material'; +import SafeSnackbar from './SafeSnackbar'; +import CommandPalette from './CommandPalette'; +import KeyboardShortcutsHelp from './KeyboardShortcutsHelp'; +import { KeyboardShortcutManager } from '../services/KeyboardShortcuts'; +import { commandRegistry } from '../services/CommandRegistry'; +import { DEFAULT_SHORTCUTS } from '../config/shortcuts'; +import ChevronLeftIcon from '@mui/icons-material/ChevronLeft'; +import ChevronRightIcon from '@mui/icons-material/ChevronRight'; +import Toolbar from './Toolbar'; +// AI Assist Imports +import AIAssistantPanel from './AI/AIAssistantPanel'; +import PromptInput from './AI/PromptInput'; +import ShapeCompletionOverlay from './AI/ShapeCompletionOverlay'; +import { useCanvasSelection } from '../hooks/useCanvasSelection'; +// AI Assist Hook +import { useAIAssistant } from '../hooks/useAIAssistant'; +import InfoOutlinedIcon from '@mui/icons-material/InfoOutlined'; +import useBrushEngine from "../hooks/useBrushEngine"; +import { + submitToDatabase, + refreshCanvas as backendRefreshCanvas, + clearBackendCanvas, + undoAction, + redoAction, + checkUndoRedoAvailability +} from '../services/canvasBackendJWT'; +import { Drawing } from '../lib/drawing'; +import { getSocket, setSocketToken } from '../services/socket'; +import { handleAuthError } from '../utils/authUtils'; +import { getUsername } from '../utils/getUsername'; +import { getAuthUser } from '../utils/getAuthUser'; +import { resetMyStacks } from '../api/rooms'; +import { TEMPLATE_LIBRARY } from '../data/templates'; +import { API_BASE } from '../config/apiConfig'; + +class UserData { + constructor(userId, username) { + this.userId = userId; + this.username = username; + this.drawings = []; + } + + addDrawing(drawing) { + this.drawings.push(drawing); + } + + clearDrawings() { + this.drawings = []; + } +} + +const DEFAULT_CANVAS_WIDTH = 3000; +const DEFAULT_CANVAS_HEIGHT = 2000; + +function Canvas({ + auth, + setUserList, + selectedUser, + setSelectedUser, + currentRoomId, + canvasRefreshTrigger = 0, + currentRoomName = "Master (not in a room)", + onExitRoom = () => { }, + onOpenSettings = null, + viewOnly = false, + isOwner = false, + roomType = "public", + walletConnected = false, + templateId = null, + onCanvasContextChange = null, +}) { + const canvasRef = useRef(null); + const snapshotRef = useRef(null); + const tempPathRef = useRef([]); + const prevDrawModeRef = useRef(null); + const clamp = (value, min, max) => Math.min(max, Math.max(min, value)); + + const currentUserRef = useRef(null); + if (currentUserRef.current === null) { + try { + const uname = + getUsername(auth) || + `anon_${Date.now()}_${Math.random().toString(36).substr(2, 5)}`; + currentUserRef.current = `${uname}|${Date.now()}`; + } catch (e) { + currentUserRef.current = `anon_${Date.now()}_${Math.random() + .toString(36) + .substr(2, 5)}`; + } + } + const currentUser = currentUserRef.current; + + const [drawing, setDrawing] = useState(false); + const [color, setColor] = useState("#000000"); + const [lineWidth, setLineWidth] = useState(5); + const [drawMode, setDrawMode] = useState("freehand"); + const [shapeType, setShapeType] = useState("circle"); + const [brushStyle] = useState("round"); + const [shapeStart, setShapeStart] = useState(null); + + const brushEngine = useBrushEngine(); + const [currentBrushType, setCurrentBrushType] = useState("normal"); + const [brushParams, setBrushParams] = useState({}); + const [selectedStamp, setSelectedStamp] = useState(null); + const [stampSettings, setStampSettings] = useState({ + size: 50, + rotation: 0, + opacity: 100, + }); + const [backendStamps, setBackendStamps] = useState([]); + const [stampPreview, setStampPreview] = useState(null); // { x, y, stamp, settings } + const stampPreviewRef = useRef(null); + const [activeFilter, setActiveFilter] = useState(null); + const [filterParams, setFilterParams] = useState({}); + const [isFilterPreview, setIsFilterPreview] = useState(false); + const filterCanvasRef = useRef(null); + const originalCanvasDataRef = useRef(null); // For preview mode undo + const preFilterCanvasStateRef = useRef(null); + + const [showColorPicker, setShowColorPicker] = useState(false); + const [isRefreshing, setIsRefreshing] = useState(false); + const [previousColor, setPreviousColor] = useState(null); + const [clearDialogOpen, setClearDialogOpen] = useState(false); + const [undoStack, setUndoStack] = useState([]); + const [redoStack, setRedoStack] = useState([]); + const [undoAvailable, setUndoAvailable] = useState(false); + const [redoAvailable, setRedoAvailable] = useState(false); + const [hasFilters, setHasFilters] = useState(false); // Track if filters exist for UI updates + + // AI Asiistant Panel States + const [aiOpen, setAiOpen] = useState(false); + const [aiBusy, setAiBusy] = useState(false); + const [aiError, setAiError] = useState(''); + const [aiGenerateService, setAiGenerateService] = useState(""); + const [showPromptInput, setShowPromptInput] = useState(false); + const [promptInputPlaceHolder, setPromptInputPlaceholder] = useState(""); + const [shapeSuggestion, setShapeSuggestion] = useState(null); + const [shapeAnchor, setShapeAnchor] = useState(null); + const [shapeOverlayOpen, setShapeOverlayOpen] = useState(false); + const [shapeCompletionEnabled, setShapeCompletionEnabled] = useState(false); + + const [templateObjects, setTemplateObjects] = useState([]); + const templateObjectsRef = useRef([]); + + useEffect(() => { + templateObjectsRef.current = templateObjects; + }, [templateObjects]); + + const canvasWidth = DEFAULT_CANVAS_WIDTH; + const canvasHeight = DEFAULT_CANVAS_HEIGHT; + + const [panOffset, setPanOffset] = useState({ x: 0, y: 0 }); + const [isPanning, setIsPanning] = useState(false); + const panStartRef = useRef({ x: 0, y: 0 }); + const panOriginRef = useRef({ x: 0, y: 0 }); + const PAN_REFRESH_COOLDOWN_MS = 2000; + const panLastRefreshRef = useRef(0); + const panRefreshSkippedRef = useRef(false); + const panEndRefreshTimerRef = useRef(null); + const pendingPanRefreshRef = useRef(false); + const [pendingDrawings, setPendingDrawings] = useState([]); + const pendingDrawingsRef = useRef([]); + + const updatePendingDrawings = (updater) => { + if (typeof updater === "function") { + setPendingDrawings((prev) => { + const next = updater(prev); + try { + pendingDrawingsRef.current = next; + } catch (e) {} + return next; + }); + } else { + setPendingDrawings(updater); + try { + pendingDrawingsRef.current = updater; + } catch (e) {} + } + }; + const refreshTimerRef = useRef(null); + const submissionQueueRef = useRef([]); + const isSubmittingRef = useRef(false); + const confirmedStrokesRef = useRef(new Set()); + const lastDrawnStateRef = useRef(null); // Track last drawn state to avoid redundant redraws + const isDrawingInProgressRef = useRef(false); // Prevent concurrent drawing operations + const offscreenCanvasRef = useRef(null); // Offscreen canvas for flicker free rendering + const forceNextRedrawRef = useRef(false); // Force next redraw even if signature matches for undo redo + const [historyMode, setHistoryMode] = useState(false); + const [historyRange, setHistoryRange] = useState(null); // {start, end} in epoch ms + const [historyDialogOpen, setHistoryDialogOpen] = useState(false); + const [historyStartInput, setHistoryStartInput] = useState(""); + const [historyEndInput, setHistoryEndInput] = useState(""); + const [isLoading, setIsLoading] = useState(false); + + const [localSnack, setLocalSnack] = useState({ + open: false, + message: "", + duration: 4000, + }); + const [confirmDestructiveOpen, setConfirmDestructiveOpen] = useState(false); + const [destructiveConfirmText, setDestructiveConfirmText] = useState(""); + const showLocalSnack = (msg, duration = 4000) => + setLocalSnack({ open: true, message: String(msg), duration }); + + // editingEnabled controls whether the user can perform mutating actions. + // When historyMode is active, a specific user is selected for replay, or + // when viewOnly is true (room is archived or user is a viewer), editing + // should be disabled. + // For secure rooms, wallet must be connected to allow editing. + const editingEnabled = !( + historyMode || + (selectedUser && selectedUser !== "") || + viewOnly || + (roomType === "secure" && !walletConnected) + ); + const closeLocalSnack = () => + setLocalSnack({ open: false, message: "", duration: 4000 }); + + // Keyboard shortcuts state + const [commandPaletteOpen, setCommandPaletteOpen] = useState(false); + const [shortcutsHelpOpen, setShortcutsHelpOpen] = useState(false); + const shortcutManagerRef = useRef(null); + + const roomUiRef = useRef({}); + const previousSelectedUserRef = useRef(null); // Track previous selectedUser to detect changes + const isRefreshingSelectedUserRef = useRef(false); // Prevent concurrent refreshes + const selectedUserRefreshQueueRef = useRef(null); // Queue the next refresh target + const selectedUserAbortControllerRef = useRef(null); // Cancel pending operations + const roomStacksRef = useRef({}); + const roomClipboardRef = useRef({}); + const roomClearedAtRef = useRef({}); + const drawAllDrawingsRef = useRef(null); // Store reference to drawAllDrawings function + const thumbnailUploadTimerRef = useRef(null); // Debounce timer for thumbnail uploads + const previousCanvasContextRef = useRef(null); // Track previous canvas context for change detection + + useEffect(() => { + if (!currentRoomId) return; + const ui = + roomUiRef.current[currentRoomId] || + JSON.parse( + localStorage.getItem(`rescanvas:toolbar:${currentRoomId}`) || "null" + ) || + {}; + if (ui.color) setColor(ui.color); + if (ui.lineWidth) setLineWidth(ui.lineWidth); + if (ui.drawMode) setDrawMode(ui.drawMode); + if (ui.shapeType) setShapeType(ui.shapeType); + if (ui.previousColor !== undefined) setPreviousColor(ui.previousColor); + if (ui.selectedStamp) setSelectedStamp(ui.selectedStamp); + if (ui.stampSettings) setStampSettings(ui.stampSettings); + if (ui.currentBrushType) { + setCurrentBrushType(ui.currentBrushType); + brushEngine.setBrushType(ui.currentBrushType); + } + if (ui.brushParams) { + setBrushParams(ui.brushParams); + brushEngine.setBrushParams(ui.brushParams); + } + roomUiRef.current[currentRoomId] = { + color: ui.color ?? color, + lineWidth: ui.lineWidth ?? lineWidth, + drawMode: ui.drawMode ?? drawMode, + shapeType: ui.shapeType ?? shapeType, + previousColor: ui.previousColor ?? previousColor, + selectedStamp: ui.selectedStamp ?? selectedStamp, + stampSettings: ui.stampSettings ?? stampSettings, + currentBrushType: ui.currentBrushType ?? currentBrushType, + brushParams: ui.brushParams ?? brushParams, + }; + const stacks = roomStacksRef.current[currentRoomId] || { + undo: [], + redo: [], + }; + setUndoStack(stacks.undo); + setRedoStack(stacks.redo); + const clip = roomClipboardRef.current[currentRoomId] || null; + if (setCutImageData) setCutImageData(clip); + }, [currentRoomId]); + + // Load template objects when templateId changes + useEffect(() => { + + if (!templateId) { + setTemplateObjects([]); + return; + } + + const template = TEMPLATE_LIBRARY.find(t => t.id === templateId); + + if (template && template.canvas && template.canvas.objects) { + setTemplateObjects(template.canvas.objects); + } else { + setTemplateObjects([]); + } + }, [templateId, currentRoomId]); + + // Force redraw whenever templateObjects change (ensures templates appear immediately) + useEffect(() => { + if (!templateObjects || templateObjects.length === 0) return; + + const timer = setTimeout(() => { + if (drawAllDrawingsRef.current) { + lastDrawnStateRef.current = null; // Force redraw by clearing cache + drawAllDrawingsRef.current(); + } else { + console.warn('drawAllDrawingsRef not ready yet'); + } + }, 100); + + return () => clearTimeout(timer); + }, [templateObjects]); + + useEffect(() => { + if (!currentRoomId) return; + const ui = { color, lineWidth, drawMode, shapeType, previousColor, selectedStamp, stampSettings, currentBrushType, brushParams }; + roomUiRef.current[currentRoomId] = ui; + try { + localStorage.setItem( + `rescanvas:toolbar:${currentRoomId}`, + JSON.stringify(ui) + ); + } catch { } + }, [currentRoomId, color, lineWidth, drawMode, shapeType, previousColor, selectedStamp, stampSettings, currentBrushType, brushParams]); + + useEffect(() => { + if (!currentRoomId) return; + const cur = roomStacksRef.current[currentRoomId] || { undo: [], redo: [] }; + cur.undo = undoStack; + roomStacksRef.current[currentRoomId] = cur; + }, [currentRoomId, undoStack]); + useEffect(() => { + if (!currentRoomId) return; + const cur = roomStacksRef.current[currentRoomId] || { undo: [], redo: [] }; + cur.redo = redoStack; + roomStacksRef.current[currentRoomId] = cur; + }, [currentRoomId, redoStack]); + + useEffect(() => { + const handleMouseUp = () => { + setIsPanning(false); + panOriginRef.current = { ...panOffset }; + try { + if (panEndRefreshTimerRef.current) { + clearTimeout(panEndRefreshTimerRef.current); + panEndRefreshTimerRef.current = null; + } + if (panRefreshSkippedRef.current) { + panRefreshSkippedRef.current = false; + mergedRefreshCanvas("pan-mouseup-skipped").finally(() => { + try { + setIsLoading(false); + } catch (e) { } + }); + } + if (pendingPanRefreshRef.current) { + pendingPanRefreshRef.current = false; + mergedRefreshCanvas("pan-mouseup-pending").finally(() => { + try { + setIsLoading(false); + } catch (e) { } + }); + } + } catch (e) { } + }; + document.addEventListener("mouseup", handleMouseUp); + return () => document.removeEventListener("mouseup", handleMouseUp); + }, [panOffset]); + + // Process submission queue to ensure strokes are submitted sequentially + const processSubmissionQueue = async () => { + if (isSubmittingRef.current || submissionQueueRef.current.length === 0) { + return; + } + + isSubmittingRef.current = true; + + while (submissionQueueRef.current.length > 0) { + const submission = submissionQueueRef.current.shift(); + try { + await submission(); + } catch (error) { + console.error("Error processing queued submission:", error); + } + } + + isSubmittingRef.current = false; + + // After processing all queued submissions, schedule a refresh to sync with backend + if (refreshTimerRef.current) clearTimeout(refreshTimerRef.current); + refreshTimerRef.current = setTimeout(() => { + mergedRefreshCanvas("post-queue").catch((e) => + console.error("Error during post-queue refresh:", e) + ); + refreshTimerRef.current = null; + }, 500); + }; + + useEffect(() => { + if (!auth?.token || !currentRoomId) return; + try { + setSocketToken(auth.token); + } catch (e) { } + + const socket = getSocket(auth.token); + + try { + socket.emit("join_room", { roomId: currentRoomId, token: auth?.token }); + } catch (e) { + socket.emit("join_room", { roomId: currentRoomId }); + } + + const scheduleRefresh = (delay = 300) => { + try { + if (refreshTimerRef.current) clearTimeout(refreshTimerRef.current); + } catch (e) { } + refreshTimerRef.current = setTimeout(() => { + mergedRefreshCanvas().catch((e) => + console.error("Error during scheduled refresh:", e) + ); + refreshTimerRef.current = null; + }, delay); + }; + + const handleNewStroke = (data) => { + try { + const myName = getUsername(auth); + if (data.user === myName) { + // This is confirmation of our own stroke + const stroke = data.stroke; + if (stroke && stroke.drawingId) { + confirmedStrokesRef.current.add(stroke.drawingId); + } + return; + } + } catch (e) { + try { + const user = getAuthUser(auth) || {}; + if (data.user === user.username) { + // This is confirmation of our own stroke + const stroke = data.stroke; + if (stroke && stroke.drawingId) { + confirmedStrokesRef.current.add(stroke.drawingId); + } + return; + } + } catch (e2) { } + } + + const stroke = data.stroke; + + // Extract metadata for advanced features (stamps, brushes, filters) + const metadata = { + brushStyle: stroke.brushStyle, + brushType: stroke.brushType, + brushParams: stroke.brushParams, + drawingType: stroke.drawingType, + stampData: stroke.stampData, + stampSettings: stroke.stampSettings, + filterType: stroke.filterType, + filterParams: stroke.filterParams, + }; + + const drawing = new Drawing( + stroke.drawingId || + `remote_${Date.now()}_${Math.random().toString(36).substr(2, 5)}`, + stroke.color || "#000000", + stroke.lineWidth || 5, + stroke.pathData || [], + stroke.ts || stroke.timestamp || Date.now(), + stroke.user || "Unknown", + metadata + ); + + try { + const clearedAt = roomClearedAtRef.current[currentRoomId]; + if ( + clearedAt && + (drawing.timestamp || drawing.ts || Date.now()) < clearedAt + ) { + return; + } + } catch (e) { } + + updatePendingDrawings((prev) => [...prev, drawing]); + + // If this is a custom stamp, add it to the stamp panel + if (drawing.drawingType === "stamp" && drawing.stampData && drawing.stampData.image) { + setBackendStamps((prevStamps) => { + const imageKey = drawing.stampData.image.substring(0, 100); + const alreadyExists = prevStamps.some(s => + s.image && s.image.substring(0, 100) === imageKey + ); + + if (!alreadyExists) { + console.log('Adding new custom stamp from Socket.IO:', drawing.stampData.name || 'Custom Stamp'); + return [...prevStamps, { + id: `stamp-${Date.now()}-${prevStamps.length}`, + name: drawing.stampData.name || 'Custom Stamp', + category: drawing.stampData.category || 'custom', + image: drawing.stampData.image, + emoji: drawing.stampData.emoji + }]; + } + return prevStamps; + }); + } + + // Use requestAnimationFrame for smoother rendering + requestAnimationFrame(() => { + drawAllDrawings(); + }); + + scheduleRefresh(350); + }; + + const handleUserJoined = (data) => { + try { + if (!data) return; + if (data.roomId !== currentRoomId) return; + console.debug("socket user_joined event", data); + if (data.username) { + showLocalSnack(`${data.username} joined the canvas.`); + } + } catch (e) { } + }; + + const handleUserLeft = (data) => { + try { + if (!data) return; + if (data.roomId !== currentRoomId) return; + console.debug("socket user_left event", data); + if (data.username) { + showLocalSnack(`${data.username} left the canvas.`); + } + } catch (e) { } + }; + + const handleStrokeUndone = (data) => { + console.log("Stroke undone event received:", data); + + forceNextRedrawRef.current = true; + lastDrawnStateRef.current = null; + + // Schedule refresh instead of immediate refresh to avoid flicker + if (refreshTimerRef.current) clearTimeout(refreshTimerRef.current); + refreshTimerRef.current = setTimeout(() => { + mergedRefreshCanvas("undo-event"); + refreshTimerRef.current = null; + }, 100); + + if (currentRoomId) { + checkUndoRedoAvailability( + auth, + setUndoAvailable, + setRedoAvailable, + currentRoomId + ); + } + }; + + const handleCanvasCleared = (data) => { + console.log("Canvas cleared event received:", data); + const clearedAt = data && data.clearedAt ? data.clearedAt : Date.now(); + if (currentRoomId) roomClearedAtRef.current[currentRoomId] = clearedAt; + + // Clear local authoritative drawings and pending drawings that predate the clear + try { + userData.clearDrawings(); + } catch (e) { } + updatePendingDrawings([]); + serverCountRef.current = 0; + + setUndoStack([]); + setRedoStack([]); + setUndoAvailable(false); + setRedoAvailable(false); + try { + if (currentRoomId) { + roomStacksRef.current[currentRoomId] = { undo: [], redo: [] }; + roomClipboardRef.current[currentRoomId] = null; + } + } catch (e) { } + + clearCanvasForRefresh(); + drawAllDrawings(); + + if (currentRoomId) { + checkUndoRedoAvailability( + auth, + setUndoAvailable, + setRedoAvailable, + currentRoomId + ); + } + }; + + socket.on("new_stroke", handleNewStroke); + socket.on("stroke_undone", handleStrokeUndone); + socket.on("canvas_cleared", handleCanvasCleared); + socket.on("user_joined", handleUserJoined); + socket.on("user_left", handleUserLeft); + socket.on("user_joined_debug", (d) => { + console.debug("socket user_joined_debug", d); + }); + + return () => { + socket.off("new_stroke", handleNewStroke); + socket.off("stroke_undone", handleStrokeUndone); + socket.off("canvas_cleared", handleCanvasCleared); + socket.off("user_joined", handleUserJoined); + socket.off("user_left", handleUserLeft); + try { + socket.emit("leave_room", { + roomId: currentRoomId, + token: auth?.token, + }); + } catch (e) { + socket.emit("leave_room", { roomId: currentRoomId }); + } + try { + if (refreshTimerRef.current) clearTimeout(refreshTimerRef.current); + } catch (e) { } + }; + }, [auth?.token, currentRoomId, auth?.user?.username]); + + useEffect(() => { + (async () => { + try { + setUndoStack([]); + setRedoStack([]); + setUndoAvailable(false); + setRedoAvailable(false); + if (currentRoomId) { + roomStacksRef.current[currentRoomId] = { undo: [], redo: [] }; + } + + // Reset selectedUser tracking when room changes + previousSelectedUserRef.current = null; + isRefreshingSelectedUserRef.current = false; + selectedUserRefreshQueueRef.current = null; + + if (auth?.token && currentRoomId) { + try { + await resetMyStacks(auth.token, currentRoomId); + } catch (e) { } + } + + if (currentRoomId) { + try { + await checkUndoRedoAvailability( + auth, + setUndoAvailable, + setRedoAvailable, + currentRoomId + ); + } catch (e) { } + } + } catch (e) { } + })(); + }, [auth?.token, currentRoomId]); + + useEffect(() => { + try { + setUndoStack([]); + setRedoStack([]); + if (currentRoomId) { + roomStacksRef.current[currentRoomId] = { undo: [], redo: [] }; + } + if (currentRoomId) { + checkUndoRedoAvailability( + auth, + setUndoAvailable, + setRedoAvailable, + currentRoomId + ).catch(() => { }); + } + } catch (e) { } + }, [auth?.token, currentRoomId]); + + // Force full refresh when selectedUser changes (drawing history selection/deselection) + useEffect(() => { + if (!currentRoomId || !auth?.token) return; + + // Serialize selectedUser for comparison (handles both string and object) + const serializeSelectedUser = (user) => { + if (!user || user === "") return ""; + if (typeof user === "string") return user; + if (typeof user === "object") + return JSON.stringify({ + user: user.user, + periodStart: user.periodStart, + }); + return String(user); + }; + + const currentSerialized = serializeSelectedUser(selectedUser); + const previousSerialized = previousSelectedUserRef.current; + + // Only refresh if selectedUser actually changed + if (currentSerialized === previousSerialized) { + return; + } + + // If a refresh is in progress, queue this change for execution after current one completes + if (isRefreshingSelectedUserRef.current) { + console.debug( + "[selectedUser] Refresh in progress, queuing new selection:", + currentSerialized + ); + selectedUserRefreshQueueRef.current = currentSerialized; + return; + } + + const performRefresh = async (targetSerialized) => { + isRefreshingSelectedUserRef.current = true; + + try { + setIsLoading(true); + + // Update the ref to mark this as the last processed value + previousSelectedUserRef.current = targetSerialized; + + // Force complete refresh from backend + userData.drawings = []; + updatePendingDrawings([]); + serverCountRef.current = 0; + lastDrawnStateRef.current = null; + + const isDeselect = !selectedUser || selectedUser === ""; + const logLabel = isDeselect + ? "selectedUser-deselect" + : "selectedUser-select"; + console.debug(`[selectedUser] Performing full refresh: ${logLabel}`, { + to: targetSerialized, + }); + + await clearCanvasForRefresh(); + await mergedRefreshCanvas(logLabel); + await drawAllDrawings(); + } catch (error) { + console.error("Error refreshing on selectedUser change:", error); + } finally { + setIsLoading(false); + isRefreshingSelectedUserRef.current = false; + + // Check if there's a queued refresh waiting + if (selectedUserRefreshQueueRef.current !== null) { + const queuedTarget = selectedUserRefreshQueueRef.current; + selectedUserRefreshQueueRef.current = null; + + // Only process queued refresh if it's different from what we just processed + if (queuedTarget !== targetSerialized) { + console.debug( + "[selectedUser] Processing queued selection:", + queuedTarget + ); + // Use setTimeout to break out of the current call stack + setTimeout(() => performRefresh(queuedTarget), 0); + } + } + } + }; + + // Start the refresh + performRefresh(currentSerialized); + }, [selectedUser, currentRoomId]); + + const clearCanvasForRefresh = async () => { + const canvas = canvasRef.current; + if (!canvas) return; // Guard against null ref during tests + + const context = canvas.getContext("2d"); + if (!context) return; // Guard against null context during tests + + context.clearRect(0, 0, canvasWidth, canvasHeight); + setUserData(initializeUserData()); + updatePendingDrawings([]); + serverCountRef.current = 0; + + // Clear selection overlay artifacts + setSelectionRect(null); + setSelectionStart(null); + + // Reset draw mode to freehand if in select mode + if (drawMode === "select") { + setDrawMode("freehand"); + } + }; + + const refreshCanvasButtonHandler = async () => { + if (isRefreshing) return; + setIsRefreshing(true); + setIsLoading(true); + try { + // Force full refresh from backend by clearing local state + userData.drawings = []; + updatePendingDrawings([]); + serverCountRef.current = 0; + lastDrawnStateRef.current = null; + + await clearCanvasForRefresh(); + await mergedRefreshCanvas("refresh-button"); + await drawAllDrawings(); + updateFilterState(); // Update filter state after refresh + } catch (error) { + console.error("Error during canvas refresh:", error); + handleAuthError(error); + } finally { + setIsRefreshing(false); + setIsLoading(false); + } + }; + + const initializeUserData = () => { + const uniqueUserId = + auth?.user?.id || + `user_${Date.now()}_${Math.random().toString(36).substr(2, 5)}`; + const username = auth?.user?.username || "MainUser"; + return new UserData(uniqueUserId, username); + }; + const [userData, setUserData] = useState(() => initializeUserData()); + const generateId = () => + `drawing_${Date.now()}_${Math.random().toString(36).substr(2, 5)}`; + const serverCountRef = useRef(0); + + // Upload canvas thumbnail for visual search embeddings + const uploadThumbnail = (roomId) => { + if (!roomId || !auth?.token) { + console.debug('Skipping thumbnail upload: no roomId or token'); + return; + } + + const canvas = canvasRef.current; + if (!canvas) { + console.debug('Skipping thumbnail upload: no canvas ref'); + return; + } + + console.log(`🎨 Uploading thumbnail for room ${roomId}...`); + + // Generate thumbnail synchronously - create small version for faster upload + let dataURL; + try { + // Create a smaller thumbnail canvas (max 800x600) to reduce file size + const maxWidth = 800; + const maxHeight = 600; + const scale = Math.min(1, maxWidth / canvas.width, maxHeight / canvas.height); + + const thumbCanvas = document.createElement('canvas'); + thumbCanvas.width = canvas.width * scale; + thumbCanvas.height = canvas.height * scale; + const thumbCtx = thumbCanvas.getContext('2d'); + thumbCtx.drawImage(canvas, 0, 0, thumbCanvas.width, thumbCanvas.height); + + // Use JPEG with lower quality for much smaller file size + dataURL = thumbCanvas.toDataURL('image/jpeg', 0.3); + console.log(`Generated thumbnail: ${thumbCanvas.width}x${thumbCanvas.height}, ${dataURL.length} chars (~${Math.round(dataURL.length * 0.75 / 1024)}KB)`); + } catch (error) { + console.error('Failed to generate canvas thumbnail:', error); + return; + } + + // DEBUG: Log the full URL and payload size + const url = `${API_BASE}/rooms/${roomId}/thumbnail`; + const payload = JSON.stringify({ thumbnail: dataURL }); + console.log(`🔍 DEBUG - About to POST to: ${url}`); + console.log(`🔍 DEBUG - Payload size: ${payload.length} bytes (${Math.round(payload.length / 1024)}KB)`); + console.log(`🔍 DEBUG - Auth token present: ${!!auth?.token}`); + + // Upload asynchronously with keepalive flag (survives page navigation) + fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${auth.token}` + }, + body: JSON.stringify({ thumbnail: dataURL }), + keepalive: true // Critical: allows request to complete even after page unload + }) + .then(response => { + if (response.ok) { + return response.json(); + } else { + console.warn(`Thumbnail upload failed: ${response.status} ${response.statusText}`); + return response.text().then(text => { + console.warn(`Error details: ${text.substring(0, 200)}`); + return null; + }); + } + }) + .then(result => { + if (result) { + console.log('✅ Thumbnail uploaded for visual search:', { + roomId: result.roomId, + size: result.thumbnailSize, + format: result.format + }); + } + }) + .catch(error => { + console.error('Failed to upload thumbnail:', error); + console.error('Error details:', { + message: error.message, + name: error.name, + stack: error.stack?.substring(0, 200) + }); + }); + }; + + // Helper function to update filter state + const updateFilterState = () => { + // Use setUserData callback to read current state accurately + setUserData((currentUserData) => { + const filterExists = currentUserData.drawings.some((d) => d.drawingType === "filter"); + const filterCount = currentUserData.drawings.filter((d) => d.drawingType === "filter").length; + console.log(`[updateFilterState] filterExists=${filterExists}, filterCount=${filterCount}`); + setHasFilters(filterExists); + return currentUserData; + }); + }; + + // Advanced Brush/Stamp/Filter Functions + const handleBrushSelect = (brushType) => { + console.log("handleBrushSelect called with:", brushType); + setCurrentBrushType(brushType); + brushEngine.setBrushType(brushType); + setDrawMode("freehand"); + console.log("Current brush type set to:", brushType); + }; + + const handleBrushParamsChange = (params) => { + setBrushParams(params); + brushEngine.setBrushParams(params); + }; + + const placeStamp = async (x, y, stamp, settings) => { + const canvas = canvasRef.current; + if (!canvas || !stamp) return; + + const context = canvas.getContext("2d"); + + // Render stamp immediately using proper context management + if (stamp.emoji) { + context.save(); + context.globalAlpha = settings.opacity / 100; + context.translate(x, y); + context.rotate((settings.rotation * Math.PI) / 180); + + const size = settings.size; + context.font = `${size}px serif`; + context.textAlign = "center"; + context.textBaseline = "middle"; + context.fillText(stamp.emoji, 0, 0); + + context.restore(); + } else if (stamp.image) { + // For image stamps, load and render synchronously using async/await + try { + const img = await new Promise((resolve, reject) => { + const image = new Image(); + image.onload = () => resolve(image); + image.onerror = () => reject(new Error("Failed to load image")); + image.src = stamp.image; + }); + + context.save(); + context.globalAlpha = settings.opacity / 100; + context.translate(x, y); + context.rotate((settings.rotation * Math.PI) / 180); + + const size = settings.size; + context.drawImage(img, -size / 2, -size / 2, size, size); + + context.restore(); + } catch (error) { + console.error("Failed to load stamp image:", stamp.image?.substring(0, 100), error); + } + } + + // Create drawing record for stamp + const stampDrawing = new Drawing( + generateId(), + color, + lineWidth, + [{ x, y }], + Date.now(), + currentUser, + { + drawingType: "stamp", + stampData: stamp, + stampSettings: settings, + } + ); + + stampDrawing.roomId = currentRoomId; + + userData.addDrawing(stampDrawing); + updatePendingDrawings((prev) => [...prev, stampDrawing]); + + // Add to undo stack + setUndoStack((prev) => [...prev, stampDrawing]); + setRedoStack([]); + + // Use submission queue to ensure stamps are submitted in order + try { + const submitTask = async () => { + try { + console.log("Submitting queued stamp:", { + drawingId: stampDrawing.drawingId, + stampData: stampDrawing.stampData, + }); + + await submitToDatabase( + stampDrawing, + auth, + { roomId: currentRoomId, roomType }, + setUndoAvailable, + setRedoAvailable + ); + + console.log("Stamp submitted successfully:", stampDrawing.drawingId); + + if (currentRoomId) { + checkUndoRedoAvailability( + auth, + setUndoAvailable, + setRedoAvailable, + currentRoomId + ); + } + } catch (error) { + console.error("Error during queued stamp submission:", error); + updatePendingDrawings((prev) => + prev.filter((d) => d.drawingId !== stampDrawing.drawingId) + ); + handleAuthError(error); + showLocalSnack("Failed to save stamp. Please try again."); + } + }; + + submissionQueueRef.current.push(submitTask); + processSubmissionQueue(); + } catch (error) { + console.error("Error preparing stamp submission:", error); + handleAuthError(error); + showLocalSnack("Failed to prepare stamp. Please try again."); + } + }; + + const handleStampSelect = (stamp, settings) => { + setSelectedStamp(stamp); + setStampSettings(settings); + setDrawMode("stamp"); + }; + + const handleStampChange = (stamp, settings) => { + setSelectedStamp(stamp); + setStampSettings(settings); + }; + + const applyFilter = async (filterType, params) => { + if (!canvasRef.current) return; + + // Always cancel preview mode first and clean up state + if (isFilterPreview) { + setIsFilterPreview(false); + } + preFilterCanvasStateRef.current = null; + originalCanvasDataRef.current = null; + + // Check if we already have a filter of this type applied + const existingFilterIndex = userData.drawings.findIndex( + (d) => d.drawingType === "filter" && d.filterType === filterType + ); + + let filterDrawing; + let isReplacement = existingFilterIndex !== -1; + + if (isReplacement) { + const existingFilter = userData.drawings[existingFilterIndex]; + existingFilter.filterParams = { ...params }; // Clone params + existingFilter.timestamp = Date.now(); + filterDrawing = existingFilter; + + // Update React state to reflect the filter parameter change + const newUserData = new UserData(userData.userId, userData.username); + newUserData.drawings = [...userData.drawings]; // Clone the array to trigger state update + setUserData(newUserData); + + // Force a complete redraw with the updated filter parameters + // This will redraw all strokes first, then apply the filter + lastDrawnStateRef.current = null; + forceNextRedrawRef.current = true; + await drawAllDrawings(); + + showLocalSnack(`Updated ${filterType} filter`); + updateFilterState(); + + // For filter updates, we need to submit the UPDATE to backend + // The backend should handle this as an update, not a new drawing + try { + await submitToDatabase( + filterDrawing, + auth, + { + roomId: currentRoomId, + roomType, + }, + setUndoAvailable, + setRedoAvailable + ); + + if (currentRoomId) { + checkUndoRedoAvailability( + auth, + setUndoAvailable, + setRedoAvailable, + currentRoomId + ); + } + } catch (error) { + console.error("Error submitting filter update:", error); + handleAuthError(error); + } + + return; // Exit early for updates + } + + // Create NEW filter record for new filter type + filterDrawing = new Drawing( + generateId(), + "#000000", + 0, + [], + Date.now(), + currentUser, + { + drawingType: "filter", + filterType, + filterParams: { ...params }, // Clone params + } + ); + + // Set filter properties directly on the drawing object + filterDrawing.drawingType = "filter"; + filterDrawing.filterType = filterType; + filterDrawing.filterParams = { ...params }; + filterDrawing.roomId = currentRoomId; + + userData.addDrawing(filterDrawing); + + // Update React state so components know about the new filter + const newUserData = new UserData(userData.userId, userData.username); + newUserData.drawings = [...userData.drawings]; // Clone array with new filter + setUserData(newUserData); + + updatePendingDrawings((prev) => [...prev, filterDrawing]); + + setUndoStack((prev) => [...prev, filterDrawing]); + setRedoStack([]); + + // Force complete redraw this will render all strokes THEN apply filter + lastDrawnStateRef.current = null; + forceNextRedrawRef.current = true; + await drawAllDrawings(); + + showLocalSnack(`Applied ${filterType} filter`); + updateFilterState(); + + try { + await submitToDatabase( + filterDrawing, + auth, + { + roomId: currentRoomId, + roomType, + }, + setUndoAvailable, + setRedoAvailable + ); + + // Check undo/redo availability after filter submission + if (currentRoomId) { + checkUndoRedoAvailability( + auth, + setUndoAvailable, + setRedoAvailable, + currentRoomId + ); + } + } catch (error) { + console.error("Error submitting filter:", error); + // On error, remove the failed filter from pending + updatePendingDrawings((prev) => + prev.filter((d) => d.drawingId !== filterDrawing.drawingId) + ); + handleAuthError(error); + } + }; + + const previewFilter = async (filterType, params) => { + const canvas = canvasRef.current; + if (!canvas) return; + + // If already in preview mode, first restore to base state + if (isFilterPreview && preFilterCanvasStateRef.current) { + const img = new Image(); + img.onload = async () => { + const context = canvas.getContext("2d"); + context.clearRect(0, 0, canvas.width, canvas.height); + context.drawImage(img, 0, 0); + + // Now apply the new preview + await applyPreviewFilter(canvas, filterType, params); + }; + img.src = preFilterCanvasStateRef.current; + return; + } + + // Store the current canvas state before preview (only once) + if (!preFilterCanvasStateRef.current) { + preFilterCanvasStateRef.current = canvas.toDataURL(); + } + + await applyPreviewFilter(canvas, filterType, params); + }; + + const applyPreviewFilter = async (canvas, filterType, params) => { + // Check if this filter type already exists in the drawings + const existingFilterIndex = userData.drawings.findIndex( + (d) => d.drawingType === "filter" && d.filterType === filterType + ); + + if (existingFilterIndex !== -1) { + // Temporarily remove this filter, redraw, then apply preview + const originalDrawings = [...userData.drawings]; + userData.drawings = userData.drawings.filter((d, i) => i !== existingFilterIndex); + + lastDrawnStateRef.current = null; + forceNextRedrawRef.current = true; + await drawAllDrawings(); + + // Restore drawings array + userData.drawings = originalDrawings; + } + + // Apply the preview filter on top of current canvas + const context = canvas.getContext("2d"); + const imageData = context.getImageData(0, 0, canvas.width, canvas.height); + const filteredImageData = applyImageFilter(imageData, filterType, params); + context.putImageData(filteredImageData, 0, 0); + + setIsFilterPreview(true); + }; + + const undoFilter = async () => { + // If in preview mode, restore from saved canvas state + if (isFilterPreview && preFilterCanvasStateRef.current) { + const canvas = canvasRef.current; + if (!canvas) return; + + const context = canvas.getContext("2d"); + const img = new Image(); + img.onload = async () => { + context.clearRect(0, 0, canvas.width, canvas.height); + context.drawImage(img, 0, 0); + setIsFilterPreview(false); + preFilterCanvasStateRef.current = null; + originalCanvasDataRef.current = null; + }; + img.src = preFilterCanvasStateRef.current; + return; + } + + // If not in preview mode, use regular undo (which properly syncs with backend) + if (!editingEnabled) { + showLocalSnack("Undo is disabled in view-only mode."); + return; + } + + if (undoStack.length === 0) { + showLocalSnack("No actions to undo."); + return; + } + + // Simply call the regular undo function, which will undo the last action + // This properly coordinates with the backend's undo system + await undo(); + }; + + const clearAllFilters = async () => { + // Clear preview state if active + if (isFilterPreview) { + setIsFilterPreview(false); + preFilterCanvasStateRef.current = null; + originalCanvasDataRef.current = null; + } + + if (!editingEnabled) { + showLocalSnack("Clear filters is disabled in view-only mode."); + return; + } + + // Find all filter drawings in userData (not just undo stack) + // Use setUserData callback to get the latest state + let filterDrawings = []; + setUserData((currentUserData) => { + const allDrawings = currentUserData.drawings || []; + filterDrawings = allDrawings.filter( + (drawing) => drawing.drawingType === "filter" + ); + console.log(`[clearAllFilters] Found ${filterDrawings.length} filters to clear`, filterDrawings); + return currentUserData; + }); + + if (filterDrawings.length === 0) { + showLocalSnack("No filters to clear."); + return; + } + + if (isRefreshing) { + showLocalSnack("Please wait for the canvas to refresh."); + return; + } + + try { + showLocalSnack(`Clearing ${filterDrawings.length} filter(s)...`); + + // Get filter IDs before removing from local state + const filterIds = filterDrawings.map(f => f.drawingId).filter(id => id); + + // Remove all filter drawings from local state immediately using proper state update + setUserData((currentUserData) => { + const newUserData = new UserData(currentUserData.userId, currentUserData.username); + newUserData.drawings = currentUserData.drawings.filter( + (d) => d.drawingType !== "filter" + ); + console.log(`[clearAllFilters] Removed ${filterDrawings.length} filters, ${newUserData.drawings.length} drawings remain`); + return newUserData; + }); + + // Remove from pendingDrawings + updatePendingDrawings((prev) => + prev.filter((d) => d.drawingType !== "filter") + ); + + // Remove from undo stack (if present) + setUndoStack((prev) => + prev.filter((d) => d.drawingType !== "filter") + ); + + // Force a complete redraw immediately to show filters are gone + lastDrawnStateRef.current = null; + forceNextRedrawRef.current = true; + await drawAllDrawings(); + + showLocalSnack(`Cleared ${filterDrawings.length} filter(s).`); + updateFilterState(); // Update filter state for UI + + // Now sync with backend - create undo markers for each filter + if (filterIds.length > 0) { + try { + // Import the API function + const { markStrokesAsUndone } = await import('../api/rooms'); + + try { + await markStrokesAsUndone(auth.token, currentRoomId, filterIds); + console.log(`Marked ${filterIds.length} filters as undone in backend`); + } catch (apiError) { + // If the API doesn't exist, fall back to calling undo multiple times + console.warn("markStrokesAsUndone API not available, using fallback"); + + // Fallback: call regular undo endpoint for each filter + const { undoRoomAction } = await import('../api/rooms'); + for (let i = 0; i < Math.min(filterDrawings.length, 10); i++) { + try { + const result = await undoRoomAction(auth.token, currentRoomId); + if (result.status === "noop") break; + await new Promise(resolve => setTimeout(resolve, 50)); + } catch (e) { + console.warn("Error calling undoRoomAction:", e); + break; + } + } + } + + await checkUndoRedoAvailability( + auth, + setUndoAvailable, + setRedoAvailable, + currentRoomId + ); + } catch (e) { + console.error("Error syncing filter removal with backend:", e); + } + } + } catch (error) { + console.error("Error clearing all filters:", error); + showLocalSnack("Failed to clear all filters. Refreshing canvas..."); + // Refresh to restore state + await refreshCanvasButtonHandler(); + } + }; + + const applyImageFilter = (imageData, filterType, params) => { + const data = imageData.data; + const filtered = new ImageData( + new Uint8ClampedArray(data), + imageData.width, + imageData.height + ); + + switch (filterType) { + case "blur": + return applyBlurFilter(filtered, params.intensity || 5); + case "hueShift": + return applyHueShiftFilter( + filtered, + params.hue || 0, + params.saturation || 0 + ); + case "chalk": + return applyChalkFilter( + filtered, + params.roughness || 50, + params.opacity || 80 + ); + case "fade": + return applyFadeFilter(filtered, params.amount || 30); + case "vintage": + return applyVintageFilter( + filtered, + params.sepia || 60, + params.vignette || 40 + ); + case "neon": + return applyNeonFilter( + filtered, + params.intensity || 15, + params.color || 180 + ); + default: + return filtered; + } + }; + + const applyBlurFilter = (imageData, intensity) => { + // Optimized separable box blur - O(n) instead of O(n²) + // This is much faster and won't crash even with higher intensity values + const data = imageData.data; + const width = imageData.width; + const height = imageData.height; + + const radius = Math.max(1, Math.floor(intensity)); + const temp = new Uint8ClampedArray(data); + const result = new Uint8ClampedArray(data); + + // Horizontal pass + for (let y = 0; y < height; y++) { + let r = 0, g = 0, b = 0, a = 0; + let count = 0; + + // Initialize window + for (let x = -radius; x <= radius; x++) { + if (x >= 0 && x < width) { + const idx = (y * width + x) * 4; + r += data[idx]; + g += data[idx + 1]; + b += data[idx + 2]; + a += data[idx + 3]; + count++; + } + } + + // Slide window across row + for (let x = 0; x < width; x++) { + const idx = (y * width + x) * 4; + temp[idx] = r / count; + temp[idx + 1] = g / count; + temp[idx + 2] = b / count; + temp[idx + 3] = a / count; + + // Remove left pixel + const leftX = x - radius; + if (leftX >= 0) { + const leftIdx = (y * width + leftX) * 4; + r -= data[leftIdx]; + g -= data[leftIdx + 1]; + b -= data[leftIdx + 2]; + a -= data[leftIdx + 3]; + count--; + } + + // Add right pixel + const rightX = x + radius + 1; + if (rightX < width) { + const rightIdx = (y * width + rightX) * 4; + r += data[rightIdx]; + g += data[rightIdx + 1]; + b += data[rightIdx + 2]; + a += data[rightIdx + 3]; + count++; + } + } + } + + // Vertical pass + for (let x = 0; x < width; x++) { + let r = 0, g = 0, b = 0, a = 0; + let count = 0; + + // Initialize window + for (let y = -radius; y <= radius; y++) { + if (y >= 0 && y < height) { + const idx = (y * width + x) * 4; + r += temp[idx]; + g += temp[idx + 1]; + b += temp[idx + 2]; + a += temp[idx + 3]; + count++; + } + } + + // Slide window down column + for (let y = 0; y < height; y++) { + const idx = (y * width + x) * 4; + result[idx] = r / count; + result[idx + 1] = g / count; + result[idx + 2] = b / count; + result[idx + 3] = a / count; + + // Remove top pixel + const topY = y - radius; + if (topY >= 0) { + const topIdx = (topY * width + x) * 4; + r -= temp[topIdx]; + g -= temp[topIdx + 1]; + b -= temp[topIdx + 2]; + a -= temp[topIdx + 3]; + count--; + } + + // Add bottom pixel + const bottomY = y + radius + 1; + if (bottomY < height) { + const bottomIdx = (bottomY * width + x) * 4; + r += temp[bottomIdx]; + g += temp[bottomIdx + 1]; + b += temp[bottomIdx + 2]; + a += temp[bottomIdx + 3]; + count++; + } + } + } + + return new ImageData(result, width, height); + }; + + const applyHueShiftFilter = (imageData, hueShift, saturationShift) => { + const data = imageData.data; + + for (let i = 0; i < data.length; i += 4) { + const r = data[i]; + const g = data[i + 1]; + const b = data[i + 2]; + + // Convert RGB to HSL + const max = Math.max(r, g, b) / 255; + const min = Math.min(r, g, b) / 255; + const diff = max - min; + const sum = max + min; + + let h = 0; + const l = sum / 2; + const s = diff === 0 ? 0 : l > 0.5 ? diff / (2 - sum) : diff / sum; + + if (diff !== 0) { + switch (max) { + case r / 255: + h = (g - b) / 255 / diff + (g < b ? 6 : 0); + break; + case g / 255: + h = (b - r) / 255 / diff + 2; + break; + case b / 255: + h = (r - g) / 255 / diff + 4; + break; + } + h /= 6; + } + + // Apply shifts + h = (h + hueShift / 360) % 1; + const newS = Math.max(0, Math.min(1, s + saturationShift / 100)); + + // Convert back to RGB + const hue2rgb = (p, q, t) => { + if (t < 0) t += 1; + if (t > 1) t -= 1; + if (t < 1 / 6) return p + (q - p) * 6 * t; + if (t < 1 / 2) return q; + if (t < 2 / 3) return p + (q - p) * (2 / 3 - t) * 6; + return p; + }; + + let newR, newG, newB; + + if (newS === 0) { + newR = newG = newB = l; + } else { + const q = l < 0.5 ? l * (1 + newS) : l + newS - l * newS; + const p = 2 * l - q; + newR = hue2rgb(p, q, h + 1 / 3); + newG = hue2rgb(p, q, h); + newB = hue2rgb(p, q, h - 1 / 3); + } + + data[i] = Math.round(newR * 255); + data[i + 1] = Math.round(newG * 255); + data[i + 2] = Math.round(newB * 255); + } + + return imageData; + }; + + const applyChalkFilter = (imageData, roughness, opacity) => { + const data = imageData.data; + + for (let i = 0; i < data.length; i += 4) { + const noise = (Math.random() - 0.5) * (roughness / 100) * 255; + const opacityFactor = opacity / 100; + + data[i] = Math.max(0, Math.min(255, data[i] + noise)) * opacityFactor; + data[i + 1] = + Math.max(0, Math.min(255, data[i + 1] + noise)) * opacityFactor; + data[i + 2] = + Math.max(0, Math.min(255, data[i + 2] + noise)) * opacityFactor; + data[i + 3] = data[i + 3] * opacityFactor; + } + + return imageData; + }; + + const applyFadeFilter = (imageData, amount) => { + const data = imageData.data; + const fadeAmount = 1 - amount / 100; + + for (let i = 0; i < data.length; i += 4) { + data[i + 3] = data[i + 3] * fadeAmount; + } + + return imageData; + }; + + const applyVintageFilter = (imageData, sepia, vignette) => { + const data = imageData.data; + const width = imageData.width; + const height = imageData.height; + const sepiaAmount = sepia / 100; + + for (let i = 0; i < data.length; i += 4) { + const r = data[i]; + const g = data[i + 1]; + const b = data[i + 2]; + + // Apply sepia + data[i] = Math.min( + 255, + (r * 0.393 + g * 0.769 + b * 0.189) * sepiaAmount + + r * (1 - sepiaAmount) + ); + data[i + 1] = Math.min( + 255, + (r * 0.349 + g * 0.686 + b * 0.168) * sepiaAmount + + g * (1 - sepiaAmount) + ); + data[i + 2] = Math.min( + 255, + (r * 0.272 + g * 0.534 + b * 0.131) * sepiaAmount + + b * (1 - sepiaAmount) + ); + + // Apply vignette + const x = (i / 4) % width; + const y = Math.floor(i / 4 / width); + const centerX = width / 2; + const centerY = height / 2; + const distance = Math.sqrt((x - centerX) ** 2 + (y - centerY) ** 2); + const maxDistance = Math.sqrt(centerX ** 2 + centerY ** 2); + const vignetteAmount = 1 - (distance / maxDistance) * (vignette / 100); + + data[i] *= vignetteAmount; + data[i + 1] *= vignetteAmount; + data[i + 2] *= vignetteAmount; + } + + return imageData; + }; + + const applyNeonFilter = (imageData, intensity, hue) => { + const data = imageData.data; + const width = imageData.width; + const height = imageData.height; + const glowIntensity = intensity / 25; // More aggressive scaling (max 50 -> 2.0) + + // Create a copy for the glow effect + const result = new Uint8ClampedArray(data); + + // Generate neon color based on hue using proper HSL to RGB conversion + const hueNormalized = hue / 360; + const neonR = Math.abs(Math.sin((hueNormalized) * Math.PI * 2)) * 255; + const neonG = Math.abs(Math.sin((hueNormalized + 0.333) * Math.PI * 2)) * 255; + const neonB = Math.abs(Math.sin((hueNormalized + 0.666) * Math.PI * 2)) * 255; + + for (let i = 0; i < data.length; i += 4) { + const alpha = data[i + 3]; + + // Only apply effect to visible pixels (any stroke) + if (alpha > 5) { + const r = data[i]; + const g = data[i + 1]; + const b = data[i + 2]; + + // Calculate brightness + const brightness = (r + g + b) / 3; + + // Apply aggressive neon glow with color tinting + const alphaFactor = alpha / 255; + const colorFactor = glowIntensity * alphaFactor; + + // Mix original color with neon color and boost brightness + const boost = 1 + (glowIntensity * 0.8); + result[i] = Math.min(255, (r * boost) + (neonR * colorFactor * 0.7)); + result[i + 1] = Math.min(255, (g * boost) + (neonG * colorFactor * 0.7)); + result[i + 2] = Math.min(255, (b * boost) + (neonB * colorFactor * 0.7)); + + // Ensure the effect is visible even on dark strokes + const minBrightness = 60 * glowIntensity; + const currentBrightness = (result[i] + result[i + 1] + result[i + 2]) / 3; + if (currentBrightness < minBrightness) { + const brightnessFactor = minBrightness / Math.max(currentBrightness, 1); + result[i] = Math.min(255, result[i] * brightnessFactor); + result[i + 1] = Math.min(255, result[i + 1] * brightnessFactor); + result[i + 2] = Math.min(255, result[i + 2] * brightnessFactor); + } + } + } + + return new ImageData(result, width, height); + }; + + const drawAllDrawings = async () => { + const currentTemplateObjects = templateObjectsRef.current || []; + + if (isDrawingInProgressRef.current) { + console.log('Drawing already in progress, skipping drawAllDrawings call'); + return; + } + + isDrawingInProgressRef.current = true; + + // Save current brush state + const savedBrushType = brushEngine ? brushEngine.brushType : null; + const savedBrushParams = brushEngine ? brushEngine.brushParams : null; + + try { + setIsLoading(true); + const canvas = canvasRef.current; + if (!canvas) { + setIsLoading(false); + isDrawingInProgressRef.current = false; + return; + } + const context = canvas.getContext("2d"); + if (!context) { + setIsLoading(false); + isDrawingInProgressRef.current = false; + return; + } + + // Include any locally-pending drawings (e.g. received via socket but + // not yet reflected by a backend refresh) so they render immediately. + const userDrawingIds = new Set((userData.drawings || []).map(d => d.drawingId)); + const uniquePendingDrawings = (pendingDrawings || []).filter( + pd => !userDrawingIds.has(pd.drawingId) + ); + + const combined = [ + ...(userData.drawings || []), + ...uniquePendingDrawings, + ]; + + // Create a state signature to detect if we need to redraw + // Include filter information to ensure redraw when filters change + const filterSignature = combined + .filter(d => d.drawingType === "filter") + .map(f => `${f.drawingId}:${f.filterType}`) + .join(','); + + const stateSignature = JSON.stringify({ + drawingCount: combined.length, + drawingIds: combined.map(d => d.drawingId).sort().join(','), + pendingCount: pendingDrawings.length, + templateCount: currentTemplateObjects?.length || 0, + templateIds: currentTemplateObjects?.map(t => `${t.type}:${t.x || t.x1 || t.cx}:${t.y || t.y1 || t.cy}`).join(',') || '', + filters: filterSignature + }); + + if (lastDrawnStateRef.current === stateSignature) { + console.log('State unchanged, skipping redraw'); + setIsLoading(false); + isDrawingInProgressRef.current = false; + return; + } + + // Clear force flag after checking it + forceNextRedrawRef.current = false; + lastDrawnStateRef.current = stateSignature; + + // for flicker free rendering + if (!offscreenCanvasRef.current || + offscreenCanvasRef.current.width !== canvasWidth || + offscreenCanvasRef.current.height !== canvasHeight + ) { + offscreenCanvasRef.current = document.createElement("canvas"); + offscreenCanvasRef.current.width = canvasWidth; + offscreenCanvasRef.current.height = canvasHeight; + } + + const offscreenContext = offscreenCanvasRef.current.getContext("2d"); + offscreenContext.imageSmoothingEnabled = false; + offscreenContext.clearRect(0, 0, canvasWidth, canvasHeight); + + // This avoids async rendering issues with image stamps + const stampsToRender = []; + + // Create and render template layer separately so it stays below all drawings + let templateCanvas = null; + if (currentTemplateObjects && currentTemplateObjects.length > 0) { + templateCanvas = document.createElement('canvas'); + templateCanvas.width = canvasWidth; + templateCanvas.height = canvasHeight; + const templateContext = templateCanvas.getContext('2d'); + templateContext.imageSmoothingEnabled = false; + + templateContext.save(); + templateContext.globalAlpha = 0.5; + + let renderedCount = 0; + for (const obj of currentTemplateObjects) { + try { + if (obj.type === 'line') { + templateContext.beginPath(); + templateContext.moveTo(obj.x1, obj.y1); + templateContext.lineTo(obj.x2, obj.y2); + templateContext.strokeStyle = obj.color || '#333'; + templateContext.lineWidth = obj.lineWidth || 2; + templateContext.stroke(); + renderedCount++; + } else if (obj.type === 'rectangle') { + templateContext.strokeStyle = obj.stroke || '#333'; + templateContext.lineWidth = obj.lineWidth || 2; + if (obj.fill && obj.fill !== 'transparent') { + templateContext.fillStyle = obj.fill; + templateContext.fillRect(obj.x, obj.y, obj.width, obj.height); + } + templateContext.strokeRect(obj.x, obj.y, obj.width, obj.height); + renderedCount++; + } else if (obj.type === 'circle') { + templateContext.beginPath(); + templateContext.arc(obj.cx, obj.cy, obj.radius, 0, Math.PI * 2); + templateContext.strokeStyle = obj.stroke || '#333'; + templateContext.lineWidth = obj.lineWidth || 2; + if (obj.fill && obj.fill !== 'transparent') { + templateContext.fillStyle = obj.fill; + templateContext.fill(); + } + templateContext.stroke(); + renderedCount++; + } else if (obj.type === 'ellipse') { + templateContext.beginPath(); + templateContext.ellipse(obj.cx, obj.cy, obj.rx, obj.ry, 0, 0, Math.PI * 2); + templateContext.strokeStyle = obj.stroke || '#333'; + templateContext.lineWidth = obj.lineWidth || 2; + if (obj.fill && obj.fill !== 'transparent') { + templateContext.fillStyle = obj.fill; + templateContext.fill(); + } + templateContext.stroke(); + renderedCount++; + } else if (obj.type === 'text') { + templateContext.fillStyle = obj.color || '#333'; + templateContext.font = `${obj.bold ? 'bold ' : ''}${obj.fontSize || 16}px Arial`; + templateContext.fillText(obj.text || '', obj.x, obj.y); + renderedCount++; + } else { + console.warn('Unknown template object type:', obj.type); + } + } catch (e) { + console.warn('Failed to render template object:', obj, e); + } + } + templateContext.restore(); + } else { + console.log('No template objects to render'); + } + + if (templateCanvas) { + offscreenContext.drawImage(templateCanvas, 0, 0); + } + + const cutOriginalIds = new Set(); + try { + combined.forEach((d) => { + if ( + d && + d.pathData && + d.pathData.tool === "cut" && + Array.isArray(d.pathData.originalStrokeIds) + ) { + d.pathData.originalStrokeIds.forEach((id) => + cutOriginalIds.add(id) + ); + } + }); + } catch (e) { } + + const sortedDrawings = combined.sort((a, b) => { + const orderA = + a.order !== undefined ? a.order : a.timestamp || a.ts || 0; + const orderB = + b.order !== undefined ? b.order : b.timestamp || b.ts || 0; + return orderA - orderB; + }); + + // Separate filter drawings from regular drawings + const regularDrawings = []; + const filterDrawings = []; + for (const drawing of sortedDrawings) { + if (drawing.drawingType === "filter") { + filterDrawings.push(drawing); + } else { + regularDrawings.push(drawing); + } + } + + // Pre-load all image stamps to ensure they render in correct z-order + const imageStampCache = new Map(); + const imageStampPromises = []; + + for (const drawing of regularDrawings) { + if (drawing.drawingType === "stamp" && drawing.stampData && drawing.stampData.image && !drawing.stampData.emoji) { + const imageUrl = drawing.stampData.image; + if (!imageStampCache.has(imageUrl)) { + const promise = new Promise((resolve) => { + const img = new Image(); + img.onload = () => { + imageStampCache.set(imageUrl, img); + resolve(); + }; + img.onerror = () => { + console.error("[drawAllDrawings] Failed to pre-load stamp image:", imageUrl.substring(0, 100)); + resolve(); // Continue even if image fails + }; + img.src = imageUrl; + }); + imageStampPromises.push(promise); + } + } + } + + // Wait for all stamp images to load before rendering + if (imageStampPromises.length > 0) { + console.log("[drawAllDrawings] Pre-loading", imageStampPromises.length, "stamp images"); + await Promise.all(imageStampPromises); + console.log("[drawAllDrawings] All stamp images loaded"); + } + + // Render drawings in chronological order. When a 'cut' record appears + // we immediately apply a destination-out erase so it removes prior content + // but does not erase strokes that are drawn after the cut. + const maskedOriginals = new Set(); + let seenAnyCut = false; + + for (const drawing of regularDrawings) { + // If this is a cut record, apply the erase to the canvas now. + if (drawing && drawing.pathData && drawing.pathData.tool === "cut") { + seenAnyCut = true; + try { + if (Array.isArray(drawing.pathData.originalStrokeIds)) { + drawing.pathData.originalStrokeIds.forEach((id) => + maskedOriginals.add(id) + ); + } + } catch (e) { } + + if (drawing.pathData && drawing.pathData.rect) { + const r = drawing.pathData.rect; + offscreenContext.save(); + try { + offscreenContext.globalCompositeOperation = "destination-out"; + offscreenContext.fillStyle = "rgba(0,0,0,1)"; + // Expand rect slightly to avoid hairline due to subpixel antialiasing + offscreenContext.fillRect( + Math.floor(r.x) - 2, + Math.floor(r.y) - 2, + Math.ceil(r.width) + 4, + Math.ceil(r.height) + 4 + ); + } finally { + offscreenContext.restore(); + } + + // Restore template layer in the cut region so templates remain visible + if (templateCanvas) { + offscreenContext.drawImage( + templateCanvas, + Math.floor(r.x) - 2, + Math.floor(r.y) - 2, + Math.ceil(r.width) + 4, + Math.ceil(r.height) + 4, + Math.floor(r.x) - 2, + Math.floor(r.y) - 2, + Math.ceil(r.width) + 4, + Math.ceil(r.height) + 4 + ); + } + } + + continue; + } + + // Skip originals that have been masked by a cut + if ( + drawing && + drawing.drawingId && + (cutOriginalIds.has(drawing.drawingId) || + maskedOriginals.has(drawing.drawingId)) + ) { + continue; + } + + // Skip temporary white "erase" helper strokes when we've seen a cut + // record; destination-out masking is authoritative and drawing white + // strokes can produce hairlines. + try { + if ( + seenAnyCut && + drawing && + drawing.color && + typeof drawing.color === "string" && + drawing.color.toLowerCase() === "#ffffff" + ) { + continue; + } + } catch (e) { } + + // Draw the drawing normally + offscreenContext.globalAlpha = 1.0; + let viewingUser = null; + let viewingPeriodStart = null; + if (selectedUser) { + if (typeof selectedUser === "string") viewingUser = selectedUser; + else if (typeof selectedUser === "object") { + viewingUser = selectedUser.user; + viewingPeriodStart = selectedUser.periodStart; + } + } + if (viewingUser && drawing.user !== viewingUser) { + offscreenContext.globalAlpha = 0.1; + } else if (viewingPeriodStart !== null) { + const ts = drawing.timestamp || drawing.order || 0; + if ( + ts < viewingPeriodStart || + ts >= viewingPeriodStart + 5 * 60 * 1000 + ) { + offscreenContext.globalAlpha = 0.1; + } + } + + // Stamps have pathData as array but need special rendering - render inline to preserve z-order + if (drawing.drawingType === "stamp" && drawing.stampData && drawing.stampSettings && Array.isArray(drawing.pathData) && drawing.pathData.length > 0) { + const stamp = drawing.stampData; + const settings = drawing.stampSettings; + const position = drawing.pathData[0]; + + try { + offscreenContext.save(); + offscreenContext.translate(position.x, position.y); + offscreenContext.rotate(((settings.rotation || 0) * Math.PI) / 180); + + const size = settings.size || 50; + + if (stamp.emoji) { + // Render emoji stamp + offscreenContext.font = `${size}px serif`; + offscreenContext.textAlign = "center"; + offscreenContext.textBaseline = "middle"; + offscreenContext.fillText(stamp.emoji, 0, 0); + console.log("[drawAllDrawings] Rendered emoji stamp inline:", stamp.emoji); + } else if (stamp.image) { + // Render image stamp using pre-loaded image + const img = imageStampCache.get(stamp.image); + if (img) { + offscreenContext.globalAlpha = (settings.opacity || 100) / 100 * offscreenContext.globalAlpha; + offscreenContext.drawImage(img, -size / 2, -size / 2, size, size); + console.log("[drawAllDrawings] Rendered image stamp inline"); + } else { + console.warn("[drawAllDrawings] Image stamp not in cache:", stamp.image?.substring(0, 100)); + } + } + + offscreenContext.restore(); + } catch (error) { + offscreenContext.restore(); + console.error("[drawAllDrawings] Error rendering stamp:", error); + } + } else if (drawing.drawingType === "stamp") { + console.warn("[drawAllDrawings] Stamp NOT rendered - missing requirements:", { + drawingId: drawing.drawingId, + drawingType: drawing.drawingType, + hasStampData: !!drawing.stampData, + hasStampSettings: !!drawing.stampSettings, + pathDataIsArray: Array.isArray(drawing.pathData), + pathDataLength: drawing.pathData ? drawing.pathData.length : 0, + pathDataType: typeof drawing.pathData, + pathDataValue: drawing.pathData, + fullDrawing: drawing + }); + } else if (Array.isArray(drawing.pathData)) { + const pts = drawing.pathData; + if (pts.length > 0) { + // Check if this is an advanced brush drawing + if (drawing.brushType && drawing.brushType !== "normal" && brushEngine) { + console.log("Rendering advanced brush in drawAllDrawings:", { + id: drawing.drawingId, + brushType: drawing.brushType, + pointCount: pts.length + }); + + // Use brush engine to render advanced brush strokes + offscreenContext.save(); + brushEngine.updateContext(offscreenContext); + + // Start the stroke at the first point + offscreenContext.beginPath(); + offscreenContext.moveTo(pts[0].x, pts[0].y); + + // Render the stroke using the brush engine with explicit brush type + brushEngine.startStroke(pts[0].x, pts[0].y); + for (let i = 1; i < pts.length; i++) { + // Use drawWithType instead of draw to bypass state dependency + brushEngine.drawWithType( + pts[i].x, + pts[i].y, + drawing.lineWidth, + drawing.color, + drawing.brushType // Pass brush type directly + ); + } + offscreenContext.restore(); + } else { + if (drawing.brushType && drawing.brushType !== "normal") { + console.log("Advanced brush found but no brushEngine:", { + id: drawing.drawingId, + brushType: drawing.brushType, + hasBrushEngine: !!brushEngine + }); + } + // Default rendering for normal brush + offscreenContext.beginPath(); + offscreenContext.moveTo(pts[0].x, pts[0].y); + for (let i = 1; i < pts.length; i++) + offscreenContext.lineTo(pts[i].x, pts[i].y); + offscreenContext.strokeStyle = drawing.color; + offscreenContext.lineWidth = drawing.lineWidth; + offscreenContext.lineCap = drawing.brushStyle || "round"; + offscreenContext.lineJoin = drawing.brushStyle || "round"; + offscreenContext.stroke(); + } + } + } else if (drawing.pathData && drawing.pathData.tool === "shape") { + if (drawing.pathData.points) { + const pts = drawing.pathData.points; + offscreenContext.save(); + offscreenContext.beginPath(); + offscreenContext.moveTo(pts[0].x, pts[0].y); + for (let i = 1; i < pts.length; i++) + offscreenContext.lineTo(pts[i].x, pts[i].y); + offscreenContext.closePath(); + offscreenContext.fillStyle = drawing.color; + offscreenContext.fill(); + offscreenContext.restore(); + } else { + const { + type, + start, + end, + brushStyle: storedBrush, + } = drawing.pathData; + offscreenContext.save(); + offscreenContext.fillStyle = drawing.color; + offscreenContext.lineWidth = drawing.lineWidth; + if (type === "circle") { + const radius = Math.sqrt( + (end.x - start.x) ** 2 + (end.y - start.y) ** 2 + ); + offscreenContext.beginPath(); + offscreenContext.arc(start.x, start.y, radius, 0, Math.PI * 2); + offscreenContext.fill(); + } else if (type === "rectangle") { + offscreenContext.fillRect( + start.x, + start.y, + end.x - start.x, + end.y - start.y + ); + } else if (type === "hexagon") { + const radius = Math.sqrt( + (end.x - start.x) ** 2 + (end.y - start.y) ** 2 + ); + offscreenContext.beginPath(); + for (let i = 0; i < 6; i++) { + const angle = (Math.PI / 3) * i; + const xPoint = start.x + radius * Math.cos(angle); + const yPoint = start.y + radius * Math.sin(angle); + if (i === 0) offscreenContext.moveTo(xPoint, yPoint); + else offscreenContext.lineTo(xPoint, yPoint); + } + offscreenContext.closePath(); + offscreenContext.fill(); + } else if (type === "line") { + offscreenContext.beginPath(); + offscreenContext.moveTo(start.x, start.y); + offscreenContext.lineTo(end.x, end.y); + offscreenContext.strokeStyle = drawing.color; + offscreenContext.lineWidth = drawing.lineWidth; + const cap = storedBrush || drawing.brushStyle || "round"; + offscreenContext.lineCap = cap; + offscreenContext.lineJoin = cap; + offscreenContext.stroke(); + } + offscreenContext.restore(); + } + } else if (drawing.pathData && drawing.pathData.tool === "image") { + const { image, x, y, width, height } = drawing.pathData; + let img = new Image(); + img.src = image; + img.onload = () => { + offscreenContext.drawImage(img, x, y, width, height); + }; + } + } + if (!selectedUser) { + // Group users by 5-minute intervals + // Use both committed drawings and pending drawings so the UI's + // user/time-group list reflects the strokes the user currently sees. + const groupMap = {}; + const groupingSource = [ + ...(userData.drawings || []), + ...(pendingDrawings || []), + ]; + groupingSource.forEach((d) => { + try { + const ts = d.timestamp || d.order || 0; + const periodStart = + Math.floor(ts / (5 * 60 * 1000)) * (5 * 60 * 1000); + if (!groupMap[periodStart]) groupMap[periodStart] = new Set(); + if (d.user) groupMap[periodStart].add(d.user); + } catch (e) { } + }); + const groups = Object.keys(groupMap).map((k) => ({ + periodStart: parseInt(k), + users: Array.from(groupMap[k]), + })); + groups.sort((a, b) => b.periodStart - a.periodStart); + if (selectedUser && selectedUser !== "") { + let stillExists = false; + if (typeof selectedUser === "string") { + for (const g of groups) { + if (g.users.includes(selectedUser)) { + stillExists = true; + break; + } + } + } else if (typeof selectedUser === "object" && selectedUser.user) { + for (const g of groups) { + if ( + g.periodStart === selectedUser.periodStart && + g.users.includes(selectedUser.user) + ) { + stillExists = true; + break; + } + } + } + + if (!stillExists) { + try { + setSelectedUser(""); + } catch (e) { + /* swallow if setter changed */ + } + } + } + + setUserList(groups); + } + + // Apply filters as post-processing after all regular drawings are rendered + if (filterDrawings.length > 0) { + console.log("[drawAllDrawings] Applying", filterDrawings.length, "filter(s)"); + for (const filterDrawing of filterDrawings) { + try { + if (filterDrawing.filterType && filterDrawing.filterParams) { + const imageData = offscreenContext.getImageData(0, 0, canvasWidth, canvasHeight); + const filteredImageData = applyImageFilter( + imageData, + filterDrawing.filterType, + filterDrawing.filterParams + ); + offscreenContext.putImageData(filteredImageData, 0, 0); + console.log("[drawAllDrawings] Applied filter:", filterDrawing.filterType); + } + } catch (e) { + console.error("[drawAllDrawings] Error applying filter:", filterDrawing.filterType, e); + } + } + } + + // Copy offscreen canvas to visible canvas atomically + console.log("[drawAllDrawings] Copying offscreen canvas to visible canvas. Total strokes rendered:", regularDrawings.length, "filters:", filterDrawings.length); + context.imageSmoothingEnabled = false; + context.clearRect(0, 0, canvasWidth, canvasHeight); + context.drawImage(offscreenCanvasRef.current, 0, 0); + console.log("[drawAllDrawings] Canvas update complete"); + } catch (e) { + console.error("Error in drawAllDrawings:", e); + } finally { + // Restore current brush state + if (brushEngine && savedBrushType) { + brushEngine.setBrushType(savedBrushType); + if (savedBrushParams) { + brushEngine.setBrushParams(savedBrushParams); + } + } + setIsLoading(false); + isDrawingInProgressRef.current = false; + } + }; + + drawAllDrawingsRef.current = drawAllDrawings; + + const undo = async () => { + if (!editingEnabled) { + showLocalSnack("Undo is disabled in view-only mode."); + return; + } + if (undoStack.length === 0) return; + if (isRefreshing) { + showLocalSnack( + "Please wait for the canvas to refresh before undoing again." + ); + return; + } + try { + await undoAction({ + auth, + currentUser: auth?.username || "anonymous", + undoStack, + setUndoStack, + setRedoStack, + userData, + drawAllDrawings, + refreshCanvasButtonHandler: refreshCanvasButtonHandler, + roomId: currentRoomId, + }); + // After undo completes, refresh undo/redo availability from server + try { + await checkUndoRedoAvailability( + auth, + setUndoAvailable, + setRedoAvailable, + currentRoomId + ); + } catch (e) { } + updateFilterState(); // Update filter state after undo + } catch (error) { + console.error("Error during undo:", error); + } + }; + + const redo = async () => { + if (!editingEnabled) { + showLocalSnack("Redo is disabled in view-only mode."); + return; + } + if (redoStack.length === 0) return; + if (isRefreshing) { + showLocalSnack( + "Please wait for the canvas to refresh before redoing again." + ); + return; + } + try { + await redoAction({ + auth, + currentUser: auth?.username || "anonymous", + redoStack, + setRedoStack, + setUndoStack, + userData, + drawAllDrawings, + refreshCanvasButtonHandler: refreshCanvasButtonHandler, + roomId: currentRoomId, + }); + // After redo completes, refresh undo/redo availability from server + try { + await checkUndoRedoAvailability( + auth, + setUndoAvailable, + setRedoAvailable, + currentRoomId + ); + } catch (e) { } + updateFilterState(); // Update filter state after redo + } catch (error) { + console.error("Error during redo:", error); + } + }; + + // Track canvas context and notify parent when it changes + useEffect(() => { + if (!onCanvasContextChange || !currentRoomId) return; + + // Calculate current canvas context + const objectCount = (userData.drawings || []).filter(d => d.drawingType !== 'filter').length + pendingDrawings.length; + + // Get unique active users from drawings + const activeUsersSet = new Set(); + [...(userData.drawings || []), ...pendingDrawings].forEach(drawing => { + if (drawing.user) { + // Extract username from "username|timestamp" format + const username = drawing.user.split('|')[0]; + activeUsersSet.add(username); + } + }); + const activeUsers = Array.from(activeUsersSet); + + const canvasContext = { + room_id: currentRoomId, + object_count: objectCount, + active_users: activeUsers, + has_filters: hasFilters, + template_id: templateId || null + }; + + // Only notify if context actually changed + const contextSignature = JSON.stringify(canvasContext); + if (previousCanvasContextRef.current !== contextSignature) { + previousCanvasContextRef.current = contextSignature; + try { + onCanvasContextChange(canvasContext); + } catch (e) { + console.error('Error calling onCanvasContextChange:', e); + } + } + }, [userData.drawings, pendingDrawings, currentRoomId, hasFilters, templateId, onCanvasContextChange]); + + // Register keyboard shortcuts and commands + useEffect(() => { + // Initialize shortcut manager + if (!shortcutManagerRef.current) { + shortcutManagerRef.current = new KeyboardShortcutManager(); + } + + const manager = shortcutManagerRef.current; + + // Register all commands with the command registry + const commands = [ + // Command Palette & Help + { + id: 'commands.palette', + label: 'Open Command Palette', + description: 'Quick access to all commands', + keywords: ['palette', 'search', 'find'], + category: 'Commands', + action: () => setCommandPaletteOpen(true), + shortcut: { key: 'k', modifiers: { ctrl: true } } + }, + { + id: 'commands.shortcuts', + label: 'Show Keyboard Shortcuts', + description: 'View all available keyboard shortcuts', + keywords: ['help', 'shortcuts', 'keys'], + category: 'Commands', + action: () => setShortcutsHelpOpen(true), + shortcut: { key: '/', modifiers: { ctrl: true } } + }, + { + id: 'commands.cancel', + label: 'Cancel / Escape', + description: 'Cancel current action or close dialogs', + keywords: ['cancel', 'escape', 'close'], + category: 'Commands', + action: () => { + if (commandPaletteOpen) setCommandPaletteOpen(false); + else if (shortcutsHelpOpen) setShortcutsHelpOpen(false); + else if (drawing) setDrawing(false); + }, + shortcut: { key: 'Escape', modifiers: {} } + }, + + // Edit Operations + { + id: 'edit.undo', + label: 'Undo', + description: 'Undo the last action', + keywords: ['undo', 'revert'], + category: 'Edit', + action: undo, + shortcut: { key: 'z', modifiers: { ctrl: true } }, + enabled: () => editingEnabled && undoStack.length > 0 + }, + { + id: 'edit.redo', + label: 'Redo', + description: 'Redo the last undone action', + keywords: ['redo', 'repeat'], + category: 'Edit', + action: redo, + shortcut: { key: 'z', modifiers: { ctrl: true, shift: true } }, + enabled: () => editingEnabled && redoStack.length > 0 + }, + + // Canvas Operations + { + id: 'canvas.clear', + label: 'Clear Canvas', + description: 'Remove all strokes from canvas', + keywords: ['clear', 'delete', 'reset'], + category: 'Canvas', + action: () => { + if (editingEnabled) { + setClearDialogOpen(true); + } else { + showLocalSnack('Canvas clearing is disabled in view-only mode'); + } + }, + shortcut: { key: 'k', modifiers: { ctrl: true, shift: true } }, + enabled: () => editingEnabled + }, + { + id: 'canvas.refresh', + label: 'Refresh Canvas', + description: 'Reload canvas from server', + keywords: ['refresh', 'reload'], + category: 'Canvas', + action: refreshCanvasButtonHandler, + shortcut: { key: 'r', modifiers: { ctrl: true } } + }, + { + id: 'canvas.settings', + label: 'Canvas Settings', + description: 'Open canvas settings', + keywords: ['settings', 'preferences'], + category: 'Canvas', + action: () => { + if (onOpenSettings) onOpenSettings(); + }, + shortcut: { key: ',', modifiers: { ctrl: true } }, + visible: () => !!onOpenSettings + }, + + // Tools + { + id: 'tool.pen', + label: 'Select Pen Tool', + description: 'Switch to freehand drawing', + keywords: ['pen', 'draw', 'brush'], + category: 'Tools', + action: () => { + if (editingEnabled) { + setDrawMode('freehand'); + showLocalSnack('Pen tool selected'); + } + }, + shortcut: { key: 'p', modifiers: {} }, + enabled: () => editingEnabled + }, + { + id: 'tool.eraser', + label: 'Select Eraser', + description: 'Switch to eraser mode', + keywords: ['eraser', 'erase', 'remove'], + category: 'Tools', + action: () => { + if (editingEnabled) { + setDrawMode('eraser'); + showLocalSnack('Eraser selected'); + } + }, + shortcut: { key: 'e', modifiers: {} }, + enabled: () => editingEnabled + }, + { + id: 'tool.rectangle', + label: 'Select Rectangle Tool', + description: 'Draw rectangles and squares', + keywords: ['rectangle', 'rect', 'square'], + category: 'Tools', + action: () => { + if (editingEnabled) { + setDrawMode('shape'); + setShapeType('rectangle'); + showLocalSnack('Rectangle tool selected'); + } + }, + shortcut: { key: 'r', modifiers: {} }, + enabled: () => editingEnabled + }, + { + id: 'tool.circle', + label: 'Select Circle Tool', + description: 'Draw circles and ellipses', + keywords: ['circle', 'oval', 'ellipse'], + category: 'Tools', + action: () => { + if (editingEnabled) { + setDrawMode('shape'); + setShapeType('circle'); + showLocalSnack('Circle tool selected'); + } + }, + shortcut: { key: 'c', modifiers: {} }, + enabled: () => editingEnabled + }, + { + id: 'tool.line', + label: 'Select Line Tool', + description: 'Draw straight lines', + keywords: ['line', 'straight'], + category: 'Tools', + action: () => { + if (editingEnabled) { + setDrawMode('shape'); + setShapeType('line'); + showLocalSnack('Line tool selected'); + } + }, + shortcut: { key: 'l', modifiers: {} }, + enabled: () => editingEnabled + } + ]; + + // Register commands with command registry + // Clear first to ensure clean state + commandRegistry.clear(); + + // Register each command (allowOverwrite for React re-renders) + commands.forEach(cmd => { + commandRegistry.register(cmd, { allowOverwrite: true }); + }); + + // Register keyboard shortcuts + manager.clear(); + commands.forEach(cmd => { + if (cmd.shortcut) { + manager.register( + cmd.shortcut.key, + cmd.shortcut.modifiers, + () => { + // Check if command is enabled before executing + if (cmd.enabled && !cmd.enabled()) { + return; + } + cmd.action(); + }, + cmd.label, + cmd.category + ); + } + }); + + // Add global keyboard event listener + const handleKeyDown = (event) => manager.handleKeyDown(event); + document.addEventListener('keydown', handleKeyDown); + + // Cleanup + return () => { + document.removeEventListener('keydown', handleKeyDown); + manager.clear(); + }; + }, [ + editingEnabled, + undoStack, + redoStack, + undo, + redo, + refreshCanvasButtonHandler, + onOpenSettings, + commandPaletteOpen, + shortcutsHelpOpen, + drawing + ]); + + const { + selectionStart, + setSelectionStart, + selectionRect, + setSelectionRect, + cutImageData, + setCutImageData, + handleCutSelection, + } = useCanvasSelection( + canvasRef, + currentUser, + userData, + generateId, + drawAllDrawings, + currentRoomId, + setUndoAvailable, + setRedoAvailable, + auth, + roomType, + showLocalSnack + ); + + // AI Assist functions + const { + textToDrawing, + generateImage, + shapeCompletion, + beautifySketch, + styleTransfer, + aiAssistLoading, + recognizeObject, + } = useAIAssistant(); + + const [selectionRecognition, setSelectionRecognition] = useState(null); + const [recognizeMode, setRecognizeMode] = useState(false); + + const handleRecognizeToggle = (enabled) => { + try { + setRecognizeMode(!!enabled); + if (enabled) { + prevDrawModeRef.current = drawMode; + setDrawMode('select'); + } else { + setDrawMode(prevDrawModeRef.current || 'freehand'); + prevDrawModeRef.current = null; + } + } catch (e) { console.error('Error toggling recognize mode', e); } + }; + + // Draw a preview of a shape (for shape mode) + const drawShapePreview = (start, end, shape, color, lineWidth) => { + if (!start || !end) return; + + const canvas = canvasRef.current; + const context = canvas.getContext("2d"); + context.save(); + context.strokeStyle = color; + context.lineWidth = lineWidth; + context.setLineDash([5, 3]); + + if (shape === "circle") { + const radius = Math.sqrt((end.x - start.x) ** 2 + (end.y - start.y) ** 2); + context.beginPath(); + context.arc(start.x, start.y, radius, 0, Math.PI * 2); + context.stroke(); + } else if (shape === "rectangle") { + context.strokeRect(start.x, start.y, end.x - start.x, end.y - start.y); + } else if (shape === "hexagon") { + const radius = Math.sqrt((end.x - start.x) ** 2 + (end.y - start.y) ** 2); + context.beginPath(); + + for (let i = 0; i < 6; i++) { + const angle = (Math.PI / 3) * i; + const xPoint = start.x + radius * Math.cos(angle); + const yPoint = start.y + radius * Math.sin(angle); + + if (i === 0) context.moveTo(xPoint, yPoint); + else context.lineTo(xPoint, yPoint); + } + context.closePath(); + context.stroke(); + } else if (shape === "line") { + context.beginPath(); + context.moveTo(start.x, start.y); + context.lineTo(end.x, end.y); + context.lineCap = brushStyle; + context.lineJoin = brushStyle; + context.stroke(); + } + + context.restore(); + }; + + // Handle paste action for cut selection + const handlePaste = async (e) => { + if (!editingEnabled) { + showLocalSnack("Editing is disabled in view-only mode."); + setDrawMode("freehand"); + return; + } + if ( + !cutImageData || + !Array.isArray(cutImageData) || + cutImageData.length === 0 + ) { + showLocalSnack("No cut selection available to paste."); + setDrawMode("freehand"); + return; + } + + const canvas = canvasRef.current; + const rectCanvas = canvas.getBoundingClientRect(); + const scaleX = canvas.width / rectCanvas.width; + const scaleY = canvas.height / rectCanvas.height; + const pasteX = (e.clientX - rectCanvas.left) * scaleX; + const pasteY = (e.clientY - rectCanvas.top) * scaleY; + + let minX = Infinity, + minY = Infinity; + + cutImageData.forEach((drawing) => { + if (Array.isArray(drawing.pathData)) { + drawing.pathData.forEach((pt) => { + minX = Math.min(minX, pt.x); + minY = Math.min(minY, pt.y); + }); + } else if (drawing.pathData && drawing.pathData.tool === "shape") { + if (drawing.pathData.points && Array.isArray(drawing.pathData.points)) { + drawing.pathData.points.forEach((pt) => { + minX = Math.min(minX, pt.x); + minY = Math.min(minY, pt.y); + }); + } else if (drawing.pathData.type === "line") { + if (drawing.pathData.start) { + minX = Math.min(minX, drawing.pathData.start.x); + minY = Math.min(minY, drawing.pathData.start.y); + } + if (drawing.pathData.end) { + minX = Math.min(minX, drawing.pathData.end.x); + minY = Math.min(minY, drawing.pathData.end.y); + } + } + } + }); + + if (minX === Infinity || minY === Infinity) { + showLocalSnack("Invalid cut data."); + return; + } + + const offsetX = pasteX - minX; + const offsetY = pasteY - minY; + let pastedDrawings = []; + + const newDrawings = cutImageData + .map((originalDrawing) => { + let newPathData; + if (Array.isArray(originalDrawing.pathData)) { + newPathData = originalDrawing.pathData.map((pt) => ({ + x: pt.x + offsetX, + y: pt.y + offsetY, + })); + } else if ( + originalDrawing.pathData && + originalDrawing.pathData.tool === "shape" + ) { + if ( + originalDrawing.pathData.points && + Array.isArray(originalDrawing.pathData.points) + ) { + const newPoints = originalDrawing.pathData.points.map((pt) => ({ + x: pt.x + offsetX, + y: pt.y + offsetY, + })); + newPathData = { ...originalDrawing.pathData, points: newPoints }; + } else if (originalDrawing.pathData.type === "line") { + const newStart = { + x: originalDrawing.pathData.start.x + offsetX, + y: originalDrawing.pathData.start.y + offsetY, + }; + const newEnd = { + x: originalDrawing.pathData.end.x + offsetX, + y: originalDrawing.pathData.end.y + offsetY, + }; + newPathData = { + ...originalDrawing.pathData, + start: newStart, + end: newEnd, + }; + } + } else { + return null; + } + + // Preserve all metadata from original drawing + const metadata = { + brushStyle: originalDrawing.brushStyle, + brushType: originalDrawing.brushType, + brushParams: originalDrawing.brushParams, + drawingType: originalDrawing.drawingType, + stampData: originalDrawing.stampData, + stampSettings: originalDrawing.stampSettings, + filterType: originalDrawing.filterType, + filterParams: originalDrawing.filterParams, + }; + + return new Drawing( + generateId(), + originalDrawing.color, + originalDrawing.lineWidth, + newPathData, + Date.now(), + currentUser, + metadata + ); + }) + .filter(Boolean); + + setIsRefreshing(true); + setRedoStack([]); + + const pasteRecordId = generateId(); + showLocalSnack(`Pasting ${newDrawings.length} item(s)... Please wait.`); + console.log("[handlePaste] Starting paste operation:", { + pasteRecordId, + drawingCount: newDrawings.length, + drawingTypes: newDrawings.map(d => d.drawingType || "stroke") + }); + + // Attach parentPasteId to each new drawing so the backend/read path can filter them + for (const nd of newDrawings) { + nd.roomId = currentRoomId; + nd.parentPasteId = pasteRecordId; + if (!nd.pathData) nd.pathData = {}; + nd.pathData.parentPasteId = pasteRecordId; + } + console.log("[handlePaste] Attached parentPasteId to all drawings:", pasteRecordId); + + // Submit all pasted drawings as replacement/child strokes but DO NOT add each to the undo stack + let submittedCount = 0; + for (const newDrawing of newDrawings) { + try { + userData.addDrawing(newDrawing); + + await submitToDatabase( + newDrawing, + auth, + { roomId: currentRoomId, roomType, skipUndoStack: true }, + setUndoAvailable, + setRedoAvailable + ); + pastedDrawings.push(newDrawing); + submittedCount++; + + showLocalSnack(`Pasting... ${submittedCount}/${newDrawings.length} items saved.`); + } catch (error) { + console.error("Failed to save drawing:", newDrawing, error); + handleAuthError(error); + } + } + + const pastedIds = pastedDrawings.map((d) => d.drawingId); + const pasteRecord = new Drawing( + pasteRecordId, + "#FFFFFF", + 1, + { tool: "paste", cut: false, pastedDrawingIds: pastedIds }, + Date.now(), + currentUser + ); + console.log("[handlePaste] Created paste record:", { + pasteRecordId, + pastedCount: pastedIds.length, + pastedIds: pastedIds.join(',') + }); + try { + // Submit the single paste-record (counts as one backend undo operation) + await submitToDatabase( + pasteRecord, + auth, + { roomId: currentRoomId, roomType }, + setUndoAvailable, + setRedoAvailable + ); + console.log("[handlePaste] Paste record submitted successfully"); + setUndoStack((prev) => [ + ...prev, + { type: "paste", pastedDrawings: pastedDrawings, backendCount: 1 }, + ]); + } catch (error) { + console.error("Failed to save paste record:", pasteRecord, error); + showLocalSnack("Paste failed to persist. Some strokes may be missing."); + } + + setIsRefreshing(false); + + // Update undo/redo availability after paste operations + if (currentRoomId) { + checkUndoRedoAvailability( + auth, + setUndoAvailable, + setRedoAvailable, + currentRoomId + ); + } + + tempPathRef.current = []; + if (pastedDrawings.length === newDrawings.length) { + drawAllDrawings(); + setCutImageData([]); + setDrawMode("freehand"); + showLocalSnack(`Paste completed! ${pastedDrawings.length} item(s) pasted successfully.`); + } else { + showLocalSnack(`Paste partially completed. ${pastedDrawings.length}/${newDrawings.length} items pasted.`); + } + }; + + const mergedRefreshCanvas = async (sourceLabel = undefined) => { + try { + if (sourceLabel) { + console.log('mergedRefreshCanvas called from:', sourceLabel, '==='); + console.debug('mergedRefreshCanvas called from:', sourceLabel); + } else { + console.log('mergedRefreshCanvas called (no label) ==='); + console.debug('mergedRefreshCanvas called'); + } + } catch (e) { } + // If currently panning, defer refresh until pan ends to avoid races and frequent backend calls. + try { + if (isPanning) { + console.debug( + "[mergedRefreshCanvas] deferring because isPanning=true, marking pendingPanRefreshRef" + ); + pendingPanRefreshRef.current = true; + return; + } + } catch (e) { } + + if (sourceLabel === "undo-event" || sourceLabel === "redo-event") { + console.log("[mergedRefreshCanvas] Forcing complete state reset for undo/redo"); + lastDrawnStateRef.current = null; + } + + setIsLoading(true); + const backendCount = await backendRefreshCanvas( + serverCountRef.current, + userData, + drawAllDrawings, + historyRange ? historyRange.start : undefined, + historyRange ? historyRange.end : undefined, + { + roomId: currentRoomId, + auth, + clearLastDrawnState: () => { + console.log("[mergedRefreshCanvas] Clearing lastDrawnStateRef to force redraw"); + lastDrawnStateRef.current = null; + } + } + ); + + // Prefer the synchronous ref mirror when available to avoid + // races between state updates and immediate refreshes. + const pendingSnapshot = (pendingDrawingsRef.current && pendingDrawingsRef.current.length > 0) + ? [...pendingDrawingsRef.current] + : [...pendingDrawings]; + + // Don't clear all pending drawings, only mark confirmed ones for removal + + serverCountRef.current = backendCount; + // Re-append any pending drawings that the backend didn't return. + const drawingMatches = (a, b) => { + if (!a || !b) return false; + if (a.drawingId && b.drawingId && a.drawingId === b.drawingId) + return true; + + try { + const sameUser = a.user === b.user; + const tsA = a.timestamp || a.ts || 0; + const tsB = b.timestamp || b.ts || 0; + const tsClose = Math.abs(tsA - tsB) < 1000; + const lenA = Array.isArray(a.pathData) + ? a.pathData.length + : a.pathData && a.pathData.points + ? a.pathData.points.length + : 0; + const lenB = Array.isArray(b.pathData) + ? b.pathData.length + : b.pathData && b.pathData.points + ? b.pathData.points.length + : 0; + const lenClose = Math.abs(lenA - lenB) <= 1; + return sameUser && tsClose && lenClose; + } catch (e) { + return false; + } + }; + + try { + const cutOriginalIds = new Set(); + (userData.drawings || []).forEach((d) => { + if ( + d.pathData && + d.pathData.tool === "cut" && + Array.isArray(d.pathData.originalStrokeIds) + ) { + d.pathData.originalStrokeIds.forEach((id) => cutOriginalIds.add(id)); + } + }); + + if (cutOriginalIds.size > 0) { + userData.drawings = (userData.drawings || []).filter( + (d) => !cutOriginalIds.has(d.drawingId) + ); + } + } catch (e) { + // best-effort + } + + // Re-append pending drawings that the backend didn't return, but + // skip any pending items older than the authoritative clearedAt timestamp + const clearedAt = currentRoomId + ? roomClearedAtRef.current[currentRoomId] + : null; + const stillPending = []; + + pendingSnapshot.forEach((pd) => { + try { + const pdTs = pd.timestamp || pd.ts || 0; + if (clearedAt && pdTs < clearedAt) { + // This pending drawing was created before a server clear; ignore it + return; + } + } catch (e) { } + + const exists = userData.drawings.find((d) => drawingMatches(d, pd)); + if (!exists) { + // Backend doesn't have it yet, keep it pending + userData.drawings.push(pd); + stillPending.push(pd); + } else { + // If pending drawing has stampData but backend version doesn't, use pending version + if (pd.drawingType === "stamp" && pd.stampData) { + const backendMatch = exists; + if (!backendMatch.stampData || !backendMatch.stampData.image && pd.stampData.image) { + console.warn("Backend stamp missing stampData, using pending version:", { + drawingId: pd.drawingId, + pendingHasStampData: !!pd.stampData, + backendHasStampData: !!backendMatch.stampData, + pendingImageLength: pd.stampData.image ? pd.stampData.image.length : 0, + backendImageLength: backendMatch.stampData && backendMatch.stampData.image ? backendMatch.stampData.image.length : 0 + }); + + // Replace backend version with pending version that has complete data + const idx = userData.drawings.findIndex((d) => drawingMatches(d, pd)); + if (idx !== -1) { + userData.drawings[idx] = pd; + } + } + } + + // Backend has it, mark as confirmed and remove from pending + if (pd.drawingId) { + confirmedStrokesRef.current.add(pd.drawingId); + } + } + }); + + // Update pending drawings to only include those still not confirmed by backend + updatePendingDrawings(stillPending); + + // CRITICAL: Deduplicate filters - only keep the LATEST of each filter type + // This prevents stacking when backend returns duplicates + const filtersByType = new Map(); + const nonFilterDrawings = []; + + (userData.drawings || []).forEach((drawing) => { + if (drawing.drawingType === "filter" && drawing.filterType) { + const existing = filtersByType.get(drawing.filterType); + // Keep the one with the latest timestamp + if (!existing || (drawing.timestamp || 0) > (existing.timestamp || 0)) { + filtersByType.set(drawing.filterType, drawing); + } + } else { + nonFilterDrawings.push(drawing); + } + }); + + // Rebuild drawings array with deduplicated filters + const deduplicatedDrawings = [ + ...nonFilterDrawings, + ...Array.from(filtersByType.values()) + ]; + + console.log(`[mergedRefreshCanvas] Deduplicated filters. Filter count: ${filtersByType.size}, Total drawings: ${deduplicatedDrawings.length}`); + + // CRITICAL: Update both the mutable userData object AND React state + // Update userData in place so the closure reference works + userData.drawings = deduplicatedDrawings; + + // Also update React state to trigger re-renders + const newUserData = new UserData(userData.userId, userData.username); + newUserData.drawings = deduplicatedDrawings; + setUserData(newUserData); + + // Extract custom stamps from all drawings and update stamp panel + extractCustomStamps(); + + // Use requestAnimationFrame for smoother rendering + requestAnimationFrame(() => { + drawAllDrawings(); + setIsLoading(false); + updateFilterState(); // Update filter state after loading drawings + }); + }; + + // Extract custom stamps from backend drawings and update StampPanel + const extractCustomStamps = () => { + try { + const customStamps = []; + const seenStamps = new Map(); // Deduplicate by image content or emoji + + (userData.drawings || []).forEach((drawing) => { + if (drawing.drawingType === "stamp" && drawing.stampData) { + const stamp = drawing.stampData; + + // Skip default emoji stamps (they're already in StampPanel) + if (stamp.emoji && !stamp.image) { + return; + } + + // For custom image stamps, create a unique key based on image content + if (stamp.image) { + const imageKey = stamp.image.substring(0, 100); // Use first 100 chars as key + + if (!seenStamps.has(imageKey)) { + seenStamps.set(imageKey, true); + customStamps.push({ + id: `stamp-${Date.now()}-${customStamps.length}`, + name: stamp.name || 'Custom Stamp', + category: stamp.category || 'custom', + image: stamp.image, + emoji: stamp.emoji + }); + } + } + } + }); + + if (customStamps.length > 0) { + console.log('Extracted custom stamps from backend:', customStamps.length); + setBackendStamps(customStamps); + } + } catch (error) { + console.error('Error extracting custom stamps:', error); + } + }; + + const startDrawingHandler = (e) => { + const canvas = canvasRef.current; + const rect = canvas.getBoundingClientRect(); + const x = e.clientX - rect.left; + const y = e.clientY - rect.top; + + if (e.button === 1) { + // Middle mouse button: start panning + setIsPanning(true); + panStartRef.current = { x: e.clientX, y: e.clientY }; + panOriginRef.current = { ...panOffset }; + setIsLoading(true); + // Throttle pan-triggered refreshes: if we recently refreshed, defer until pan end + try { + const now = Date.now(); + const diff = now - panLastRefreshRef.current; + console.debug( + `[pan] now=${now} lastRefresh=${panLastRefreshRef.current} diff=${diff} cooldown=${PAN_REFRESH_COOLDOWN_MS}` + ); + if (diff > PAN_REFRESH_COOLDOWN_MS) { + panLastRefreshRef.current = now; + console.debug("[pan] triggering immediate mergedRefreshCanvas"); + mergedRefreshCanvas("pan-start").finally(() => setIsLoading(false)); + } else { + // Mark that we skipped the immediate refresh and schedule a deferred refresh on mouseup + panRefreshSkippedRef.current = true; + console.debug( + "[pan] skipped immediate refresh; scheduling deferred refresh on mouseup" + ); + if (panEndRefreshTimerRef.current) + clearTimeout(panEndRefreshTimerRef.current); + panEndRefreshTimerRef.current = setTimeout(() => { + if (panRefreshSkippedRef.current) { + panRefreshSkippedRef.current = false; + panLastRefreshRef.current = Date.now(); + console.debug("[pan] deferred timer firing mergedRefreshCanvas"); + mergedRefreshCanvas("pan-deferred").finally(() => + setIsLoading(false) + ); + } + panEndRefreshTimerRef.current = null; + }, Math.max(200, PAN_REFRESH_COOLDOWN_MS - diff)); + setIsLoading(false); + } + } catch (e) { + mergedRefreshCanvas().finally(() => setIsLoading(false)); + } + return; + } + + if (!editingEnabled) return; + + if (drawMode === "eraser" || drawMode === "freehand") { + const context = canvas.getContext("2d"); + context.strokeStyle = color; + context.lineWidth = lineWidth; + context.lineCap = brushStyle; + context.lineJoin = brushStyle; + + // Initialize brush engine for advanced brushes + if (brushEngine) { + brushEngine.updateContext(context); + brushEngine.startStroke(x, y); + + // For normal brush, we still need the standard path setup + if (currentBrushType === "normal") { + context.beginPath(); + context.moveTo(x, y); + } + } else { + // Fallback if no brush engine + context.beginPath(); + context.moveTo(x, y); + } + + tempPathRef.current = [{ x, y }]; + setDrawing(true); + } else if (drawMode === "shape") { + setShapeStart({ x, y }); + setDrawing(true); + + const dataURL = canvas.toDataURL(); + let snapshotImg = new Image(); + + snapshotImg.src = dataURL; + snapshotRef.current = snapshotImg; + } else if (drawMode === "select") { + setSelectionStart({ x, y }); + setSelectionRect(null); + setDrawing(true); + + const dataURL = canvas.toDataURL(); + let snapshotImg = new Image(); + + snapshotImg.src = dataURL; + snapshotRef.current = snapshotImg; + } else if (drawMode === "paste") { + handlePaste(e); + } else if (drawMode === "stamp") { + // Start stamp preview on mousedown (will place on mouseup) + if (selectedStamp && stampSettings) { + setStampPreview({ x, y, stamp: selectedStamp, settings: stampSettings }); + stampPreviewRef.current = { x, y, stamp: selectedStamp, settings: stampSettings }; + setDrawing(true); // Enable dragging + } + } + }; + + const handlePan = (e) => { + if (!isPanning) return; + + // If the middle button is no longer pressed, stop panning. + if (!(e.buttons & 4)) { + setIsPanning(false); + panOriginRef.current = { ...panOffset }; + return; + } + + const deltaX = e.clientX - panStartRef.current.x; + const deltaY = e.clientY - panStartRef.current.y; + let newX = panOriginRef.current.x + deltaX; + let newY = panOriginRef.current.y + deltaY; + const containerWidth = window.innerWidth; + const containerHeight = window.innerHeight; + + // Calculate minimum allowed offsets so that the canvas edge is not exceeded. + // Our canvas is fixed at canvasWidth and canvasHeight. + const minX = containerWidth - canvasWidth; // This will be negative if canvasWidth > containerWidth + const minY = containerHeight - canvasHeight; + + newX = clamp(newX, minX, 0); + newY = clamp(newY, minY, 0); + + setPanOffset({ + x: newX, + y: newY, + }); + }; + + const drawHandler = (e) => { + if (isPanning) { + handlePan(e); + return; + } + if (!editingEnabled) return; // prevent drawing but allow other handlers like panning to proceed + if (!drawing) return; + + const canvas = canvasRef.current; + const rect = canvas.getBoundingClientRect(); + const x = e.clientX - rect.left; + const y = e.clientY - rect.top; + + console.log( + "Drawing with brush type:", + currentBrushType, + "drawMode:", + drawMode + ); + + // Update stamp preview position during drag + if (drawMode === "stamp" && stampPreviewRef.current) { + setStampPreview({ ...stampPreviewRef.current, x, y }); + stampPreviewRef.current = { ...stampPreviewRef.current, x, y }; + return; + } + + if (drawMode === "eraser" || drawMode === "freehand") { + const context = canvas.getContext("2d"); + + // Use advanced brush engine if available + if (brushEngine && currentBrushType !== "normal") { + console.log("Drawing with advanced brush engine:", currentBrushType); + // Ensure context is up to date + brushEngine.updateContext(context); + + // Ensure brush engine has current state + if (brushEngine.brushType !== currentBrushType) { + brushEngine.setBrushType(currentBrushType); + } + if ( + JSON.stringify(brushEngine.brushParams) !== + JSON.stringify(brushParams) + ) { + brushEngine.setBrushParams(brushParams); + } + + brushEngine.draw(x, y, lineWidth, color); + } else { + console.log("Drawing with normal brush"); + // Default drawing behavior + context.lineTo(x, y); + context.stroke(); + context.beginPath(); + context.moveTo(x, y); + } + + tempPathRef.current.push({ x, y }); + } else if (drawMode === "shape" && drawing) { + // update shape preview with adjusted coordinates + if (snapshotRef.current && snapshotRef.current.complete) { + const context = canvas.getContext("2d"); + context.clearRect(0, 0, canvasWidth, canvasHeight); + context.drawImage(snapshotRef.current, 0, 0); + } + + drawShapePreview(shapeStart, { x, y }, shapeType, color, lineWidth); + } else if (drawMode === "select" && drawing) { + setSelectionRect({ start: selectionStart, end: { x, y } }); + + if (snapshotRef.current && snapshotRef.current.complete) { + const context = canvas.getContext("2d"); + context.clearRect(0, 0, canvasWidth, canvasHeight); + context.drawImage(snapshotRef.current, 0, 0); + } + + const context = canvas.getContext("2d"); + context.save(); + context.strokeStyle = "blue"; + context.lineWidth = 1; + context.setLineDash([6, 3]); + + const s = selectionStart; + const selX = Math.min(s.x, x); + const selY = Math.min(s.y, y); + const selWidth = Math.abs(x - s.x); + const selHeight = Math.abs(y - s.y); + + context.strokeRect(selX, selY, selWidth, selHeight); + context.restore(); + } + }; + + const stopDrawingHandler = async (e) => { + if (isPanning && e.button === 1) { + setIsPanning(false); + return; + } + if (!drawing) return; + setDrawing(false); + + if (!editingEnabled) { + tempPathRef.current = []; + return; + } + + snapshotRef.current = null; + const canvas = canvasRef.current; + const rect = canvas.getBoundingClientRect(); + const finalX = e.clientX - rect.left; + const finalY = e.clientY - rect.top; + + if (drawMode === "eraser" || drawMode === "freehand") { + const newDrawing = new Drawing( + `drawing_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`, + color, + lineWidth, + tempPathRef.current, + Date.now(), + currentUser, + { + brushStyle: brushStyle, + brushType: currentBrushType, + brushParams: brushParams, + drawingType: "stroke", + } + ); + newDrawing.roomId = currentRoomId; + newDrawing.brushType = currentBrushType; + newDrawing.brushParams = brushParams; + + setUndoStack((prev) => [...prev, newDrawing]); + setRedoStack([]); + + try { + userData.addDrawing(newDrawing); + // Add to pending drawings for immediate display (optimistic UI) + updatePendingDrawings((prev) => [...prev, newDrawing]); + + // Use requestAnimationFrame for immediate, smooth redraw + requestAnimationFrame(() => { + drawAllDrawings(); + }); + + // Queue the submission instead of submitting immediately + const submitTask = async () => { + try { + console.log("Submitting queued stroke:", { + drawingId: newDrawing.drawingId, + pathLength: tempPathRef.current.length, + }); + + await submitToDatabase( + newDrawing, + auth, + { + roomId: currentRoomId, + roomType, + }, + setUndoAvailable, + setRedoAvailable + ); + + // Don't remove from pending here - let mergedRefreshCanvas or socket confirmation handle it + + if (currentRoomId) { + checkUndoRedoAvailability( + auth, + setUndoAvailable, + setRedoAvailable, + currentRoomId + ); + } + } catch (error) { + console.error("Error during queued freehand submission:", error); + // On error, remove the failed stroke from pending + updatePendingDrawings((prev) => + prev.filter((d) => d.drawingId !== newDrawing.drawingId) + ); + handleAuthError(error); + } + }; + + submissionQueueRef.current.push(submitTask); + processSubmissionQueue(); + } catch (error) { + console.error("Error preparing freehand stroke:", error); + handleAuthError(error); + } finally { + setIsRefreshing(false); + } + tempPathRef.current = []; + + // If shape completion mode is ON, ask for a suggestion + if (shapeCompletionEnabled) { + handleShapeAutoCompletion(); + } + } else if (drawMode === "shape") { + if (!shapeStart) { + return; + } + + const finalEnd = { x: finalX, y: finalY }; + const context = canvas.getContext("2d"); + + context.save(); + context.fillStyle = color; + context.lineWidth = lineWidth; + context.setLineDash([]); + if (shapeType === "circle") { + const radius = Math.sqrt( + (finalEnd.x - shapeStart.x) ** 2 + (finalEnd.y - shapeStart.y) ** 2 + ); + + context.beginPath(); + context.arc(shapeStart.x, shapeStart.y, radius, 0, Math.PI * 2); + context.fill(); + } else if (shapeType === "rectangle") { + context.fillRect( + shapeStart.x, + shapeStart.y, + finalEnd.x - shapeStart.x, + finalEnd.y - shapeStart.y + ); + } else if (shapeType === "hexagon") { + const radius = Math.sqrt( + (finalEnd.x - shapeStart.x) ** 2 + (finalEnd.y - shapeStart.y) ** 2 + ); + context.beginPath(); + for (let i = 0; i < 6; i++) { + const angle = (Math.PI / 3) * i; + const xPoint = shapeStart.x + radius * Math.cos(angle); + const yPoint = shapeStart.y + radius * Math.sin(angle); + + if (i === 0) context.moveTo(xPoint, yPoint); + else context.lineTo(xPoint, yPoint); + } + + context.closePath(); + context.fill(); + } else if (shapeType === "line") { + context.beginPath(); + context.moveTo(shapeStart.x, shapeStart.y); + context.lineTo(finalEnd.x, finalEnd.y); + context.strokeStyle = color; + context.lineWidth = lineWidth; + context.lineCap = brushStyle; + context.lineJoin = brushStyle; + context.stroke(); + } + context.restore(); + + const shapeDrawingData = { + tool: "shape", + type: shapeType, + start: shapeStart, + end: finalEnd, + brushStyle: shapeType === "line" ? brushStyle : undefined, + }; + + const newDrawing = new Drawing( + `drawing_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`, + color, + lineWidth, + shapeDrawingData, + Date.now(), + currentUser, + { + brushStyle: shapeType === "line" ? brushStyle : "round", + brushType: currentBrushType, + brushParams: brushParams, + drawingType: "shape", + } + ); + newDrawing.roomId = currentRoomId; + + userData.addDrawing(newDrawing); + updatePendingDrawings((prev) => [...prev, newDrawing]); + + // Use requestAnimationFrame for smooth shape rendering + requestAnimationFrame(() => { + drawAllDrawings(); + }); + + setUndoStack((prev) => [...prev, newDrawing]); + setRedoStack([]); + + // Queue the submission + const submitTask = async () => { + try { + await submitToDatabase( + newDrawing, + auth, + { + roomId: currentRoomId, + roomType, + }, + setUndoAvailable, + setRedoAvailable + ); + + // Don't remove from pending here - let mergedRefreshCanvas or socket confirmation handle it + + // Update undo/redo availability after shape submission + if (currentRoomId) { + checkUndoRedoAvailability( + auth, + setUndoAvailable, + setRedoAvailable, + currentRoomId + ); + } + } catch (error) { + console.error("Error during queued shape submission:", error); + // On error, remove the failed stroke from pending + updatePendingDrawings((prev) => + prev.filter((d) => d.drawingId !== newDrawing.drawingId) + ); + handleAuthError(error); + } + }; + + submissionQueueRef.current.push(submitTask); + processSubmissionQueue(); + + if (shapeCompletionEnabled) { + handleShapeAutoCompletion(); + } + + setShapeStart(null); + } else if (drawMode === "select") { + setDrawing(false); + + try { + await mergedRefreshCanvas(); + } catch (error) { + console.error("Error during select submission or refresh:", error); + } finally { + setIsRefreshing(false); + } + + // If user completed a selection, run a lightweight recognition on the + // drawings that intersect the selection rect and show a temporary badge. + try { + if (selectionRect && typeof recognizeObject === 'function') { + const { start, end } = selectionRect; + const rectX = Math.min(start.x, end.x); + const rectY = Math.min(start.y, end.y); + const rectW = Math.abs(end.x - start.x); + const rectH = Math.abs(end.y - start.y); + + // Gather intersecting drawings (simple heuristics) + const selected = []; + (userData.drawings || []).forEach((d) => { + try { + const pd = d.pathData || d; + if (!pd) return; + + if (Array.isArray(pd.points) || Array.isArray(pd)) { + const pts = Array.isArray(pd.points) ? pd.points : pd; + if (pts.some((p) => p.x >= rectX && p.x <= rectX + rectW && p.y >= rectY && p.y <= rectY + rectH)) { + selected.push(d); + return; + } + } + + if (pd.tool === 'shape') { + // Bounding-box check for shapes + const s = pd.start || {}; + const en = pd.end || {}; + if (s.x !== undefined && en.x !== undefined) { + const minX = Math.min(s.x, en.x); + const maxX = Math.max(s.x, en.x); + const minY = Math.min(s.y, en.y); + const maxY = Math.max(s.y, en.y); + if (!(maxX < rectX || minX > rectX + rectW || maxY < rectY || minY > rectY + rectH)) { + selected.push(d); + return; + } + } + if (pd.points && Array.isArray(pd.points)) { + if (pd.points.some((p) => p.x >= rectX && p.x <= rectX + rectW && p.y >= rectY && p.y <= rectY + rectH)) { + selected.push(d); + return; + } + } + } + } catch (inner) { /* ignore malformed shapes */ } + }); + + // Call recognition API with the selected subset + const bounds = { width: canvasRef.current?.width || 0, height: canvasRef.current?.height || 0 }; + const res = await recognizeObject(selected, { x: rectX, y: rectY, width: rectW, height: rectH }, bounds); + + if (res && !res.error) { + setSelectionRecognition({ label: res.label || 'unknown', confidence: res.confidence || 0, explanation: res.explanation || '' }); + // Clear after a short delay + setTimeout(() => setSelectionRecognition(null), 6000); + } else { + setSelectionRecognition({ label: 'unknown', confidence: 0, explanation: '' }); + setTimeout(() => setSelectionRecognition(null), 3000); + } + } + } catch (err) { + console.error('Recognition failed:', err); + } + + mergedRefreshCanvas(); + } else if (drawMode === "stamp" && stampPreviewRef.current) { + // Place stamp at final position on mouseup + const { x, y, stamp, settings } = stampPreviewRef.current; + await placeStamp(x, y, stamp, settings); + + // Clear preview + setStampPreview(null); + stampPreviewRef.current = null; + } + }; + + const openHistoryDialog = () => { + setSelectedUser(""); + setHistoryDialogOpen(true); + }; + + const handleApplyHistory = async (startMs, endMs) => { + // startMs and endMs are epoch ms. If not provided, read from inputs. + const start = + startMs !== undefined + ? startMs + : historyStartInput + ? new Date(historyStartInput).getTime() + : NaN; + const end = + endMs !== undefined + ? endMs + : historyEndInput + ? new Date(historyEndInput).getTime() + : NaN; + + if (isNaN(start) || isNaN(end)) { + showLocalSnack( + "Please select both start and end date/time before applying History Recall." + ); + return; + } + if (start > end) { + showLocalSnack("Invalid time range selected. Make sure start <= end."); + return; + } + + // Deselect any selected user when entering history recall + setSelectedUser(""); + setHistoryRange({ start, end }); + setIsLoading(true); + + // Try to load drawings for the requested time range + await clearCanvasForRefresh(); + // set a temporary historyRange so mergedRefreshCanvas will use it + setHistoryRange({ start, end }); + try { + const backendCount = await backendRefreshCanvas( + serverCountRef.current, + userData, + drawAllDrawings, + start, + end, + { roomId: currentRoomId, auth } + ); + serverCountRef.current = backendCount; + // If no drawings loaded, inform user and rollback historyRange + if (!userData.drawings || userData.drawings.length === 0) { + setHistoryRange(null); + showLocalSnack( + "No drawings were found in that date/time range. Please select another range or exit history recall mode." + ); + return; + } + setHistoryMode(true); + setHistoryDialogOpen(false); + } catch (e) { + console.error("Error applying history range:", e); + setHistoryRange(null); + showLocalSnack( + "An error occurred while loading history. See console for details." + ); + } finally { + setIsLoading(false); + } + }; + + // Auto-refresh when the active room changes + useEffect(() => { + // wipe local cache so we don't flash previous room's strokes + userData.drawings = []; + setIsRefreshing(true); + + // clear what's on screen immediately + try { + if (canvasRef.current) { + const ctx = canvasRef.current.getContext("2d"); + if (ctx) { + ctx.clearRect( + 0, + 0, + canvasRef.current.width, + canvasRef.current.height + ); + } + drawAllDrawings(); + } + } catch { } + + // reload for the new room + (async () => { + try { + await mergedRefreshCanvas(); // already room-aware + } finally { + setIsRefreshing(false); + } + })(); + }, [currentRoomId, canvasRefreshTrigger]); + + const exitHistoryMode = async () => { + // Deselect any selected user when leaving history mode + setSelectedUser(""); + setHistoryMode(false); + setHistoryRange(null); + setIsLoading(true); + try { + await clearCanvasForRefresh(); + serverCountRef.current = await backendRefreshCanvas( + serverCountRef.current, + userData, + drawAllDrawings, + undefined, + undefined, + { roomId: currentRoomId, auth } + ); + } finally { + setIsLoading(false); + } + }; + + const clearCanvas = async () => { + if (!editingEnabled) { + showLocalSnack("Cannot clear canvas in view-only mode."); + return; + } + const canvas = canvasRef.current; + const context = canvas.getContext("2d"); + + context.clearRect(0, 0, canvasWidth, canvasHeight); + + setUserData(initializeUserData()); + setUndoStack([]); + setRedoStack([]); + updatePendingDrawings([]); + serverCountRef.current = 0; + }; + + const handleExportCanvas = async () => { + if (!currentRoomId) { + showLocalSnack("Cannot export: not in a room"); + return; + } + + try { + setIsLoading(true); + showLocalSnack("Exporting canvas data..."); + + const { exportRoomCanvas } = await import('../api/rooms'); + console.log('[Export] Calling API with roomId:', currentRoomId); + console.log('[Export] Auth token present:', !!auth?.token); + + const exportData = await exportRoomCanvas(auth?.token, currentRoomId); + + console.log('[Export] Received exportData:', { + exists: !!exportData, + type: typeof exportData, + keys: exportData ? Object.keys(exportData) : [], + hasStrokes: exportData ? !!exportData.strokes : false, + strokeCount: exportData ? exportData.strokeCount : 'N/A' + }); + + if (!exportData) { + console.error('[Export] exportData is null or undefined'); + showLocalSnack("Export failed: no data returned from server"); + return; + } + + if (!exportData.strokes) { + console.error('[Export] exportData.strokes is missing:', exportData); + showLocalSnack(`Export failed: no strokes in response (got ${exportData.strokeCount || 0} count)`); + return; + } + + // Create a downloadable JSON file + const dataStr = JSON.stringify(exportData, null, 2); + const dataBlob = new Blob([dataStr], { type: 'application/json' }); + const url = URL.createObjectURL(dataBlob); + const link = document.createElement('a'); + link.href = url; + link.download = `${exportData.roomName || 'canvas'}_export_${Date.now()}.json`; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + URL.revokeObjectURL(url); + + showLocalSnack(`Exported ${exportData.strokeCount} strokes successfully`); + console.log('[Export] Success - downloaded file'); + } catch (error) { + console.error("[Export] Error caught:", error); + console.error("[Export] Error stack:", error.stack); + showLocalSnack(`Export failed: ${error.message || 'Unknown error'}`); + } finally { + setIsLoading(false); + } + }; + + const handleImportCanvas = async () => { + if (!currentRoomId) { + showLocalSnack("Cannot import: not in a room"); + return; + } + + if (!editingEnabled) { + showLocalSnack("Cannot import in view-only mode"); + return; + } + + // Create a file input element + const input = document.createElement('input'); + input.type = 'file'; + input.accept = 'application/json,.json'; + + input.onchange = async (e) => { + const file = e.target.files[0]; + if (!file) return; + + try { + setIsLoading(true); + showLocalSnack("Reading import file..."); + + const text = await file.text(); + const importData = JSON.parse(text); + + if (!importData.strokes || !Array.isArray(importData.strokes)) { + showLocalSnack("Invalid import file: missing strokes array"); + return; + } + + // Ask user if they want to clear existing canvas + const clearExisting = window.confirm( + `Import ${importData.strokes.length} strokes?\n\n` + + `Click OK to replace current canvas, or Cancel to merge with existing drawings.` + ); + + showLocalSnack(`Importing ${importData.strokes.length} strokes...`); + + const { importRoomCanvas } = await import('../api/rooms'); + const result = await importRoomCanvas(auth?.token, currentRoomId, importData, clearExisting); + + if (result.status === 'success') { + showLocalSnack( + `Import complete: ${result.imported} imported, ${result.failed} failed`, + 6000 + ); + + // Refresh canvas to show imported data + setTimeout(async () => { + try { + await clearCanvasForRefresh(); + await mergedRefreshCanvas("post-import"); + } catch (error) { + console.error("Error refreshing after import:", error); + } + }, 500); + } else { + showLocalSnack(`Import failed: ${result.message || 'Unknown error'}`); + } + } catch (error) { + console.error("Import error:", error); + showLocalSnack(`Import failed: ${error.message || 'Invalid file format'}`); + } finally { + setIsLoading(false); + } + }; + + input.click(); + }; + + const toggleColorPicker = (event) => { + const viewportHeight = window.innerHeight; + const pickerHeight = 350; + const rect = event.target.getBoundingClientRect(); + const pickerElement = document.querySelector(".Canvas-color-picker"); + + setShowColorPicker(!showColorPicker); + + if (rect.bottom + pickerHeight > viewportHeight && pickerElement) { + pickerElement.classList.add("Canvas-color-picker--adjust-bottom"); + } else if (pickerElement) { + pickerElement.classList.remove("Canvas-color-picker--adjust-bottom"); + } + }; + + const closeColorPicker = () => { + setShowColorPicker(false); + }; + + useEffect(() => { + setIsRefreshing(true); + clearCanvasForRefresh(); + + mergedRefreshCanvas().then(() => { + setTimeout(() => { + setIsRefreshing(false); + }, 500); + }); + }, [selectedUser]); + + useEffect(() => { + setUndoAvailable(undoStack.length > 0); + setRedoAvailable(redoStack.length > 0); + }, [undoStack, redoStack]); + + // Upload thumbnail when leaving the canvas (component unmount or room change) + useEffect(() => { + const roomIdSnapshot = currentRoomId; + + return () => { + // Upload thumbnail on cleanup (when navigating away) + if (roomIdSnapshot) { + uploadThumbnail(roomIdSnapshot); + } + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [currentRoomId]); + + // Add AI-generated objects to canvas and backend + const addAIGeneratedObjects = async (objects) => { + if (!Array.isArray(objects) || objects.length === 0) { + return; + } + + const created = []; + + for (const obj of objects) { + // Support image objects (raster) returned by style/image endpoints + let metadata = {}; + let pathData = obj.pathData; + + if (obj.drawingType === 'image' || obj.imageDataUrl) { + metadata.drawingType = 'image'; + metadata.stampData = { + imageDataUrl: obj.imageDataUrl || obj.image || null, + x: obj.x ?? 0, + y: obj.y ?? 0, + width: obj.width ?? canvasWidth, + height: obj.height ?? canvasHeight, + }; + + // Provide minimal pathData so the renderer can locate the image + pathData = { + tool: 'shape', + type: 'rectangle', + start: { x: metadata.stampData.x, y: metadata.stampData.y }, + end: { x: metadata.stampData.x + metadata.stampData.width, y: metadata.stampData.y + metadata.stampData.height }, + }; + } + + const newDrawing = new Drawing( + generateId(), + obj.color || '#000000', + obj.lineWidth ?? 2, + pathData, + Date.now(), + currentUser, + metadata + ); + newDrawing.roomId = currentRoomId; + + userData.addDrawing(newDrawing); + updatePendingDrawings(prev => [...prev, newDrawing]); + created.push(newDrawing); + + submissionQueueRef.current.push(async () => { + try { + await submitToDatabase( + newDrawing, + auth, + { roomId: currentRoomId, roomType, skipUndoStack: true }, + setUndoAvailable, + setRedoAvailable + ); + if (currentRoomId) { + checkUndoRedoAvailability(auth, setUndoAvailable, setRedoAvailable, currentRoomId); + } + } catch (e) { + console.error("AI object save failed:", e); + // setPendingDrawings(prev => prev.filter(d => d.drawingId !== newDrawing.drawingId)); + handleAuthError(e); + } + }); + } + + lastDrawnStateRef.current = null; + requestAnimationFrame(() => { drawAllDrawings(); }); + processSubmissionQueue(); + + setRedoStack([]); + }; + + function getVisibleCanvasBounds(padding=50) { + const canvas = canvasRef.current; + if (!canvas) return null; + + const rect = canvas.getBoundingClientRect(); + const vx0 = 0, vy0 = 0; + const vx1 = window.innerWidth; + const vy1 = window.innerHeight; + + const ix0 = Math.max(rect.left, vx0); + const iy0 = Math.max(rect.top, vy0); + const ix1 = Math.min(rect.right, vx1); + const iy1 = Math.min(rect.bottom,vy1); + + if (ix1 <= ix0 || iy1 <= iy0) return { x: 0, y: 0, width: 0, height: 0 }; + + const scaleX = canvas.width / rect.width; + const scaleY = canvas.height / rect.height; + + // Convert intersection to canvas space + const x = (ix0 - rect.left) * scaleX; + const y = (iy0 - rect.top) * scaleY; + const width = (ix1 - ix0) * scaleX; + const height = (iy1 - iy0) * scaleY; + + // Apply inner padding + const padX = padding * scaleX; + const padY = padding * scaleY; + + const paddedX = Math.max(0, x + padX); + const paddedY = Math.max(0, y + padY); + const paddedWidth = Math.max(0, width - padX * 2); + const paddedHeight = Math.max(0, height - padY * 2); + + return { + x: Math.floor(paddedX), + y: Math.floor(paddedY), + width: Math.floor(Math.min(canvas.width - paddedX, paddedWidth)), + height: Math.floor(Math.min(canvas.height - paddedY, paddedHeight)), + }; + } + + const extractAIErrorMessage = (payload) => { + if (!payload) return 'Unknown error'; + if (typeof payload === 'string') return payload; + + const candidate = payload.detail ?? payload.error ?? payload.message ?? payload; + if (typeof candidate === 'string') return candidate; + + if (candidate && typeof candidate === 'object') { + if (candidate.detail) { + const nested = extractAIErrorMessage(candidate.detail); + if (nested) return nested; + } + if (candidate.message) return candidate.message; + try { + return JSON.stringify(candidate); + } catch (jsonError) { + console.error('Failed to stringify AI error payload:', jsonError); + } + } + + return 'Unknown error'; + }; + + const handlePromptInputSubmition = async (prompt) => { + try { + if (aiGenerateService === 'drawing') { + const canvasBounds = getVisibleCanvasBounds(); + const canvasState = { + drawings: [...userData.drawings, ...pendingDrawings], + bounds: { + width: (canvasBounds?.width || canvasWidth), + height: (canvasBounds?.height || canvasHeight) + }, + }; + + const resp = await textToDrawing(prompt, canvasState); + const payload = typeof resp === 'string' ? JSON.parse(resp) : resp; + + if (payload && payload.error) { + // Backend returned an error + const errorMsg = extractAIErrorMessage(payload); + showLocalSnack(`AI sketch generation failed: ${errorMsg}`); + } else if (payload && Array.isArray(payload.objects)) { + await addAIGeneratedObjects(payload.objects); + showLocalSnack("AI objects rendered to canvas."); + } else { + showLocalSnack(`An error occurred while generating the sketch. Response: ${JSON.stringify(payload).substring(0, 100)}`); + } + + } else if (aiGenerateService === 'image') { + const resp = await generateImage(prompt, 512, 512); + const payload = resp && resp.imageDataUrl ? resp : null; + + if (payload && payload.imageDataUrl) { + // Insert image as a full-canvas object (simple placement) + const imgObj = { + drawingType: 'image', + imageDataUrl: payload.imageDataUrl, + x: 0, + y: 0, + width: canvasWidth, + height: canvasHeight, + }; + await addAIGeneratedObjects([imgObj]); + showLocalSnack('Generated image added to canvas.'); + } else { + showLocalSnack('Image generation failed.'); + } + + } else if (aiGenerateService === 'style') { + const canvasBounds = getVisibleCanvasBounds(); + const canvasState = { + drawings: [...userData.drawings, ...pendingDrawings], + bounds: { + width: (canvasBounds?.width || canvasWidth), + height: (canvasBounds?.height || canvasHeight) + }, + }; + + const resp = await styleTransfer(canvasState, prompt); + const payload = resp; + + if (payload && Array.isArray(payload.objects)) { + await addAIGeneratedObjects(payload.objects); + showLocalSnack('Style transfer applied to canvas.'); + } else { + const errMsg = payload ? extractAIErrorMessage(payload) : 'unknown error'; + console.error('Style transfer failed, payload:', payload); + showLocalSnack(`Style transfer failed: ${errMsg}`); + } + } + } catch (e) { + console.error('AI prompt handling error:', e); + showLocalSnack('Unexpected error handling AI request.'); + } + }; + + const handleShapeAutoCompletion = async () => { + if (!editingEnabled) { + showLocalSnack("Shape completion is disabled in view-only mode."); + return; + } + + try { + const canvasBounds = getVisibleCanvasBounds(); + const canvasState = { + drawings: [...userData.drawings, ...pendingDrawings], + bounds: { + width: (canvasBounds?.width || canvasWidth), + height: (canvasBounds?.height || canvasHeight) + }, + }; + const suggestion = await shapeCompletion(canvasState); + console.log("suggestion: ", suggestion); + if (!suggestion || suggestion.error || !suggestion.object) { + showLocalSnack("AI could not infer a shape."); + return; + } + + const { pathData } = suggestion.object || {}; + const anchor = computeSuggestionAnchor(pathData, canvasWidth, canvasHeight); + + setShapeSuggestion(suggestion); + setShapeAnchor(anchor); + } catch (e) { + console.error("Shape completion error:", e); + showLocalSnack("Unexpected error during shape completion."); + } + }; + + const handleShapeCompletionToggle = (enabled) => { + setShapeCompletionEnabled(enabled); + if (!enabled) { + setShapeSuggestion(null); + setShapeAnchor(null); + } + }; + + function computeSuggestionAnchor(pathData, canvasWidth, canvasHeight) { + if (!pathData) { + return { x: canvasWidth / 2, y: canvasHeight / 2 }; + } + + if (Array.isArray(pathData.points) && pathData.points.length > 0) { + let minX = pathData.points[0].x; + let maxX = pathData.points[0].x; + let minY = pathData.points[0].y; + let maxY = pathData.points[0].y; + + for (const p of pathData.points) { + minX = Math.min(minX, p.x); + maxX = Math.max(maxX, p.x); + minY = Math.min(minY, p.y); + maxY = Math.max(maxY, p.y); + } + + return { + x: (minX + maxX) / 2, + y: (minY + maxY) / 2, + }; + } + + if (pathData.start && pathData.end) { + return { + x: (pathData.start.x + pathData.end.x) / 2, + y: (pathData.start.y + pathData.end.y) / 2, + }; + } + + return { x: canvasWidth / 2, y: canvasHeight / 2 }; + } + +const acceptShapeSuggestion = async () => { + if (!shapeSuggestion?.object) return; + + await addAIGeneratedObjects([shapeSuggestion.object]); + + // Clear overlay + setShapeSuggestion(null); + setShapeAnchor(null); +}; + + const rejectShapeSuggestion = () => { + setShapeSuggestion(null); + setShapeAnchor(null); + }; + + const buildCanvasStateForAI = () => { + const canvasBounds = getVisibleCanvasBounds(); + const width = canvasBounds?.width || 1000; + const height = canvasBounds?.height || 1000; + + const allDrawings = [ + ...(userData?.drawings || []), + ...(pendingDrawings || []), + ].filter((d) => d.drawingType !== "filter"); + + const objects = allDrawings.map((d) => ({ + id: d.drawingId, + color: d.color, + lineWidth: d.lineWidth, + pathData: d.pathData, + brushType: d.brushType, + brushStyle: d.brushStyle, + drawingType: d.drawingType, + })); + + return { width, height, objects }; + }; + + const handleBeautifyCanvas = async () => { + if (!userData) return; + if (aiAssistLoading) return; + + if (!editingEnabled) { + showLocalSnack("Editing is disabled in view-only mode."); + return; + } + + try { + const canvasState = buildCanvasStateForAI(); + const result = await beautifySketch(canvasState); + + if (!result || !Array.isArray(result.objects) || result.objects.length === 0) { + showLocalSnack("Beautify failed. Please try again."); + return; + } + + const beautifiedObjects = result.objects; + + // Clears the canvas state + await clearCanvas(); + try { + const resp = await clearBackendCanvas({ + roomId: currentRoomId, + auth, + }); + + if (resp && resp.clearedAt && currentRoomId) { + roomClearedAtRef.current[currentRoomId] = resp.clearedAt; + } + } catch (e) { + console.error("Failed to clear backend:", e); + } + + try { + await checkUndoRedoAvailability( + auth, + setUndoAvailable, + setRedoAvailable, + currentRoomId + ); + } catch (e) { } + setUserList([]); + try { + setSelectedUser(""); + } catch (e) { + /* ignore if setter missing */ + } + + await addAIGeneratedObjects(beautifiedObjects); + + showLocalSnack("Sketch beautified"); + } catch (err) { + console.log("[Beautify] Error:", err); + showLocalSnack("Beautify failed. Please try again."); + } + }; + + + const [showToolbar, setShowToolbar] = useState(true); + const [hoverToolbar, setHoverToolbar] = useState(false); + + return ( +
+ {/* Top header: room name + optional history range + exit button */} + + + {currentRoomName || "Master (not in a room)"} + + + {historyMode && historyRange && ( + + {new Date(historyRange.start).toLocaleString()} —{" "} + {new Date(historyRange.end).toLocaleString()} + + )} + + {currentRoomId && ( + + )} + + + {selectionRecognition && selectionRect && ( + (() => { + const s = selectionRect.start; + const e = selectionRect.end; + if (!s || !e) return null; + const left = Math.min(s.x, e.x); + const top = Math.min(s.y, e.y) - 36; // show badge above selection + return ( + + + {selectionRecognition.label} {selectionRecognition.confidence ? `(${Math.round(selectionRecognition.confidence*100)}%)` : ''} + + + ); + })() + )} + + {/* Archived overlay banner - visible when viewOnly (archived or explicit viewer) */} + {viewOnly && ( + + + + Archived — View Only + + {/* Owner-only destructive delete button placed under the banner */} + {isOwner && ( + + + + )} + + + )} + + {/* Wallet disconnected banner - visible when secure room wallet is not connected */} + {roomType === "secure" && !walletConnected && ( + + + + ⚠ Wallet Not Connected — Canvas Locked + + + + )} + + {/* Confirm Destructive Delete dialog (owner-only) */} + { + setConfirmDestructiveOpen(false); + setDestructiveConfirmText(""); + }} + > + Permanently delete room + + + This will permanently delete this room and all its data for every + user. This action is irreversible. + + + To confirm, type DELETE below. + + setDestructiveConfirmText(e.target.value)} + placeholder="Type DELETE to confirm" + sx={{ mt: 1 }} + /> + + + + + + + + + + {/* Stamp preview overlay */} + {stampPreview && ( + + {stampPreview.stamp.emoji ? ( + + {stampPreview.stamp.emoji} + + ) : stampPreview.stamp.image ? ( + Stamp preview + ) : null} + + )} + + setHoverToolbar(true)} + onMouseLeave={() => setHoverToolbar(false)} + > + setShowToolbar((v) => !v)} + sx={{ + position: "absolute", + right: showToolbar ? 0 : -20, + top: "50%", + transform: "translateY(-50%)", + + width: 20, + height: 60, + display: "flex", + alignItems: "center", + justifyContent: "center", + + opacity: hoverToolbar ? 1 : 0, + transition: "opacity 0.2s", + bgcolor: "rgba(0,0,0,0.2)", + cursor: "pointer", + zIndex: 1001, + }} + > + + {showToolbar ? ( + + ) : ( + + )} + + + { + if (!editingEnabled) { + showLocalSnack("Cut is disabled in view-only mode."); + return; + } + showLocalSnack("Cutting selection... This may take a moment."); + try { + const result = await handleCutSelection(); + if (result && result.compositeCutAction) { + setUndoStack((prev) => [...prev, result.compositeCutAction]); + } + setIsRefreshing(true); + showLocalSnack("Syncing cut operation..."); + try { + await mergedRefreshCanvas(); + showLocalSnack("Cut completed successfully!"); + } catch (e) { + console.error("Error syncing cut with server:", e); + showLocalSnack("Cut completed, but sync failed. Try refreshing."); + } finally { + setIsRefreshing(false); + } + } catch (e) { + console.error("Error during cut:", e); + showLocalSnack("Cut operation failed. Please try again."); + } + }} + cutImageData={cutImageData} + setClearDialogOpen={setClearDialogOpen} + /* Export/Import handlers */ + handleExportCanvas={handleExportCanvas} + handleImportCanvas={handleImportCanvas} + /* Advanced brush/stamp/filter props */ + currentBrushType={currentBrushType} + onBrushSelect={handleBrushSelect} + onBrushParamsChange={handleBrushParamsChange} + selectedStamp={selectedStamp} + onStampSelect={handleStampSelect} + onStampChange={handleStampChange} + backendStamps={backendStamps} + onFilterApply={applyFilter} + onFilterPreview={previewFilter} + onFilterUndo={undoFilter} + onClearAllFilters={clearAllFilters} + canUndoFilter={ + !!originalCanvasDataRef.current || + undoStack.some((drawing) => drawing.drawingType === "filter") + } + canClearFilters={hasFilters} + appliedFilters={ + (() => { + const filters = userData.drawings.filter((drawing) => drawing.drawingType === "filter"); + console.log(`[Canvas render] Passing ${filters.length} applied filters to Toolbar`, filters); + return filters; + })() + } + /* History Recall props (required so the toolbar can open/change/exit history mode) */ + openHistoryDialog={openHistoryDialog} + exitHistoryMode={exitHistoryMode} + historyMode={historyMode} + controlsDisabled={!editingEnabled} + onOpenSettings={onOpenSettings} + + // handle showing the AI Assitant Panel + onToggleAI={() => setAiOpen(!aiOpen)} + /> + + setAiOpen(false)} + isBusy={aiAssistLoading} + error={aiError} + showPromptInput={(showPrompt, obj) => { + setShowPromptInput(showPrompt); + setPromptInputPlaceholder(obj.placeholder); + setAiGenerateService(obj.type); + }} + onShapeCompletionToggle={handleShapeCompletionToggle} + onBeautify={() => handleBeautifyCanvas()} + onRecognizeToggle={handleRecognizeToggle} + /> + + + {/* AI Assistant Prompt Input */} + + + {isRefreshing && ( +
+
+
+ )} + + {/* History Recall Dialog */} + setHistoryDialogOpen(false)} + aria-labelledby="history-recall-dialog" + > + + History Recall - Select Date/Time Range + + + + Choose a start and end date/time to recall drawings from + ResilientDB. Only drawings within the selected range will be loaded. + + + setHistoryStartInput(e.target.value)} + InputLabelProps={{ shrink: true }} + /> + setHistoryEndInput(e.target.value)} + InputLabelProps={{ shrink: true }} + /> + + + + + + + + + + + + + {historyMode + ? "History Mode Enabled — Canvas Editing Disabled" + : selectedUser && selectedUser !== "" + ? "Viewing Past Drawing History of Selected User — Canvas Editing Disabled" + : ""} + + + + + {/* Loading overlay: fades in/out while drawings load */} + + + + Loading Drawings... + + + + setClearDialogOpen(false)}> + Clear Canvas + + + Are you sure you want to clear the canvas for everyone? + + + + + + + + + {/* Command Palette - Quick command search and execution */} + setCommandPaletteOpen(false)} + commands={commandRegistry.getAll()} + onExecute={(command) => { + try { + command.action(); + } catch (error) { + console.error('[Canvas] Error executing command:', error); + showLocalSnack('Error executing command'); + } + }} + /> + + {/* Keyboard Shortcuts Help Dialog */} + setShortcutsHelpOpen(false)} + shortcuts={shortcutManagerRef.current?.getAllShortcuts() || []} + /> + + + + {/* Rendering AI suggestions */} + +
+ ); +} + +export default Canvas; diff --git a/frontend/src/components/Chat/AIAssistantChat.jsx b/frontend/src/components/Chat/AIAssistantChat.jsx new file mode 100644 index 00000000..c649f55a --- /dev/null +++ b/frontend/src/components/Chat/AIAssistantChat.jsx @@ -0,0 +1,58 @@ +import React, { useEffect, useRef } from 'react'; +import { Box, Typography, Paper } from '@mui/material'; +import SmartToyIcon from '@mui/icons-material/SmartToy'; +import ChatBubble from './ChatBubble'; +import ChatInput from './ChatInput'; +import '../../styles/Chat.css'; + +/** + * AIAssistantChat - Main chat window component for AI assistant + * + * @param {string} roomId - The room ID where the chat is happening + * @param {Array} messages - Chat messages array (controlled from parent) + * @param {boolean} isLoading - Loading state (controlled from parent) + * @param {Function} onSendMessage - Callback to send message (controlled from parent) + * @param {Object} canvasContext - Current canvas context (object count, active users, etc.) + */ +const AIAssistantChat = ({ roomId, messages, isLoading, onSendMessage, canvasContext }) => { + const messagesEndRef = useRef(null); + + // Auto-scroll to bottom when new messages arrive + useEffect(() => { + messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); + }, [messages]); + + const handleSend = (text) => { + onSendMessage(text, roomId, canvasContext); + }; + + return ( + + + + + AI Assistant + + + + + {messages.length === 0 ? ( + + + Start a conversation with the AI assistant! + + + ) : ( + messages.map((message, index) => ( + + )) + )} +
+ + + + + ); +}; + +export default AIAssistantChat; diff --git a/frontend/src/components/Chat/ChatBubble.jsx b/frontend/src/components/Chat/ChatBubble.jsx new file mode 100644 index 00000000..fe07d64f --- /dev/null +++ b/frontend/src/components/Chat/ChatBubble.jsx @@ -0,0 +1,73 @@ +import React from 'react'; +import { Box, Paper, Typography, Avatar } from '@mui/material'; +import PersonIcon from '@mui/icons-material/Person'; +import SmartToyIcon from '@mui/icons-material/SmartToy'; +import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; + +/** + * ChatBubble - Individual message bubble component + * + * @param {Object} message - Message object containing sender and text + * @param {string} message.sender - Either 'user' or 'bot' + * @param {string} message.text - The message text content + * @param {boolean} [message.isError] - Whether this is an error message + */ +const ChatBubble = ({ message }) => { + const { sender, text, isError } = message; + const isUser = sender === 'user'; + + return ( + + {!isUser && ( + + {isError ? : } + + )} + + + + {text} + + + + {isUser && ( + + + + )} + + ); +}; + +export default ChatBubble; diff --git a/frontend/src/components/Chat/ChatInput.jsx b/frontend/src/components/Chat/ChatInput.jsx new file mode 100644 index 00000000..e7d0611c --- /dev/null +++ b/frontend/src/components/Chat/ChatInput.jsx @@ -0,0 +1,65 @@ +import React, { useState } from 'react'; +import { Box, TextField, IconButton, CircularProgress } from '@mui/material'; +import SendIcon from '@mui/icons-material/Send'; + +/** + * ChatInput - Input component for sending messages to the chatbot + * + * @param {Function} onSend - Callback function to handle message submission + * @param {boolean} isLoading - Whether a message is currently being processed + */ +const ChatInput = ({ onSend, isLoading }) => { + const [inputText, setInputText] = useState(''); + + const handleSubmit = (e) => { + e.preventDefault(); + + if (inputText.trim()) { + onSend(inputText); + setInputText(''); + } + }; + + return ( + + setInputText(e.target.value)} + placeholder="Type your message..." + disabled={isLoading} + variant="outlined" + size="small" + sx={{ + '& .MuiOutlinedInput-root': { + borderRadius: '20px', + } + }} + /> + + {isLoading ? ( + + ) : ( + + )} + + + ); +}; + +export default ChatInput; diff --git a/frontend/src/components/Layout.jsx b/frontend/src/components/Layout.jsx index 5b767d22..3dc8fabf 100644 --- a/frontend/src/components/Layout.jsx +++ b/frontend/src/components/Layout.jsx @@ -288,10 +288,10 @@ export default function Layout() { message = error; } } catch (e) { - console.warn('Error parsing rejection:', e); - } + console.warn('Error parsing rejection:', e); + } - setGlobalSnack({ open: true, message, duration: 5000 }); + setGlobalSnack({ open: true, message, duration: 5000 }); }; window.addEventListener('unhandledrejection', handleUnhandledRejection); diff --git a/frontend/src/components/Search/AISearchPanel.jsx b/frontend/src/components/Search/AISearchPanel.jsx new file mode 100644 index 00000000..ed0a3493 --- /dev/null +++ b/frontend/src/components/Search/AISearchPanel.jsx @@ -0,0 +1,98 @@ +import React, { useState } from 'react'; +import { Box, TextField, Button, Stack, Paper, Typography, List, ListItem, Divider, CircularProgress, Alert } from '@mui/material'; +import apiClient from '../../api/apiClient'; +import RouterLinkWrapper from '../RouterLinkWrapper'; +import VisualSearchUpload from './VisualSearchUpload'; + +export default function AISearchPanel({ auth }) { + const [imageB64, setImageB64] = useState(null); + const [query, setQuery] = useState(''); + const [uploadedFilename, setUploadedFilename] = useState(null); + const [loading, setLoading] = useState(false); + const [results, setResults] = useState([]); + const [error, setError] = useState(null); + + const doSearch = async () => { + setLoading(true); + setError(null); + try { + if (!imageB64 && (!query || !query.trim())) { + setError('Please provide a description or upload an image to search.'); + setResults([]); + return; + } + const payload = {}; + if (query && query.trim()) payload.q = query.trim(); + if (imageB64) payload.image_b64 = imageB64; + const res = await apiClient.post('/api/v1/search/ai', payload); + if (!res) { + setResults([]); + setError('No response from server'); + } else if (res.status && res.status !== 'ok') { + setResults([]); + setError(res.message || 'Search failed'); + } else { + setResults((res && res.results) || []); + } + } catch (e) { + console.error('Search failed', e); + const msg = (e && e.message) || 'Network or server error'; + setError(msg); + setResults([]); + } finally { + setLoading(false); + } + }; + + return ( + + + AI Search + + setQuery(e.target.value)} + fullWidth + multiline + minRows={2} + /> + + setImageB64(b64)} onFileName={(fn) => setUploadedFilename(fn)} /> + + + + + {uploadedFilename && ( + Uploaded: {uploadedFilename} + )} + {loading && } + {error && {error}} + + + + + Results + + {results.length === 0 && No results} + {results.map(r => ( + + + + {r.name} + {r.ownerName || ''} + + + score: {typeof r.score === 'number' ? r.score.toFixed(2) : '-'} + + + {r.snippet && {r.snippet}} + + ))} + + + + + ); +} diff --git a/frontend/src/components/Search/VisualSearchUpload.jsx b/frontend/src/components/Search/VisualSearchUpload.jsx new file mode 100644 index 00000000..d9878eb1 --- /dev/null +++ b/frontend/src/components/Search/VisualSearchUpload.jsx @@ -0,0 +1,45 @@ +import React from 'react'; +import { Button, Typography } from '@mui/material'; + +/** + * VisualSearchUpload + * Props: + * - onImageBase64(base64String) + * - onFileName(filename) + * - accept (string) optional file accept string + */ +export default function VisualSearchUpload({ onImageBase64, onFileName, accept = 'image/*' }) { + const fileInputRef = React.useRef(null); + const [filename, setFilename] = React.useState(null); + + const handleFile = (file) => { + if (!file) return; + setFilename(file.name || null); + if (typeof onFileName === 'function') onFileName(file.name || null); + const reader = new FileReader(); + reader.onload = (e) => { + const dataUrl = e.target.result || ''; + const b64 = dataUrl.split(',')[1] || ''; + if (typeof onImageBase64 === 'function') onImageBase64(b64); + }; + reader.readAsDataURL(file); + }; + + return ( + <> + handleFile(e.target.files && e.target.files[0])} + /> + + {filename && ( + {filename} + )} + + ); +} diff --git a/frontend/src/components/Toolbar.js b/frontend/src/components/Toolbar.js index 59bb103c..cda9c178 100644 --- a/frontend/src/components/Toolbar.js +++ b/frontend/src/components/Toolbar.js @@ -16,6 +16,7 @@ import UploadIcon from "@mui/icons-material/Upload"; import BrushIcon from "@mui/icons-material/Brush"; import PaletteIcon from "@mui/icons-material/Palette"; import StarIcon from "@mui/icons-material/Star"; +import PsychologyIcon from "@mui/icons-material/Psychology"; import DrawModeMenu from "../lib/drawModeMenu"; import ShapeMenu from "../lib/shapeMenu"; import BrushPanel from "./BrushEditor/BrushPanel"; @@ -76,7 +77,8 @@ const Toolbar = ({ onClearAllFilters, canUndoFilter, canClearFilters, - appliedFilters + appliedFilters, + onToggleAI }) => { const [tool, setTool] = useState(null); const [anchorEl, setAnchorEl] = useState(null); @@ -396,6 +398,18 @@ const Toolbar = ({ )} + + + + + + + +
); }; diff --git a/frontend/src/hooks/useAIAssistant.js b/frontend/src/hooks/useAIAssistant.js new file mode 100644 index 00000000..c74d7dc5 --- /dev/null +++ b/frontend/src/hooks/useAIAssistant.js @@ -0,0 +1,63 @@ +import { useState } from "react"; + +export function useAIAssistant() { + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [result, setResult] = useState(null); + + const callAIAssistant = async (endpoint, body) => { + setLoading(true); + setError(null); + + try { + // During local development the React dev server serves the UI on a + // different port (usually 3000) which will return 404 for backend + // API routes. Use an explicit backend base URL when running locally. + const isLocalhost = typeof window !== 'undefined' && ( + window.location.hostname === 'localhost' || window.location.hostname === '127.0.0.1' + ); + const BACKEND_BASE = process.env.REACT_APP_API_URL || (isLocalhost ? 'http://localhost:10010' : ''); + + const res = await fetch(`${BACKEND_BASE}/api/ai_assistant/${endpoint}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + + const data = await res.json().catch(() => null); + if (!res.ok) { + const err = data || new Error(`Request failed: ${res.status}`); + throw err; + } + + setResult(data); + return data; + } catch (err) { + setError(err?.message || err); + console.error("AI assistant error:", err); + return { error: err }; + } finally { + setLoading(false); + } + }; + + // Wrapper methods for each route + const textToDrawing = (prompt, canvasState) => callAIAssistant("drawing", { prompt, canvasState }); + const shapeCompletion = (canvasState) => callAIAssistant("complete", { canvasState }); + const generateImage = (prompt, width = 512, height = 512, style = 'default') => callAIAssistant("image", { prompt, width, height, style }); + const beautifySketch = (canvasState) => callAIAssistant("beautify", { canvasState }); + const styleTransfer = (canvasState, stylePrompt) => callAIAssistant("style", { canvasState, stylePrompt }); + const recognizeObject = (canvasObjects, box, bounds) => callAIAssistant("recognize", { canvasObjects, box, bounds }); + + return { + aiAssistLoading: loading, + aiAssistError: error, + aiAssistResult: result, + textToDrawing, + shapeCompletion, + generateImage, + beautifySketch, + styleTransfer, + recognizeObject, + }; +} diff --git a/frontend/src/hooks/useChatbot.js b/frontend/src/hooks/useChatbot.js new file mode 100644 index 00000000..590d07c0 --- /dev/null +++ b/frontend/src/hooks/useChatbot.js @@ -0,0 +1,113 @@ +import { useState, useCallback, useEffect } from 'react'; +import { postChatMessage, getChatHistory } from '../api/chatbotApi'; + +/** + * Custom hook for managing chatbot interactions + * + * Manages conversation state and handles sending/receiving messages + * with the AI chatbot service. Loads chat history from backend on mount. + * + * @param {string} roomId - The room ID to load history for + * @returns {Object} Chatbot state and functions + * @returns {Array} messages - Array of chat messages with sender and text + * @returns {boolean} isLoading - Whether a message is currently being processed + * @returns {Function} sendMessage - Function to send a message to the bot + * @returns {Function} resetMessages - Function to clear all messages + */ +export const useChatbot = (roomId) => { + const [messages, setMessages] = useState([]); + const [isLoading, setIsLoading] = useState(false); + const [historyLoaded, setHistoryLoaded] = useState(false); + + /** + * Load chat history from backend when roomId changes + */ + useEffect(() => { + if (!roomId || historyLoaded) return; + + const loadHistory = async () => { + try { + console.log('[Chatbot] Loading chat history for room:', roomId); + const response = await getChatHistory(roomId, 50); + const history = response.history || []; + + // Convert backend format to frontend format + const formattedMessages = history.map(msg => ({ + sender: msg.sender, + text: msg.message, + timestamp: msg.timestamp + })); + + setMessages(formattedMessages); + setHistoryLoaded(true); + console.log('[Chatbot] Loaded', formattedMessages.length, 'messages from history'); + } catch (error) { + console.error('[Chatbot] Error loading chat history:', error); + // Don't block the UI if history fails to load + setHistoryLoaded(true); + } + }; + + loadHistory(); + }, [roomId, historyLoaded]); + + /** + * Reset/clear all chat messages + */ + const resetMessages = useCallback(() => { + setMessages([]); + setHistoryLoaded(false); + }, []); + + /** + * Send a message to the chatbot + * + * @param {string} userMessage - The user's message text + * @param {string} roomId - The room ID where the conversation is happening + * @param {Object} canvasContext - Current canvas state for context-aware responses + */ + const sendMessage = useCallback(async (userMessage, roomId, canvasContext = null) => { + setIsLoading(true); + + // Add user message to chat immediately + setMessages(prev => [...prev, { sender: 'user', text: userMessage }]); + + try { + // Send message to backend API with canvas context + console.log('[Chatbot] Sending message:', { roomId, userMessage, canvasContext }); + const response = await postChatMessage(roomId, userMessage, messages, canvasContext); + console.log('[Chatbot] Received response:', response); + const botReply = response.reply; + + // Add bot's reply to chat + setMessages(prev => [...prev, { sender: 'bot', text: botReply }]); + } catch (error) { + console.error('[Chatbot] API error:', error); + console.error('[Chatbot] Error details:', { + message: error.message, + status: error.status, + data: error.data, + stack: error.stack + }); + + // Add error message to chat for user visibility + setMessages(prev => [ + ...prev, + { + sender: 'bot', + text: 'Sorry, I encountered an error. Please try again.', + isError: true + } + ]); + } finally { + setIsLoading(false); + } + }, [messages]); + + return { + messages, + isLoading, + sendMessage, + resetMessages + }; +}; diff --git a/frontend/src/pages/Dashboard.jsx b/frontend/src/pages/Dashboard.jsx index 3acc7139..a7403483 100644 --- a/frontend/src/pages/Dashboard.jsx +++ b/frontend/src/pages/Dashboard.jsx @@ -11,6 +11,7 @@ import Autocomplete from '@mui/material/Autocomplete'; import TemplateGallery from '../components/TemplateGallery'; import TemplateLoader from '../services/templateLoader'; import { listRooms, createRoom, shareRoom, listInvites, acceptInvite, declineInvite, updateRoom, suggestUsers, suggestRooms, getRoomMembers } from '../api/rooms'; +import AISearchPanel from '../components/Search/AISearchPanel'; import { getUsername } from '../utils/getUsername'; import { useNavigate, Link } from 'react-router-dom'; import RouterLinkWrapper from '../components/RouterLinkWrapper'; @@ -612,6 +613,7 @@ export default function Dashboard({ auth }) { {/* Pending invites */} + Pending Invites diff --git a/frontend/src/pages/Room.jsx b/frontend/src/pages/Room.jsx index 60e784be..2c1c7132 100644 --- a/frontend/src/pages/Room.jsx +++ b/frontend/src/pages/Room.jsx @@ -9,12 +9,15 @@ import { getRoomDetails, getRoomStrokes } from '../api/rooms'; import { getUsername } from '../utils/getUsername'; import Canvas from '../components/Canvas'; import WalletConnector from '../components/WalletConnector'; +import AIAssistantChat from '../components/Chat/AIAssistantChat'; +import { useChatbot } from '../hooks/useChatbot'; import { handleAuthError } from '../utils/authUtils'; import { formatErrorMessage } from '../utils/errorHandling'; import { getSocket, setSocketToken } from '../services/socket'; import ChevronLeftIcon from '@mui/icons-material/ChevronLeft'; import ChevronRightIcon from '@mui/icons-material/ChevronRight'; import SettingsIcon from '@mui/icons-material/Settings'; +import SmartToyIcon from '@mui/icons-material/SmartToy'; import theme from '../config/theme'; export default function Room({ auth }) { @@ -30,6 +33,20 @@ export default function Room({ auth }) { const [expandedGroups, setExpandedGroups] = useState([]); const [showUserList, setShowUserList] = useState(true); const [hovering, setHovering] = useState(false); + const [showChatbot, setShowChatbot] = useState(false); + + // Chatbot state - persists when chatbot is minimized, resets when leaving room + // History is loaded from backend on mount + const { messages, isLoading, sendMessage, resetMessages } = useChatbot(roomId); + + // Canvas context for AI chatbot + const [canvasContext, setCanvasContext] = useState({ + room_id: roomId, + object_count: 0, + active_users: [], + has_filters: false, + template_id: null + }); const [helpOpen, setHelpOpen] = useState(false); const [blogOpen, setBlogOpen] = useState(false); @@ -181,8 +198,10 @@ export default function Room({ auth }) { sock.off("room_deleted", onRoomDeleted); sock.emit("leave_room", { roomId }); } catch (_) { } + // Reset chatbot messages when leaving the room + resetMessages(); }; - }, [roomId, auth?.token, navigate, load]); + }, [roomId, auth?.token, navigate, load, resetMessages]); if (loading) return ; // If the room is archived, only the owner should be able to edit; others view-only @@ -251,6 +270,7 @@ export default function Room({ auth }) { walletConnected={walletConnected} templateId={info?.templateId} onOpenSettings={((info && ((info.myRole || 'editor') !== 'viewer')) ? (() => navigate(`/rooms/${roomId}/settings`)) : null)} + onCanvasContextChange={setCanvasContext} /> @@ -441,6 +461,57 @@ export default function Room({ auth }) { )} + + {/* Chatbot Toggle Button - Bottom Right */} + + setShowChatbot(!showChatbot)} + sx={{ + width: 56, + height: 56, + bgcolor: 'primary.main', + color: 'white', + '&:hover': { + bgcolor: 'primary.dark', + }, + boxShadow: 3, + }} + > + + + + + {/* AI Assistant Chat Window - Toggleable */} + {showChatbot && ( + + + + )} {/* Legacy per-room footer removed; Layout provides the global footer now. */} diff --git a/frontend/src/services/canvasBackendJWT.js b/frontend/src/services/canvasBackendJWT.js index 5a8e50ea..c80a60b0 100644 --- a/frontend/src/services/canvasBackendJWT.js +++ b/frontend/src/services/canvasBackendJWT.js @@ -798,7 +798,7 @@ export const redoAction = async ({ drawAllDrawings(); // No need for full refresh after paste redo - local state is already correct - shouldRefreshFromBackend = false; + shouldRefreshFromBackend = false; } else { userData.drawings.push(lastUndone); diff --git a/frontend/src/styles/Chat.css b/frontend/src/styles/Chat.css new file mode 100644 index 00000000..8e3bb77b --- /dev/null +++ b/frontend/src/styles/Chat.css @@ -0,0 +1,63 @@ +/* Chat Component Styles - Complementing Material-UI */ + +/* Chat Container */ +.ai-assistant-chat { + display: flex; + flex-direction: column; + height: 100%; + overflow: hidden; +} + +/* Chat Header */ +.chat-header { + padding: 16px; + background-color: #f5f5f5; + border-bottom: 1px solid #e0e0e0; +} + +/* Messages Container */ +.chat-messages { + flex: 1; + overflow-y: auto; + padding: 16px; + display: flex; + flex-direction: column; +} + +/* Empty State */ +.chat-empty-state { + display: flex; + align-items: center; + justify-content: center; + height: 100%; + text-align: center; +} + +/* Chat Input Form */ +.chat-input-form { + display: flex; + padding: 16px; + background-color: #f9f9f9; + border-top: 1px solid #e0e0e0; + align-items: center; + gap: 8px; +} + +/* Scrollbar Styling */ +.chat-messages::-webkit-scrollbar { + width: 6px; +} + +.chat-messages::-webkit-scrollbar-track { + background: #f1f1f1; + border-radius: 3px; +} + +.chat-messages::-webkit-scrollbar-thumb { + background: #bbb; + border-radius: 3px; +} + +.chat-messages::-webkit-scrollbar-thumb:hover { + background: #888; +} diff --git a/frontend/src/styles/ai-assistant.css b/frontend/src/styles/ai-assistant.css new file mode 100644 index 00000000..261fa49d --- /dev/null +++ b/frontend/src/styles/ai-assistant.css @@ -0,0 +1,45 @@ +.ai-assistant-panel-container { + display: flex; + min-width: 250px; + padding: 0 10px; + margin-top: 10px; + align-items: center; + justify-content: space-between; + background-color: #25D8C5; + border-top-right-radius: 15px; + border-bottom-right-radius: 15px; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.2); +} + +.ai-assistant-panel-container-open { + position: absolute; + left: 0; + top: -75px; + transition-duration: .4s; +} + +.ai-assistant-panel-container-close { + position: absolute; + left: -300px; + top: -75px; + transition-duration: .4s; +} + +.ai-asisstant-panel-item { + padding: 0 5px; + margin: 5px; + border-radius: 8px; +} + +.ai-asisstant-panel-item-active { + padding: 0 5px; + margin: 5px; + border-radius: 8px; + background-color: rgba(0, 0, 0, .1); +} + +.ai-asisstant-panel-item:hover { + background-color: rgba(0, 0, 0, .05); + transition-duration: .3s; +} +